Meg's Mirrors

zig

1	pt: Zcu.PerThread,
2	target: *const std.Target,
3	air: Air,
4	nav_index: InternPool.Nav.Index,
5
6	// Blocks
7	def_order: std.array_hash_map.Auto(Air.Inst.Index, void),
8	blocks: std.array_hash_map.Auto(Air.Inst.Index, Block),
9	loops: std.array_hash_map.Auto(Air.Inst.Index, Loop),
10	active_loops: std.ArrayList(Loop.Index),
11	loop_live: struct {
12	set: std.array_hash_map.Auto(struct { Loop.Index, Air.Inst.Index }, void),
13	list: std.ArrayList(Air.Inst.Index),
14	},
15	dom_start: u32,
16	dom_len: u32,
17	dom: std.ArrayList(DomInt),
18
19	// Wip Mir
20	saved_registers: std.enums.EnumSet(Register.Alias),
21	instructions: std.ArrayList(codegen.aarch64.encoding.Instruction),
22	literals: std.ArrayList(u32),
23	nav_relocs: std.ArrayList(codegen.aarch64.Mir.Reloc.Nav),
24	uav_relocs: std.ArrayList(codegen.aarch64.Mir.Reloc.Uav),
25	lazy_relocs: std.ArrayList(codegen.aarch64.Mir.Reloc.Lazy),
26	global_relocs: std.ArrayList(codegen.aarch64.Mir.Reloc.Global),
27	literal_relocs: std.ArrayList(codegen.aarch64.Mir.Reloc.Literal),
28
29	// Stack Frame
30	returns: bool,
31	va_list: union(enum) {
32	other: Value.Indirect,
33	sysv: struct {
34	__stack: Value.Indirect,
35	__gr_top: Value.Indirect,
36	__vr_top: Value.Indirect,
37	__gr_offs: i32,
38	__vr_offs: i32,
39	},
40	},
41	stack_size: u24,
42	stack_align: InternPool.Alignment,
43
44	// Value Tracking
45	live_registers: LiveRegisters,
46	live_values: std.AutoHashMapUnmanaged(Air.Inst.Index, Value.Index),
47	values: std.ArrayList(Value),
48
49	pub const LiveRegisters = std.enums.EnumArray(Register.Alias, Value.Index);
50
51	pub const Block = struct {
52	live_registers: LiveRegisters,
53	target_label: u32,
54
55	pub const main: Air.Inst.Index = @fromBackingInt(@intCast(
56	std.math.maxInt(@typeInfo(Air.Inst.Index).@"enum".tag_type),
57	));
58
59	fn branch(target_block: *const Block, isel: *Select) !void {
60	if (isel.instructions.items.len > target_block.target_label) {
61	try isel.emit(.b(@intCast((isel.instructions.items.len + 1 - target_block.target_label) << 2)));
62	}
63	try isel.merge(&target_block.live_registers, .{});
64	}
65	};
66
67	pub const Loop = struct {
68	def_order: u32,
69	dom: u32,
70	depth: u32,
71	live: u32,
72	live_registers: LiveRegisters,
73	repeat_list: u32,
74
75	pub const invalid: Air.Inst.Index = @fromBackingInt(@intCast(
76	std.math.maxInt(@typeInfo(Air.Inst.Index).@"enum".tag_type),
77	));
78
79	pub const Index = enum(u32) {
80	_,
81
82	fn inst(li: Loop.Index, isel: *Select) Air.Inst.Index {
83	return isel.loops.keys()[@backingInt(li)];
84	}
85
86	fn get(li: Loop.Index, isel: *Select) *Loop {
87	return &isel.loops.values()[@backingInt(li)];
88	}
89	};
90
91	pub const empty_list: u32 = std.math.maxInt(u32);
92
93	fn branch(target_loop: *Loop, isel: *Select) !void {
94	try isel.instructions.ensureUnusedCapacity(isel.pt.zcu.gpa, 1);
95	const repeat_list_tail = target_loop.repeat_list;
96	target_loop.repeat_list = @intCast(isel.instructions.items.len);
97	isel.instructions.appendAssumeCapacity(@bitCast(repeat_list_tail));
98	try isel.merge(&target_loop.live_registers, .{});
99	}
100	};
101
102	pub fn deinit(isel: *Select) void {
103	const gpa = isel.pt.zcu.gpa;
104
105	isel.def_order.deinit(gpa);
106	isel.blocks.deinit(gpa);
107	isel.loops.deinit(gpa);
108	isel.active_loops.deinit(gpa);
109	isel.loop_live.set.deinit(gpa);
110	isel.loop_live.list.deinit(gpa);
111	isel.dom.deinit(gpa);
112
113	isel.instructions.deinit(gpa);
114	isel.literals.deinit(gpa);
115	isel.nav_relocs.deinit(gpa);
116	isel.uav_relocs.deinit(gpa);
117	isel.lazy_relocs.deinit(gpa);
118	isel.global_relocs.deinit(gpa);
119	isel.literal_relocs.deinit(gpa);
120
121	isel.live_values.deinit(gpa);
122	isel.values.deinit(gpa);
123
124	isel.* = undefined;
125	}
126
127	pub fn analyze(isel: *Select, air_body: []const Air.Inst.Index) !void {
128	const zcu = isel.pt.zcu;
129	const ip = &zcu.intern_pool;
130	const gpa = zcu.gpa;
131	const air_tags = isel.air.instructions.items(.tag);
132	const air_data = isel.air.instructions.items(.data);
133	var air_body_index: usize = 0;
134	var air_inst_index = air_body[air_body_index];
135	const initial_def_order_len = isel.def_order.count();
136	air_tag: switch (air_tags[@backingInt(air_inst_index)]) {
137	// No "scalarize" legalizations are enabled, so these instructions never appear.
138	.legalize_vec_elem_val => unreachable,
139	.legalize_vec_store_elem => unreachable,
140	// No soft float legalizations are enabled.
141	.legalize_compiler_rt_call => unreachable,
142
143	.arg,
144	.ret_addr,
145	.frame_addr,
146	.err_return_trace,
147	.save_err_return_trace_index,
148	.runtime_nav_ptr,
149	.c_va_start,
150	=> {
151	try isel.def_order.putNoClobber(gpa, air_inst_index, {});
152
153	air_body_index += 1;
154	air_inst_index = air_body[air_body_index];
155	continue :air_tag air_tags[@backingInt(air_inst_index)];
156	},
157	.add,
158	.add_safe,
159	.add_optimized,
160	.add_wrap,
161	.add_sat,
162	.sub,
163	.sub_safe,
164	.sub_optimized,
165	.sub_wrap,
166	.sub_sat,
167	.mul,
168	.mul_safe,
169	.mul_optimized,
170	.mul_wrap,
171	.mul_sat,
172	.div_float,
173	.div_float_optimized,
174	.div_trunc,
175	.div_trunc_optimized,
176	.div_floor,
177	.div_floor_optimized,
178	.div_ceil,
179	.div_ceil_optimized,
180	.div_exact,
181	.div_exact_optimized,
182	.rem,
183	.rem_optimized,
184	.mod,
185	.mod_optimized,
186	.max,
187	.min,
188	.bit_and,
189	.bit_or,
190	.shr,
191	.shr_exact,
192	.shl,
193	.shl_exact,
194	.shl_sat,
195	.xor,
196	.cmp_lt,
197	.cmp_lt_optimized,
198	.cmp_lte,
199	.cmp_lte_optimized,
200	.cmp_eq,
201	.cmp_eq_optimized,
202	.cmp_gte,
203	.cmp_gte_optimized,
204	.cmp_gt,
205	.cmp_gt_optimized,
206	.cmp_neq,
207	.cmp_neq_optimized,
208	.array_elem_val,
209	.slice_elem_val,
210	.ptr_elem_val,
211	=> {
212	const bin_op = air_data[@backingInt(air_inst_index)].bin_op;
213
214	try isel.analyzeUse(bin_op.lhs);
215	try isel.analyzeUse(bin_op.rhs);
216	try isel.def_order.putNoClobber(gpa, air_inst_index, {});
217
218	air_body_index += 1;
219	air_inst_index = air_body[air_body_index];
220	continue :air_tag air_tags[@backingInt(air_inst_index)];
221	},
222	.ptr_add,
223	.ptr_sub,
224	.add_with_overflow,
225	.sub_with_overflow,
226	.mul_with_overflow,
227	.shl_with_overflow,
228	.slice,
229	.slice_elem_ptr,
230	.ptr_elem_ptr,
231	=> {
232	const ty_pl = air_data[@backingInt(air_inst_index)].ty_pl;
233	const bin_op = isel.air.extraData(Air.Bin, ty_pl.payload).data;
234
235	try isel.analyzeUse(bin_op.lhs);
236	try isel.analyzeUse(bin_op.rhs);
237	try isel.def_order.putNoClobber(gpa, air_inst_index, {});
238
239	air_body_index += 1;
240	air_inst_index = air_body[air_body_index];
241	continue :air_tag air_tags[@backingInt(air_inst_index)];
242	},
243	.alloc => {
244	const ty = air_data[@backingInt(air_inst_index)].ty;
245
246	isel.stack_align = isel.stack_align.maxStrict(ty.ptrAlignment(zcu));
247	try isel.def_order.putNoClobber(gpa, air_inst_index, {});
248
249	air_body_index += 1;
250	air_inst_index = air_body[air_body_index];
251	continue :air_tag air_tags[@backingInt(air_inst_index)];
252	},
253	.inferred_alloc,
254	.inferred_alloc_comptime,
255	.wasm_memory_size,
256	.wasm_memory_grow,
257	.work_item_id,
258	.work_group_size,
259	.work_group_id,
260	.spirv_runtime_array_len,
261	.array_to_vector,
262	=> unreachable,
263	.ret_ptr => {
264	const ty = air_data[@backingInt(air_inst_index)].ty;
265
266	if (isel.live_values.get(Block.main)) |ret_vi| switch (ret_vi.parent(isel)) {
267	.unallocated, .stack_slot => isel.stack_align = isel.stack_align.maxStrict(ty.ptrAlignment(zcu)),
268	.value, .constant => unreachable,
269	.address => |address_vi| try isel.live_values.putNoClobber(gpa, air_inst_index, address_vi.ref(isel)),
270	};
271	try isel.def_order.putNoClobber(gpa, air_inst_index, {});
272
273	air_body_index += 1;
274	air_inst_index = air_body[air_body_index];
275	continue :air_tag air_tags[@backingInt(air_inst_index)];
276	},
277	.assembly => {
278	const ty_pl = air_data[@backingInt(air_inst_index)].ty_pl;
279	const unwrapped_asm = isel.air.unwrapAsm(air_inst_index);
280
281	for (unwrapped_asm.outputs) |operand| if (operand != .none) try isel.analyzeUse(operand);
282	if (ty_pl.ty.ip_index != .void_type) try isel.def_order.putNoClobber(gpa, air_inst_index, {});
283
284	air_body_index += 1;
285	air_inst_index = air_body[air_body_index];
286	continue :air_tag air_tags[@backingInt(air_inst_index)];
287	},
288	.not,
289	.clz,
290	.ctz,
291	.popcount,
292	.byte_swap,
293	.bit_reverse,
294	.abs,
295	.load,
296	.fptrunc,
297	.fpext,
298	.int_cast,
299	.int_cast_safe,
300	.trunc,
301	.optional_payload,
302	.optional_payload_ptr,
303	.optional_payload_ptr_set,
304	.wrap_optional,
305	.unwrap_errunion_payload,
306	.unwrap_errunion_err,
307	.unwrap_errunion_payload_ptr,
308	.unwrap_errunion_err_ptr,
309	.errunion_payload_ptr_set,
310	.wrap_errunion_payload,
311	.wrap_errunion_err,
312	.struct_field_ptr_index_0,
313	.struct_field_ptr_index_1,
314	.struct_field_ptr_index_2,
315	.struct_field_ptr_index_3,
316	.get_union_tag,
317	.ptr_slice_len_ptr,
318	.ptr_slice_ptr_ptr,
319	.array_to_slice,
320	.int_from_float,
321	.int_from_float_optimized,
322	.int_from_float_safe,
323	.int_from_float_optimized_safe,
324	.float_from_int,
325	.splat,
326	.error_set_has_value,
327	.addrspace_cast,
328	.c_va_arg,
329	.c_va_copy,
330	=> {
331	const ty_op = air_data[@backingInt(air_inst_index)].ty_op;
332
333	try isel.analyzeUse(ty_op.operand);
334	try isel.def_order.putNoClobber(gpa, air_inst_index, {});
335
336	air_body_index += 1;
337	air_inst_index = air_body[air_body_index];
338	continue :air_tag air_tags[@backingInt(air_inst_index)];
339	},
340	.bit_cast,
341	.ptr_cast,
342	.ptr_from_int,
343	.int_from_ptr,
344	.error_cast,
345	.error_from_int,
346	.int_from_error,
347	.union_from_enum,
348	=> {
349	const ty_op = air_data[@backingInt(air_inst_index)].ty_op;
350	maybe_noop: {
351	if (ty_op.ty.ip_index != isel.air.typeOf(ty_op.operand, ip).toIntern()) break :maybe_noop;
352	if (true) break :maybe_noop;
353	if (ty_op.operand.toIndex()) |src_air_inst_index| {
354	if (isel.hints.get(src_air_inst_index)) |hint_vpsi| {
355	try isel.hints.putNoClobber(gpa, air_inst_index, hint_vpsi);
356	}
357	}
358	}
359	try isel.analyzeUse(ty_op.operand);
360	try isel.def_order.putNoClobber(gpa, air_inst_index, {});
361
362	air_body_index += 1;
363	air_inst_index = air_body[air_body_index];
364	continue :air_tag air_tags[@backingInt(air_inst_index)];
365	},
366	.bit_cast_safe => unreachable, // legalized
367	inline .block, .dbg_inline_block => |air_tag| {
368	const air_body_block = switch (air_tag) {
369	else => comptime unreachable,
370	.block => isel.air.unwrapBlock(air_inst_index),
371	.dbg_inline_block => isel.air.unwrapDbgBlock(air_inst_index),
372	};
373
374	const result_ty = air_body_block.ty.toIntern();
375
376	if (result_ty == .noreturn_type) {
377	try isel.analyze(air_body_block.body);
378
379	air_body_index += 1;
380	break :air_tag;
381	}
382
383	assert(!(try isel.blocks.getOrPut(gpa, air_inst_index)).found_existing);
384	try isel.analyze(air_body_block.body);
385	const block_entry = isel.blocks.pop().?;
386	assert(block_entry.key == air_inst_index);
387
388	if (result_ty != .void_type) try isel.def_order.putNoClobber(gpa, air_inst_index, {});
389
390	air_body_index += 1;
391	air_inst_index = air_body[air_body_index];
392	continue :air_tag air_tags[@backingInt(air_inst_index)];
393	},
394	.loop => {
395	const air_body_block = isel.air.unwrapBlock(air_inst_index);
396
397	const initial_dom_start = isel.dom_start;
398	const initial_dom_len = isel.dom_len;
399	isel.dom_start = @intCast(isel.dom.items.len);
400	isel.dom_len = @intCast(isel.blocks.count());
401	try isel.active_loops.append(gpa, @fromBackingInt(@intCast(isel.loops.count())));
402	try isel.loops.putNoClobber(gpa, air_inst_index, .{
403	.def_order = @intCast(isel.def_order.count()),
404	.dom = isel.dom_start,
405	.depth = isel.dom_len,
406	.live = 0,
407	.live_registers = undefined,
408	.repeat_list = undefined,
409	});
410	try isel.dom.appendNTimes(gpa, 0, @divCeil(isel.dom_len, @bitSizeOf(DomInt)));
411	try isel.analyze(air_body_block.body);
412	for (
413	isel.dom.items[initial_dom_start..].ptr,
414	isel.dom.items[isel.dom_start..][0..@divCeil(initial_dom_len, @bitSizeOf(DomInt))],
415	) |*initial_dom, loop_dom| initial_dom.* |= loop_dom;
416	isel.dom_start = initial_dom_start;
417	isel.dom_len = initial_dom_len;
418	assert(isel.active_loops.pop().?.inst(isel) == air_inst_index);
419
420	air_body_index += 1;
421	},
422	.repeat, .trap, .unreach => air_body_index += 1,
423	.br => {
424	const br = air_data[@backingInt(air_inst_index)].br;
425	const block_index = isel.blocks.getIndex(br.block_inst).?;
426	if (block_index < isel.dom_len) isel.dom.items[isel.dom_start + block_index / @bitSizeOf(DomInt)] |= @as(DomInt, 1) << @truncate(block_index);
427	try isel.analyzeUse(br.operand);
428
429	air_body_index += 1;
430	},
431	.breakpoint, .dbg_stmt, .dbg_empty_stmt, .dbg_var_ptr, .dbg_var_val, .dbg_arg_inline, .c_va_end => {
432	air_body_index += 1;
433	air_inst_index = air_body[air_body_index];
434	continue :air_tag air_tags[@backingInt(air_inst_index)];
435	},
436	.call,
437	.call_always_tail,
438	.call_never_tail,
439	.call_never_inline,
440	=> {
441	const air_call = isel.air.unwrapCall(air_inst_index);
442	const args = air_call.args;
443	isel.saved_registers.insert(.lr);
444	const callee_ty = isel.air.typeOf(air_call.callee, ip);
445	const func_info = switch (ip.indexToKey(callee_ty.toIntern())) {
446	else => unreachable,
447	.func_type => |func_type| func_type,
448	.ptr_type => |ptr_type| ip.indexToKey(ptr_type.child).func_type,
449	};
450
451	try isel.analyzeUse(air_call.callee);
452	var param_it: CallAbiIterator = .init;
453	for (args, 0..) |arg, arg_index| {
454	const restore_values_len = isel.values.items.len;
455	defer isel.values.shrinkRetainingCapacity(restore_values_len);
456	const param_vi = param_vi: {
457	const param_ty = isel.air.typeOf(arg, ip);
458	if (arg_index >= func_info.param_types.len) {
459	assert(func_info.is_var_args);
460	switch (isel.va_list) {
461	.other => break :param_vi try param_it.nonSysvVarArg(isel, param_ty),
462	.sysv => {},
463	}
464	}
465	break :param_vi try param_it.param(isel, param_ty);
466	} orelse continue;
467	defer param_vi.deref(isel);
468	const passed_vi = switch (param_vi.parent(isel)) {
469	.unallocated, .stack_slot => param_vi,
470	.value, .constant => unreachable,
471	.address => |address_vi| address_vi,
472	};
473	switch (passed_vi.parent(isel)) {
474	.unallocated => {},
475	.stack_slot => |stack_slot| {
476	assert(stack_slot.base == .sp);
477	isel.stack_size = @max(
478	isel.stack_size,
479	stack_slot.offset + @as(u24, @intCast(passed_vi.size(isel))),
480	);
481	},
482	.value, .constant, .address => unreachable,
483	}
484
485	try isel.analyzeUse(arg);
486	}
487
488	var ret_it: CallAbiIterator = .init;
489	if (try ret_it.ret(isel, isel.air.typeOfIndex(air_inst_index, ip))) |ret_vi| {
490	tracking_log.debug("${d} <- %{d}", .{ @backingInt(ret_vi), @backingInt(air_inst_index) });
491	switch (ret_vi.parent(isel)) {
492	.unallocated, .stack_slot => {},
493	.value, .constant => unreachable,
494	.address => |address_vi| {
495	defer address_vi.deref(isel);
496	const ret_value = ret_vi.get(isel);
497	ret_value.flags.parent_tag = .unallocated;
498	ret_value.parent_payload = .{ .unallocated = {} };
499	},
500	}
501	try isel.live_values.putNoClobber(gpa, air_inst_index, ret_vi);
502
503	try isel.def_order.putNoClobber(gpa, air_inst_index, {});
504	}
505
506	air_body_index += 1;
507	air_inst_index = air_body[air_body_index];
508	continue :air_tag air_tags[@backingInt(air_inst_index)];
509	},
510	.sqrt,
511	.sin,
512	.cos,
513	.tan,
514	.exp,
515	.exp2,
516	.log,
517	.log2,
518	.log10,
519	.floor,
520	.ceil,
521	.round,
522	.trunc_float,
523	.neg,
524	.neg_optimized,
525	.is_null,
526	.is_non_null,
527	.is_null_ptr,
528	.is_non_null_ptr,
529	.is_err,
530	.is_non_err,
531	.is_err_ptr,
532	.is_non_err_ptr,
533	.is_named_enum_value,
534	.tag_name,
535	.error_name,
536	.cmp_lte_errors_len,
537	=> {
538	const un_op = air_data[@backingInt(air_inst_index)].un_op;
539
540	try isel.analyzeUse(un_op);
541	try isel.def_order.putNoClobber(gpa, air_inst_index, {});
542
543	air_body_index += 1;
544	air_inst_index = air_body[air_body_index];
545	continue :air_tag air_tags[@backingInt(air_inst_index)];
546	},
547	.cmp_vector, .cmp_vector_optimized => {
548	const ty_pl = air_data[@backingInt(air_inst_index)].ty_pl;
549	const extra = isel.air.extraData(Air.VectorCmp, ty_pl.payload).data;
550
551	try isel.analyzeUse(extra.lhs);
552	try isel.analyzeUse(extra.rhs);
553	try isel.def_order.putNoClobber(gpa, air_inst_index, {});
554
555	air_body_index += 1;
556	air_inst_index = air_body[air_body_index];
557	continue :air_tag air_tags[@backingInt(air_inst_index)];
558	},
559	.cond_br => {
560	const cond_br = isel.air.unwrapCondBr(air_inst_index);
561
562	try isel.analyzeUse(cond_br.condition);
563
564	try isel.analyze(cond_br.then_body);
565	try isel.analyze(cond_br.else_body);
566
567	air_body_index += 1;
568	},
569	.switch_br => {
570	const switch_br = isel.air.unwrapSwitch(air_inst_index);
571
572	try isel.analyzeUse(switch_br.operand);
573
574	var cases_it = switch_br.iterateCases();
575	while (cases_it.next()) |case| try isel.analyze(case.body);
576	if (switch_br.else_body_len > 0) try isel.analyze(cases_it.elseBody());
577
578	air_body_index += 1;
579	},
580	.loop_switch_br => {
581	const switch_br = isel.air.unwrapSwitch(air_inst_index);
582
583	const initial_dom_start = isel.dom_start;
584	const initial_dom_len = isel.dom_len;
585	isel.dom_start = @intCast(isel.dom.items.len);
586	isel.dom_len = @intCast(isel.blocks.count());
587	try isel.active_loops.append(gpa, @fromBackingInt(@intCast(isel.loops.count())));
588	try isel.loops.putNoClobber(gpa, air_inst_index, .{
589	.def_order = @intCast(isel.def_order.count()),
590	.dom = isel.dom_start,
591	.depth = isel.dom_len,
592	.live = 0,
593	.live_registers = undefined,
594	.repeat_list = undefined,
595	});
596	try isel.dom.appendNTimes(gpa, 0, @divCeil(isel.dom_len, @bitSizeOf(DomInt)));
597
598	var cases_it = switch_br.iterateCases();
599	while (cases_it.next()) |case| try isel.analyze(case.body);
600	if (switch_br.else_body_len > 0) try isel.analyze(cases_it.elseBody());
601
602	for (
603	isel.dom.items[initial_dom_start..].ptr,
604	isel.dom.items[isel.dom_start..][0..@divCeil(initial_dom_len, @bitSizeOf(DomInt))],
605	) |*initial_dom, loop_dom| initial_dom.* |= loop_dom;
606	isel.dom_start = initial_dom_start;
607	isel.dom_len = initial_dom_len;
608	assert(isel.active_loops.pop().?.inst(isel) == air_inst_index);
609
610	air_body_index += 1;
611	},
612	.switch_dispatch => {
613	const br = air_data[@backingInt(air_inst_index)].br;
614
615	try isel.analyzeUse(br.operand);
616
617	air_body_index += 1;
618	},
619	.@"try", .try_cold => {
620	const unwrapped_try = isel.air.unwrapTry(air_inst_index);
621
622	try isel.analyzeUse(unwrapped_try.error_union);
623	try isel.analyze(unwrapped_try.else_body);
624	try isel.def_order.putNoClobber(gpa, air_inst_index, {});
625
626	air_body_index += 1;
627	air_inst_index = air_body[air_body_index];
628	continue :air_tag air_tags[@backingInt(air_inst_index)];
629	},
630	.try_ptr, .try_ptr_cold => {
631	const unwrapped_try = isel.air.unwrapTryPtr(air_inst_index);
632
633	try isel.analyzeUse(unwrapped_try.error_union_ptr);
634	try isel.analyze(unwrapped_try.else_body);
635	try isel.def_order.putNoClobber(gpa, air_inst_index, {});
636
637	air_body_index += 1;
638	air_inst_index = air_body[air_body_index];
639	continue :air_tag air_tags[@backingInt(air_inst_index)];
640	},
641	.ret, .ret_safe, .ret_load => {
642	const un_op = air_data[@backingInt(air_inst_index)].un_op;
643	isel.returns = true;
644
645	const block_index = 0;
646	assert(isel.blocks.keys()[block_index] == Block.main);
647	if (isel.dom_len > 0) isel.dom.items[isel.dom_start] |= 1 << block_index;
648
649	try isel.analyzeUse(un_op);
650
651	air_body_index += 1;
652	},
653	.store,
654	.store_safe,
655	.set_union_tag,
656	.memset,
657	.memset_safe,
658	.memcpy,
659	.memmove,
660	.atomic_store_unordered,
661	.atomic_store_monotonic,
662	.atomic_store_release,
663	.atomic_store_seq_cst,
664	=> {
665	const bin_op = air_data[@backingInt(air_inst_index)].bin_op;
666
667	try isel.analyzeUse(bin_op.lhs);
668	try isel.analyzeUse(bin_op.rhs);
669
670	air_body_index += 1;
671	air_inst_index = air_body[air_body_index];
672	continue :air_tag air_tags[@backingInt(air_inst_index)];
673	},
674	.struct_field_ptr, .agg_field_val => {
675	const ty_pl = air_data[@backingInt(air_inst_index)].ty_pl;
676	const extra = isel.air.extraData(Air.StructField, ty_pl.payload).data;
677
678	try isel.analyzeUse(extra.struct_operand);
679	try isel.def_order.putNoClobber(gpa, air_inst_index, {});
680
681	air_body_index += 1;
682	air_inst_index = air_body[air_body_index];
683	continue :air_tag air_tags[@backingInt(air_inst_index)];
684	},
685	.slice_len => {
686	const ty_op = air_data[@backingInt(air_inst_index)].ty_op;
687
688	try isel.analyzeUse(ty_op.operand);
689	try isel.def_order.putNoClobber(gpa, air_inst_index, {});
690
691	const slice_vi = try isel.use(ty_op.operand);
692	var len_part_it = slice_vi.field(isel.air.typeOf(ty_op.operand, ip), 8, 8);
693	if (try len_part_it.only(isel)) |len_part_vi|
694	try isel.live_values.putNoClobber(gpa, air_inst_index, len_part_vi.ref(isel));
695
696	air_body_index += 1;
697	air_inst_index = air_body[air_body_index];
698	continue :air_tag air_tags[@backingInt(air_inst_index)];
699	},
700	.slice_ptr => {
701	const ty_op = air_data[@backingInt(air_inst_index)].ty_op;
702
703	try isel.analyzeUse(ty_op.operand);
704	try isel.def_order.putNoClobber(gpa, air_inst_index, {});
705
706	const slice_vi = try isel.use(ty_op.operand);
707	var ptr_part_it = slice_vi.field(isel.air.typeOf(ty_op.operand, ip), 0, 8);
708	if (try ptr_part_it.only(isel)) |ptr_part_vi|
709	try isel.live_values.putNoClobber(gpa, air_inst_index, ptr_part_vi.ref(isel));
710
711	air_body_index += 1;
712	air_inst_index = air_body[air_body_index];
713	continue :air_tag air_tags[@backingInt(air_inst_index)];
714	},
715	.reduce, .reduce_optimized => {
716	const reduce = air_data[@backingInt(air_inst_index)].reduce;
717
718	try isel.analyzeUse(reduce.operand);
719	try isel.def_order.putNoClobber(gpa, air_inst_index, {});
720
721	air_body_index += 1;
722	air_inst_index = air_body[air_body_index];
723	continue :air_tag air_tags[@backingInt(air_inst_index)];
724	},
725	.shuffle_one => {
726	const extra = isel.air.unwrapShuffleOne(zcu, air_inst_index);
727
728	try isel.analyzeUse(extra.operand);
729	try isel.def_order.putNoClobber(gpa, air_inst_index, {});
730
731	air_body_index += 1;
732	air_inst_index = air_body[air_body_index];
733	continue :air_tag air_tags[@backingInt(air_inst_index)];
734	},
735	.shuffle_two => {
736	const extra = isel.air.unwrapShuffleTwo(zcu, air_inst_index);
737
738	try isel.analyzeUse(extra.operand_a);
739	try isel.analyzeUse(extra.operand_b);
740	try isel.def_order.putNoClobber(gpa, air_inst_index, {});
741
742	air_body_index += 1;
743	air_inst_index = air_body[air_body_index];
744	continue :air_tag air_tags[@backingInt(air_inst_index)];
745	},
746	.select, .mul_add => {
747	const pl_op = air_data[@backingInt(air_inst_index)].pl_op;
748	const bin_op = isel.air.extraData(Air.Bin, pl_op.payload).data;
749
750	try isel.analyzeUse(pl_op.operand);
751	try isel.analyzeUse(bin_op.lhs);
752	try isel.analyzeUse(bin_op.rhs);
753	try isel.def_order.putNoClobber(gpa, air_inst_index, {});
754
755	air_body_index += 1;
756	air_inst_index = air_body[air_body_index];
757	continue :air_tag air_tags[@backingInt(air_inst_index)];
758	},
759	.cmpxchg_weak, .cmpxchg_strong => {
760	const ty_pl = air_data[@backingInt(air_inst_index)].ty_pl;
761	const extra = isel.air.extraData(Air.Cmpxchg, ty_pl.payload).data;
762
763	try isel.analyzeUse(extra.ptr);
764	try isel.analyzeUse(extra.expected_value);
765	try isel.analyzeUse(extra.new_value);
766	try isel.def_order.putNoClobber(gpa, air_inst_index, {});
767
768	air_body_index += 1;
769	air_inst_index = air_body[air_body_index];
770	continue :air_tag air_tags[@backingInt(air_inst_index)];
771	},
772	.atomic_load => {
773	const atomic_load = air_data[@backingInt(air_inst_index)].atomic_load;
774
775	try isel.analyzeUse(atomic_load.ptr);
776	try isel.def_order.putNoClobber(gpa, air_inst_index, {});
777
778	air_body_index += 1;
779	air_inst_index = air_body[air_body_index];
780	continue :air_tag air_tags[@backingInt(air_inst_index)];
781	},
782	.atomic_rmw => {
783	const pl_op = air_data[@backingInt(air_inst_index)].pl_op;
784	const extra = isel.air.extraData(Air.AtomicRmw, pl_op.payload).data;
785
786	try isel.analyzeUse(extra.operand);
787	try isel.def_order.putNoClobber(gpa, air_inst_index, {});
788
789	air_body_index += 1;
790	air_inst_index = air_body[air_body_index];
791	continue :air_tag air_tags[@backingInt(air_inst_index)];
792	},
793	.aggregate_init => {
794	const ty_pl = air_data[@backingInt(air_inst_index)].ty_pl;
795	const elements: []const Air.Inst.Ref = @ptrCast(isel.air.extra.items[ty_pl.payload..][0..@intCast(ty_pl.ty.arrayLen(zcu))]);
796
797	for (elements) |element| try isel.analyzeUse(element);
798	try isel.def_order.putNoClobber(gpa, air_inst_index, {});
799
800	air_body_index += 1;
801	air_inst_index = air_body[air_body_index];
802	continue :air_tag air_tags[@backingInt(air_inst_index)];
803	},
804	.union_init => {
805	const ty_pl = air_data[@backingInt(air_inst_index)].ty_pl;
806	const extra = isel.air.extraData(Air.UnionInit, ty_pl.payload).data;
807
808	try isel.analyzeUse(extra.init);
809	try isel.def_order.putNoClobber(gpa, air_inst_index, {});
810
811	air_body_index += 1;
812	air_inst_index = air_body[air_body_index];
813	continue :air_tag air_tags[@backingInt(air_inst_index)];
814	},
815	.prefetch => {
816	const prefetch = air_data[@backingInt(air_inst_index)].prefetch;
817
818	try isel.analyzeUse(prefetch.ptr);
819
820	air_body_index += 1;
821	air_inst_index = air_body[air_body_index];
822	continue :air_tag air_tags[@backingInt(air_inst_index)];
823	},
824	.field_parent_ptr => {
825	const ty_pl = air_data[@backingInt(air_inst_index)].ty_pl;
826	const extra = isel.air.extraData(Air.FieldParentPtr, ty_pl.payload).data;
827
828	try isel.analyzeUse(extra.field_ptr);
829	try isel.def_order.putNoClobber(gpa, air_inst_index, {});
830
831	air_body_index += 1;
832	air_inst_index = air_body[air_body_index];
833	continue :air_tag air_tags[@backingInt(air_inst_index)];
834	},
835	.set_err_return_trace => {
836	const un_op = air_data[@backingInt(air_inst_index)].un_op;
837
838	try isel.analyzeUse(un_op);
839
840	air_body_index += 1;
841	air_inst_index = air_body[air_body_index];
842	continue :air_tag air_tags[@backingInt(air_inst_index)];
843	},
844	}
845	assert(air_body_index == air_body.len);
846	isel.def_order.shrinkRetainingCapacity(initial_def_order_len);
847	}
848
849	fn analyzeUse(isel: *Select, air_ref: Air.Inst.Ref) !void {
850	const air_inst_index = air_ref.toIndex() orelse return;
851	const def_order_index = isel.def_order.getIndex(air_inst_index).?;
852
853	// Loop liveness
854	var active_loop_index = isel.active_loops.items.len;
855	while (active_loop_index > 0) {
856	const prev_active_loop_index = active_loop_index - 1;
857	const active_loop = isel.active_loops.items[prev_active_loop_index];
858	if (def_order_index >= active_loop.get(isel).def_order) break;
859	active_loop_index = prev_active_loop_index;
860	}
861	if (active_loop_index < isel.active_loops.items.len) {
862	const active_loop = isel.active_loops.items[active_loop_index];
863	const loop_live_gop =
864	try isel.loop_live.set.getOrPut(isel.pt.zcu.gpa, .{ active_loop, air_inst_index });
865	if (!loop_live_gop.found_existing) active_loop.get(isel).live += 1;
866	}
867	}
868
869	pub fn finishAnalysis(isel: *Select) !void {
870	const gpa = isel.pt.zcu.gpa;
871
872	// Loop Liveness
873	if (isel.loops.count() > 0) {
874	try isel.loops.ensureUnusedCapacity(gpa, 1);
875
876	const loop_live_len: u32 = @intCast(isel.loop_live.set.count());
877	if (loop_live_len > 0) {
878	try isel.loop_live.list.resize(gpa, loop_live_len);
879
880	const loops = isel.loops.values();
881	for (loops[1..], loops[0 .. loops.len - 1]) |*loop, prev_loop| loop.live += prev_loop.live;
882	assert(loops[loops.len - 1].live == loop_live_len);
883
884	for (isel.loop_live.set.keys()) |entry| {
885	const loop, const inst = entry;
886	const loop_live = &loop.get(isel).live;
887	loop_live.* -= 1;
888	isel.loop_live.list.items[loop_live.*] = inst;
889	}
890	assert(loops[0].live == 0);
891	}
892
893	const invalid_gop = isel.loops.getOrPutAssumeCapacity(Loop.invalid);
894	assert(!invalid_gop.found_existing);
895	invalid_gop.value_ptr.live = loop_live_len;
896	}
897	}
898
899	pub fn body(isel: *Select, air_body: []const Air.Inst.Index) codegen.Error!void {
900	const zcu = isel.pt.zcu;
901	const ip = &zcu.intern_pool;
902	const gpa = zcu.gpa;
903
904	{
905	var live_reg_it = isel.live_registers.iterator();
906	while (live_reg_it.next()) |live_reg_entry| switch (live_reg_entry.value.*) {
907	_ => {
908	const ra = &live_reg_entry.value.get(isel).location_payload.small.register;
909	assert(ra.* == live_reg_entry.key);
910	ra.* = .zr;
911	live_reg_entry.value.* = .free;
912	},
913	.allocating => live_reg_entry.value.* = .free,
914	.free => {},
915	};
916	}
917
918	var air: struct {
919	isel: *Select,
920	tag_items: []const Air.Inst.Tag,
921	data_items: []const Air.Inst.Data,
922	body: []const Air.Inst.Index,
923	body_index: u32,
924	inst_index: Air.Inst.Index,
925
926	fn tag(it: *@This(), inst_index: Air.Inst.Index) Air.Inst.Tag {
927	return it.tag_items[@backingInt(inst_index)];
928	}
929
930	fn data(it: *@This(), inst_index: Air.Inst.Index) Air.Inst.Data {
931	return it.data_items[@backingInt(inst_index)];
932	}
933
934	fn next(it: *@This()) ?Air.Inst.Tag {
935	if (it.body_index == 0) {
936	@branchHint(.unlikely);
937	return null;
938	}
939	it.body_index -= 1;
940	it.inst_index = it.body[it.body_index];
941	wip_mir_log.debug("{f}", .{it.fmtAir(it.inst_index)});
942	return it.tag(it.inst_index);
943	}
944
945	fn fmtAir(it: @This(), inst: Air.Inst.Index) struct {
946	isel: *Select,
947	inst: Air.Inst.Index,
948	pub fn format(fmt_air: @This(), writer: *std.Io.Writer) std.Io.Writer.Error!void {
949	fmt_air.isel.air.writeInst(writer, fmt_air.inst, fmt_air.isel.pt, null);
950	}
951	} {
952	return .{ .isel = it.isel, .inst = inst };
953	}
954	} = .{
955	.isel = isel,
956	.tag_items = isel.air.instructions.items(.tag),
957	.data_items = isel.air.instructions.items(.data),
958	.body = air_body,
959	.body_index = @intCast(air_body.len),
960	.inst_index = undefined,
961	};
962	air_tag: switch (air.next().?) {
963	else => |air_tag| return isel.fail("unimplemented {t}", .{air_tag}),
964
965	// No "scalarize" legalizations are enabled, so these instructions never appear.
966	.legalize_vec_elem_val => unreachable,
967	.legalize_vec_store_elem => unreachable,
968
969	.arg => {
970	const arg_vi = isel.live_values.fetchRemove(air.inst_index).?.value;
971	defer arg_vi.deref(isel);
972	switch (arg_vi.parent(isel)) {
973	.unallocated, .stack_slot => if (arg_vi.hint(isel)) |arg_ra| {
974	try arg_vi.defLiveIn(isel, arg_ra, comptime &.initFill(.free));
975	} else {
976	var arg_part_it = arg_vi.parts(isel);
977	while (arg_part_it.next()) |arg_part| {
978	try arg_part.defLiveIn(isel, arg_part.hint(isel).?, comptime &.initFill(.free));
979	}
980	},
981	.value, .constant => unreachable,
982	.address => |address_vi| try address_vi.defLiveIn(isel, address_vi.hint(isel).?, comptime &.initFill(.free)),
983	}
984	if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
985	},
986	.add, .add_safe, .add_optimized, .add_wrap, .sub, .sub_safe, .sub_optimized, .sub_wrap => |air_tag| {
987	if (isel.live_values.fetchRemove(air.inst_index)) |res_vi| unused: {
988	defer res_vi.value.deref(isel);
989
990	const bin_op = air.data(air.inst_index).bin_op;
991	const ty = isel.air.typeOf(bin_op.lhs, ip);
992	if (!ty.isRuntimeFloat()) try res_vi.value.addOrSubtract(isel, ty, try isel.use(bin_op.lhs), switch (air_tag) {
993	else => unreachable,
994	.add, .add_safe, .add_wrap => .add,
995	.sub, .sub_safe, .sub_wrap => .sub,
996	}, try isel.use(bin_op.rhs), .{
997	.overflow = switch (air_tag) {
998	else => unreachable,
999	.add, .sub => .@"unreachable",
1000	.add_safe, .sub_safe => .{ .panic = .integer_overflow },
1001	.add_wrap, .sub_wrap => .wrap,
1002	},
1003	}) else switch (ty.floatBits(isel.target)) {
1004	else => unreachable,
1005	16, 32, 64 => |bits| {
1006	const res_ra = try res_vi.value.defReg(isel) orelse break :unused;
1007	const need_fcvt = switch (bits) {
1008	else => unreachable,
1009	16 => !isel.target.cpu.has(.aarch64, .fullfp16),
1010	32, 64 => false,
1011	};
1012	if (need_fcvt) try isel.emit(.fcvt(res_ra.h(), res_ra.s()));
1013	const lhs_vi = try isel.use(bin_op.lhs);
1014	const rhs_vi = try isel.use(bin_op.rhs);
1015	const lhs_mat = try lhs_vi.matReg(isel);
1016	const rhs_mat = try rhs_vi.matReg(isel);
1017	const lhs_ra = if (need_fcvt) try isel.allocVecReg() else lhs_mat.ra;
1018	defer if (need_fcvt) isel.freeReg(lhs_ra);
1019	const rhs_ra = if (need_fcvt) try isel.allocVecReg() else rhs_mat.ra;
1020	defer if (need_fcvt) isel.freeReg(rhs_ra);
1021	try isel.emit(bits: switch (bits) {
1022	else => unreachable,
1023	16 => if (need_fcvt) continue :bits 32 else switch (air_tag) {
1024	else => unreachable,
1025	.add, .add_optimized => .fadd(res_ra.h(), lhs_ra.h(), rhs_ra.h()),
1026	.sub, .sub_optimized => .fsub(res_ra.h(), lhs_ra.h(), rhs_ra.h()),
1027	},
1028	32 => switch (air_tag) {
1029	else => unreachable,
1030	.add, .add_optimized => .fadd(res_ra.s(), lhs_ra.s(), rhs_ra.s()),
1031	.sub, .sub_optimized => .fsub(res_ra.s(), lhs_ra.s(), rhs_ra.s()),
1032	},
1033	64 => switch (air_tag) {
1034	else => unreachable,
1035	.add, .add_optimized => .fadd(res_ra.d(), lhs_ra.d(), rhs_ra.d()),
1036	.sub, .sub_optimized => .fsub(res_ra.d(), lhs_ra.d(), rhs_ra.d()),
1037	},
1038	});
1039	if (need_fcvt) {
1040	try isel.emit(.fcvt(rhs_ra.s(), rhs_mat.ra.h()));
1041	try isel.emit(.fcvt(lhs_ra.s(), lhs_mat.ra.h()));
1042	}
1043	try rhs_mat.finish(isel);
1044	try lhs_mat.finish(isel);
1045	},
1046	80, 128 => |bits| {
1047	try call.prepareReturn(isel);
1048	switch (bits) {
1049	else => unreachable,
1050	16, 32, 64, 128 => try call.returnLiveIn(isel, res_vi.value, .v0),
1051	80 => {
1052	var res_hi16_it = res_vi.value.field(ty, 8, 8);
1053	const res_hi16_vi = try res_hi16_it.only(isel);
1054	try call.returnLiveIn(isel, res_hi16_vi.?, .r1);
1055	var res_lo64_it = res_vi.value.field(ty, 0, 8);
1056	const res_lo64_vi = try res_lo64_it.only(isel);
1057	try call.returnLiveIn(isel, res_lo64_vi.?, .r0);
1058	},
1059	}
1060	try call.finishReturn(isel);
1061
1062	try call.prepareCallee(isel);
1063	try isel.global_relocs.append(gpa, .{
1064	.name = switch (air_tag) {
1065	else => unreachable,
1066	.add, .add_optimized => switch (bits) {
1067	else => unreachable,
1068	16 => "__addhf3",
1069	32 => "__addsf3",
1070	64 => "__adddf3",
1071	80 => "__addxf3",
1072	128 => "__addtf3",
1073	},
1074	.sub, .sub_optimized => switch (bits) {
1075	else => unreachable,
1076	16 => "__subhf3",
1077	32 => "__subsf3",
1078	64 => "__subdf3",
1079	80 => "__subxf3",
1080	128 => "__subtf3",
1081	},
1082	},
1083	.reloc = .{ .label = @intCast(isel.instructions.items.len) },
1084	});
1085	try isel.emit(.bl(0));
1086	try call.finishCallee(isel);
1087
1088	try call.prepareParams(isel);
1089	const lhs_vi = try isel.use(bin_op.lhs);
1090	const rhs_vi = try isel.use(bin_op.rhs);
1091	switch (bits) {
1092	else => unreachable,
1093	16, 32, 64, 128 => {
1094	try call.paramLiveOut(isel, rhs_vi, .v1);
1095	try call.paramLiveOut(isel, lhs_vi, .v0);
1096	},
1097	80 => {
1098	var rhs_hi16_it = rhs_vi.field(ty, 8, 8);
1099	const rhs_hi16_vi = try rhs_hi16_it.only(isel);
1100	try call.paramLiveOut(isel, rhs_hi16_vi.?, .r3);
1101	var rhs_lo64_it = rhs_vi.field(ty, 0, 8);
1102	const rhs_lo64_vi = try rhs_lo64_it.only(isel);
1103	try call.paramLiveOut(isel, rhs_lo64_vi.?, .r2);
1104	var lhs_hi16_it = lhs_vi.field(ty, 8, 8);
1105	const lhs_hi16_vi = try lhs_hi16_it.only(isel);
1106	try call.paramLiveOut(isel, lhs_hi16_vi.?, .r1);
1107	var lhs_lo64_it = lhs_vi.field(ty, 0, 8);
1108	const lhs_lo64_vi = try lhs_lo64_it.only(isel);
1109	try call.paramLiveOut(isel, lhs_lo64_vi.?, .r0);
1110	},
1111	}
1112	try call.finishParams(isel);
1113	},
1114	}
1115	}
1116	if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
1117	},
1118	.add_sat, .sub_sat => |air_tag| {
1119	if (isel.live_values.fetchRemove(air.inst_index)) |res_vi| unused: {
1120	defer res_vi.value.deref(isel);
1121
1122	const bin_op = air.data(air.inst_index).bin_op;
1123	const ty = isel.air.typeOf(bin_op.lhs, ip);
1124	if (!ty.isAbiInt(zcu)) return isel.fail("bad {t} {f}", .{ air_tag, isel.fmtType(ty) });
1125	const int_info = ty.intInfo(zcu);
1126	switch (int_info.bits) {
1127	0 => unreachable,
1128	32, 64 => |bits| switch (int_info.signedness) {
1129	.signed => return isel.fail("bad {t} {f}", .{ air_tag, isel.fmtType(ty) }),
1130	.unsigned => {
1131	const res_ra = try res_vi.value.defReg(isel) orelse break :unused;
1132	const lhs_vi = try isel.use(bin_op.lhs);
1133	const rhs_vi = try isel.use(bin_op.rhs);
1134	const lhs_mat = try lhs_vi.matReg(isel);
1135	const rhs_mat = try rhs_vi.matReg(isel);
1136	const unsat_res_ra = try isel.allocIntReg();
1137	defer isel.freeReg(unsat_res_ra);
1138	switch (air_tag) {
1139	else => unreachable,
1140	.add_sat => switch (bits) {
1141	else => unreachable,
1142	32 => {
1143	try isel.emit(.csinv(res_ra.w(), unsat_res_ra.w(), .wzr, .invert(.cs)));
1144	try isel.emit(.adds(unsat_res_ra.w(), lhs_mat.ra.w(), .{ .register = rhs_mat.ra.w() }));
1145	},
1146	64 => {
1147	try isel.emit(.csinv(res_ra.x(), unsat_res_ra.x(), .xzr, .invert(.cs)));
1148	try isel.emit(.adds(unsat_res_ra.x(), lhs_mat.ra.x(), .{ .register = rhs_mat.ra.x() }));
1149	},
1150	},
1151	.sub_sat => switch (bits) {
1152	else => unreachable,
1153	32 => {
1154	try isel.emit(.csel(res_ra.w(), unsat_res_ra.w(), .wzr, .invert(.cc)));
1155	try isel.emit(.subs(unsat_res_ra.w(), lhs_mat.ra.w(), .{ .register = rhs_mat.ra.w() }));
1156	},
1157	64 => {
1158	try isel.emit(.csel(res_ra.x(), unsat_res_ra.x(), .xzr, .invert(.cc)));
1159	try isel.emit(.subs(unsat_res_ra.x(), lhs_mat.ra.x(), .{ .register = rhs_mat.ra.x() }));
1160	},
1161	},
1162	}
1163	try rhs_mat.finish(isel);
1164	try lhs_mat.finish(isel);
1165	},
1166	},
1167	else => return isel.fail("too big {t} {f}", .{ air_tag, isel.fmtType(ty) }),
1168	}
1169	}
1170	if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
1171	},
1172	.mul, .mul_optimized, .mul_wrap => |air_tag| {
1173	if (isel.live_values.fetchRemove(air.inst_index)) |res_vi| unused: {
1174	defer res_vi.value.deref(isel);
1175
1176	const bin_op = air.data(air.inst_index).bin_op;
1177	const ty = isel.air.typeOf(bin_op.lhs, ip);
1178	if (!ty.isRuntimeFloat()) {
1179	if (!ty.isAbiInt(zcu)) return isel.fail("bad {t} {f}", .{ air_tag, isel.fmtType(ty) });
1180	const int_info = ty.intInfo(zcu);
1181	switch (int_info.bits) {
1182	0 => unreachable,
1183	1 => {
1184	const res_ra = try res_vi.value.defReg(isel) orelse break :unused;
1185	switch (int_info.signedness) {
1186	.signed => switch (air_tag) {
1187	else => unreachable,
1188	.mul => break :unused try isel.emit(.orr(res_ra.w(), .wzr, .{ .register = .wzr })),
1189	.mul_wrap => {},
1190	},
1191	.unsigned => {},
1192	}
1193	const lhs_vi = try isel.use(bin_op.lhs);
1194	const rhs_vi = try isel.use(bin_op.rhs);
1195	const lhs_mat = try lhs_vi.matReg(isel);
1196	const rhs_mat = try rhs_vi.matReg(isel);
1197	try isel.emit(.@"and"(res_ra.w(), lhs_mat.ra.w(), .{ .register = rhs_mat.ra.w() }));
1198	try rhs_mat.finish(isel);
1199	try lhs_mat.finish(isel);
1200	},
1201	2...32 => |bits| {
1202	const res_ra = try res_vi.value.defReg(isel) orelse break :unused;
1203	switch (air_tag) {
1204	else => unreachable,
1205	.mul => {},
1206	.mul_wrap => switch (bits) {
1207	else => unreachable,
1208	1...31 => try isel.emit(switch (int_info.signedness) {
1209	.signed => .sbfm(res_ra.w(), res_ra.w(), .{
1210	.N = .word,
1211	.immr = 0,
1212	.imms = @intCast(bits - 1),
1213	}),
1214	.unsigned => .ubfm(res_ra.w(), res_ra.w(), .{
1215	.N = .word,
1216	.immr = 0,
1217	.imms = @intCast(bits - 1),
1218	}),
1219	}),
1220	32 => {},
1221	},
1222	}
1223	const lhs_vi = try isel.use(bin_op.lhs);
1224	const rhs_vi = try isel.use(bin_op.rhs);
1225	const lhs_mat = try lhs_vi.matReg(isel);
1226	const rhs_mat = try rhs_vi.matReg(isel);
1227	try isel.emit(.madd(res_ra.w(), lhs_mat.ra.w(), rhs_mat.ra.w(), .wzr));
1228	try rhs_mat.finish(isel);
1229	try lhs_mat.finish(isel);
1230	},
1231	33...64 => |bits| {
1232	const res_ra = try res_vi.value.defReg(isel) orelse break :unused;
1233	switch (air_tag) {
1234	else => unreachable,
1235	.mul => {},
1236	.mul_wrap => switch (bits) {
1237	else => unreachable,
1238	33...63 => try isel.emit(switch (int_info.signedness) {
1239	.signed => .sbfm(res_ra.x(), res_ra.x(), .{
1240	.N = .doubleword,
1241	.immr = 0,
1242	.imms = @intCast(bits - 1),
1243	}),
1244	.unsigned => .ubfm(res_ra.x(), res_ra.x(), .{
1245	.N = .doubleword,
1246	.immr = 0,
1247	.imms = @intCast(bits - 1),
1248	}),
1249	}),
1250	64 => {},
1251	},
1252	}
1253	const lhs_vi = try isel.use(bin_op.lhs);
1254	const rhs_vi = try isel.use(bin_op.rhs);
1255	const lhs_mat = try lhs_vi.matReg(isel);
1256	const rhs_mat = try rhs_vi.matReg(isel);
1257	try isel.emit(.madd(res_ra.x(), lhs_mat.ra.x(), rhs_mat.ra.x(), .xzr));
1258	try rhs_mat.finish(isel);
1259	try lhs_mat.finish(isel);
1260	},
1261	65...128 => |bits| {
1262	var res_hi64_it = res_vi.value.field(ty, 8, 8);
1263	const res_hi64_vi = try res_hi64_it.only(isel);
1264	const res_hi64_ra = try res_hi64_vi.?.defReg(isel);
1265	var res_lo64_it = res_vi.value.field(ty, 0, 8);
1266	const res_lo64_vi = try res_lo64_it.only(isel);
1267	const res_lo64_ra = try res_lo64_vi.?.defReg(isel);
1268	if (res_hi64_ra == null and res_lo64_ra == null) break :unused;
1269	if (res_hi64_ra) |res_ra| switch (air_tag) {
1270	else => unreachable,
1271	.mul => {},
1272	.mul_wrap => switch (bits) {
1273	else => unreachable,
1274	65...127 => try isel.emit(switch (int_info.signedness) {
1275	.signed => .sbfm(res_ra.x(), res_ra.x(), .{
1276	.N = .doubleword,
1277	.immr = 0,
1278	.imms = @intCast(bits - 1),
1279	}),
1280	.unsigned => .ubfm(res_ra.x(), res_ra.x(), .{
1281	.N = .doubleword,
1282	.immr = 0,
1283	.imms = @intCast(bits - 1),
1284	}),
1285	}),
1286	128 => {},
1287	},
1288	};
1289	const lhs_vi = try isel.use(bin_op.lhs);
1290	const rhs_vi = try isel.use(bin_op.rhs);
1291	const lhs_lo64_mat, const rhs_lo64_mat = lo64_mat: {
1292	const res_hi64_lock: RegLock = if (res_hi64_ra != null and res_lo64_ra != null)
1293	isel.lockReg(res_hi64_ra.?)
1294	else
1295	.empty;
1296	defer res_hi64_lock.unlock(isel);
1297
1298	var rhs_lo64_it = rhs_vi.field(ty, 0, 8);
1299	const rhs_lo64_vi = try rhs_lo64_it.only(isel);
1300	const rhs_lo64_mat = try rhs_lo64_vi.?.matReg(isel);
1301	var lhs_lo64_it = lhs_vi.field(ty, 0, 8);
1302	const lhs_lo64_vi = try lhs_lo64_it.only(isel);
1303	const lhs_lo64_mat = try lhs_lo64_vi.?.matReg(isel);
1304	break :lo64_mat .{ lhs_lo64_mat, rhs_lo64_mat };
1305	};
1306	if (res_lo64_ra) |res_ra| try isel.emit(.madd(res_ra.x(), lhs_lo64_mat.ra.x(), rhs_lo64_mat.ra.x(), .xzr));
1307	if (res_hi64_ra) |res_ra| {
1308	var rhs_hi64_it = rhs_vi.field(ty, 8, 8);
1309	const rhs_hi64_vi = try rhs_hi64_it.only(isel);
1310	const rhs_hi64_mat = try rhs_hi64_vi.?.matReg(isel);
1311	var lhs_hi64_it = lhs_vi.field(ty, 8, 8);
1312	const lhs_hi64_vi = try lhs_hi64_it.only(isel);
1313	const lhs_hi64_mat = try lhs_hi64_vi.?.matReg(isel);
1314	const acc_ra = try isel.allocIntReg();
1315	defer isel.freeReg(acc_ra);
1316	try isel.emit(.madd(res_ra.x(), lhs_hi64_mat.ra.x(), rhs_lo64_mat.ra.x(), acc_ra.x()));
1317	try isel.emit(.madd(acc_ra.x(), lhs_lo64_mat.ra.x(), rhs_hi64_mat.ra.x(), acc_ra.x()));
1318	try isel.emit(.umulh(acc_ra.x(), lhs_lo64_mat.ra.x(), rhs_lo64_mat.ra.x()));
1319	try rhs_hi64_mat.finish(isel);
1320	try lhs_hi64_mat.finish(isel);
1321	}
1322	try rhs_lo64_mat.finish(isel);
1323	try lhs_lo64_mat.finish(isel);
1324	},
1325	else => return isel.fail("too big {t} {f}", .{ air_tag, isel.fmtType(ty) }),
1326	}
1327	} else switch (ty.floatBits(isel.target)) {
1328	else => unreachable,
1329	16, 32, 64 => |bits| {
1330	const res_ra = try res_vi.value.defReg(isel) orelse break :unused;
1331	const need_fcvt = switch (bits) {
1332	else => unreachable,
1333	16 => !isel.target.cpu.has(.aarch64, .fullfp16),
1334	32, 64 => false,
1335	};
1336	if (need_fcvt) try isel.emit(.fcvt(res_ra.h(), res_ra.s()));
1337	const lhs_vi = try isel.use(bin_op.lhs);
1338	const rhs_vi = try isel.use(bin_op.rhs);
1339	const lhs_mat = try lhs_vi.matReg(isel);
1340	const rhs_mat = try rhs_vi.matReg(isel);
1341	const lhs_ra = if (need_fcvt) try isel.allocVecReg() else lhs_mat.ra;
1342	defer if (need_fcvt) isel.freeReg(lhs_ra);
1343	const rhs_ra = if (need_fcvt) try isel.allocVecReg() else rhs_mat.ra;
1344	defer if (need_fcvt) isel.freeReg(rhs_ra);
1345	try isel.emit(bits: switch (bits) {
1346	else => unreachable,
1347	16 => if (need_fcvt)
1348	continue :bits 32
1349	else
1350	.fmul(res_ra.h(), lhs_ra.h(), rhs_ra.h()),
1351	32 => .fmul(res_ra.s(), lhs_ra.s(), rhs_ra.s()),
1352	64 => .fmul(res_ra.d(), lhs_ra.d(), rhs_ra.d()),
1353	});
1354	if (need_fcvt) {
1355	try isel.emit(.fcvt(rhs_ra.s(), rhs_mat.ra.h()));
1356	try isel.emit(.fcvt(lhs_ra.s(), lhs_mat.ra.h()));
1357	}
1358	try rhs_mat.finish(isel);
1359	try lhs_mat.finish(isel);
1360	},
1361	80, 128 => |bits| {
1362	try call.prepareReturn(isel);
1363	switch (bits) {
1364	else => unreachable,
1365	16, 32, 64, 128 => try call.returnLiveIn(isel, res_vi.value, .v0),
1366	80 => {
1367	var res_hi16_it = res_vi.value.field(ty, 8, 8);
1368	const res_hi16_vi = try res_hi16_it.only(isel);
1369	try call.returnLiveIn(isel, res_hi16_vi.?, .r1);
1370	var res_lo64_it = res_vi.value.field(ty, 0, 8);
1371	const res_lo64_vi = try res_lo64_it.only(isel);
1372	try call.returnLiveIn(isel, res_lo64_vi.?, .r0);
1373	},
1374	}
1375	try call.finishReturn(isel);
1376
1377	try call.prepareCallee(isel);
1378	try isel.global_relocs.append(gpa, .{
1379	.name = switch (bits) {
1380	else => unreachable,
1381	16 => "__mulhf3",
1382	32 => "__mulsf3",
1383	64 => "__muldf3",
1384	80 => "__mulxf3",
1385	128 => "__multf3",
1386	},
1387	.reloc = .{ .label = @intCast(isel.instructions.items.len) },
1388	});
1389	try isel.emit(.bl(0));
1390	try call.finishCallee(isel);
1391
1392	try call.prepareParams(isel);
1393	const lhs_vi = try isel.use(bin_op.lhs);
1394	const rhs_vi = try isel.use(bin_op.rhs);
1395	switch (bits) {
1396	else => unreachable,
1397	16, 32, 64, 128 => {
1398	try call.paramLiveOut(isel, rhs_vi, .v1);
1399	try call.paramLiveOut(isel, lhs_vi, .v0);
1400	},
1401	80 => {
1402	var rhs_hi16_it = rhs_vi.field(ty, 8, 8);
1403	const rhs_hi16_vi = try rhs_hi16_it.only(isel);
1404	try call.paramLiveOut(isel, rhs_hi16_vi.?, .r3);
1405	var rhs_lo64_it = rhs_vi.field(ty, 0, 8);
1406	const rhs_lo64_vi = try rhs_lo64_it.only(isel);
1407	try call.paramLiveOut(isel, rhs_lo64_vi.?, .r2);
1408	var lhs_hi16_it = lhs_vi.field(ty, 8, 8);
1409	const lhs_hi16_vi = try lhs_hi16_it.only(isel);
1410	try call.paramLiveOut(isel, lhs_hi16_vi.?, .r1);
1411	var lhs_lo64_it = lhs_vi.field(ty, 0, 8);
1412	const lhs_lo64_vi = try lhs_lo64_it.only(isel);
1413	try call.paramLiveOut(isel, lhs_lo64_vi.?, .r0);
1414	},
1415	}
1416	try call.finishParams(isel);
1417	},
1418	}
1419	}
1420	if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
1421	},
1422	.mul_safe => |air_tag| {
1423	if (isel.live_values.fetchRemove(air.inst_index)) |res_vi| unused: {
1424	defer res_vi.value.deref(isel);
1425
1426	const bin_op = air.data(air.inst_index).bin_op;
1427	const ty = isel.air.typeOf(bin_op.lhs, ip);
1428	if (!ty.isAbiInt(zcu)) return isel.fail("bad {t} {f}", .{ air_tag, isel.fmtType(ty) });
1429	const int_info = ty.intInfo(zcu);
1430	switch (int_info.signedness) {
1431	.signed => switch (int_info.bits) {
1432	0 => unreachable,
1433	1 => {
1434	const res_ra = try res_vi.value.defReg(isel) orelse break :unused;
1435	const lhs_vi = try isel.use(bin_op.lhs);
1436	const rhs_vi = try isel.use(bin_op.rhs);
1437	const lhs_mat = try lhs_vi.matReg(isel);
1438	const rhs_mat = try rhs_vi.matReg(isel);
1439	try isel.emit(.orr(res_ra.w(), lhs_mat.ra.w(), .{ .register = rhs_mat.ra.w() }));
1440	const skip_label = isel.instructions.items.len;
1441	try isel.emitPanic(.integer_overflow);
1442	try isel.emit(.@"b."(
1443	.invert(.ne),
1444	@intCast((isel.instructions.items.len + 1 - skip_label) << 2),
1445	));
1446	try isel.emit(.ands(.wzr, lhs_mat.ra.w(), .{ .register = rhs_mat.ra.w() }));
1447	try rhs_mat.finish(isel);
1448	try lhs_mat.finish(isel);
1449	},
1450	else => return isel.fail("too big {t} {f}", .{ air_tag, isel.fmtType(ty) }),
1451	},
1452	.unsigned => switch (int_info.bits) {
1453	0 => unreachable,
1454	1 => {
1455	const res_ra = try res_vi.value.defReg(isel) orelse break :unused;
1456	const lhs_vi = try isel.use(bin_op.lhs);
1457	const rhs_vi = try isel.use(bin_op.rhs);
1458	const lhs_mat = try lhs_vi.matReg(isel);
1459	const rhs_mat = try rhs_vi.matReg(isel);
1460	try isel.emit(.@"and"(res_ra.w(), lhs_mat.ra.w(), .{ .register = rhs_mat.ra.w() }));
1461	try rhs_mat.finish(isel);
1462	try lhs_mat.finish(isel);
1463	},
1464	2...16 => |bits| {
1465	const res_ra = try res_vi.value.defReg(isel) orelse break :unused;
1466	const lhs_vi = try isel.use(bin_op.lhs);
1467	const rhs_vi = try isel.use(bin_op.rhs);
1468	const lhs_mat = try lhs_vi.matReg(isel);
1469	const rhs_mat = try rhs_vi.matReg(isel);
1470	const skip_label = isel.instructions.items.len;
1471	try isel.emitPanic(.integer_overflow);
1472	try isel.emit(.@"b."(
1473	.eq,
1474	@intCast((isel.instructions.items.len + 1 - skip_label) << 2),
1475	));
1476	try isel.emit(.ands(.wzr, res_ra.w(), .{ .immediate = .{
1477	.N = .word,
1478	.immr = @intCast(32 - bits),
1479	.imms = @intCast(32 - bits - 1),
1480	} }));
1481	try isel.emit(.madd(res_ra.w(), lhs_mat.ra.w(), rhs_mat.ra.w(), .wzr));
1482	try rhs_mat.finish(isel);
1483	try lhs_mat.finish(isel);
1484	},
1485	17...32 => |bits| {
1486	const res_ra = try res_vi.value.defReg(isel) orelse break :unused;
1487	const lhs_vi = try isel.use(bin_op.lhs);
1488	const rhs_vi = try isel.use(bin_op.rhs);
1489	const lhs_mat = try lhs_vi.matReg(isel);
1490	const rhs_mat = try rhs_vi.matReg(isel);
1491	const skip_label = isel.instructions.items.len;
1492	try isel.emitPanic(.integer_overflow);
1493	try isel.emit(.@"b."(
1494	.eq,
1495	@intCast((isel.instructions.items.len + 1 - skip_label) << 2),
1496	));
1497	try isel.emit(.ands(.xzr, res_ra.x(), .{ .immediate = .{
1498	.N = .doubleword,
1499	.immr = @intCast(64 - bits),
1500	.imms = @intCast(64 - bits - 1),
1501	} }));
1502	try isel.emit(.umaddl(res_ra.x(), lhs_mat.ra.w(), rhs_mat.ra.w(), .xzr));
1503	try rhs_mat.finish(isel);
1504	try lhs_mat.finish(isel);
1505	},
1506	33...63 => |bits| {
1507	const lo64_ra = try res_vi.value.defReg(isel) orelse break :unused;
1508	const lhs_vi = try isel.use(bin_op.lhs);
1509	const rhs_vi = try isel.use(bin_op.rhs);
1510	const lhs_mat = try lhs_vi.matReg(isel);
1511	const rhs_mat = try rhs_vi.matReg(isel);
1512	const hi64_ra = hi64_ra: {
1513	const lo64_lock = isel.tryLockReg(lo64_ra);
1514	defer lo64_lock.unlock(isel);
1515	break :hi64_ra try isel.allocIntReg();
1516	};
1517	defer isel.freeReg(hi64_ra);
1518	const skip_label = isel.instructions.items.len;
1519	try isel.emitPanic(.integer_overflow);
1520	try isel.emit(.cbz(
1521	hi64_ra.x(),
1522	@intCast((isel.instructions.items.len + 1 - skip_label) << 2),
1523	));
1524	try isel.emit(.orr(hi64_ra.x(), hi64_ra.x(), .{ .shifted_register = .{
1525	.register = lo64_ra.x(),
1526	.shift = .{ .lsr = @intCast(bits) },
1527	} }));
1528	try isel.emit(.madd(lo64_ra.x(), lhs_mat.ra.x(), rhs_mat.ra.x(), .xzr));
1529	try isel.emit(.umulh(hi64_ra.x(), lhs_mat.ra.x(), rhs_mat.ra.x()));
1530	try rhs_mat.finish(isel);
1531	try lhs_mat.finish(isel);
1532	},
1533	64 => {
1534	const res_ra = try res_vi.value.defReg(isel) orelse break :unused;
1535	const lhs_vi = try isel.use(bin_op.lhs);
1536	const rhs_vi = try isel.use(bin_op.rhs);
1537	const lhs_mat = try lhs_vi.matReg(isel);
1538	const rhs_mat = try rhs_vi.matReg(isel);
1539	try isel.emit(.madd(res_ra.x(), lhs_mat.ra.x(), rhs_mat.ra.x(), .xzr));
1540	const hi64_ra = try isel.allocIntReg();
1541	defer isel.freeReg(hi64_ra);
1542	const skip_label = isel.instructions.items.len;
1543	try isel.emitPanic(.integer_overflow);
1544	try isel.emit(.cbz(
1545	hi64_ra.x(),
1546	@intCast((isel.instructions.items.len + 1 - skip_label) << 2),
1547	));
1548	try isel.emit(.umulh(hi64_ra.x(), lhs_mat.ra.x(), rhs_mat.ra.x()));
1549	try rhs_mat.finish(isel);
1550	try lhs_mat.finish(isel);
1551	},
1552	65...128 => return isel.fail("bad {t} {f}", .{ air_tag, isel.fmtType(ty) }),
1553	else => return isel.fail("too big {t} {f}", .{ air_tag, isel.fmtType(ty) }),
1554	},
1555	}
1556	}
1557	if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
1558	},
1559	.mul_sat => |air_tag| {
1560	if (isel.live_values.fetchRemove(air.inst_index)) |res_vi| unused: {
1561	defer res_vi.value.deref(isel);
1562
1563	const bin_op = air.data(air.inst_index).bin_op;
1564	const ty = isel.air.typeOf(bin_op.lhs, ip);
1565	if (!ty.isAbiInt(zcu)) return isel.fail("bad {t} {f}", .{ air_tag, isel.fmtType(ty) });
1566	const int_info = ty.intInfo(zcu);
1567	switch (int_info.bits) {
1568	0 => unreachable,
1569	1 => {
1570	const res_ra = try res_vi.value.defReg(isel) orelse break :unused;
1571	switch (int_info.signedness) {
1572	.signed => try isel.emit(.orr(res_ra.w(), .wzr, .{ .register = .wzr })),
1573	.unsigned => {
1574	const lhs_vi = try isel.use(bin_op.lhs);
1575	const rhs_vi = try isel.use(bin_op.rhs);
1576	const lhs_mat = try lhs_vi.matReg(isel);
1577	const rhs_mat = try rhs_vi.matReg(isel);
1578	try isel.emit(.@"and"(res_ra.w(), lhs_mat.ra.w(), .{ .register = rhs_mat.ra.w() }));
1579	try rhs_mat.finish(isel);
1580	try lhs_mat.finish(isel);
1581	},
1582	}
1583	},
1584	2...32 => |bits| {
1585	const res_ra = try res_vi.value.defReg(isel) orelse break :unused;
1586	const saturated_ra = switch (int_info.signedness) {
1587	.signed => try isel.allocIntReg(),
1588	.unsigned => switch (bits) {
1589	else => unreachable,
1590	2...31 => try isel.allocIntReg(),
1591	32 => .zr,
1592	},
1593	};
1594	defer if (saturated_ra != .zr) isel.freeReg(saturated_ra);
1595	const unwrapped_ra = try isel.allocIntReg();
1596	defer isel.freeReg(unwrapped_ra);
1597	try isel.emit(switch (saturated_ra) {
1598	else => .csel(res_ra.w(), unwrapped_ra.w(), saturated_ra.w(), .eq),
1599	.zr => .csinv(res_ra.w(), unwrapped_ra.w(), saturated_ra.w(), .eq),
1600	});
1601	switch (bits) {
1602	else => unreachable,
1603	2...7, 9...15, 17...31 => switch (int_info.signedness) {
1604	.signed => {
1605	const wrapped_ra = try isel.allocIntReg();
1606	defer isel.freeReg(wrapped_ra);
1607	switch (bits) {
1608	else => unreachable,
1609	1...7, 9...15 => {
1610	try isel.emit(.subs(.wzr, unwrapped_ra.w(), .{ .register = wrapped_ra.w() }));
1611	try isel.emit(.sbfm(wrapped_ra.w(), unwrapped_ra.w(), .{
1612	.N = .word,
1613	.immr = 0,
1614	.imms = @intCast(bits - 1),
1615	}));
1616	},
1617	17...31 => {
1618	try isel.emit(.subs(.xzr, unwrapped_ra.x(), .{ .register = wrapped_ra.x() }));
1619	try isel.emit(.sbfm(wrapped_ra.x(), unwrapped_ra.x(), .{
1620	.N = .doubleword,
1621	.immr = 0,
1622	.imms = @intCast(bits - 1),
1623	}));
1624	},
1625	}
1626	},
1627	.unsigned => switch (bits) {
1628	else => unreachable,
1629	1...7, 9...15 => try isel.emit(.ands(.wzr, unwrapped_ra.w(), .{ .immediate = .{
1630	.N = .word,
1631	.immr = @intCast(32 - bits),
1632	.imms = @intCast(32 - bits - 1),
1633	} })),
1634	17...31 => try isel.emit(.ands(.xzr, unwrapped_ra.x(), .{ .immediate = .{
1635	.N = .doubleword,
1636	.immr = @intCast(64 - bits),
1637	.imms = @intCast(64 - bits - 1),
1638	} })),
1639	},
1640	},
1641	8 => try isel.emit(.subs(.wzr, unwrapped_ra.w(), .{ .extended_register = .{
1642	.register = unwrapped_ra.w(),
1643	.extend = switch (int_info.signedness) {
1644	.signed => .{ .sxtb = 0 },
1645	.unsigned => .{ .uxtb = 0 },
1646	},
1647	} })),
1648	16 => try isel.emit(.subs(.wzr, unwrapped_ra.w(), .{ .extended_register = .{
1649	.register = unwrapped_ra.w(),
1650	.extend = switch (int_info.signedness) {
1651	.signed => .{ .sxth = 0 },
1652	.unsigned => .{ .uxth = 0 },
1653	},
1654	} })),
1655	32 => try isel.emit(.subs(.xzr, unwrapped_ra.x(), .{ .extended_register = .{
1656	.register = unwrapped_ra.w(),
1657	.extend = switch (int_info.signedness) {
1658	.signed => .{ .sxtw = 0 },
1659	.unsigned => .{ .uxtw = 0 },
1660	},
1661	} })),
1662	}
1663	const lhs_vi = try isel.use(bin_op.lhs);
1664	const rhs_vi = try isel.use(bin_op.rhs);
1665	const lhs_mat = try lhs_vi.matReg(isel);
1666	const rhs_mat = try rhs_vi.matReg(isel);
1667	switch (int_info.signedness) {
1668	.signed => {
1669	try isel.emit(.eor(saturated_ra.w(), saturated_ra.w(), .{ .immediate = .{
1670	.N = .word,
1671	.immr = 0,
1672	.imms = @intCast(bits - 1 - 1),
1673	} }));
1674	try isel.emit(.sbfm(saturated_ra.w(), saturated_ra.w(), .{
1675	.N = .word,
1676	.immr = @intCast(bits - 1),
1677	.imms = @intCast(bits - 1 + 1 - 1),
1678	}));
1679	try isel.emit(.eor(saturated_ra.w(), lhs_mat.ra.w(), .{ .register = rhs_mat.ra.w() }));
1680	},
1681	.unsigned => switch (bits) {
1682	else => unreachable,
1683	2...31 => try isel.movImmediate(saturated_ra.w(), @as(u32, std.math.maxInt(u32)) >> @intCast(32 - bits)),
1684	32 => {},
1685	},
1686	}
1687	switch (bits) {
1688	else => unreachable,
1689	2...16 => try isel.emit(.madd(unwrapped_ra.w(), lhs_mat.ra.w(), rhs_mat.ra.w(), .wzr)),
1690	17...32 => switch (int_info.signedness) {
1691	.signed => try isel.emit(.smaddl(unwrapped_ra.x(), lhs_mat.ra.w(), rhs_mat.ra.w(), .xzr)),
1692	.unsigned => try isel.emit(.umaddl(unwrapped_ra.x(), lhs_mat.ra.w(), rhs_mat.ra.w(), .xzr)),
1693	},
1694	}
1695	try rhs_mat.finish(isel);
1696	try lhs_mat.finish(isel);
1697	},
1698	33...64 => |bits| {
1699	const res_ra = try res_vi.value.defReg(isel) orelse break :unused;
1700	const saturated_ra = switch (int_info.signedness) {
1701	.signed => try isel.allocIntReg(),
1702	.unsigned => switch (bits) {
1703	else => unreachable,
1704	33...63 => try isel.allocIntReg(),
1705	64 => .zr,
1706	},
1707	};
1708	defer if (saturated_ra != .zr) isel.freeReg(saturated_ra);
1709	const unwrapped_lo64_ra = try isel.allocIntReg();
1710	defer isel.freeReg(unwrapped_lo64_ra);
1711	const unwrapped_hi64_ra = try isel.allocIntReg();
1712	defer isel.freeReg(unwrapped_hi64_ra);
1713	try isel.emit(switch (saturated_ra) {
1714	else => .csel(res_ra.x(), unwrapped_lo64_ra.x(), saturated_ra.x(), .eq),
1715	.zr => .csinv(res_ra.x(), unwrapped_lo64_ra.x(), saturated_ra.x(), .eq),
1716	});
1717	switch (int_info.signedness) {
1718	.signed => switch (bits) {
1719	else => unreachable,
1720	32...63 => {
1721	const wrapped_lo64_ra = try isel.allocIntReg();
1722	defer isel.freeReg(wrapped_lo64_ra);
1723	try isel.emit(.ccmp(
1724	unwrapped_lo64_ra.x(),
1725	.{ .register = wrapped_lo64_ra.x() },
1726	.{ .n = false, .z = false, .c = false, .v = false },
1727	.eq,
1728	));
1729	try isel.emit(.subs(.xzr, unwrapped_hi64_ra.x(), .{ .shifted_register = .{
1730	.register = unwrapped_lo64_ra.x(),
1731	.shift = .{ .asr = 63 },
1732	} }));
1733	try isel.emit(.sbfm(wrapped_lo64_ra.x(), unwrapped_lo64_ra.x(), .{
1734	.N = .doubleword,
1735	.immr = 0,
1736	.imms = @intCast(bits - 1),
1737	}));
1738	},
1739	64 => try isel.emit(.subs(.xzr, unwrapped_hi64_ra.x(), .{ .shifted_register = .{
1740	.register = unwrapped_lo64_ra.x(),
1741	.shift = .{ .asr = @intCast(bits - 1) },
1742	} })),
1743	},
1744	.unsigned => switch (bits) {
1745	else => unreachable,
1746	32...63 => {
1747	const overflow_ra = try isel.allocIntReg();
1748	defer isel.freeReg(overflow_ra);
1749	try isel.emit(.subs(.xzr, overflow_ra.x(), .{ .immediate = 0 }));
1750	try isel.emit(.orr(overflow_ra.x(), unwrapped_hi64_ra.x(), .{ .shifted_register = .{
1751	.register = unwrapped_lo64_ra.x(),
1752	.shift = .{ .lsr = @intCast(bits) },
1753	} }));
1754	},
1755	64 => try isel.emit(.subs(.xzr, unwrapped_hi64_ra.x(), .{ .immediate = 0 })),
1756	},
1757	}
1758	const lhs_vi = try isel.use(bin_op.lhs);
1759	const rhs_vi = try isel.use(bin_op.rhs);
1760	const lhs_mat = try lhs_vi.matReg(isel);
1761	const rhs_mat = try rhs_vi.matReg(isel);
1762	switch (int_info.signedness) {
1763	.signed => {
1764	try isel.emit(.eor(saturated_ra.x(), saturated_ra.x(), .{ .immediate = .{
1765	.N = .doubleword,
1766	.immr = 0,
1767	.imms = @intCast(bits - 1 - 1),
1768	} }));
1769	try isel.emit(.sbfm(saturated_ra.x(), saturated_ra.x(), .{
1770	.N = .doubleword,
1771	.immr = @intCast(bits - 1),
1772	.imms = @intCast(bits - 1 + 1 - 1),
1773	}));
1774	try isel.emit(.eor(saturated_ra.x(), lhs_mat.ra.x(), .{ .register = rhs_mat.ra.x() }));
1775	try isel.emit(.madd(unwrapped_lo64_ra.x(), lhs_mat.ra.x(), rhs_mat.ra.x(), .xzr));
1776	try isel.emit(.smulh(unwrapped_hi64_ra.x(), lhs_mat.ra.x(), rhs_mat.ra.x()));
1777	},
1778	.unsigned => {
1779	switch (bits) {
1780	else => unreachable,
1781	32...63 => try isel.movImmediate(saturated_ra.x(), @as(u64, std.math.maxInt(u64)) >> @intCast(64 - bits)),
1782	64 => {},
1783	}
1784	try isel.emit(.madd(unwrapped_lo64_ra.x(), lhs_mat.ra.x(), rhs_mat.ra.x(), .xzr));
1785	try isel.emit(.umulh(unwrapped_hi64_ra.x(), lhs_mat.ra.x(), rhs_mat.ra.x()));
1786	},
1787	}
1788	try rhs_mat.finish(isel);
1789	try lhs_mat.finish(isel);
1790	},
1791	else => return isel.fail("too big {t} {f}", .{ air_tag, isel.fmtType(ty) }),
1792	}
1793	}
1794	if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
1795	},
1796	.div_float, .div_float_optimized => {
1797	if (isel.live_values.fetchRemove(air.inst_index)) |res_vi| unused: {
1798	defer res_vi.value.deref(isel);
1799
1800	const bin_op = air.data(air.inst_index).bin_op;
1801	const ty = isel.air.typeOf(bin_op.lhs, ip);
1802	switch (ty.floatBits(isel.target)) {
1803	else => unreachable,
1804	16, 32, 64 => |bits| {
1805	const res_ra = try res_vi.value.defReg(isel) orelse break :unused;
1806	const need_fcvt = switch (bits) {
1807	else => unreachable,
1808	16 => !isel.target.cpu.has(.aarch64, .fullfp16),
1809	32, 64 => false,
1810	};
1811	if (need_fcvt) try isel.emit(.fcvt(res_ra.h(), res_ra.s()));
1812	const lhs_vi = try isel.use(bin_op.lhs);
1813	const rhs_vi = try isel.use(bin_op.rhs);
1814	const lhs_mat = try lhs_vi.matReg(isel);
1815	const rhs_mat = try rhs_vi.matReg(isel);
1816	const lhs_ra = if (need_fcvt) try isel.allocVecReg() else lhs_mat.ra;
1817	defer if (need_fcvt) isel.freeReg(lhs_ra);
1818	const rhs_ra = if (need_fcvt) try isel.allocVecReg() else rhs_mat.ra;
1819	defer if (need_fcvt) isel.freeReg(rhs_ra);
1820	try isel.emit(bits: switch (bits) {
1821	else => unreachable,
1822	16 => if (need_fcvt)
1823	continue :bits 32
1824	else
1825	.fdiv(res_ra.h(), lhs_ra.h(), rhs_ra.h()),
1826	32 => .fdiv(res_ra.s(), lhs_ra.s(), rhs_ra.s()),
1827	64 => .fdiv(res_ra.d(), lhs_ra.d(), rhs_ra.d()),
1828	});
1829	if (need_fcvt) {
1830	try isel.emit(.fcvt(rhs_ra.s(), rhs_mat.ra.h()));
1831	try isel.emit(.fcvt(lhs_ra.s(), lhs_mat.ra.h()));
1832	}
1833	try rhs_mat.finish(isel);
1834	try lhs_mat.finish(isel);
1835	},
1836	80, 128 => |bits| {
1837	try call.prepareReturn(isel);
1838	switch (bits) {
1839	else => unreachable,
1840	16, 32, 64, 128 => try call.returnLiveIn(isel, res_vi.value, .v0),
1841	80 => {
1842	var res_hi16_it = res_vi.value.field(ty, 8, 8);
1843	const res_hi16_vi = try res_hi16_it.only(isel);
1844	try call.returnLiveIn(isel, res_hi16_vi.?, .r1);
1845	var res_lo64_it = res_vi.value.field(ty, 0, 8);
1846	const res_lo64_vi = try res_lo64_it.only(isel);
1847	try call.returnLiveIn(isel, res_lo64_vi.?, .r0);
1848	},
1849	}
1850	try call.finishReturn(isel);
1851
1852	try call.prepareCallee(isel);
1853	try isel.global_relocs.append(gpa, .{
1854	.name = switch (bits) {
1855	else => unreachable,
1856	16 => "__divhf3",
1857	32 => "__divsf3",
1858	64 => "__divdf3",
1859	80 => "__divxf3",
1860	128 => "__divtf3",
1861	},
1862	.reloc = .{ .label = @intCast(isel.instructions.items.len) },
1863	});
1864	try isel.emit(.bl(0));
1865	try call.finishCallee(isel);
1866
1867	try call.prepareParams(isel);
1868	const lhs_vi = try isel.use(bin_op.lhs);
1869	const rhs_vi = try isel.use(bin_op.rhs);
1870	switch (bits) {
1871	else => unreachable,
1872	16, 32, 64, 128 => {
1873	try call.paramLiveOut(isel, rhs_vi, .v1);
1874	try call.paramLiveOut(isel, lhs_vi, .v0);
1875	},
1876	80 => {
1877	var rhs_hi16_it = rhs_vi.field(ty, 8, 8);
1878	const rhs_hi16_vi = try rhs_hi16_it.only(isel);
1879	try call.paramLiveOut(isel, rhs_hi16_vi.?, .r3);
1880	var rhs_lo64_it = rhs_vi.field(ty, 0, 8);
1881	const rhs_lo64_vi = try rhs_lo64_it.only(isel);
1882	try call.paramLiveOut(isel, rhs_lo64_vi.?, .r2);
1883	var lhs_hi16_it = lhs_vi.field(ty, 8, 8);
1884	const lhs_hi16_vi = try lhs_hi16_it.only(isel);
1885	try call.paramLiveOut(isel, lhs_hi16_vi.?, .r1);
1886	var lhs_lo64_it = lhs_vi.field(ty, 0, 8);
1887	const lhs_lo64_vi = try lhs_lo64_it.only(isel);
1888	try call.paramLiveOut(isel, lhs_lo64_vi.?, .r0);
1889	},
1890	}
1891	try call.finishParams(isel);
1892	},
1893	}
1894	}
1895	if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
1896	},
1897	.div_trunc, .div_trunc_optimized, .div_floor, .div_floor_optimized, .div_exact, .div_exact_optimized => |air_tag| {
1898	if (isel.live_values.fetchRemove(air.inst_index)) |res_vi| unused: {
1899	defer res_vi.value.deref(isel);
1900
1901	const bin_op = air.data(air.inst_index).bin_op;
1902	const ty = isel.air.typeOf(bin_op.lhs, ip);
1903	if (!ty.isRuntimeFloat()) {
1904	if (!ty.isAbiInt(zcu)) return isel.fail("bad {t} {f}", .{ air_tag, isel.fmtType(ty) });
1905	const int_info = ty.intInfo(zcu);
1906	switch (int_info.bits) {
1907	0 => unreachable,
1908	1...64 => |bits| {
1909	const res_ra = try res_vi.value.defReg(isel) orelse break :unused;
1910	const lhs_vi = try isel.use(bin_op.lhs);
1911	const rhs_vi = try isel.use(bin_op.rhs);
1912	const lhs_mat = try lhs_vi.matReg(isel);
1913	const rhs_mat = try rhs_vi.matReg(isel);
1914	const div_ra = div_ra: switch (air_tag) {
1915	else => unreachable,
1916	.div_trunc, .div_exact => res_ra,
1917	.div_floor => switch (int_info.signedness) {
1918	.signed => {
1919	const div_ra = try isel.allocIntReg();
1920	errdefer isel.freeReg(div_ra);
1921	const rem_ra = try isel.allocIntReg();
1922	defer isel.freeReg(rem_ra);
1923	switch (bits) {
1924	else => unreachable,
1925	1...32 => {
1926	try isel.emit(.csel(res_ra.w(), div_ra.w(), rem_ra.w(), .pl));
1927	try isel.emit(.sub(rem_ra.w(), div_ra.w(), .{ .immediate = 1 }));
1928	try isel.emit(.ccmp(
1929	rem_ra.w(),
1930	.{ .immediate = 0 },
1931	.{ .n = false, .z = false, .c = false, .v = false },
1932	.ne,
1933	));
1934	try isel.emit(.eor(rem_ra.w(), rem_ra.w(), .{ .register = rhs_mat.ra.w() }));
1935	try isel.emit(.subs(.wzr, rem_ra.w(), .{ .immediate = 0 }));
1936	try isel.emit(.msub(rem_ra.w(), div_ra.w(), rhs_mat.ra.w(), lhs_mat.ra.w()));
1937	},
1938	33...64 => {
1939	try isel.emit(.csel(res_ra.x(), div_ra.x(), rem_ra.x(), .pl));
1940	try isel.emit(.sub(rem_ra.x(), div_ra.x(), .{ .immediate = 1 }));
1941	try isel.emit(.ccmp(
1942	rem_ra.x(),
1943	.{ .immediate = 0 },
1944	.{ .n = false, .z = false, .c = false, .v = false },
1945	.ne,
1946	));
1947	try isel.emit(.eor(rem_ra.x(), rem_ra.x(), .{ .register = rhs_mat.ra.x() }));
1948	try isel.emit(.subs(.xzr, rem_ra.x(), .{ .immediate = 0 }));
1949	try isel.emit(.msub(rem_ra.x(), div_ra.x(), rhs_mat.ra.x(), lhs_mat.ra.x()));
1950	},
1951	}
1952	break :div_ra div_ra;
1953	},
1954	.unsigned => res_ra,
1955	},
1956	};
1957	defer if (div_ra != res_ra) isel.freeReg(div_ra);
1958	try isel.emit(switch (bits) {
1959	else => unreachable,
1960	1...32 => switch (int_info.signedness) {
1961	.signed => .sdiv(div_ra.w(), lhs_mat.ra.w(), rhs_mat.ra.w()),
1962	.unsigned => .udiv(div_ra.w(), lhs_mat.ra.w(), rhs_mat.ra.w()),
1963	},
1964	33...64 => switch (int_info.signedness) {
1965	.signed => .sdiv(div_ra.x(), lhs_mat.ra.x(), rhs_mat.ra.x()),
1966	.unsigned => .udiv(div_ra.x(), lhs_mat.ra.x(), rhs_mat.ra.x()),
1967	},
1968	});
1969	try rhs_mat.finish(isel);
1970	try lhs_mat.finish(isel);
1971	},
1972	65...128 => {
1973	switch (air_tag) {
1974	else => unreachable,
1975	.div_trunc, .div_exact => {},
1976	.div_floor => switch (int_info.signedness) {
1977	.signed => return isel.fail("unimplemented {t}", .{air_tag}),
1978	.unsigned => {},
1979	},
1980	}
1981
1982	try call.prepareReturn(isel);
1983	var res_hi64_it = res_vi.value.field(ty, 8, 8);
1984	const res_hi64_vi = try res_hi64_it.only(isel);
1985	try call.returnLiveIn(isel, res_hi64_vi.?, .r1);
1986	var res_lo64_it = res_vi.value.field(ty, 0, 8);
1987	const res_lo64_vi = try res_lo64_it.only(isel);
1988	try call.returnLiveIn(isel, res_lo64_vi.?, .r0);
1989	try call.finishReturn(isel);
1990
1991	try call.prepareCallee(isel);
1992	try isel.global_relocs.append(gpa, .{
1993	.name = switch (int_info.signedness) {
1994	.signed => "__divti3",
1995	.unsigned => "__udivti3",
1996	},
1997	.reloc = .{ .label = @intCast(isel.instructions.items.len) },
1998	});
1999	try isel.emit(.bl(0));
2000	try call.finishCallee(isel);
2001
2002	try call.prepareParams(isel);
2003	const lhs_vi = try isel.use(bin_op.lhs);
2004	const rhs_vi = try isel.use(bin_op.rhs);
2005	var rhs_hi64_it = rhs_vi.field(ty, 8, 8);
2006	const rhs_hi64_vi = try rhs_hi64_it.only(isel);
2007	try call.paramLiveOut(isel, rhs_hi64_vi.?, .r3);
2008	var rhs_lo64_it = rhs_vi.field(ty, 0, 8);
2009	const rhs_lo64_vi = try rhs_lo64_it.only(isel);
2010	try call.paramLiveOut(isel, rhs_lo64_vi.?, .r2);
2011	var lhs_hi64_it = lhs_vi.field(ty, 8, 8);
2012	const lhs_hi64_vi = try lhs_hi64_it.only(isel);
2013	try call.paramLiveOut(isel, lhs_hi64_vi.?, .r1);
2014	var lhs_lo64_it = lhs_vi.field(ty, 0, 8);
2015	const lhs_lo64_vi = try lhs_lo64_it.only(isel);
2016	try call.paramLiveOut(isel, lhs_lo64_vi.?, .r0);
2017	try call.finishParams(isel);
2018	},
2019	else => return isel.fail("too big {t} {f}", .{ air_tag, isel.fmtType(ty) }),
2020	}
2021	} else switch (ty.floatBits(isel.target)) {
2022	else => unreachable,
2023	16, 32, 64 => |bits| {
2024	const res_ra = try res_vi.value.defReg(isel) orelse break :unused;
2025	const need_fcvt = switch (bits) {
2026	else => unreachable,
2027	16 => !isel.target.cpu.has(.aarch64, .fullfp16),
2028	32, 64 => false,
2029	};
2030	if (need_fcvt) try isel.emit(.fcvt(res_ra.h(), res_ra.s()));
2031	const lhs_vi = try isel.use(bin_op.lhs);
2032	const rhs_vi = try isel.use(bin_op.rhs);
2033	const lhs_mat = try lhs_vi.matReg(isel);
2034	const rhs_mat = try rhs_vi.matReg(isel);
2035	const lhs_ra = if (need_fcvt) try isel.allocVecReg() else lhs_mat.ra;
2036	defer if (need_fcvt) isel.freeReg(lhs_ra);
2037	const rhs_ra = if (need_fcvt) try isel.allocVecReg() else rhs_mat.ra;
2038	defer if (need_fcvt) isel.freeReg(rhs_ra);
2039	bits: switch (bits) {
2040	else => unreachable,
2041	16 => if (need_fcvt) continue :bits 32 else {
2042	switch (air_tag) {
2043	else => unreachable,
2044	.div_trunc, .div_trunc_optimized => try isel.emit(.frintz(res_ra.h(), res_ra.h())),
2045	.div_floor, .div_floor_optimized => try isel.emit(.frintm(res_ra.h(), res_ra.h())),
2046	.div_exact, .div_exact_optimized => {},
2047	}
2048	try isel.emit(.fdiv(res_ra.h(), lhs_ra.h(), rhs_ra.h()));
2049	},
2050	32 => {
2051	switch (air_tag) {
2052	else => unreachable,
2053	.div_trunc, .div_trunc_optimized => try isel.emit(.frintz(res_ra.s(), res_ra.s())),
2054	.div_floor, .div_floor_optimized => try isel.emit(.frintm(res_ra.s(), res_ra.s())),
2055	.div_exact, .div_exact_optimized => {},
2056	}
2057	try isel.emit(.fdiv(res_ra.s(), lhs_ra.s(), rhs_ra.s()));
2058	},
2059	64 => {
2060	switch (air_tag) {
2061	else => unreachable,
2062	.div_trunc, .div_trunc_optimized => try isel.emit(.frintz(res_ra.d(), res_ra.d())),
2063	.div_floor, .div_floor_optimized => try isel.emit(.frintm(res_ra.d(), res_ra.d())),
2064	.div_exact, .div_exact_optimized => {},
2065	}
2066	try isel.emit(.fdiv(res_ra.d(), lhs_ra.d(), rhs_ra.d()));
2067	},
2068	}
2069	if (need_fcvt) {
2070	try isel.emit(.fcvt(rhs_ra.s(), rhs_mat.ra.h()));
2071	try isel.emit(.fcvt(lhs_ra.s(), lhs_mat.ra.h()));
2072	}
2073	try rhs_mat.finish(isel);
2074	try lhs_mat.finish(isel);
2075	},
2076	80, 128 => |bits| {
2077	try call.prepareReturn(isel);
2078	switch (bits) {
2079	else => unreachable,
2080	16, 32, 64, 128 => try call.returnLiveIn(isel, res_vi.value, .v0),
2081	80 => {
2082	var res_hi16_it = res_vi.value.field(ty, 8, 8);
2083	const res_hi16_vi = try res_hi16_it.only(isel);
2084	try call.returnLiveIn(isel, res_hi16_vi.?, .r1);
2085	var res_lo64_it = res_vi.value.field(ty, 0, 8);
2086	const res_lo64_vi = try res_lo64_it.only(isel);
2087	try call.returnLiveIn(isel, res_lo64_vi.?, .r0);
2088	},
2089	}
2090	try call.finishReturn(isel);
2091
2092	try call.prepareCallee(isel);
2093	switch (air_tag) {
2094	else => unreachable,
2095	.div_trunc, .div_trunc_optimized => {
2096	try isel.global_relocs.append(gpa, .{
2097	.name = switch (bits) {
2098	else => unreachable,
2099	16 => "__trunch",
2100	32 => "truncf",
2101	64 => "trunc",
2102	80 => "__truncx",
2103	128 => "truncf128",
2104	},
2105	.reloc = .{ .label = @intCast(isel.instructions.items.len) },
2106	});
2107	try isel.emit(.bl(0));
2108	},
2109	.div_floor, .div_floor_optimized => {
2110	try isel.global_relocs.append(gpa, .{
2111	.name = switch (bits) {
2112	else => unreachable,
2113	16 => "__floorh",
2114	32 => "floorf",
2115	64 => "floor",
2116	80 => "__floorx",
2117	128 => "floorf128",
2118	},
2119	.reloc = .{ .label = @intCast(isel.instructions.items.len) },
2120	});
2121	try isel.emit(.bl(0));
2122	},
2123	.div_exact, .div_exact_optimized => {},
2124	}
2125	try isel.global_relocs.append(gpa, .{
2126	.name = switch (bits) {
2127	else => unreachable,
2128	16 => "__divhf3",
2129	32 => "__divsf3",
2130	64 => "__divdf3",
2131	80 => "__divxf3",
2132	128 => "__divtf3",
2133	},
2134	.reloc = .{ .label = @intCast(isel.instructions.items.len) },
2135	});
2136	try isel.emit(.bl(0));
2137	try call.finishCallee(isel);
2138
2139	try call.prepareParams(isel);
2140	const lhs_vi = try isel.use(bin_op.lhs);
2141	const rhs_vi = try isel.use(bin_op.rhs);
2142	switch (bits) {
2143	else => unreachable,
2144	16, 32, 64, 128 => {
2145	try call.paramLiveOut(isel, rhs_vi, .v1);
2146	try call.paramLiveOut(isel, lhs_vi, .v0);
2147	},
2148	80 => {
2149	var rhs_hi16_it = rhs_vi.field(ty, 8, 8);
2150	const rhs_hi16_vi = try rhs_hi16_it.only(isel);
2151	try call.paramLiveOut(isel, rhs_hi16_vi.?, .r3);
2152	var rhs_lo64_it = rhs_vi.field(ty, 0, 8);
2153	const rhs_lo64_vi = try rhs_lo64_it.only(isel);
2154	try call.paramLiveOut(isel, rhs_lo64_vi.?, .r2);
2155	var lhs_hi16_it = lhs_vi.field(ty, 8, 8);
2156	const lhs_hi16_vi = try lhs_hi16_it.only(isel);
2157	try call.paramLiveOut(isel, lhs_hi16_vi.?, .r1);
2158	var lhs_lo64_it = lhs_vi.field(ty, 0, 8);
2159	const lhs_lo64_vi = try lhs_lo64_it.only(isel);
2160	try call.paramLiveOut(isel, lhs_lo64_vi.?, .r0);
2161	},
2162	}
2163	try call.finishParams(isel);
2164	},
2165	}
2166	}
2167	if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
2168	},
2169	.rem, .rem_optimized, .mod, .mod_optimized => |air_tag| {
2170	if (isel.live_values.fetchRemove(air.inst_index)) |res_vi| unused: {
2171	defer res_vi.value.deref(isel);
2172
2173	const bin_op = air.data(air.inst_index).bin_op;
2174	const ty = isel.air.typeOf(bin_op.lhs, ip);
2175	if (!ty.isRuntimeFloat()) {
2176	if (!ty.isAbiInt(zcu)) return isel.fail("bad {t} {f}", .{ air_tag, isel.fmtType(ty) });
2177	const int_info = ty.intInfo(zcu);
2178	if (int_info.bits > 64) return isel.fail("too big {t} {f}", .{ air_tag, isel.fmtType(ty) });
2179
2180	const res_ra = try res_vi.value.defReg(isel) orelse break :unused;
2181	const lhs_vi = try isel.use(bin_op.lhs);
2182	const rhs_vi = try isel.use(bin_op.rhs);
2183	const lhs_mat = try lhs_vi.matReg(isel);
2184	const rhs_mat = try rhs_vi.matReg(isel);
2185	const div_ra = try isel.allocIntReg();
2186	defer isel.freeReg(div_ra);
2187	const rem_ra = rem_ra: switch (air_tag) {
2188	else => unreachable,
2189	.rem => res_ra,
2190	.mod => switch (int_info.signedness) {
2191	.signed => {
2192	const rem_ra = try isel.allocIntReg();
2193	errdefer isel.freeReg(rem_ra);
2194	switch (int_info.bits) {
2195	else => unreachable,
2196	1...32 => {
2197	try isel.emit(.csel(res_ra.w(), rem_ra.w(), div_ra.w(), .pl));
2198	try isel.emit(.add(div_ra.w(), rem_ra.w(), .{ .register = rhs_mat.ra.w() }));
2199	try isel.emit(.ccmp(
2200	div_ra.w(),
2201	.{ .immediate = 0 },
2202	.{ .n = false, .z = false, .c = false, .v = false },
2203	.ne,
2204	));
2205	try isel.emit(.eor(div_ra.w(), rem_ra.w(), .{ .register = rhs_mat.ra.w() }));
2206	try isel.emit(.subs(.wzr, rem_ra.w(), .{ .immediate = 0 }));
2207	},
2208	33...64 => {
2209	try isel.emit(.csel(res_ra.x(), rem_ra.x(), div_ra.x(), .pl));
2210	try isel.emit(.add(div_ra.x(), rem_ra.x(), .{ .register = rhs_mat.ra.x() }));
2211	try isel.emit(.ccmp(
2212	div_ra.x(),
2213	.{ .immediate = 0 },
2214	.{ .n = false, .z = false, .c = false, .v = false },
2215	.ne,
2216	));
2217	try isel.emit(.eor(div_ra.x(), rem_ra.x(), .{ .register = rhs_mat.ra.x() }));
2218	try isel.emit(.subs(.xzr, rem_ra.x(), .{ .immediate = 0 }));
2219	},
2220	}
2221	break :rem_ra rem_ra;
2222	},
2223	.unsigned => res_ra,
2224	},
2225	};
2226	defer if (rem_ra != res_ra) isel.freeReg(rem_ra);
2227	switch (int_info.bits) {
2228	else => unreachable,
2229	1...32 => {
2230	try isel.emit(.msub(rem_ra.w(), div_ra.w(), rhs_mat.ra.w(), lhs_mat.ra.w()));
2231	try isel.emit(switch (int_info.signedness) {
2232	.signed => .sdiv(div_ra.w(), lhs_mat.ra.w(), rhs_mat.ra.w()),
2233	.unsigned => .udiv(div_ra.w(), lhs_mat.ra.w(), rhs_mat.ra.w()),
2234	});
2235	},
2236	33...64 => {
2237	try isel.emit(.msub(rem_ra.x(), div_ra.x(), rhs_mat.ra.x(), lhs_mat.ra.x()));
2238	try isel.emit(switch (int_info.signedness) {
2239	.signed => .sdiv(div_ra.x(), lhs_mat.ra.x(), rhs_mat.ra.x()),
2240	.unsigned => .udiv(div_ra.x(), lhs_mat.ra.x(), rhs_mat.ra.x()),
2241	});
2242	},
2243	}
2244	try rhs_mat.finish(isel);
2245	try lhs_mat.finish(isel);
2246	} else {
2247	const bits = ty.floatBits(isel.target);
2248	switch (air_tag) {
2249	else => unreachable,
2250	.rem, .rem_optimized => {
2251	if (!res_vi.value.isUsed(isel)) break :unused;
2252	try call.prepareReturn(isel);
2253	switch (bits) {
2254	else => unreachable,
2255	16, 32, 64, 128 => try call.returnLiveIn(isel, res_vi.value, .v0),
2256	80 => {
2257	var res_hi16_it = res_vi.value.field(ty, 8, 8);
2258	const res_hi16_vi = try res_hi16_it.only(isel);
2259	try call.returnLiveIn(isel, res_hi16_vi.?, .r1);
2260	var res_lo64_it = res_vi.value.field(ty, 0, 8);
2261	const res_lo64_vi = try res_lo64_it.only(isel);
2262	try call.returnLiveIn(isel, res_lo64_vi.?, .r0);
2263	},
2264	}
2265	try call.finishReturn(isel);
2266	},
2267	.mod, .mod_optimized => switch (bits) {
2268	else => unreachable,
2269	16, 32, 64 => {
2270	const res_ra = try res_vi.value.defReg(isel) orelse break :unused;
2271	try call.prepareReturn(isel);
2272	const rem_ra: Register.Alias = .v0;
2273	const temp1_ra: Register.Alias = .v1;
2274	const temp2_ra: Register.Alias = switch (res_ra) {
2275	rem_ra, temp1_ra => .v2,
2276	else => res_ra,
2277	};
2278	const need_fcvt = switch (bits) {
2279	else => unreachable,
2280	16 => !isel.target.cpu.has(.aarch64, .fullfp16),
2281	32, 64 => false,
2282	};
2283	if (need_fcvt) try isel.emit(.fcvt(res_ra.h(), res_ra.s()));
2284	try isel.emit(switch (res_ra) {
2285	rem_ra => .bif(res_ra.@"8b"(), temp2_ra.@"8b"(), temp1_ra.@"8b"()),
2286	temp1_ra => .bsl(res_ra.@"8b"(), rem_ra.@"8b"(), temp2_ra.@"8b"()),
2287	else => .bit(res_ra.@"8b"(), rem_ra.@"8b"(), temp1_ra.@"8b"()),
2288	});
2289	const rhs_vi = try isel.use(bin_op.rhs);
2290	const rhs_mat = try rhs_vi.matReg(isel);
2291	try isel.emit(bits: switch (bits) {
2292	else => unreachable,
2293	16 => if (need_fcvt)
2294	continue :bits 32
2295	else
2296	.fadd(temp2_ra.h(), rem_ra.h(), rhs_mat.ra.h()),
2297	32 => .fadd(temp2_ra.s(), rem_ra.s(), rhs_mat.ra.s()),
2298	64 => .fadd(temp2_ra.d(), rem_ra.d(), rhs_mat.ra.d()),
2299	});
2300	if (need_fcvt) {
2301	try isel.emit(.fcvt(rhs_mat.ra.s(), rhs_mat.ra.h()));
2302	try isel.emit(.fcvt(rem_ra.s(), rem_ra.h()));
2303	}
2304	try isel.emit(.orr(temp1_ra.@"8b"(), temp1_ra.@"8b"(), .{
2305	.register = temp2_ra.@"8b"(),
2306	}));
2307	try isel.emit(switch (bits) {
2308	else => unreachable,
2309	16 => .cmge(temp1_ra.@"4h"(), temp1_ra.@"4h"(), .zero),
2310	32 => .cmge(temp1_ra.@"2s"(), temp1_ra.@"2s"(), .zero),
2311	64 => .cmge(temp1_ra.d(), temp1_ra.d(), .zero),
2312	});
2313	try isel.emit(switch (bits) {
2314	else => unreachable,
2315	16 => .fcmeq(temp2_ra.h(), rem_ra.h(), .zero),
2316	32 => .fcmeq(temp2_ra.s(), rem_ra.s(), .zero),
2317	64 => .fcmeq(temp2_ra.d(), rem_ra.d(), .zero),
2318	});
2319	try isel.emit(.eor(temp1_ra.@"8b"(), rem_ra.@"8b"(), .{
2320	.register = rhs_mat.ra.@"8b"(),
2321	}));
2322	try rhs_mat.finish(isel);
2323	try call.finishReturn(isel);
2324	},
2325	80, 128 => {
2326	if (!res_vi.value.isUsed(isel)) break :unused;
2327	try call.prepareReturn(isel);
2328	switch (bits) {
2329	else => unreachable,
2330	16, 32, 64, 128 => try call.returnLiveIn(isel, res_vi.value, .v0),
2331	80 => {
2332	var res_hi16_it = res_vi.value.field(ty, 8, 8);
2333	const res_hi16_vi = try res_hi16_it.only(isel);
2334	try call.returnLiveIn(isel, res_hi16_vi.?, .r1);
2335	var res_lo64_it = res_vi.value.field(ty, 0, 8);
2336	const res_lo64_vi = try res_lo64_it.only(isel);
2337	try call.returnLiveIn(isel, res_lo64_vi.?, .r0);
2338	},
2339	}
2340	const skip_label = isel.instructions.items.len;
2341	try isel.global_relocs.append(gpa, .{
2342	.name = switch (bits) {
2343	else => unreachable,
2344	16 => "__addhf3",
2345	32 => "__addsf3",
2346	64 => "__adddf3",
2347	80 => "__addxf3",
2348	128 => "__addtf3",
2349	},
2350	.reloc = .{ .label = @intCast(isel.instructions.items.len) },
2351	});
2352	try isel.emit(.bl(0));
2353	const rhs_vi = try isel.use(bin_op.rhs);
2354	switch (bits) {
2355	else => unreachable,
2356	80 => {
2357	const lhs_lo64_ra: Register.Alias = .r0;
2358	const lhs_hi16_ra: Register.Alias = .r1;
2359	const rhs_lo64_ra: Register.Alias = .r2;
2360	const rhs_hi16_ra: Register.Alias = .r3;
2361	const temp_ra: Register.Alias = .r4;
2362	var rhs_hi16_it = rhs_vi.field(ty, 8, 8);
2363	const rhs_hi16_vi = try rhs_hi16_it.only(isel);
2364	try call.paramLiveOut(isel, rhs_hi16_vi.?, rhs_hi16_ra);
2365	var rhs_lo64_it = rhs_vi.field(ty, 0, 8);
2366	const rhs_lo64_vi = try rhs_lo64_it.only(isel);
2367	try call.paramLiveOut(isel, rhs_lo64_vi.?, rhs_lo64_ra);
2368	try isel.emit(.cbz(
2369	temp_ra.x(),
2370	@intCast((isel.instructions.items.len + 1 - skip_label) << 2),
2371	));
2372	try isel.emit(.orr(temp_ra.x(), lhs_lo64_ra.x(), .{ .shifted_register = .{
2373	.register = lhs_hi16_ra.x(),
2374	.shift = .{ .lsl = 64 - 15 },
2375	} }));
2376	try isel.emit(.tbz(
2377	temp_ra.w(),
2378	15,
2379	@intCast((isel.instructions.items.len + 1 - skip_label) << 2),
2380	));
2381	try isel.emit(.eor(temp_ra.w(), lhs_hi16_ra.w(), .{
2382	.register = rhs_hi16_ra.w(),
2383	}));
2384	},
2385	128 => {
2386	const lhs_ra: Register.Alias = .v0;
2387	const rhs_ra: Register.Alias = .v1;
2388	const temp1_ra: Register.Alias = .r0;
2389	const temp2_ra: Register.Alias = .r1;
2390	try call.paramLiveOut(isel, rhs_vi, rhs_ra);
2391	try isel.emit(.@"b."(
2392	.pl,
2393	@intCast((isel.instructions.items.len + 1 - skip_label) << 2),
2394	));
2395	try isel.emit(.cbz(
2396	temp1_ra.x(),
2397	@intCast((isel.instructions.items.len + 1 - skip_label) << 2),
2398	));
2399	try isel.emit(.orr(temp1_ra.x(), temp1_ra.x(), .{ .shifted_register = .{
2400	.register = temp2_ra.x(),
2401	.shift = .{ .lsl = 1 },
2402	} }));
2403	try isel.emit(.fmov(temp1_ra.x(), .{
2404	.register = rhs_ra.d(),
2405	}));
2406	try isel.emit(.tbz(
2407	temp1_ra.x(),
2408	63,
2409	@intCast((isel.instructions.items.len + 1 - skip_label) << 2),
2410	));
2411	try isel.emit(.eor(temp1_ra.x(), temp1_ra.x(), .{
2412	.register = temp2_ra.x(),
2413	}));
2414	try isel.emit(.fmov(temp2_ra.x(), .{
2415	.register = rhs_ra.@"d[]"(1),
2416	}));
2417	try isel.emit(.fmov(temp1_ra.x(), .{
2418	.register = lhs_ra.@"d[]"(1),
2419	}));
2420	},
2421	}
2422	try call.finishReturn(isel);
2423	},
2424	},
2425	}
2426
2427	try call.prepareCallee(isel);
2428	try isel.global_relocs.append(gpa, .{
2429	.name = switch (bits) {
2430	else => unreachable,
2431	16 => "__fmodh",
2432	32 => "fmodf",
2433	64 => "fmod",
2434	80 => "__fmodx",
2435	128 => "fmodf128",
2436	},
2437	.reloc = .{ .label = @intCast(isel.instructions.items.len) },
2438	});
2439	try isel.emit(.bl(0));
2440	try call.finishCallee(isel);
2441
2442	try call.prepareParams(isel);
2443	const lhs_vi = try isel.use(bin_op.lhs);
2444	const rhs_vi = try isel.use(bin_op.rhs);
2445	switch (bits) {
2446	else => unreachable,
2447	16, 32, 64, 128 => {
2448	try call.paramLiveOut(isel, rhs_vi, .v1);
2449	try call.paramLiveOut(isel, lhs_vi, .v0);
2450	},
2451	80 => {
2452	var rhs_hi16_it = rhs_vi.field(ty, 8, 8);
2453	const rhs_hi16_vi = try rhs_hi16_it.only(isel);
2454	try call.paramLiveOut(isel, rhs_hi16_vi.?, .r3);
2455	var rhs_lo64_it = rhs_vi.field(ty, 0, 8);
2456	const rhs_lo64_vi = try rhs_lo64_it.only(isel);
2457	try call.paramLiveOut(isel, rhs_lo64_vi.?, .r2);
2458	var lhs_hi16_it = lhs_vi.field(ty, 8, 8);
2459	const lhs_hi16_vi = try lhs_hi16_it.only(isel);
2460	try call.paramLiveOut(isel, lhs_hi16_vi.?, .r1);
2461	var lhs_lo64_it = lhs_vi.field(ty, 0, 8);
2462	const lhs_lo64_vi = try lhs_lo64_it.only(isel);
2463	try call.paramLiveOut(isel, lhs_lo64_vi.?, .r0);
2464	},
2465	}
2466	try call.finishParams(isel);
2467	}
2468	}
2469	if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
2470	},
2471	.ptr_add, .ptr_sub => |air_tag| {
2472	if (isel.live_values.fetchRemove(air.inst_index)) |res_vi| unused: {
2473	defer res_vi.value.deref(isel);
2474	const res_ra = try res_vi.value.defReg(isel) orelse break :unused;
2475
2476	const ty_pl = air.data(air.inst_index).ty_pl;
2477	const bin_op = isel.air.extraData(Air.Bin, ty_pl.payload).data;
2478	const elem_size = ty_pl.ty.childType(zcu).abiSize(zcu);
2479
2480	const base_vi = try isel.use(bin_op.lhs);
2481	var base_part_it = base_vi.field(ty_pl.ty, 0, 8);
2482	const base_part_vi = try base_part_it.only(isel);
2483	const base_part_mat = try base_part_vi.?.matReg(isel);
2484	const index_vi = try isel.use(bin_op.rhs);
2485	try isel.elemPtr(res_ra, base_part_mat.ra, switch (air_tag) {
2486	else => unreachable,
2487	.ptr_add => .add,
2488	.ptr_sub => .sub,
2489	}, elem_size, index_vi);
2490	try base_part_mat.finish(isel);
2491	}
2492	if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
2493	},
2494	.max, .min => |air_tag| {
2495	if (isel.live_values.fetchRemove(air.inst_index)) |res_vi| unused: {
2496	defer res_vi.value.deref(isel);
2497
2498	const bin_op = air.data(air.inst_index).bin_op;
2499	const ty = isel.air.typeOf(bin_op.lhs, ip);
2500	if (!ty.isRuntimeFloat()) {
2501	if (!ty.isAbiInt(zcu)) return isel.fail("bad {t} {f}", .{ air_tag, isel.fmtType(ty) });
2502	const int_info = ty.intInfo(zcu);
2503	if (int_info.bits > 64) return isel.fail("too big {t} {f}", .{ air_tag, isel.fmtType(ty) });
2504
2505	const res_ra = try res_vi.value.defReg(isel) orelse break :unused;
2506	const lhs_vi = try isel.use(bin_op.lhs);
2507	const rhs_vi = try isel.use(bin_op.rhs);
2508	const lhs_mat = try lhs_vi.matReg(isel);
2509	const rhs_mat = try rhs_vi.matReg(isel);
2510	const cond: codegen.aarch64.encoding.ConditionCode = switch (air_tag) {
2511	else => unreachable,
2512	.max => switch (int_info.signedness) {
2513	.signed => .ge,
2514	.unsigned => .hs,
2515	},
2516	.min => switch (int_info.signedness) {
2517	.signed => .lt,
2518	.unsigned => .lo,
2519	},
2520	};
2521	switch (int_info.bits) {
2522	else => unreachable,
2523	1...32 => {
2524	try isel.emit(.csel(res_ra.w(), lhs_mat.ra.w(), rhs_mat.ra.w(), cond));
2525	try isel.emit(.subs(.wzr, lhs_mat.ra.w(), .{ .register = rhs_mat.ra.w() }));
2526	},
2527	33...64 => {
2528	try isel.emit(.csel(res_ra.x(), lhs_mat.ra.x(), rhs_mat.ra.x(), cond));
2529	try isel.emit(.subs(.xzr, lhs_mat.ra.x(), .{ .register = rhs_mat.ra.x() }));
2530	},
2531	}
2532	try rhs_mat.finish(isel);
2533	try lhs_mat.finish(isel);
2534	} else switch (ty.floatBits(isel.target)) {
2535	else => unreachable,
2536	16, 32, 64 => |bits| {
2537	const res_ra = try res_vi.value.defReg(isel) orelse break :unused;
2538	const need_fcvt = switch (bits) {
2539	else => unreachable,
2540	16 => !isel.target.cpu.has(.aarch64, .fullfp16),
2541	32, 64 => false,
2542	};
2543	if (need_fcvt) try isel.emit(.fcvt(res_ra.h(), res_ra.s()));
2544	const lhs_vi = try isel.use(bin_op.lhs);
2545	const rhs_vi = try isel.use(bin_op.rhs);
2546	const lhs_mat = try lhs_vi.matReg(isel);
2547	const rhs_mat = try rhs_vi.matReg(isel);
2548	const lhs_ra = if (need_fcvt) try isel.allocVecReg() else lhs_mat.ra;
2549	defer if (need_fcvt) isel.freeReg(lhs_ra);
2550	const rhs_ra = if (need_fcvt) try isel.allocVecReg() else rhs_mat.ra;
2551	defer if (need_fcvt) isel.freeReg(rhs_ra);
2552	try isel.emit(bits: switch (bits) {
2553	else => unreachable,
2554	16 => if (need_fcvt) continue :bits 32 else switch (air_tag) {
2555	else => unreachable,
2556	.max => .fmaxnm(res_ra.h(), lhs_ra.h(), rhs_ra.h()),
2557	.min => .fminnm(res_ra.h(), lhs_ra.h(), rhs_ra.h()),
2558	},
2559	32 => switch (air_tag) {
2560	else => unreachable,
2561	.max => .fmaxnm(res_ra.s(), lhs_ra.s(), rhs_ra.s()),
2562	.min => .fminnm(res_ra.s(), lhs_ra.s(), rhs_ra.s()),
2563	},
2564	64 => switch (air_tag) {
2565	else => unreachable,
2566	.max => .fmaxnm(res_ra.d(), lhs_ra.d(), rhs_ra.d()),
2567	.min => .fminnm(res_ra.d(), lhs_ra.d(), rhs_ra.d()),
2568	},
2569	});
2570	if (need_fcvt) {
2571	try isel.emit(.fcvt(rhs_ra.s(), rhs_mat.ra.h()));
2572	try isel.emit(.fcvt(lhs_ra.s(), lhs_mat.ra.h()));
2573	}
2574	try rhs_mat.finish(isel);
2575	try lhs_mat.finish(isel);
2576	},
2577	80, 128 => |bits| {
2578	try call.prepareReturn(isel);
2579	switch (bits) {
2580	else => unreachable,
2581	16, 32, 64, 128 => try call.returnLiveIn(isel, res_vi.value, .v0),
2582	80 => {
2583	var res_hi16_it = res_vi.value.field(ty, 8, 8);
2584	const res_hi16_vi = try res_hi16_it.only(isel);
2585	try call.returnLiveIn(isel, res_hi16_vi.?, .r1);
2586	var res_lo64_it = res_vi.value.field(ty, 0, 8);
2587	const res_lo64_vi = try res_lo64_it.only(isel);
2588	try call.returnLiveIn(isel, res_lo64_vi.?, .r0);
2589	},
2590	}
2591	try call.finishReturn(isel);
2592
2593	try call.prepareCallee(isel);
2594	try isel.global_relocs.append(gpa, .{
2595	.name = switch (air_tag) {
2596	else => unreachable,
2597	.max => switch (bits) {
2598	else => unreachable,
2599	16 => "__fmaxh",
2600	32 => "fmaxf",
2601	64 => "fmax",
2602	80 => "__fmaxx",
2603	128 => "fmaxf128",
2604	},
2605	.min => switch (bits) {
2606	else => unreachable,
2607	16 => "__fminh",
2608	32 => "fminf",
2609	64 => "fmin",
2610	80 => "__fminx",
2611	128 => "fminf128",
2612	},
2613	},
2614	.reloc = .{ .label = @intCast(isel.instructions.items.len) },
2615	});
2616	try isel.emit(.bl(0));
2617	try call.finishCallee(isel);
2618
2619	try call.prepareParams(isel);
2620	const lhs_vi = try isel.use(bin_op.lhs);
2621	const rhs_vi = try isel.use(bin_op.rhs);
2622	switch (bits) {
2623	else => unreachable,
2624	16, 32, 64, 128 => {
2625	try call.paramLiveOut(isel, rhs_vi, .v1);
2626	try call.paramLiveOut(isel, lhs_vi, .v0);
2627	},
2628	80 => {
2629	var rhs_hi16_it = rhs_vi.field(ty, 8, 8);
2630	const rhs_hi16_vi = try rhs_hi16_it.only(isel);
2631	try call.paramLiveOut(isel, rhs_hi16_vi.?, .r3);
2632	var rhs_lo64_it = rhs_vi.field(ty, 0, 8);
2633	const rhs_lo64_vi = try rhs_lo64_it.only(isel);
2634	try call.paramLiveOut(isel, rhs_lo64_vi.?, .r2);
2635	var lhs_hi16_it = lhs_vi.field(ty, 8, 8);
2636	const lhs_hi16_vi = try lhs_hi16_it.only(isel);
2637	try call.paramLiveOut(isel, lhs_hi16_vi.?, .r1);
2638	var lhs_lo64_it = lhs_vi.field(ty, 0, 8);
2639	const lhs_lo64_vi = try lhs_lo64_it.only(isel);
2640	try call.paramLiveOut(isel, lhs_lo64_vi.?, .r0);
2641	},
2642	}
2643	try call.finishParams(isel);
2644	},
2645	}
2646	}
2647	if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
2648	},
2649	.add_with_overflow, .sub_with_overflow => |air_tag| {
2650	if (isel.live_values.fetchRemove(air.inst_index)) |res_vi| {
2651	defer res_vi.value.deref(isel);
2652
2653	const ty_pl = air.data(air.inst_index).ty_pl;
2654	const bin_op = isel.air.extraData(Air.Bin, ty_pl.payload).data;
2655	const ty = isel.air.typeOf(bin_op.lhs, ip);
2656	const lhs_vi = try isel.use(bin_op.lhs);
2657	const rhs_vi = try isel.use(bin_op.rhs);
2658	const ty_size = lhs_vi.size(isel);
2659	var overflow_it = res_vi.value.field(ty_pl.ty, ty_size, 1);
2660	const overflow_vi = try overflow_it.only(isel);
2661	var wrapped_it = res_vi.value.field(ty_pl.ty, 0, ty_size);
2662	const wrapped_vi = try wrapped_it.only(isel);
2663	try wrapped_vi.?.addOrSubtract(isel, ty, lhs_vi, switch (air_tag) {
2664	else => unreachable,
2665	.add_with_overflow => .add,
2666	.sub_with_overflow => .sub,
2667	}, rhs_vi, .{
2668	.overflow = if (try overflow_vi.?.defReg(isel)) |overflow_ra| .{ .ra = overflow_ra } else .wrap,
2669	});
2670	}
2671	if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
2672	},
2673	.alloc, .ret_ptr => |air_tag| {
2674	if (isel.live_values.fetchRemove(air.inst_index)) |ptr_vi| unused: {
2675	defer ptr_vi.value.deref(isel);
2676	switch (air_tag) {
2677	else => unreachable,
2678	.alloc => {},
2679	.ret_ptr => if (isel.live_values.get(Block.main)) |ret_vi| switch (ret_vi.parent(isel)) {
2680	.unallocated, .stack_slot => {},
2681	.value, .constant => unreachable,
2682	.address => break :unused,
2683	},
2684	}
2685	const ptr_ra = try ptr_vi.value.defReg(isel) orelse break :unused;
2686
2687	const ty = air.data(air.inst_index).ty;
2688	const slot_size = ty.childType(zcu).abiSize(zcu);
2689	const slot_align = ty.ptrAlignment(zcu);
2690	const slot_offset = slot_align.forward(isel.stack_size);
2691	isel.stack_size = @intCast(slot_offset + slot_size);
2692	const lo12: u12 = @truncate(slot_offset >> 0);
2693	const hi12: u12 = @intCast(slot_offset >> 12);
2694	if (hi12 > 0) try isel.emit(.add(
2695	ptr_ra.x(),
2696	if (lo12 > 0) ptr_ra.x() else .sp,
2697	.{ .shifted_immediate = .{ .immediate = hi12, .lsl = .@"12" } },
2698	));
2699	if (lo12 > 0 or hi12 == 0) try isel.emit(.add(ptr_ra.x(), .sp, .{ .immediate = lo12 }));
2700	}
2701	if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
2702	},
2703	.inferred_alloc, .inferred_alloc_comptime => unreachable,
2704	.assembly => {
2705	const ty_pl = air.data(air.inst_index).ty_pl;
2706	const unwrapped_asm = isel.air.unwrapAsm(air.inst_index);
2707	const inputs = unwrapped_asm.inputs;
2708
2709	var as: codegen.aarch64.Assemble = .{
2710	.source = undefined,
2711	.operands = .empty,
2712	};
2713	defer as.operands.deinit(gpa);
2714
2715	var it = unwrapped_asm.iterateOutputs();
2716	while (it.next()) |output| {
2717	const constraint = output.constraint;
2718	const name = output.name;
2719
2720	switch (output.operand) {
2721	else => return isel.fail("invalid constraint: '{s}'", .{constraint}),
2722	.none => if (std.mem.startsWith(u8, constraint, "={") and std.mem.endsWith(u8, constraint, "}")) {
2723	const output_reg = Register.parse(constraint["={".len .. constraint.len - "}".len]) orelse
2724	return isel.fail("invalid constraint: '{s}'", .{constraint});
2725	const output_ra = output_reg.alias;
2726	if (isel.live_values.fetchRemove(air.inst_index)) |output_vi| {
2727	defer output_vi.value.deref(isel);
2728	try output_vi.value.defLiveIn(isel, output_reg.alias, comptime &.initFill(.free));
2729	isel.freeReg(output_ra);
2730	}
2731	if (!std.mem.eql(u8, name, "_")) {
2732	const operand_gop = try as.operands.getOrPut(gpa, name);
2733	if (operand_gop.found_existing) return isel.fail("duplicate output name: '{s}'", .{name});
2734	operand_gop.value_ptr.* = .{ .register = switch (ty_pl.ty.abiSize(zcu)) {
2735	0 => unreachable,
2736	1...4 => output_ra.w(),
2737	5...8 => output_ra.x(),
2738	else => return isel.fail("too big output type: '{f}'", .{isel.fmtType(ty_pl.ty)}),
2739	} };
2740	}
2741	} else if (std.mem.eql(u8, constraint, "=r")) {
2742	const output_ra = if (isel.live_values.fetchRemove(air.inst_index)) |output_vi| output_ra: {
2743	defer output_vi.value.deref(isel);
2744	break :output_ra try output_vi.value.defReg(isel) orelse try isel.allocIntReg();
2745	} else try isel.allocIntReg();
2746	if (!std.mem.eql(u8, name, "_")) {
2747	const operand_gop = try as.operands.getOrPut(gpa, name);
2748	if (operand_gop.found_existing) return isel.fail("duplicate output name: '{s}'", .{name});
2749	operand_gop.value_ptr.* = .{ .register = switch (ty_pl.ty.abiSize(zcu)) {
2750	0 => unreachable,
2751	1...4 => output_ra.w(),
2752	5...8 => output_ra.x(),
2753	else => return isel.fail("too big output type: '{f}'", .{isel.fmtType(ty_pl.ty)}),
2754	} };
2755	}
2756	} else return isel.fail("invalid constraint: '{s}'", .{constraint}),
2757	}
2758	}
2759
2760	const input_mats = try gpa.alloc(Value.Materialize, inputs.len);
2761	var index: u32 = 0;
2762	defer gpa.free(input_mats);
2763	it = unwrapped_asm.iterateInputs();
2764	while (it.next()) |input| : (index += 1) {
2765	const constraint = input.constraint;
2766	const name = input.name;
2767
2768	if (std.mem.startsWith(u8, constraint, "{") and std.mem.endsWith(u8, constraint, "}")) {
2769	const input_reg = Register.parse(constraint["{".len .. constraint.len - "}".len]) orelse
2770	return isel.fail("invalid constraint: '{s}'", .{constraint});
2771	input_mats[index] = .{ .vi = try isel.use(input.operand), .ra = input_reg.alias };
2772	if (!std.mem.eql(u8, name, "_")) {
2773	const operand_gop = try as.operands.getOrPut(gpa, name);
2774	if (operand_gop.found_existing) return isel.fail("duplicate input name: '{s}'", .{name});
2775	const input_ty = isel.air.typeOf(input.operand, ip);
2776	operand_gop.value_ptr.* = .{ .register = switch (input_ty.abiSize(zcu)) {
2777	0 => unreachable,
2778	1...4 => input_reg.alias.w(),
2779	5...8 => input_reg.alias.x(),
2780	else => return isel.fail("too big input type: '{f}'", .{
2781	isel.fmtType(isel.air.typeOf(input.operand, ip)),
2782	}),
2783	} };
2784	}
2785	} else if (std.mem.eql(u8, constraint, "r")) {
2786	const input_vi = try isel.use(input.operand);
2787	input_mats[index] = try input_vi.matReg(isel);
2788	if (!std.mem.eql(u8, name, "_")) {
2789	const operand_gop = try as.operands.getOrPut(gpa, name);
2790	if (operand_gop.found_existing) return isel.fail("duplicate input name: '{s}'", .{name});
2791	operand_gop.value_ptr.* = .{ .register = switch (input_vi.size(isel)) {
2792	0 => unreachable,
2793	1...4 => input_mats[index].ra.w(),
2794	5...8 => input_mats[index].ra.x(),
2795	else => return isel.fail("too big input type: '{f}'", .{
2796	isel.fmtType(isel.air.typeOf(input.operand, ip)),
2797	}),
2798	} };
2799	}
2800	} else if (std.mem.eql(u8, name, "_")) {
2801	input_mats[index].vi = try isel.use(input.operand);
2802	} else return isel.fail("invalid constraint: '{s}'", .{constraint});
2803	}
2804
2805	const clobbers_val: Constant = .fromInterned(unwrapped_asm.clobbers);
2806	const clobbers_ty = clobbers_val.typeOf(zcu);
2807	var clobbers_bigint_buf: Constant.BigIntSpace = undefined;
2808	const clobbers_bigint = clobbers_val.toBigInt(&clobbers_bigint_buf, zcu);
2809	for (0..clobbers_ty.structFieldCount(zcu)) |field_index| {
2810	assert(clobbers_ty.fieldType(field_index, zcu).toIntern() == .bool_type);
2811	const limb_bits = @bitSizeOf(std.math.big.Limb);
2812	if (field_index / limb_bits >= clobbers_bigint.limbs.len) continue; // field is false
2813	switch (@as(u1, @truncate(clobbers_bigint.limbs[field_index / limb_bits] >> @intCast(field_index % limb_bits)))) {
2814	0 => continue, // field is false
2815	1 => {}, // field is true
2816	}
2817	const clobber_name = clobbers_ty.structFieldName(field_index, zcu).toSlice(ip).?;
2818	if (std.mem.eql(u8, clobber_name, "memory")) continue;
2819	if (std.mem.eql(u8, clobber_name, "nzcv")) continue;
2820	const clobber_reg = Register.parse(clobber_name) orelse
2821	return isel.fail("unable to parse clobber: '{s}'", .{clobber_name});
2822	const live_vi = isel.live_registers.getPtr(clobber_reg.alias);
2823	switch (live_vi.*) {
2824	_ => {},
2825	.allocating => return isel.fail("clobbered twice: '{s}'", .{clobber_name}),
2826	.free => live_vi.* = .allocating,
2827	}
2828	}
2829	for (0..clobbers_ty.structFieldCount(zcu)) |field_index| {
2830	const limb_bits = @bitSizeOf(std.math.big.Limb);
2831	if (field_index / limb_bits >= clobbers_bigint.limbs.len) continue; // field is false
2832	switch (@as(u1, @truncate(clobbers_bigint.limbs[field_index / limb_bits] >> @intCast(field_index % limb_bits)))) {
2833	0 => continue, // field is false
2834	1 => {}, // field is true
2835	}
2836	const clobber_name = clobbers_ty.structFieldName(field_index, zcu).toSlice(ip).?;
2837	if (std.mem.eql(u8, clobber_name, "memory")) continue;
2838	if (std.mem.eql(u8, clobber_name, "nzcv")) continue;
2839	const clobber_ra = Register.parse(clobber_name).?.alias;
2840	const live_vi = isel.live_registers.getPtr(clobber_ra);
2841	switch (live_vi.*) {
2842	_ => {
2843	if (!try isel.fill(clobber_ra))
2844	return isel.fail("unable to clobber: '{s}'", .{clobber_name});
2845	assert(live_vi.* == .free);
2846	live_vi.* = .allocating;
2847	},
2848	.allocating => {},
2849	.free => unreachable,
2850	}
2851	}
2852
2853	as.source = unwrapped_asm.source;
2854	const asm_start = isel.instructions.items.len;
2855	while (as.nextInstruction() catch |err| switch (err) {
2856	error.InvalidSyntax => {
2857	const remaining_source = std.mem.span(as.source);
2858	return isel.fail("unable to assemble: '{s}'", .{std.mem.trim(
2859	u8,
2860	as.source[0 .. std.mem.findScalar(u8, remaining_source, '\n') orelse remaining_source.len],
2861	&std.ascii.whitespace,
2862	)});
2863	},
2864	}) |instruction| try isel.emit(instruction);
2865	std.mem.reverse(codegen.aarch64.encoding.Instruction, isel.instructions.items[asm_start..]);
2866
2867	it = unwrapped_asm.iterateInputs();
2868	index = 0;
2869	while (it.next()) |input| : (index += 1) {
2870	const constraint = input.constraint;
2871	const name = input.name;
2872
2873	if (std.mem.startsWith(u8, constraint, "{") and std.mem.endsWith(u8, constraint, "}")) {
2874	try input_mats[index].vi.liveOut(isel, input_mats[index].ra);
2875	} else if (std.mem.eql(u8, constraint, "r")) {
2876	try input_mats[index].finish(isel);
2877	} else if (std.mem.eql(u8, name, "_")) {
2878	try input_mats[index].vi.mat(isel);
2879	} else unreachable;
2880	}
2881
2882	for (0..clobbers_ty.structFieldCount(zcu)) |field_index| {
2883	const limb_bits = @bitSizeOf(std.math.big.Limb);
2884	if (field_index / limb_bits >= clobbers_bigint.limbs.len) continue; // field is false
2885	switch (@as(u1, @truncate(clobbers_bigint.limbs[field_index / limb_bits] >> @intCast(field_index % limb_bits)))) {
2886	0 => continue, // field is false
2887	1 => {}, // field is true
2888	}
2889	const clobber_name = clobbers_ty.structFieldName(field_index, zcu).toSlice(ip).?;
2890	if (std.mem.eql(u8, clobber_name, "memory")) continue;
2891	if (std.mem.eql(u8, clobber_name, "cc")) continue;
2892	isel.freeReg(Register.parse(clobber_name).?.alias);
2893	}
2894
2895	if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
2896	},
2897	.bit_and, .bit_or, .xor => |air_tag| {
2898	if (isel.live_values.fetchRemove(air.inst_index)) |res_vi| {
2899	defer res_vi.value.deref(isel);
2900
2901	const bin_op = air.data(air.inst_index).bin_op;
2902	const ty = isel.air.typeOf(bin_op.lhs, ip);
2903	const int_info: std.lang.Type.Int = if (ty.toIntern() == .bool_type)
2904	.{ .signedness = .unsigned, .bits = 1 }
2905	else if (ty.isAbiInt(zcu))
2906	ty.intInfo(zcu)
2907	else
2908	return isel.fail("bad {t} {f}", .{ air_tag, isel.fmtType(ty) });
2909	if (int_info.bits > 128) return isel.fail("too big {t} {f}", .{ air_tag, isel.fmtType(ty) });
2910
2911	const lhs_vi = try isel.use(bin_op.lhs);
2912	const rhs_vi = try isel.use(bin_op.rhs);
2913	var offset = res_vi.value.size(isel);
2914	while (offset > 0) {
2915	const size = @min(offset, 8);
2916	offset -= size;
2917	var res_part_it = res_vi.value.field(ty, offset, size);
2918	const res_part_vi = try res_part_it.only(isel);
2919	const res_part_ra = try res_part_vi.?.defReg(isel) orelse continue;
2920	var lhs_part_it = lhs_vi.field(ty, offset, size);
2921	const lhs_part_vi = try lhs_part_it.only(isel);
2922	const lhs_part_mat = try lhs_part_vi.?.matReg(isel);
2923	var rhs_part_it = rhs_vi.field(ty, offset, size);
2924	const rhs_part_vi = try rhs_part_it.only(isel);
2925	const rhs_part_mat = try rhs_part_vi.?.matReg(isel);
2926	try isel.emit(switch (air_tag) {
2927	else => unreachable,
2928	.bit_and => switch (size) {
2929	else => unreachable,
2930	1, 2, 4 => .@"and"(res_part_ra.w(), lhs_part_mat.ra.w(), .{ .register = rhs_part_mat.ra.w() }),
2931	8 => .@"and"(res_part_ra.x(), lhs_part_mat.ra.x(), .{ .register = rhs_part_mat.ra.x() }),
2932	},
2933	.bit_or => switch (size) {
2934	else => unreachable,
2935	1, 2, 4 => .orr(res_part_ra.w(), lhs_part_mat.ra.w(), .{ .register = rhs_part_mat.ra.w() }),
2936	8 => .orr(res_part_ra.x(), lhs_part_mat.ra.x(), .{ .register = rhs_part_mat.ra.x() }),
2937	},
2938	.xor => switch (size) {
2939	else => unreachable,
2940	1, 2, 4 => .eor(res_part_ra.w(), lhs_part_mat.ra.w(), .{ .register = rhs_part_mat.ra.w() }),
2941	8 => .eor(res_part_ra.x(), lhs_part_mat.ra.x(), .{ .register = rhs_part_mat.ra.x() }),
2942	},
2943	});
2944	try rhs_part_mat.finish(isel);
2945	try lhs_part_mat.finish(isel);
2946	}
2947	}
2948	if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
2949	},
2950	.shr, .shr_exact, .shl, .shl_exact => |air_tag| {
2951	if (isel.live_values.fetchRemove(air.inst_index)) |res_vi| unused: {
2952	defer res_vi.value.deref(isel);
2953
2954	const bin_op = air.data(air.inst_index).bin_op;
2955	const ty = isel.air.typeOf(bin_op.lhs, ip);
2956	if (!ty.isAbiInt(zcu)) return isel.fail("bad {t} {f}", .{ air_tag, isel.fmtType(ty) });
2957	const int_info = ty.intInfo(zcu);
2958	switch (int_info.bits) {
2959	0 => unreachable,
2960	1...64 => |bits| {
2961	const res_ra = try res_vi.value.defReg(isel) orelse break :unused;
2962	switch (air_tag) {
2963	else => unreachable,
2964	.shr, .shr_exact, .shl_exact => {},
2965	.shl => switch (bits) {
2966	else => unreachable,
2967	1...31 => try isel.emit(switch (int_info.signedness) {
2968	.signed => .sbfm(res_ra.w(), res_ra.w(), .{
2969	.N = .word,
2970	.immr = 0,
2971	.imms = @intCast(bits - 1),
2972	}),
2973	.unsigned => .ubfm(res_ra.w(), res_ra.w(), .{
2974	.N = .word,
2975	.immr = 0,
2976	.imms = @intCast(bits - 1),
2977	}),
2978	}),
2979	32 => {},
2980	33...63 => try isel.emit(switch (int_info.signedness) {
2981	.signed => .sbfm(res_ra.x(), res_ra.x(), .{
2982	.N = .doubleword,
2983	.immr = 0,
2984	.imms = @intCast(bits - 1),
2985	}),
2986	.unsigned => .ubfm(res_ra.x(), res_ra.x(), .{
2987	.N = .doubleword,
2988	.immr = 0,
2989	.imms = @intCast(bits - 1),
2990	}),
2991	}),
2992	64 => {},
2993	},
2994	}
2995
2996	const lhs_vi = try isel.use(bin_op.lhs);
2997	const rhs_vi = try isel.use(bin_op.rhs);
2998	const lhs_mat = try lhs_vi.matReg(isel);
2999	const rhs_mat = try rhs_vi.matReg(isel);
3000	try isel.emit(switch (air_tag) {
3001	else => unreachable,
3002	.shr, .shr_exact => switch (bits) {
3003	else => unreachable,
3004	1...32 => switch (int_info.signedness) {
3005	.signed => .asrv(res_ra.w(), lhs_mat.ra.w(), rhs_mat.ra.w()),
3006	.unsigned => .lsrv(res_ra.w(), lhs_mat.ra.w(), rhs_mat.ra.w()),
3007	},
3008	33...64 => switch (int_info.signedness) {
3009	.signed => .asrv(res_ra.x(), lhs_mat.ra.x(), rhs_mat.ra.x()),
3010	.unsigned => .lsrv(res_ra.x(), lhs_mat.ra.x(), rhs_mat.ra.x()),
3011	},
3012	},
3013	.shl, .shl_exact => switch (bits) {
3014	else => unreachable,
3015	1...32 => .lslv(res_ra.w(), lhs_mat.ra.w(), rhs_mat.ra.w()),
3016	33...64 => .lslv(res_ra.x(), lhs_mat.ra.x(), rhs_mat.ra.x()),
3017	},
3018	});
3019	try rhs_mat.finish(isel);
3020	try lhs_mat.finish(isel);
3021	},
3022	65...128 => |bits| {
3023	var res_hi64_it = res_vi.value.field(ty, 8, 8);
3024	const res_hi64_vi = try res_hi64_it.only(isel);
3025	const res_hi64_ra = try res_hi64_vi.?.defReg(isel);
3026	var res_lo64_it = res_vi.value.field(ty, 0, 8);
3027	const res_lo64_vi = try res_lo64_it.only(isel);
3028	const res_lo64_ra = try res_lo64_vi.?.defReg(isel);
3029	if (res_hi64_ra == null and res_lo64_ra == null) break :unused;
3030	if (res_hi64_ra) |res_ra| switch (air_tag) {
3031	else => unreachable,
3032	.shr, .shr_exact, .shl_exact => {},
3033	.shl => switch (bits) {
3034	else => unreachable,
3035	65...127 => try isel.emit(switch (int_info.signedness) {
3036	.signed => .sbfm(res_ra.x(), res_ra.x(), .{
3037	.N = .doubleword,
3038	.immr = 0,
3039	.imms = @intCast(bits - 64 - 1),
3040	}),
3041	.unsigned => .ubfm(res_ra.x(), res_ra.x(), .{
3042	.N = .doubleword,
3043	.immr = 0,
3044	.imms = @intCast(bits - 64 - 1),
3045	}),
3046	}),
3047	128 => {},
3048	},
3049	};
3050
3051	const lhs_vi = try isel.use(bin_op.lhs);
3052	const lhs_hi64_mat = lhs_hi64_mat: {
3053	const res_lock: RegLock = switch (air_tag) {
3054	else => unreachable,
3055	.shr, .shr_exact => switch (int_info.signedness) {
3056	.signed => if (res_lo64_ra) |res_ra| isel.lockReg(res_ra) else .empty,
3057	.unsigned => .empty,
3058	},
3059	.shl, .shl_exact => .empty,
3060	};
3061	defer res_lock.unlock(isel);
3062	var lhs_hi64_it = lhs_vi.field(ty, 8, 8);
3063	const lhs_hi64_vi = try lhs_hi64_it.only(isel);
3064	break :lhs_hi64_mat try lhs_hi64_vi.?.matReg(isel);
3065	};
3066	var lhs_lo64_it = lhs_vi.field(ty, 0, 8);
3067	const lhs_lo64_vi = try lhs_lo64_it.only(isel);
3068	const lhs_lo64_mat = try lhs_lo64_vi.?.matReg(isel);
3069	const rhs_vi = try isel.use(bin_op.rhs);
3070	const rhs_mat = try rhs_vi.matReg(isel);
3071	const lo64_ra = lo64_ra: {
3072	const res_lock: RegLock = switch (air_tag) {
3073	else => unreachable,
3074	.shr, .shr_exact => switch (int_info.signedness) {
3075	.signed => if (res_lo64_ra) |res_ra| isel.tryLockReg(res_ra) else .empty,
3076	.unsigned => .empty,
3077	},
3078	.shl, .shl_exact => if (res_hi64_ra) |res_ra| isel.tryLockReg(res_ra) else .empty,
3079	};
3080	defer res_lock.unlock(isel);
3081	break :lo64_ra try isel.allocIntReg();
3082	};
3083	defer isel.freeReg(lo64_ra);
3084	const hi64_ra = hi64_ra: {
3085	const res_lock: RegLock = switch (air_tag) {
3086	else => unreachable,
3087	.shr, .shr_exact => if (res_lo64_ra) |res_ra| isel.tryLockReg(res_ra) else .empty,
3088	.shl, .shl_exact => .empty,
3089	};
3090	defer res_lock.unlock(isel);
3091	break :hi64_ra try isel.allocIntReg();
3092	};
3093	defer isel.freeReg(hi64_ra);
3094	switch (air_tag) {
3095	else => unreachable,
3096	.shr, .shr_exact => {
3097	if (res_hi64_ra) |res_ra| switch (int_info.signedness) {
3098	.signed => {
3099	try isel.emit(.csel(res_ra.x(), hi64_ra.x(), lo64_ra.x(), .eq));
3100	try isel.emit(.sbfm(lo64_ra.x(), lhs_hi64_mat.ra.x(), .{
3101	.N = .doubleword,
3102	.immr = @intCast(bits - 64 - 1),
3103	.imms = @intCast(bits - 64 - 1),
3104	}));
3105	},
3106	.unsigned => try isel.emit(.csel(res_ra.x(), hi64_ra.x(), .xzr, .eq)),
3107	};
3108	if (res_lo64_ra) |res_ra| try isel.emit(.csel(res_ra.x(), lo64_ra.x(), hi64_ra.x(), .eq));
3109	switch (int_info.signedness) {
3110	.signed => try isel.emit(.asrv(hi64_ra.x(), lhs_hi64_mat.ra.x(), rhs_mat.ra.x())),
3111	.unsigned => try isel.emit(.lsrv(hi64_ra.x(), lhs_hi64_mat.ra.x(), rhs_mat.ra.x())),
3112	}
3113	},
3114	.shl, .shl_exact => {
3115	if (res_lo64_ra) |res_ra| try isel.emit(.csel(res_ra.x(), lo64_ra.x(), .xzr, .eq));
3116	if (res_hi64_ra) |res_ra| try isel.emit(.csel(res_ra.x(), hi64_ra.x(), lo64_ra.x(), .eq));
3117	try isel.emit(.lslv(lo64_ra.x(), lhs_lo64_mat.ra.x(), rhs_mat.ra.x()));
3118	},
3119	}
3120	try isel.emit(.ands(.wzr, rhs_mat.ra.w(), .{ .immediate = .{ .N = .word, .immr = 32 - 6, .imms = 0 } }));
3121	switch (air_tag) {
3122	else => unreachable,
3123	.shr, .shr_exact => if (res_lo64_ra) |_| {
3124	try isel.emit(.orr(
3125	lo64_ra.x(),
3126	lo64_ra.x(),
3127	.{ .shifted_register = .{ .register = hi64_ra.x(), .shift = .{ .lsl = 1 } } },
3128	));
3129	try isel.emit(.lslv(hi64_ra.x(), lhs_hi64_mat.ra.x(), hi64_ra.x()));
3130	try isel.emit(.lsrv(lo64_ra.x(), lhs_lo64_mat.ra.x(), rhs_mat.ra.x()));
3131	try isel.emit(.orn(hi64_ra.w(), .wzr, .{ .register = rhs_mat.ra.w() }));
3132	},
3133	.shl, .shl_exact => if (res_hi64_ra) |_| {
3134	try isel.emit(.orr(
3135	hi64_ra.x(),
3136	hi64_ra.x(),
3137	.{ .shifted_register = .{ .register = lo64_ra.x(), .shift = .{ .lsr = 1 } } },
3138	));
3139	try isel.emit(.lsrv(lo64_ra.x(), lhs_lo64_mat.ra.x(), lo64_ra.x()));
3140	try isel.emit(.lslv(hi64_ra.x(), lhs_hi64_mat.ra.x(), rhs_mat.ra.x()));
3141	try isel.emit(.orn(lo64_ra.w(), .wzr, .{ .register = rhs_mat.ra.w() }));
3142	},
3143	}
3144	try rhs_mat.finish(isel);
3145	try lhs_lo64_mat.finish(isel);
3146	try lhs_hi64_mat.finish(isel);
3147	break :unused;
3148	},
3149	else => return isel.fail("too big {t} {f}", .{ air_tag, isel.fmtType(ty) }),
3150	}
3151	}
3152	if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
3153	},
3154	.not => |air_tag| {
3155	if (isel.live_values.fetchRemove(air.inst_index)) |res_vi| {
3156	defer res_vi.value.deref(isel);
3157
3158	const ty_op = air.data(air.inst_index).ty_op;
3159	const ty = ty_op.ty;
3160	const int_info: std.lang.Type.Int = int_info: {
3161	if (ty_op.ty.ip_index == .bool_type) break :int_info .{ .signedness = .unsigned, .bits = 1 };
3162	if (!ty.isAbiInt(zcu)) return isel.fail("bad {t} {f}", .{ air_tag, isel.fmtType(ty) });
3163	break :int_info ty.intInfo(zcu);
3164	};
3165	if (int_info.bits > 128) return isel.fail("too big {t} {f}", .{ air_tag, isel.fmtType(ty) });
3166
3167	const src_vi = try isel.use(ty_op.operand);
3168	var offset = res_vi.value.size(isel);
3169	while (offset > 0) {
3170	const size = @min(offset, 8);
3171	offset -= size;
3172	var res_part_it = res_vi.value.field(ty, offset, size);
3173	const res_part_vi = try res_part_it.only(isel);
3174	const res_part_ra = try res_part_vi.?.defReg(isel) orelse continue;
3175	var src_part_it = src_vi.field(ty, offset, size);
3176	const src_part_vi = try src_part_it.only(isel);
3177	const src_part_mat = try src_part_vi.?.matReg(isel);
3178	try isel.emit(switch (int_info.signedness) {
3179	.signed => switch (size) {
3180	else => unreachable,
3181	1, 2, 4 => .orn(res_part_ra.w(), .wzr, .{ .register = src_part_mat.ra.w() }),
3182	8 => .orn(res_part_ra.x(), .xzr, .{ .register = src_part_mat.ra.x() }),
3183	},
3184	.unsigned => switch (@min(int_info.bits - 8 * offset, 64)) {
3185	else => unreachable,
3186	1...31 => |bits| .eor(res_part_ra.w(), src_part_mat.ra.w(), .{ .immediate = .{
3187	.N = .word,
3188	.immr = 0,
3189	.imms = @intCast(bits - 1),
3190	} }),
3191	32 => .orn(res_part_ra.w(), .wzr, .{ .register = src_part_mat.ra.w() }),
3192	33...63 => |bits| .eor(res_part_ra.x(), src_part_mat.ra.x(), .{ .immediate = .{
3193	.N = .doubleword,
3194	.immr = 0,
3195	.imms = @intCast(bits - 1),
3196	} }),
3197	64 => .orn(res_part_ra.x(), .xzr, .{ .register = src_part_mat.ra.x() }),
3198	},
3199	});
3200	try src_part_mat.finish(isel);
3201	}
3202	}
3203	if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
3204	},
3205	.bit_cast,
3206	.bit_cast_safe, // TODO safety check
3207	.ptr_cast,
3208	.ptr_from_int,
3209	.int_from_ptr,
3210	.error_cast,
3211	.error_from_int,
3212	.int_from_error,
3213	.union_from_enum,
3214	=> |air_tag| {
3215	if (isel.live_values.fetchRemove(air.inst_index)) |dst_vi| unused: {
3216	defer dst_vi.value.deref(isel);
3217	const ty_op = air.data(air.inst_index).ty_op;
3218	const dst_ty = ty_op.ty;
3219	const dst_tag = dst_ty.zigTypeTag(zcu);
3220	const src_ty = isel.air.typeOf(ty_op.operand, ip);
3221	const src_tag = src_ty.zigTypeTag(zcu);
3222	if (dst_ty.isAbiInt(zcu) and (src_tag == .bool or src_ty.isAbiInt(zcu))) {
3223	const dst_int_info = dst_ty.intInfo(zcu);
3224	const src_int_info: std.lang.Type.Int = if (src_tag == .bool) .{ .signedness = undefined, .bits = 1 } else src_ty.intInfo(zcu);
3225	assert(dst_int_info.bits == src_int_info.bits);
3226	if (dst_tag != .@"struct" and src_tag != .@"struct" and src_tag != .bool and dst_int_info.signedness == src_int_info.signedness) {
3227	try dst_vi.value.move(isel, ty_op.operand);
3228	} else switch (dst_int_info.bits) {
3229	0 => unreachable,
3230	1...31 => |dst_bits| {
3231	const dst_ra = try dst_vi.value.defReg(isel) orelse break :unused;
3232	const src_vi = try isel.use(ty_op.operand);
3233	const src_mat = try src_vi.matReg(isel);
3234	try isel.emit(switch (dst_int_info.signedness) {
3235	.signed => .sbfm(dst_ra.w(), src_mat.ra.w(), .{
3236	.N = .word,
3237	.immr = 0,
3238	.imms = @intCast(dst_bits - 1),
3239	}),
3240	.unsigned => .ubfm(dst_ra.w(), src_mat.ra.w(), .{
3241	.N = .word,
3242	.immr = 0,
3243	.imms = @intCast(dst_bits - 1),
3244	}),
3245	});
3246	try src_mat.finish(isel);
3247	},
3248	32 => try dst_vi.value.move(isel, ty_op.operand),
3249	33...63 => |dst_bits| {
3250	const dst_ra = try dst_vi.value.defReg(isel) orelse break :unused;
3251	const src_vi = try isel.use(ty_op.operand);
3252	const src_mat = try src_vi.matReg(isel);
3253	try isel.emit(switch (dst_int_info.signedness) {
3254	.signed => .sbfm(dst_ra.x(), src_mat.ra.x(), .{
3255	.N = .doubleword,
3256	.immr = 0,
3257	.imms = @intCast(dst_bits - 1),
3258	}),
3259	.unsigned => .ubfm(dst_ra.x(), src_mat.ra.x(), .{
3260	.N = .doubleword,
3261	.immr = 0,
3262	.imms = @intCast(dst_bits - 1),
3263	}),
3264	});
3265	try src_mat.finish(isel);
3266	},
3267	64 => try dst_vi.value.move(isel, ty_op.operand),
3268	65...127 => |dst_bits| {
3269	const src_vi = try isel.use(ty_op.operand);
3270	var dst_hi64_it = dst_vi.value.field(dst_ty, 8, 8);
3271	const dst_hi64_vi = try dst_hi64_it.only(isel);
3272	if (try dst_hi64_vi.?.defReg(isel)) |dst_hi64_ra| {
3273	var src_hi64_it = src_vi.field(src_ty, 8, 8);
3274	const src_hi64_vi = try src_hi64_it.only(isel);
3275	const src_hi64_mat = try src_hi64_vi.?.matReg(isel);
3276	try isel.emit(switch (dst_int_info.signedness) {
3277	.signed => .sbfm(dst_hi64_ra.x(), src_hi64_mat.ra.x(), .{
3278	.N = .doubleword,
3279	.immr = 0,
3280	.imms = @intCast(dst_bits - 64 - 1),
3281	}),
3282	.unsigned => .ubfm(dst_hi64_ra.x(), src_hi64_mat.ra.x(), .{
3283	.N = .doubleword,
3284	.immr = 0,
3285	.imms = @intCast(dst_bits - 64 - 1),
3286	}),
3287	});
3288	try src_hi64_mat.finish(isel);
3289	}
3290	var dst_lo64_it = dst_vi.value.field(dst_ty, 0, 8);
3291	const dst_lo64_vi = try dst_lo64_it.only(isel);
3292	if (try dst_lo64_vi.?.defReg(isel)) |dst_lo64_ra| {
3293	var src_lo64_it = src_vi.field(src_ty, 0, 8);
3294	const src_lo64_vi = try src_lo64_it.only(isel);
3295	try src_lo64_vi.?.liveOut(isel, dst_lo64_ra);
3296	}
3297	},
3298	128 => try dst_vi.value.move(isel, ty_op.operand),
3299	else => return isel.fail("bad {t} {f} {f}", .{ air_tag, isel.fmtType(dst_ty), isel.fmtType(src_ty) }),
3300	}
3301	} else if ((dst_ty.isPtrAtRuntime(zcu) or dst_ty.isAbiInt(zcu)) and (src_ty.isPtrAtRuntime(zcu) or src_ty.isAbiInt(zcu))) {
3302	try dst_vi.value.move(isel, ty_op.operand);
3303	} else if (dst_ty.isSliceAtRuntime(zcu) and src_ty.isSliceAtRuntime(zcu)) {
3304	try dst_vi.value.move(isel, ty_op.operand);
3305	} else if (dst_tag == .error_union and src_tag == .error_union) {
3306	assert(dst_ty.errorUnionSet(zcu).hasRuntimeBits(zcu) ==
3307	src_ty.errorUnionSet(zcu).hasRuntimeBits(zcu));
3308	if (dst_ty.errorUnionPayload(zcu).toIntern() == src_ty.errorUnionPayload(zcu).toIntern()) {
3309	try dst_vi.value.move(isel, ty_op.operand);
3310	} else return isel.fail("bad {t} {f} {f}", .{ air_tag, isel.fmtType(dst_ty), isel.fmtType(src_ty) });
3311	} else if (dst_tag == .float and src_tag == .float) {
3312	assert(dst_ty.floatBits(isel.target) == src_ty.floatBits(isel.target));
3313	try dst_vi.value.move(isel, ty_op.operand);
3314	} else if (dst_ty.isAbiInt(zcu) and src_tag == .float) {
3315	const dst_int_info = dst_ty.intInfo(zcu);
3316	assert(dst_int_info.bits == src_ty.floatBits(isel.target));
3317	switch (dst_int_info.bits) {
3318	else => unreachable,
3319	16 => {
3320	const dst_ra = try dst_vi.value.defReg(isel) orelse break :unused;
3321	const src_vi = try isel.use(ty_op.operand);
3322	const src_mat = try src_vi.matReg(isel);
3323	switch (dst_int_info.signedness) {
3324	.signed => try isel.emit(.smov(dst_ra.w(), src_mat.ra.@"h[]"(0))),
3325	.unsigned => try isel.emit(if (isel.target.cpu.has(.aarch64, .fullfp16))
3326	.fmov(dst_ra.w(), .{ .register = src_mat.ra.h() })
3327	else
3328	.umov(dst_ra.w(), src_mat.ra.@"h[]"(0))),
3329	}
3330	try src_mat.finish(isel);
3331	},
3332	32 => {
3333	const dst_ra = try dst_vi.value.defReg(isel) orelse break :unused;
3334	const src_vi = try isel.use(ty_op.operand);
3335	const src_mat = try src_vi.matReg(isel);
3336	try isel.emit(.fmov(dst_ra.w(), .{ .register = src_mat.ra.s() }));
3337	try src_mat.finish(isel);
3338	},
3339	64 => {
3340	const dst_ra = try dst_vi.value.defReg(isel) orelse break :unused;
3341	const src_vi = try isel.use(ty_op.operand);
3342	const src_mat = try src_vi.matReg(isel);
3343	try isel.emit(.fmov(dst_ra.x(), .{ .register = src_mat.ra.d() }));
3344	try src_mat.finish(isel);
3345	},
3346	80 => switch (dst_int_info.signedness) {
3347	.signed => {
3348	const src_vi = try isel.use(ty_op.operand);
3349	var dst_hi16_it = dst_vi.value.field(dst_ty, 8, 8);
3350	const dst_hi16_vi = try dst_hi16_it.only(isel);
3351	if (try dst_hi16_vi.?.defReg(isel)) |dst_hi16_ra| {
3352	var src_hi16_it = src_vi.field(src_ty, 8, 8);
3353	const src_hi16_vi = try src_hi16_it.only(isel);
3354	const src_hi16_mat = try src_hi16_vi.?.matReg(isel);
3355	try isel.emit(.sbfm(dst_hi16_ra.x(), src_hi16_mat.ra.x(), .{
3356	.N = .doubleword,
3357	.immr = 0,
3358	.imms = 16 - 1,
3359	}));
3360	try src_hi16_mat.finish(isel);
3361	}
3362	var dst_lo64_it = dst_vi.value.field(dst_ty, 0, 8);
3363	const dst_lo64_vi = try dst_lo64_it.only(isel);
3364	if (try dst_lo64_vi.?.defReg(isel)) |dst_lo64_ra| {
3365	var src_lo64_it = src_vi.field(src_ty, 0, 8);
3366	const src_lo64_vi = try src_lo64_it.only(isel);
3367	try src_lo64_vi.?.liveOut(isel, dst_lo64_ra);
3368	}
3369	},
3370	else => try dst_vi.value.move(isel, ty_op.operand),
3371	},
3372	128 => {
3373	const src_vi = try isel.use(ty_op.operand);
3374	const src_mat = try src_vi.matReg(isel);
3375	var dst_hi64_it = dst_vi.value.field(dst_ty, 8, 8);
3376	const dst_hi64_vi = try dst_hi64_it.only(isel);
3377	if (try dst_hi64_vi.?.defReg(isel)) |dst_hi64_ra| try isel.emit(.fmov(dst_hi64_ra.x(), .{ .register = src_mat.ra.@"d[]"(1) }));
3378	var dst_lo64_it = dst_vi.value.field(dst_ty, 0, 8);
3379	const dst_lo64_vi = try dst_lo64_it.only(isel);
3380	if (try dst_lo64_vi.?.defReg(isel)) |dst_lo64_ra| try isel.emit(.fmov(dst_lo64_ra.x(), .{ .register = src_mat.ra.d() }));
3381	try src_mat.finish(isel);
3382	},
3383	}
3384	} else if (dst_tag == .float and src_ty.isAbiInt(zcu)) {
3385	const src_int_info = src_ty.intInfo(zcu);
3386	assert(dst_ty.floatBits(isel.target) == src_int_info.bits);
3387	switch (src_int_info.bits) {
3388	else => unreachable,
3389	16 => {
3390	const dst_ra = try dst_vi.value.defReg(isel) orelse break :unused;
3391	const src_vi = try isel.use(ty_op.operand);
3392	const src_mat = try src_vi.matReg(isel);
3393	try isel.emit(.fmov(
3394	if (isel.target.cpu.has(.aarch64, .fullfp16)) dst_ra.h() else dst_ra.s(),
3395	.{ .register = src_mat.ra.w() },
3396	));
3397	try src_mat.finish(isel);
3398	},
3399	32 => {
3400	const dst_ra = try dst_vi.value.defReg(isel) orelse break :unused;
3401	const src_vi = try isel.use(ty_op.operand);
3402	const src_mat = try src_vi.matReg(isel);
3403	try isel.emit(.fmov(dst_ra.s(), .{ .register = src_mat.ra.w() }));
3404	try src_mat.finish(isel);
3405	},
3406	64 => {
3407	const dst_ra = try dst_vi.value.defReg(isel) orelse break :unused;
3408	const src_vi = try isel.use(ty_op.operand);
3409	const src_mat = try src_vi.matReg(isel);
3410	try isel.emit(.fmov(dst_ra.d(), .{ .register = src_mat.ra.x() }));
3411	try src_mat.finish(isel);
3412	},
3413	80 => switch (src_int_info.signedness) {
3414	.signed => {
3415	const src_vi = try isel.use(ty_op.operand);
3416	var dst_hi16_it = dst_vi.value.field(dst_ty, 8, 8);
3417	const dst_hi16_vi = try dst_hi16_it.only(isel);
3418	if (try dst_hi16_vi.?.defReg(isel)) |dst_hi16_ra| {
3419	var src_hi16_it = src_vi.field(src_ty, 8, 8);
3420	const src_hi16_vi = try src_hi16_it.only(isel);
3421	const src_hi16_mat = try src_hi16_vi.?.matReg(isel);
3422	try isel.emit(.ubfm(dst_hi16_ra.x(), src_hi16_mat.ra.x(), .{
3423	.N = .doubleword,
3424	.immr = 0,
3425	.imms = 16 - 1,
3426	}));
3427	try src_hi16_mat.finish(isel);
3428	}
3429	var dst_lo64_it = dst_vi.value.field(dst_ty, 0, 8);
3430	const dst_lo64_vi = try dst_lo64_it.only(isel);
3431	if (try dst_lo64_vi.?.defReg(isel)) |dst_lo64_ra| {
3432	var src_lo64_it = src_vi.field(src_ty, 0, 8);
3433	const src_lo64_vi = try src_lo64_it.only(isel);
3434	try src_lo64_vi.?.liveOut(isel, dst_lo64_ra);
3435	}
3436	},
3437	else => try dst_vi.value.move(isel, ty_op.operand),
3438	},
3439	128 => {
3440	const dst_ra = try dst_vi.value.defReg(isel) orelse break :unused;
3441	const src_vi = try isel.use(ty_op.operand);
3442	var src_hi64_it = src_vi.field(src_ty, 8, 8);
3443	const src_hi64_vi = try src_hi64_it.only(isel);
3444	const src_hi64_mat = try src_hi64_vi.?.matReg(isel);
3445	try isel.emit(.fmov(dst_ra.@"d[]"(1), .{ .register = src_hi64_mat.ra.x() }));
3446	try src_hi64_mat.finish(isel);
3447	var src_lo64_it = src_vi.field(src_ty, 0, 8);
3448	const src_lo64_vi = try src_lo64_it.only(isel);
3449	const src_lo64_mat = try src_lo64_vi.?.matReg(isel);
3450	try isel.emit(.fmov(dst_ra.d(), .{ .register = src_lo64_mat.ra.x() }));
3451	try src_lo64_mat.finish(isel);
3452	},
3453	}
3454	} else if (dst_ty.isAbiInt(zcu) and src_tag == .array and src_ty.childType(zcu).isAbiInt(zcu)) {
3455	const dst_int_info = dst_ty.intInfo(zcu);
3456	const src_child_int_info = src_ty.childType(zcu).intInfo(zcu);
3457	const src_len = src_ty.arrayLenIncludingSentinel(zcu);
3458	assert(dst_int_info.bits == src_child_int_info.bits * src_len);
3459	const src_child_size = src_ty.childType(zcu).abiSize(zcu);
3460	if (8 * src_child_size == src_child_int_info.bits) {
3461	try dst_vi.value.defAddr(isel, dst_ty, .{ .wrap = dst_int_info }) orelse break :unused;
3462
3463	try call.prepareReturn(isel);
3464	try call.finishReturn(isel);
3465
3466	try call.prepareCallee(isel);
3467	try isel.global_relocs.append(gpa, .{
3468	.name = "memcpy",
3469	.reloc = .{ .label = @intCast(isel.instructions.items.len) },
3470	});
3471	try isel.emit(.bl(0));
3472	try call.finishCallee(isel);
3473
3474	try call.prepareParams(isel);
3475	const src_vi = try isel.use(ty_op.operand);
3476	try isel.movImmediate(.x2, src_child_size * src_len);
3477	try call.paramAddress(isel, src_vi, .r1);
3478	try call.paramAddress(isel, dst_vi.value, .r0);
3479	try call.finishParams(isel);
3480	} else return isel.fail("bad {t} {f} {f}", .{ air_tag, isel.fmtType(dst_ty), isel.fmtType(src_ty) });
3481	} else if (dst_tag == .array and dst_ty.childType(zcu).isAbiInt(zcu) and src_ty.isAbiInt(zcu)) {
3482	const dst_child_int_info = dst_ty.childType(zcu).intInfo(zcu);
3483	const src_int_info = src_ty.intInfo(zcu);
3484	const dst_len = dst_ty.arrayLenIncludingSentinel(zcu);
3485	assert(dst_child_int_info.bits * dst_len == src_int_info.bits);
3486	const dst_child_size = dst_ty.childType(zcu).abiSize(zcu);
3487	if (8 * dst_child_size == dst_child_int_info.bits) {
3488	try dst_vi.value.defAddr(isel, dst_ty, .{}) orelse break :unused;
3489
3490	try call.prepareReturn(isel);
3491	try call.finishReturn(isel);
3492
3493	try call.prepareCallee(isel);
3494	try isel.global_relocs.append(gpa, .{
3495	.name = "memcpy",
3496	.reloc = .{ .label = @intCast(isel.instructions.items.len) },
3497	});
3498	try isel.emit(.bl(0));
3499	try call.finishCallee(isel);
3500
3501	try call.prepareParams(isel);
3502	const src_vi = try isel.use(ty_op.operand);
3503	try isel.movImmediate(.x2, dst_child_size * dst_len);
3504	try call.paramAddress(isel, src_vi, .r1);
3505	try call.paramAddress(isel, dst_vi.value, .r0);
3506	try call.finishParams(isel);
3507	} else return isel.fail("bad {t} {f} {f}", .{ air_tag, isel.fmtType(dst_ty), isel.fmtType(src_ty) });
3508	} else return isel.fail("bad {t} {f} {f}", .{ air_tag, isel.fmtType(dst_ty), isel.fmtType(src_ty) });
3509	}
3510	if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
3511	},
3512	.block => {
3513	const unwrapped_block = isel.air.unwrapBlock(air.inst_index);
3514	try isel.block(
3515	air.inst_index,
3516	unwrapped_block.ty,
3517	unwrapped_block.body,
3518	);
3519	if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
3520	},
3521	.loop => {
3522	const unwrapped_block = isel.air.unwrapBlock(air.inst_index);
3523	const loops = isel.loops.values();
3524	const loop_index = isel.loops.getIndex(air.inst_index).?;
3525	const loop = &loops[loop_index];
3526
3527	tracking_log.debug("{f}", .{
3528	isel.fmtDom(air.inst_index, loop.dom, @intCast(isel.blocks.count())),
3529	});
3530	tracking_log.debug("{f}", .{isel.fmtLoopLive(air.inst_index)});
3531	assert(loop.depth == isel.blocks.count());
3532
3533	if (false) {
3534	// loops are dumb...
3535	for (isel.loop_live.list.items[loop.live..loops[loop_index + 1].live]) |live_inst| {
3536	const live_vi = try isel.use(live_inst.toRef());
3537	try live_vi.mat(isel);
3538	}
3539
3540	// IT'S DOM TIME!!!
3541	for (isel.blocks.values(), 0..) |*dom_block, dom_index| {
3542	if (@as(u1, @truncate(isel.dom.items[
3543	loop.dom + dom_index / @bitSizeOf(DomInt)
3544	] >> @truncate(dom_index))) == 0) continue;
3545	var live_reg_it = dom_block.live_registers.iterator();
3546	while (live_reg_it.next()) |live_reg_entry| switch (live_reg_entry.value.*) {
3547	_ => |live_vi| try live_vi.mat(isel),
3548	.allocating => unreachable,
3549	.free => {},
3550	};
3551	}
3552	}
3553
3554	loop.live_registers = isel.live_registers;
3555	loop.repeat_list = Loop.empty_list;
3556	try isel.body(unwrapped_block.body);
3557	try isel.merge(&loop.live_registers, .{ .fill_extra = true });
3558
3559	var repeat_label = loop.repeat_list;
3560	assert(repeat_label != Loop.empty_list);
3561	while (repeat_label != Loop.empty_list) {
3562	const instruction = &isel.instructions.items[repeat_label];
3563	const next_repeat_label = instruction.*;
3564	instruction.* = .b(-@as(i28, @intCast((isel.instructions.items.len - 1 - repeat_label) << 2)));
3565	repeat_label = @bitCast(next_repeat_label);
3566	}
3567
3568	if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
3569	},
3570	.repeat => {
3571	const repeat = air.data(air.inst_index).repeat;
3572	try isel.loops.getPtr(repeat.loop_inst).?.branch(isel);
3573	if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
3574	},
3575	.br => {
3576	const br = air.data(air.inst_index).br;
3577	try isel.blocks.getPtr(br.block_inst).?.branch(isel);
3578	if (isel.live_values.get(br.block_inst)) |dst_vi| try dst_vi.move(isel, br.operand);
3579	if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
3580	},
3581	.trap => {
3582	try isel.emit(.brk(0x1));
3583	if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
3584	},
3585	.breakpoint => {
3586	try isel.emit(.brk(0xf000));
3587	if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
3588	},
3589	.ret_addr => {
3590	if (isel.live_values.fetchRemove(air.inst_index)) |addr_vi| unused: {
3591	defer addr_vi.value.deref(isel);
3592	const addr_ra = try addr_vi.value.defReg(isel) orelse break :unused;
3593	try isel.emit(.ldr(addr_ra.x(), .{ .unsigned_offset = .{ .base = .fp, .offset = 8 } }));
3594	}
3595	if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
3596	},
3597	.frame_addr => {
3598	if (isel.live_values.fetchRemove(air.inst_index)) |addr_vi| unused: {
3599	defer addr_vi.value.deref(isel);
3600	const addr_ra = try addr_vi.value.defReg(isel) orelse break :unused;
3601	try isel.emit(.orr(addr_ra.x(), .xzr, .{ .register = .fp }));
3602	}
3603	if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
3604	},
3605	.call => {
3606	const air_call = isel.air.unwrapCall(air.inst_index);
3607	const args = air_call.args;
3608	const callee_ty = isel.air.typeOf(air_call.callee, ip);
3609	const func_info = switch (ip.indexToKey(callee_ty.toIntern())) {
3610	else => unreachable,
3611	.func_type => |func_type| func_type,
3612	.ptr_type => |ptr_type| ip.indexToKey(ptr_type.child).func_type,
3613	};
3614
3615	try call.prepareReturn(isel);
3616	const maybe_def_ret_vi = isel.live_values.fetchRemove(air.inst_index);
3617	var maybe_ret_addr_vi: ?Value.Index = null;
3618	if (maybe_def_ret_vi) |def_ret_vi| {
3619	defer def_ret_vi.value.deref(isel);
3620
3621	var ret_it: CallAbiIterator = .init;
3622	const ret_vi = try ret_it.ret(isel, isel.air.typeOfIndex(air.inst_index, ip));
3623	defer ret_vi.?.deref(isel);
3624	switch (ret_vi.?.parent(isel)) {
3625	.unallocated, .stack_slot => if (ret_vi.?.hint(isel)) |ret_ra| {
3626	try call.returnLiveIn(isel, def_ret_vi.value, ret_ra);
3627	} else {
3628	var def_ret_part_it = def_ret_vi.value.parts(isel);
3629	var ret_part_it = ret_vi.?.parts(isel);
3630	while (def_ret_part_it.next()) |ret_part_vi| {
3631	try call.returnLiveIn(isel, ret_part_vi, ret_part_it.next().?.hint(isel).?);
3632	}
3633	},
3634	.value, .constant => unreachable,
3635	.address => |address_vi| {
3636	maybe_ret_addr_vi = address_vi;
3637	_ = try def_ret_vi.value.defAddr(isel, isel.air.typeOfIndex(air.inst_index, ip), .{
3638	.expected_live_registers = &call.caller_saved_regs,
3639	});
3640	},
3641	}
3642	}
3643	try call.finishReturn(isel);
3644
3645	try call.prepareCallee(isel);
3646	if (air_call.callee.toInterned()) |ct_callee| {
3647	try isel.nav_relocs.append(gpa, switch (ip.indexToKey(ct_callee)) {
3648	else => unreachable,
3649	inline .@"extern", .func => |func| .{
3650	.nav = func.owner_nav,
3651	.reloc = .{ .label = @intCast(isel.instructions.items.len) },
3652	},
3653	.ptr => |ptr| .{
3654	.nav = ptr.base_addr.nav,
3655	.reloc = .{
3656	.label = @intCast(isel.instructions.items.len),
3657	.addend = ptr.byte_offset,
3658	},
3659	},
3660	});
3661	try isel.emit(.bl(0));
3662	} else {
3663	const callee_vi = try isel.use(air_call.callee);
3664	const callee_mat = try callee_vi.matReg(isel);
3665	try isel.emit(.blr(callee_mat.ra.x()));
3666	try callee_mat.finish(isel);
3667	}
3668	try call.finishCallee(isel);
3669
3670	try call.prepareParams(isel);
3671	if (maybe_ret_addr_vi) |ret_addr_vi| try call.paramAddress(
3672	isel,
3673	maybe_def_ret_vi.?.value,
3674	ret_addr_vi.hint(isel).?,
3675	);
3676	var param_it: CallAbiIterator = .init;
3677	for (args, 0..) |arg, arg_index| {
3678	const param_ty = isel.air.typeOf(arg, ip);
3679	const param_vi = param_vi: {
3680	if (arg_index >= func_info.param_types.len) {
3681	assert(func_info.is_var_args);
3682	switch (isel.va_list) {
3683	.other => break :param_vi try param_it.nonSysvVarArg(isel, param_ty),
3684	.sysv => {},
3685	}
3686	}
3687	break :param_vi try param_it.param(isel, param_ty);
3688	} orelse continue;
3689	defer param_vi.deref(isel);
3690	const arg_vi = try isel.use(arg);
3691	switch (param_vi.parent(isel)) {
3692	.unallocated => if (param_vi.hint(isel)) |param_ra| {
3693	try call.paramLiveOut(isel, arg_vi, param_ra);
3694	} else {
3695	var param_part_it = param_vi.parts(isel);
3696	var arg_part_it = arg_vi.parts(isel);
3697	if (arg_part_it.only()) |_| {
3698	try isel.values.ensureUnusedCapacity(gpa, param_part_it.remaining);
3699	arg_vi.setParts(isel, param_part_it.remaining);
3700	while (param_part_it.next()) |param_part_vi| _ = arg_vi.addPart(
3701	isel,
3702	param_part_vi.get(isel).offset_from_parent,
3703	param_part_vi.size(isel),
3704	);
3705	param_part_it = param_vi.parts(isel);
3706	arg_part_it = arg_vi.parts(isel);
3707	}
3708	while (param_part_it.next()) |param_part_vi| {
3709	const arg_part_vi = arg_part_it.next().?;
3710	assert(arg_part_vi.get(isel).offset_from_parent ==
3711	param_part_vi.get(isel).offset_from_parent);
3712	assert(arg_part_vi.size(isel) == param_part_vi.size(isel));
3713	try call.paramLiveOut(isel, arg_part_vi, param_part_vi.hint(isel).?);
3714	}
3715	},
3716	.stack_slot => |stack_slot| try arg_vi.store(isel, param_ty, stack_slot.base, .{
3717	.offset = @intCast(stack_slot.offset),
3718	}),
3719	.value, .constant => unreachable,
3720	.address => |address_vi| try call.paramAddress(isel, arg_vi, address_vi.hint(isel).?),
3721	}
3722	}
3723	try call.finishParams(isel);
3724
3725	if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
3726	},
3727	.clz => |air_tag| {
3728	if (isel.live_values.fetchRemove(air.inst_index)) |res_vi| unused: {
3729	defer res_vi.value.deref(isel);
3730
3731	const ty_op = air.data(air.inst_index).ty_op;
3732	const ty = isel.air.typeOf(ty_op.operand, ip);
3733	if (!ty.isAbiInt(zcu)) return isel.fail("bad {t} {f}", .{ air_tag, isel.fmtType(ty) });
3734	const int_info = ty.intInfo(zcu);
3735	switch (int_info.bits) {
3736	0 => unreachable,
3737	1...64 => {
3738	const res_ra = try res_vi.value.defReg(isel) orelse break :unused;
3739	const src_vi = try isel.use(ty_op.operand);
3740	const src_mat = try src_vi.matReg(isel);
3741	try isel.clzLimb(res_ra, int_info, src_mat.ra);
3742	try src_mat.finish(isel);
3743	},
3744	65...128 => |bits| {
3745	const res_ra = try res_vi.value.defReg(isel) orelse break :unused;
3746	const src_vi = try isel.use(ty_op.operand);
3747	var src_hi64_it = src_vi.field(ty, 8, 8);
3748	const src_hi64_vi = try src_hi64_it.only(isel);
3749	const src_hi64_mat = try src_hi64_vi.?.matReg(isel);
3750	var src_lo64_it = src_vi.field(ty, 0, 8);
3751	const src_lo64_vi = try src_lo64_it.only(isel);
3752	const src_lo64_mat = try src_lo64_vi.?.matReg(isel);
3753	const lo64_ra = try isel.allocIntReg();
3754	defer isel.freeReg(lo64_ra);
3755	const hi64_ra = try isel.allocIntReg();
3756	defer isel.freeReg(hi64_ra);
3757	try isel.emit(.csel(res_ra.w(), lo64_ra.w(), hi64_ra.w(), .eq));
3758	try isel.emit(.add(lo64_ra.w(), lo64_ra.w(), .{ .immediate = @intCast(bits - 64) }));
3759	try isel.emit(.subs(.xzr, src_hi64_mat.ra.x(), .{ .immediate = 0 }));
3760	try isel.clzLimb(hi64_ra, .{ .signedness = int_info.signedness, .bits = bits - 64 }, src_hi64_mat.ra);
3761	try isel.clzLimb(lo64_ra, .{ .signedness = .unsigned, .bits = 64 }, src_lo64_mat.ra);
3762	try src_hi64_mat.finish(isel);
3763	try src_lo64_mat.finish(isel);
3764	},
3765	else => return isel.fail("too big {t} {f}", .{ air_tag, isel.fmtType(ty) }),
3766	}
3767	}
3768	if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
3769	},
3770	.ctz => |air_tag| {
3771	if (isel.live_values.fetchRemove(air.inst_index)) |res_vi| unused: {
3772	defer res_vi.value.deref(isel);
3773
3774	const ty_op = air.data(air.inst_index).ty_op;
3775	const ty = isel.air.typeOf(ty_op.operand, ip);
3776	if (!ty.isAbiInt(zcu)) return isel.fail("bad {t} {f}", .{ air_tag, isel.fmtType(ty) });
3777	const int_info = ty.intInfo(zcu);
3778	switch (int_info.bits) {
3779	0 => unreachable,
3780	1...64 => {
3781	const res_ra = try res_vi.value.defReg(isel) orelse break :unused;
3782	const src_vi = try isel.use(ty_op.operand);
3783	const src_mat = try src_vi.matReg(isel);
3784	try isel.ctzLimb(res_ra, int_info, src_mat.ra);
3785	try src_mat.finish(isel);
3786	},
3787	65...128 => |bits| {
3788	const res_ra = try res_vi.value.defReg(isel) orelse break :unused;
3789	const src_vi = try isel.use(ty_op.operand);
3790	var src_hi64_it = src_vi.field(ty, 8, 8);
3791	const src_hi64_vi = try src_hi64_it.only(isel);
3792	const src_hi64_mat = try src_hi64_vi.?.matReg(isel);
3793	var src_lo64_it = src_vi.field(ty, 0, 8);
3794	const src_lo64_vi = try src_lo64_it.only(isel);
3795	const src_lo64_mat = try src_lo64_vi.?.matReg(isel);
3796	const lo64_ra = try isel.allocIntReg();
3797	defer isel.freeReg(lo64_ra);
3798	const hi64_ra = try isel.allocIntReg();
3799	defer isel.freeReg(hi64_ra);
3800	try isel.emit(.csel(res_ra.w(), lo64_ra.w(), hi64_ra.w(), .ne));
3801	try isel.emit(.add(hi64_ra.w(), hi64_ra.w(), .{ .immediate = 64 }));
3802	try isel.emit(.subs(.xzr, src_lo64_mat.ra.x(), .{ .immediate = 0 }));
3803	try isel.ctzLimb(hi64_ra, .{ .signedness = .unsigned, .bits = 64 }, src_hi64_mat.ra);
3804	try isel.ctzLimb(lo64_ra, .{ .signedness = int_info.signedness, .bits = bits - 64 }, src_lo64_mat.ra);
3805	try src_hi64_mat.finish(isel);
3806	try src_lo64_mat.finish(isel);
3807	},
3808	else => return isel.fail("too big {t} {f}", .{ air_tag, isel.fmtType(ty) }),
3809	}
3810	}
3811	if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
3812	},
3813	.popcount => |air_tag| {
3814	if (isel.live_values.fetchRemove(air.inst_index)) |res_vi| unused: {
3815	defer res_vi.value.deref(isel);
3816
3817	const ty_op = air.data(air.inst_index).ty_op;
3818	const ty = isel.air.typeOf(ty_op.operand, ip);
3819	if (!ty.isAbiInt(zcu)) return isel.fail("bad {t} {f}", .{ air_tag, isel.fmtType(ty) });
3820	const int_info = ty.intInfo(zcu);
3821	if (int_info.bits > 64) return isel.fail("too big {t} {f}", .{ air_tag, isel.fmtType(ty) });
3822
3823	const res_ra = try res_vi.value.defReg(isel) orelse break :unused;
3824	const src_vi = try isel.use(ty_op.operand);
3825	const src_mat = try src_vi.matReg(isel);
3826	const vec_ra = try isel.allocVecReg();
3827	defer isel.freeReg(vec_ra);
3828	try isel.emit(.umov(res_ra.w(), vec_ra.@"b[]"(0)));
3829	switch (int_info.bits) {
3830	else => unreachable,
3831	1...8 => {},
3832	9...16 => try isel.emit(.addp(vec_ra.@"8b"(), vec_ra.@"8b"(), .{ .vector = vec_ra.@"8b"() })),
3833	17...64 => try isel.emit(.addv(vec_ra.b(), vec_ra.@"8b"())),
3834	}
3835	try isel.emit(.cnt(vec_ra.@"8b"(), vec_ra.@"8b"()));
3836	switch (int_info.bits) {
3837	else => unreachable,
3838	1...31 => |bits| switch (int_info.signedness) {
3839	.signed => {
3840	try isel.emit(.fmov(vec_ra.s(), .{ .register = res_ra.w() }));
3841	try isel.emit(.ubfm(res_ra.w(), src_mat.ra.w(), .{
3842	.N = .word,
3843	.immr = 0,
3844	.imms = @intCast(bits - 1),
3845	}));
3846	},
3847	.unsigned => try isel.emit(.fmov(vec_ra.s(), .{ .register = src_mat.ra.w() })),
3848	},
3849	32 => try isel.emit(.fmov(vec_ra.s(), .{ .register = src_mat.ra.w() })),
3850	33...63 => |bits| switch (int_info.signedness) {
3851	.signed => {
3852	try isel.emit(.fmov(vec_ra.d(), .{ .register = res_ra.x() }));
3853	try isel.emit(.ubfm(res_ra.x(), src_mat.ra.x(), .{
3854	.N = .doubleword,
3855	.immr = 0,
3856	.imms = @intCast(bits - 1),
3857	}));
3858	},
3859	.unsigned => try isel.emit(.fmov(vec_ra.d(), .{ .register = src_mat.ra.x() })),
3860	},
3861	64 => try isel.emit(.fmov(vec_ra.d(), .{ .register = src_mat.ra.x() })),
3862	}
3863	try src_mat.finish(isel);
3864	}
3865	if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
3866	},
3867	.byte_swap => |air_tag| {
3868	if (isel.live_values.fetchRemove(air.inst_index)) |res_vi| unused: {
3869	defer res_vi.value.deref(isel);
3870
3871	const ty_op = air.data(air.inst_index).ty_op;
3872	const ty = ty_op.ty;
3873	if (!ty.isAbiInt(zcu)) return isel.fail("bad {t} {f}", .{ air_tag, isel.fmtType(ty) });
3874	const int_info = ty.intInfo(zcu);
3875	if (int_info.bits > 64) return isel.fail("too big {t} {f}", .{ air_tag, isel.fmtType(ty) });
3876
3877	if (int_info.bits == 8) break :unused try res_vi.value.move(isel, ty_op.operand);
3878	const res_ra = try res_vi.value.defReg(isel) orelse break :unused;
3879	const src_vi = try isel.use(ty_op.operand);
3880	const src_mat = try src_vi.matReg(isel);
3881	switch (int_info.bits) {
3882	else => unreachable,
3883	16 => switch (int_info.signedness) {
3884	.signed => {
3885	try isel.emit(.sbfm(res_ra.w(), res_ra.w(), .{
3886	.N = .word,
3887	.immr = 32 - 16,
3888	.imms = 32 - 1,
3889	}));
3890	try isel.emit(.rev(res_ra.w(), src_mat.ra.w()));
3891	},
3892	.unsigned => try isel.emit(.rev16(res_ra.w(), src_mat.ra.w())),
3893	},
3894	24 => {
3895	switch (int_info.signedness) {
3896	.signed => try isel.emit(.sbfm(res_ra.w(), res_ra.w(), .{
3897	.N = .word,
3898	.immr = 32 - 24,
3899	.imms = 32 - 1,
3900	})),
3901	.unsigned => try isel.emit(.ubfm(res_ra.w(), res_ra.w(), .{
3902	.N = .word,
3903	.immr = 32 - 24,
3904	.imms = 32 - 1,
3905	})),
3906	}
3907	try isel.emit(.rev(res_ra.w(), src_mat.ra.w()));
3908	},
3909	32 => try isel.emit(.rev(res_ra.w(), src_mat.ra.w())),
3910	40, 48, 56 => |bits| {
3911	switch (int_info.signedness) {
3912	.signed => try isel.emit(.sbfm(res_ra.x(), res_ra.x(), .{
3913	.N = .doubleword,
3914	.immr = @intCast(64 - bits),
3915	.imms = 64 - 1,
3916	})),
3917	.unsigned => try isel.emit(.ubfm(res_ra.x(), res_ra.x(), .{
3918	.N = .doubleword,
3919	.immr = @intCast(64 - bits),
3920	.imms = 64 - 1,
3921	})),
3922	}
3923	try isel.emit(.rev(res_ra.x(), src_mat.ra.x()));
3924	},
3925	64 => try isel.emit(.rev(res_ra.x(), src_mat.ra.x())),
3926	}
3927	try src_mat.finish(isel);
3928	}
3929	if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
3930	},
3931	.bit_reverse => |air_tag| {
3932	if (isel.live_values.fetchRemove(air.inst_index)) |res_vi| unused: {
3933	defer res_vi.value.deref(isel);
3934
3935	const ty_op = air.data(air.inst_index).ty_op;
3936	const ty = ty_op.ty;
3937	if (!ty.isAbiInt(zcu)) return isel.fail("bad {t} {f}", .{ air_tag, isel.fmtType(ty) });
3938	const int_info = ty.intInfo(zcu);
3939	if (int_info.bits > 64) return isel.fail("too big {t} {f}", .{ air_tag, isel.fmtType(ty) });
3940
3941	const res_ra = try res_vi.value.defReg(isel) orelse break :unused;
3942	const src_vi = try isel.use(ty_op.operand);
3943	const src_mat = try src_vi.matReg(isel);
3944	switch (int_info.bits) {
3945	else => unreachable,
3946	1...31 => |bits| {
3947	switch (int_info.signedness) {
3948	.signed => try isel.emit(.sbfm(res_ra.w(), res_ra.w(), .{
3949	.N = .word,
3950	.immr = @intCast(32 - bits),
3951	.imms = 32 - 1,
3952	})),
3953	.unsigned => try isel.emit(.ubfm(res_ra.w(), res_ra.w(), .{
3954	.N = .word,
3955	.immr = @intCast(32 - bits),
3956	.imms = 32 - 1,
3957	})),
3958	}
3959	try isel.emit(.rbit(res_ra.w(), src_mat.ra.w()));
3960	},
3961	32 => try isel.emit(.rbit(res_ra.w(), src_mat.ra.w())),
3962	33...63 => |bits| {
3963	switch (int_info.signedness) {
3964	.signed => try isel.emit(.sbfm(res_ra.x(), res_ra.x(), .{
3965	.N = .doubleword,
3966	.immr = @intCast(64 - bits),
3967	.imms = 64 - 1,
3968	})),
3969	.unsigned => try isel.emit(.ubfm(res_ra.x(), res_ra.x(), .{
3970	.N = .doubleword,
3971	.immr = @intCast(64 - bits),
3972	.imms = 64 - 1,
3973	})),
3974	}
3975	try isel.emit(.rbit(res_ra.x(), src_mat.ra.x()));
3976	},
3977	64 => try isel.emit(.rbit(res_ra.x(), src_mat.ra.x())),
3978	}
3979	try src_mat.finish(isel);
3980	}
3981	if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
3982	},
3983	.sqrt, .floor, .ceil, .round, .trunc_float => |air_tag| {
3984	if (isel.live_values.fetchRemove(air.inst_index)) |res_vi| unused: {
3985	defer res_vi.value.deref(isel);
3986
3987	const un_op = air.data(air.inst_index).un_op;
3988	const ty = isel.air.typeOf(un_op, ip);
3989	switch (ty.floatBits(isel.target)) {
3990	else => unreachable,
3991	16, 32, 64 => |bits| {
3992	const res_ra = try res_vi.value.defReg(isel) orelse break :unused;
3993	const need_fcvt = switch (bits) {
3994	else => unreachable,
3995	16 => !isel.target.cpu.has(.aarch64, .fullfp16),
3996	32, 64 => false,
3997	};
3998	if (need_fcvt) try isel.emit(.fcvt(res_ra.h(), res_ra.s()));
3999	const src_vi = try isel.use(un_op);
4000	const src_mat = try src_vi.matReg(isel);
4001	const src_ra = if (need_fcvt) try isel.allocVecReg() else src_mat.ra;
4002	defer if (need_fcvt) isel.freeReg(src_ra);
4003	try isel.emit(bits: switch (bits) {
4004	else => unreachable,
4005	16 => if (need_fcvt) continue :bits 32 else switch (air_tag) {
4006	else => unreachable,
4007	.sqrt => .fsqrt(res_ra.h(), src_ra.h()),
4008	.floor => .frintm(res_ra.h(), src_ra.h()),
4009	.ceil => .frintp(res_ra.h(), src_ra.h()),
4010	.round => .frinta(res_ra.h(), src_ra.h()),
4011	.trunc_float => .frintz(res_ra.h(), src_ra.h()),
4012	},
4013	32 => switch (air_tag) {
4014	else => unreachable,
4015	.sqrt => .fsqrt(res_ra.s(), src_ra.s()),
4016	.floor => .frintm(res_ra.s(), src_ra.s()),
4017	.ceil => .frintp(res_ra.s(), src_ra.s()),
4018	.round => .frinta(res_ra.s(), src_ra.s()),
4019	.trunc_float => .frintz(res_ra.s(), src_ra.s()),
4020	},
4021	64 => switch (air_tag) {
4022	else => unreachable,
4023	.sqrt => .fsqrt(res_ra.d(), src_ra.d()),
4024	.floor => .frintm(res_ra.d(), src_ra.d()),
4025	.ceil => .frintp(res_ra.d(), src_ra.d()),
4026	.round => .frinta(res_ra.d(), src_ra.d()),
4027	.trunc_float => .frintz(res_ra.d(), src_ra.d()),
4028	},
4029	});
4030	if (need_fcvt) try isel.emit(.fcvt(src_ra.s(), src_mat.ra.h()));
4031	try src_mat.finish(isel);
4032	},
4033	80, 128 => |bits| {
4034	try call.prepareReturn(isel);
4035	switch (bits) {
4036	else => unreachable,
4037	16, 32, 64, 128 => try call.returnLiveIn(isel, res_vi.value, .v0),
4038	80 => {
4039	var res_hi16_it = res_vi.value.field(ty, 8, 8);
4040	const res_hi16_vi = try res_hi16_it.only(isel);
4041	try call.returnLiveIn(isel, res_hi16_vi.?, .r1);
4042	var res_lo64_it = res_vi.value.field(ty, 0, 8);
4043	const res_lo64_vi = try res_lo64_it.only(isel);
4044	try call.returnLiveIn(isel, res_lo64_vi.?, .r0);
4045	},
4046	}
4047	try call.finishReturn(isel);
4048
4049	try call.prepareCallee(isel);
4050	try isel.global_relocs.append(gpa, .{
4051	.name = switch (air_tag) {
4052	else => unreachable,
4053	.sqrt => switch (bits) {
4054	else => unreachable,
4055	16 => "__sqrth",
4056	32 => "sqrtf",
4057	64 => "sqrt",
4058	80 => "__sqrtx",
4059	128 => "sqrtf128",
4060	},
4061	.floor => switch (bits) {
4062	else => unreachable,
4063	16 => "__floorh",
4064	32 => "floorf",
4065	64 => "floor",
4066	80 => "__floorx",
4067	128 => "floorf128",
4068	},
4069	.ceil => switch (bits) {
4070	else => unreachable,
4071	16 => "__ceilh",
4072	32 => "ceilf",
4073	64 => "ceil",
4074	80 => "__ceilx",
4075	128 => "ceilf128",
4076	},
4077	.round => switch (bits) {
4078	else => unreachable,
4079	16 => "__roundh",
4080	32 => "roundf",
4081	64 => "round",
4082	80 => "__roundx",
4083	128 => "roundf128",
4084	},
4085	.trunc_float => switch (bits) {
4086	else => unreachable,
4087	16 => "__trunch",
4088	32 => "truncf",
4089	64 => "trunc",
4090	80 => "__truncx",
4091	128 => "truncf128",
4092	},
4093	},
4094	.reloc = .{ .label = @intCast(isel.instructions.items.len) },
4095	});
4096	try isel.emit(.bl(0));
4097	try call.finishCallee(isel);
4098
4099	try call.prepareParams(isel);
4100	const src_vi = try isel.use(un_op);
4101	switch (bits) {
4102	else => unreachable,
4103	16, 32, 64, 128 => try call.paramLiveOut(isel, src_vi, .v0),
4104	80 => {
4105	var src_hi16_it = src_vi.field(ty, 8, 8);
4106	const src_hi16_vi = try src_hi16_it.only(isel);
4107	try call.paramLiveOut(isel, src_hi16_vi.?, .r1);
4108	var src_lo64_it = src_vi.field(ty, 0, 8);
4109	const src_lo64_vi = try src_lo64_it.only(isel);
4110	try call.paramLiveOut(isel, src_lo64_vi.?, .r0);
4111	},
4112	}
4113	try call.finishParams(isel);
4114	},
4115	}
4116	}
4117	if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
4118	},
4119	.sin, .cos, .tan, .exp, .exp2, .log, .log2, .log10 => |air_tag| {
4120	if (isel.live_values.fetchRemove(air.inst_index)) |res_vi| {
4121	defer res_vi.value.deref(isel);
4122
4123	const un_op = air.data(air.inst_index).un_op;
4124	const ty = isel.air.typeOf(un_op, ip);
4125	const bits = ty.floatBits(isel.target);
4126	try call.prepareReturn(isel);
4127	switch (bits) {
4128	else => unreachable,
4129	16, 32, 64, 128 => try call.returnLiveIn(isel, res_vi.value, .v0),
4130	80 => {
4131	var res_hi16_it = res_vi.value.field(ty, 8, 8);
4132	const res_hi16_vi = try res_hi16_it.only(isel);
4133	try call.returnLiveIn(isel, res_hi16_vi.?, .r1);
4134	var res_lo64_it = res_vi.value.field(ty, 0, 8);
4135	const res_lo64_vi = try res_lo64_it.only(isel);
4136	try call.returnLiveIn(isel, res_lo64_vi.?, .r0);
4137	},
4138	}
4139	try call.finishReturn(isel);
4140
4141	try call.prepareCallee(isel);
4142	try isel.global_relocs.append(gpa, .{
4143	.name = switch (air_tag) {
4144	else => unreachable,
4145	.sin => switch (bits) {
4146	else => unreachable,
4147	16 => "__sinh",
4148	32 => "sinf",
4149	64 => "sin",
4150	80 => "__sinx",
4151	128 => "sinf128",
4152	},
4153	.cos => switch (bits) {
4154	else => unreachable,
4155	16 => "__cosh",
4156	32 => "cosf",
4157	64 => "cos",
4158	80 => "__cosx",
4159	128 => "cosf128",
4160	},
4161	.tan => switch (bits) {
4162	else => unreachable,
4163	16 => "__tanh",
4164	32 => "tanf",
4165	64 => "tan",
4166	80 => "__tanx",
4167	128 => "tanf128",
4168	},
4169	.exp => switch (bits) {
4170	else => unreachable,
4171	16 => "__exph",
4172	32 => "expf",
4173	64 => "exp",
4174	80 => "__expx",
4175	128 => "expf128",
4176	},
4177	.exp2 => switch (bits) {
4178	else => unreachable,
4179	16 => "__exp2h",
4180	32 => "exp2f",
4181	64 => "exp2",
4182	80 => "__exp2x",
4183	128 => "exp2f128",
4184	},
4185	.log => switch (bits) {
4186	else => unreachable,
4187	16 => "__logh",
4188	32 => "logf",
4189	64 => "log",
4190	80 => "__logx",
4191	128 => "logf128",
4192	},
4193	.log2 => switch (bits) {
4194	else => unreachable,
4195	16 => "__log2h",
4196	32 => "log2f",
4197	64 => "log2",
4198	80 => "__log2x",
4199	128 => "log2f128",
4200	},
4201	.log10 => switch (bits) {
4202	else => unreachable,
4203	16 => "__log10h",
4204	32 => "log10f",
4205	64 => "log10",
4206	80 => "__log10x",
4207	128 => "log10f128",
4208	},
4209	},
4210	.reloc = .{ .label = @intCast(isel.instructions.items.len) },
4211	});
4212	try isel.emit(.bl(0));
4213	try call.finishCallee(isel);
4214
4215	try call.prepareParams(isel);
4216	const src_vi = try isel.use(un_op);
4217	switch (bits) {
4218	else => unreachable,
4219	16, 32, 64, 128 => try call.paramLiveOut(isel, src_vi, .v0),
4220	80 => {
4221	var src_hi16_it = src_vi.field(ty, 8, 8);
4222	const src_hi16_vi = try src_hi16_it.only(isel);
4223	try call.paramLiveOut(isel, src_hi16_vi.?, .r1);
4224	var src_lo64_it = src_vi.field(ty, 0, 8);
4225	const src_lo64_vi = try src_lo64_it.only(isel);
4226	try call.paramLiveOut(isel, src_lo64_vi.?, .r0);
4227	},
4228	}
4229	try call.finishParams(isel);
4230	}
4231	if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
4232	},
4233	.abs => |air_tag| {
4234	if (isel.live_values.fetchRemove(air.inst_index)) |res_vi| unused: {
4235	defer res_vi.value.deref(isel);
4236
4237	const ty_op = air.data(air.inst_index).ty_op;
4238	const ty = ty_op.ty;
4239	if (!ty.isRuntimeFloat()) {
4240	if (!ty.isAbiInt(zcu)) return isel.fail("bad {t} {f}", .{ air_tag, isel.fmtType(ty) });
4241	switch (ty.intInfo(zcu).bits) {
4242	0 => unreachable,
4243	1...32 => {
4244	const res_ra = try res_vi.value.defReg(isel) orelse break :unused;
4245	const src_vi = try isel.use(ty_op.operand);
4246	const src_mat = try src_vi.matReg(isel);
4247	try isel.emit(.csneg(res_ra.w(), src_mat.ra.w(), src_mat.ra.w(), .pl));
4248	try isel.emit(.subs(.wzr, src_mat.ra.w(), .{ .immediate = 0 }));
4249	try src_mat.finish(isel);
4250	},
4251	33...64 => {
4252	const res_ra = try res_vi.value.defReg(isel) orelse break :unused;
4253	const src_vi = try isel.use(ty_op.operand);
4254	const src_mat = try src_vi.matReg(isel);
4255	try isel.emit(.csneg(res_ra.x(), src_mat.ra.x(), src_mat.ra.x(), .pl));
4256	try isel.emit(.subs(.xzr, src_mat.ra.x(), .{ .immediate = 0 }));
4257	try src_mat.finish(isel);
4258	},
4259	65...128 => {
4260	var res_hi64_it = res_vi.value.field(ty, 8, 8);
4261	const res_hi64_vi = try res_hi64_it.only(isel);
4262	const res_hi64_ra = try res_hi64_vi.?.defReg(isel);
4263	var res_lo64_it = res_vi.value.field(ty, 0, 8);
4264	const res_lo64_vi = try res_lo64_it.only(isel);
4265	const res_lo64_ra = try res_lo64_vi.?.defReg(isel);
4266	if (res_hi64_ra == null and res_lo64_ra == null) break :unused;
4267	const src_ty = isel.air.typeOf(ty_op.operand, ip);
4268	const src_vi = try isel.use(ty_op.operand);
4269	var src_hi64_it = src_vi.field(src_ty, 8, 8);
4270	const src_hi64_vi = try src_hi64_it.only(isel);
4271	const src_hi64_mat = try src_hi64_vi.?.matReg(isel);
4272	var src_lo64_it = src_vi.field(src_ty, 0, 8);
4273	const src_lo64_vi = try src_lo64_it.only(isel);
4274	const src_lo64_mat = try src_lo64_vi.?.matReg(isel);
4275	const lo64_ra = try isel.allocIntReg();
4276	defer isel.freeReg(lo64_ra);
4277	const hi64_ra, const mask_ra = alloc_ras: {
4278	const res_lo64_lock: RegLock = if (res_lo64_ra) |res_ra| isel.tryLockReg(res_ra) else .empty;
4279	defer res_lo64_lock.unlock(isel);
4280	break :alloc_ras .{ try isel.allocIntReg(), try isel.allocIntReg() };
4281	};
4282	defer {
4283	isel.freeReg(hi64_ra);
4284	isel.freeReg(mask_ra);
4285	}
4286	if (res_hi64_ra) |res_ra| try isel.emit(.sbc(res_ra.x(), hi64_ra.x(), mask_ra.x()));
4287	try isel.emit(.subs(
4288	if (res_lo64_ra) |res_ra| res_ra.x() else .xzr,
4289	lo64_ra.x(),
4290	.{ .register = mask_ra.x() },
4291	));
4292	if (res_hi64_ra) |_| try isel.emit(.eor(hi64_ra.x(), src_hi64_mat.ra.x(), .{ .register = mask_ra.x() }));
4293	try isel.emit(.eor(lo64_ra.x(), src_lo64_mat.ra.x(), .{ .register = mask_ra.x() }));
4294	try isel.emit(.sbfm(mask_ra.x(), src_hi64_mat.ra.x(), .{
4295	.N = .doubleword,
4296	.immr = 64 - 1,
4297	.imms = 64 - 1,
4298	}));
4299	try src_lo64_mat.finish(isel);
4300	try src_hi64_mat.finish(isel);
4301	},
4302	else => return isel.fail("too big {t} {f}", .{ air_tag, isel.fmtType(ty) }),
4303	}
4304	} else switch (ty.floatBits(isel.target)) {
4305	else => unreachable,
4306	16 => {
4307	const res_ra = try res_vi.value.defReg(isel) orelse break :unused;
4308	const src_vi = try isel.use(ty_op.operand);
4309	const src_mat = try src_vi.matReg(isel);
4310	try isel.emit(if (isel.target.cpu.has(.aarch64, .fullfp16))
4311	.fabs(res_ra.h(), src_mat.ra.h())
4312	else
4313	.bic(res_ra.@"4h"(), res_ra.@"4h"(), .{ .shifted_immediate = .{
4314	.immediate = 0b10000000,
4315	.lsl = 8,
4316	} }));
4317	try src_mat.finish(isel);
4318	},
4319	32 => {
4320	const res_ra = try res_vi.value.defReg(isel) orelse break :unused;
4321	const src_vi = try isel.use(ty_op.operand);
4322	const src_mat = try src_vi.matReg(isel);
4323	try isel.emit(.fabs(res_ra.s(), src_mat.ra.s()));
4324	try src_mat.finish(isel);
4325	},
4326	64 => {
4327	const res_ra = try res_vi.value.defReg(isel) orelse break :unused;
4328	const src_vi = try isel.use(ty_op.operand);
4329	const src_mat = try src_vi.matReg(isel);
4330	try isel.emit(.fabs(res_ra.d(), src_mat.ra.d()));
4331	try src_mat.finish(isel);
4332	},
4333	80 => {
4334	const src_vi = try isel.use(ty_op.operand);
4335	var res_hi16_it = res_vi.value.field(ty, 8, 8);
4336	const res_hi16_vi = try res_hi16_it.only(isel);
4337	if (try res_hi16_vi.?.defReg(isel)) |res_hi16_ra| {
4338	var src_hi16_it = src_vi.field(ty, 8, 8);
4339	const src_hi16_vi = try src_hi16_it.only(isel);
4340	const src_hi16_mat = try src_hi16_vi.?.matReg(isel);
4341	try isel.emit(.@"and"(res_hi16_ra.w(), src_hi16_mat.ra.w(), .{ .immediate = .{
4342	.N = .word,
4343	.immr = 0,
4344	.imms = 15 - 1,
4345	} }));
4346	try src_hi16_mat.finish(isel);
4347	}
4348	var res_lo64_it = res_vi.value.field(ty, 0, 8);
4349	const res_lo64_vi = try res_lo64_it.only(isel);
4350	if (try res_lo64_vi.?.defReg(isel)) |res_lo64_ra| {
4351	var src_lo64_it = src_vi.field(ty, 0, 8);
4352	const src_lo64_vi = try src_lo64_it.only(isel);
4353	try src_lo64_vi.?.liveOut(isel, res_lo64_ra);
4354	}
4355	},
4356	128 => {
4357	const res_ra = try res_vi.value.defReg(isel) orelse break :unused;
4358	const src_vi = try isel.use(ty_op.operand);
4359	const src_mat = try src_vi.matReg(isel);
4360	const neg_zero_ra = try isel.allocVecReg();
4361	defer isel.freeReg(neg_zero_ra);
4362	try isel.emit(.bic(res_ra.@"16b"(), src_mat.ra.@"16b"(), .{ .register = neg_zero_ra.@"16b"() }));
4363	try isel.literals.appendNTimes(gpa, 0, -%isel.literals.items.len % 4);
4364	try isel.literal_relocs.append(gpa, .{
4365	.label = @intCast(isel.instructions.items.len),
4366	});
4367	try isel.emit(.ldr(neg_zero_ra.q(), .{
4368	.literal = @intCast((isel.instructions.items.len + 1 + isel.literals.items.len) << 2),
4369	}));
4370	try isel.emitLiteral(&(@as([15]u8, @splat(0)) ++ .{0x80}));
4371	try src_mat.finish(isel);
4372	},
4373	}
4374	}
4375	if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
4376	},
4377	.neg, .neg_optimized => {
4378	if (isel.live_values.fetchRemove(air.inst_index)) |res_vi| unused: {
4379	defer res_vi.value.deref(isel);
4380
4381	const un_op = air.data(air.inst_index).un_op;
4382	const ty = isel.air.typeOf(un_op, ip);
4383	switch (ty.floatBits(isel.target)) {
4384	else => unreachable,
4385	16 => {
4386	const res_ra = try res_vi.value.defReg(isel) orelse break :unused;
4387	const src_vi = try isel.use(un_op);
4388	const src_mat = try src_vi.matReg(isel);
4389	if (isel.target.cpu.has(.aarch64, .fullfp16)) {
4390	try isel.emit(.fneg(res_ra.h(), src_mat.ra.h()));
4391	} else {
4392	const neg_zero_ra = try isel.allocVecReg();
4393	defer isel.freeReg(neg_zero_ra);
4394	try isel.emit(.eor(res_ra.@"8b"(), res_ra.@"8b"(), .{ .register = neg_zero_ra.@"8b"() }));
4395	try isel.emit(.movi(neg_zero_ra.@"4h"(), 0b10000000, .{ .lsl = 8 }));
4396	}
4397	try src_mat.finish(isel);
4398	},
4399	32 => {
4400	const res_ra = try res_vi.value.defReg(isel) orelse break :unused;
4401	const src_vi = try isel.use(un_op);
4402	const src_mat = try src_vi.matReg(isel);
4403	try isel.emit(.fneg(res_ra.s(), src_mat.ra.s()));
4404	try src_mat.finish(isel);
4405	},
4406	64 => {
4407	const res_ra = try res_vi.value.defReg(isel) orelse break :unused;
4408	const src_vi = try isel.use(un_op);
4409	const src_mat = try src_vi.matReg(isel);
4410	try isel.emit(.fneg(res_ra.d(), src_mat.ra.d()));
4411	try src_mat.finish(isel);
4412	},
4413	80 => {
4414	const src_vi = try isel.use(un_op);
4415	var res_hi16_it = res_vi.value.field(ty, 8, 8);
4416	const res_hi16_vi = try res_hi16_it.only(isel);
4417	if (try res_hi16_vi.?.defReg(isel)) |res_hi16_ra| {
4418	var src_hi16_it = src_vi.field(ty, 8, 8);
4419	const src_hi16_vi = try src_hi16_it.only(isel);
4420	const src_hi16_mat = try src_hi16_vi.?.matReg(isel);
4421	try isel.emit(.eor(res_hi16_ra.w(), src_hi16_mat.ra.w(), .{ .immediate = .{
4422	.N = .word,
4423	.immr = 32 - 15,
4424	.imms = 1 - 1,
4425	} }));
4426	try src_hi16_mat.finish(isel);
4427	}
4428	var res_lo64_it = res_vi.value.field(ty, 0, 8);
4429	const res_lo64_vi = try res_lo64_it.only(isel);
4430	if (try res_lo64_vi.?.defReg(isel)) |res_lo64_ra| {
4431	var src_lo64_it = src_vi.field(ty, 0, 8);
4432	const src_lo64_vi = try src_lo64_it.only(isel);
4433	try src_lo64_vi.?.liveOut(isel, res_lo64_ra);
4434	}
4435	},
4436	128 => {
4437	const res_ra = try res_vi.value.defReg(isel) orelse break :unused;
4438	const src_vi = try isel.use(un_op);
4439	const src_mat = try src_vi.matReg(isel);
4440	const neg_zero_ra = try isel.allocVecReg();
4441	defer isel.freeReg(neg_zero_ra);
4442	try isel.emit(.eor(res_ra.@"16b"(), src_mat.ra.@"16b"(), .{ .register = neg_zero_ra.@"16b"() }));
4443	try isel.literals.appendNTimes(gpa, 0, -%isel.literals.items.len % 4);
4444	try isel.literal_relocs.append(gpa, .{
4445	.label = @intCast(isel.instructions.items.len),
4446	});
4447	try isel.emit(.ldr(neg_zero_ra.q(), .{
4448	.literal = @intCast((isel.instructions.items.len + 1 + isel.literals.items.len) << 2),
4449	}));
4450	try isel.emitLiteral(&(@as([15]u8, @splat(0)) ++ .{0x80}));
4451	try src_mat.finish(isel);
4452	},
4453	}
4454	}
4455	if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
4456	},
4457	.cmp_lt, .cmp_lte, .cmp_eq, .cmp_gte, .cmp_gt, .cmp_neq => |air_tag| {
4458	if (isel.live_values.fetchRemove(air.inst_index)) |res_vi| unused: {
4459	defer res_vi.value.deref(isel);
4460
4461	const bin_op = air.data(air.inst_index).bin_op;
4462	const ty = isel.air.typeOf(bin_op.lhs, ip);
4463	switch (ip.indexToKey(ty.toIntern())) {
4464	else => {},
4465	.opt_type => |payload_ty| switch (air_tag) {
4466	else => unreachable,
4467	.cmp_eq, .cmp_neq => if (!ty.optionalReprIsPayload(zcu)) {
4468	const lhs_vi = try isel.use(bin_op.lhs);
4469	const rhs_vi = try isel.use(bin_op.rhs);
4470	const payload_size = ZigType.abiSize(.fromInterned(payload_ty), zcu);
4471	var lhs_payload_part_it = lhs_vi.field(ty, 0, payload_size);
4472	const lhs_payload_part_vi = try lhs_payload_part_it.only(isel);
4473	var rhs_payload_part_it = rhs_vi.field(ty, 0, payload_size);
4474	const rhs_payload_part_vi = try rhs_payload_part_it.only(isel);
4475	const cmp_info = try isel.cmp(
4476	try res_vi.value.defReg(isel) orelse break :unused,
4477	.fromInterned(payload_ty),
4478	lhs_payload_part_vi.?,
4479	air_tag.toCmpOp().?,
4480	rhs_payload_part_vi.?,
4481	);
4482	try isel.emit(.@"b."(
4483	.vc,
4484	@intCast((isel.instructions.items.len + 1 - cmp_info.cset_label) << 2),
4485	));
4486	var lhs_has_value_part_it = lhs_vi.field(ty, payload_size, 1);
4487	const lhs_has_value_part_vi = try lhs_has_value_part_it.only(isel);
4488	const lhs_has_value_part_mat = try lhs_has_value_part_vi.?.matReg(isel);
4489	var rhs_has_value_part_it = rhs_vi.field(ty, payload_size, 1);
4490	const rhs_has_value_part_vi = try rhs_has_value_part_it.only(isel);
4491	const rhs_has_value_part_mat = try rhs_has_value_part_vi.?.matReg(isel);
4492	try isel.emit(.ccmp(
4493	lhs_has_value_part_mat.ra.w(),
4494	.{ .register = rhs_has_value_part_mat.ra.w() },
4495	.{ .n = false, .z = false, .c = false, .v = true },
4496	.eq,
4497	));
4498	try isel.emit(.ands(
4499	.wzr,
4500	lhs_has_value_part_mat.ra.w(),
4501	.{ .register = rhs_has_value_part_mat.ra.w() },
4502	));
4503	try rhs_has_value_part_mat.finish(isel);
4504	try lhs_has_value_part_mat.finish(isel);
4505	break :unused;
4506	},
4507	},
4508	}
4509	_ = try isel.cmp(
4510	try res_vi.value.defReg(isel) orelse break :unused,
4511	ty,
4512	try isel.use(bin_op.lhs),
4513	air_tag.toCmpOp().?,
4514	try isel.use(bin_op.rhs),
4515	);
4516	}
4517	if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
4518	},
4519	.cond_br => {
4520	const cond_br = isel.air.unwrapCondBr(air.inst_index);
4521
4522	try isel.body(cond_br.then_body);
4523	const else_label = isel.instructions.items.len;
4524	const else_live_registers = isel.live_registers;
4525	try isel.body(cond_br.else_body);
4526	try isel.merge(&else_live_registers, .{});
4527
4528	const cond_vi = try isel.use(cond_br.condition);
4529	const cond_mat = try cond_vi.matReg(isel);
4530	try isel.emit(.tbz(
4531	cond_mat.ra.x(),
4532	0,
4533	@intCast((isel.instructions.items.len + 1 - else_label) << 2),
4534	));
4535	try cond_mat.finish(isel);
4536
4537	if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
4538	},
4539	.switch_br => {
4540	const switch_br = isel.air.unwrapSwitch(air.inst_index);
4541	const cond_ty = isel.air.typeOf(switch_br.operand, ip);
4542	const cond_int_info: std.lang.Type.Int = if (cond_ty.toIntern() == .bool_type)
4543	.{ .signedness = .unsigned, .bits = 1 }
4544	else if (cond_ty.isAbiInt(zcu))
4545	cond_ty.intInfo(zcu)
4546	else
4547	return isel.fail("bad switch cond {f}", .{isel.fmtType(cond_ty)});
4548
4549	var final_case = true;
4550	if (switch_br.else_body_len > 0) {
4551	var cases_it = switch_br.iterateCases();
4552	while (cases_it.next()) |_| {}
4553	try isel.body(cases_it.elseBody());
4554	assert(final_case);
4555	final_case = false;
4556	}
4557	const zero_reg: Register = switch (cond_int_info.bits) {
4558	else => unreachable,
4559	1...32 => .wzr,
4560	33...64 => .xzr,
4561	};
4562	var cond_mat: ?Value.Materialize = null;
4563	var cond_reg: Register = undefined;
4564	var cases_it = switch_br.iterateCases();
4565	while (cases_it.next()) |case| {
4566	const next_label = isel.instructions.items.len;
4567	const next_live_registers = isel.live_registers;
4568	try isel.body(case.body);
4569	if (final_case) {
4570	final_case = false;
4571	continue;
4572	}
4573	try isel.merge(&next_live_registers, .{});
4574	if (cond_mat == null) {
4575	var cond_vi = try isel.use(switch_br.operand);
4576	cond_mat = try cond_vi.matReg(isel);
4577	cond_reg = switch (cond_int_info.bits) {
4578	else => unreachable,
4579	1...32 => cond_mat.?.ra.w(),
4580	33...64 => cond_mat.?.ra.x(),
4581	};
4582	}
4583	if (case.ranges.len == 0 and case.items.len == 1 and Constant.fromInterned(
4584	case.items[0].toInterned().?,
4585	).compareHetero(.eq, .zero_comptime_int, zcu)) {
4586	try isel.emit(.cbnz(
4587	cond_reg,
4588	@intCast((isel.instructions.items.len + 1 - next_label) << 2),
4589	));
4590	continue;
4591	}
4592	try isel.emit(.@"b."(
4593	.invert(switch (case.ranges.len) {
4594	0 => .eq,
4595	else => .ls,
4596	}),
4597	@intCast((isel.instructions.items.len + 1 - next_label) << 2),
4598	));
4599	var case_range_index = case.ranges.len;
4600	while (case_range_index > 0) {
4601	case_range_index -= 1;
4602
4603	const low_val: Constant = .fromInterned(case.ranges[case_range_index][0].toInterned().?);
4604	var low_bigint_space: Constant.BigIntSpace = undefined;
4605	const low_bigint = low_val.toBigInt(&low_bigint_space, zcu);
4606	const low_int: i64 = if (low_bigint.positive) @bitCast(
4607	low_bigint.toInt(u64) catch
4608	return isel.fail("too big case range start: {f}", .{isel.fmtConstant(low_val)}),
4609	) else low_bigint.toInt(i64) catch
4610	return isel.fail("too big case range start: {f}", .{isel.fmtConstant(low_val)});
4611
4612	const high_val: Constant = .fromInterned(case.ranges[case_range_index][1].toInterned().?);
4613	var high_bigint_space: Constant.BigIntSpace = undefined;
4614	const high_bigint = high_val.toBigInt(&high_bigint_space, zcu);
4615	const high_int: i64 = if (high_bigint.positive) @bitCast(
4616	high_bigint.toInt(u64) catch
4617	return isel.fail("too big case range end: {f}", .{isel.fmtConstant(high_val)}),
4618	) else high_bigint.toInt(i64) catch
4619	return isel.fail("too big case range end: {f}", .{isel.fmtConstant(high_val)});
4620
4621	const adjusted_ra = switch (low_int) {
4622	0 => cond_mat.?.ra,
4623	else => try isel.allocIntReg(),
4624	};
4625	defer if (adjusted_ra != cond_mat.?.ra) isel.freeReg(adjusted_ra);
4626	const adjusted_reg = switch (cond_int_info.bits) {
4627	else => unreachable,
4628	1...32 => adjusted_ra.w(),
4629	33...64 => adjusted_ra.x(),
4630	};
4631	const delta_int = high_int -% low_int;
4632	if (case_range_index | case.items.len > 0) {
4633	if (std.math.cast(u5, delta_int)) |pos_imm| try isel.emit(.ccmp(
4634	adjusted_reg,
4635	.{ .immediate = pos_imm },
4636	.{ .n = false, .z = true, .c = false, .v = false },
4637	if (case_range_index > 0) .hi else .ne,
4638	)) else if (std.math.cast(u5, -delta_int)) |neg_imm| try isel.emit(.ccmn(
4639	adjusted_reg,
4640	.{ .immediate = neg_imm },
4641	.{ .n = false, .z = true, .c = false, .v = false },
4642	if (case_range_index > 0) .hi else .ne,
4643	)) else {
4644	const imm_ra = try isel.allocIntReg();
4645	defer isel.freeReg(imm_ra);
4646	const imm_reg = switch (cond_int_info.bits) {
4647	else => unreachable,
4648	1...32 => imm_ra.w(),
4649	33...64 => imm_ra.x(),
4650	};
4651	try isel.emit(.ccmp(
4652	cond_reg,
4653	.{ .register = imm_reg },
4654	.{ .n = false, .z = true, .c = false, .v = false },
4655	if (case_range_index > 0) .hi else .ne,
4656	));
4657	try isel.movImmediate(imm_reg, @bitCast(delta_int));
4658	}
4659	} else {
4660	if (std.math.cast(u12, delta_int)) |pos_imm| try isel.emit(.subs(
4661	zero_reg,
4662	adjusted_reg,
4663	.{ .immediate = pos_imm },
4664	)) else if (std.math.cast(u12, -delta_int)) |neg_imm| try isel.emit(.adds(
4665	zero_reg,
4666	adjusted_reg,
4667	.{ .immediate = neg_imm },
4668	)) else if (if (@as(i12, @truncate(delta_int)) == 0)
4669	std.math.cast(u12, delta_int >> 12)
4670	else
4671	null) |pos_imm_lsr_12| try isel.emit(.subs(
4672	zero_reg,
4673	adjusted_reg,
4674	.{ .shifted_immediate = .{ .immediate = pos_imm_lsr_12, .lsl = .@"12" } },
4675	)) else if (if (@as(i12, @truncate(-delta_int)) == 0)
4676	std.math.cast(u12, -delta_int >> 12)
4677	else
4678	null) |neg_imm_lsr_12| try isel.emit(.adds(
4679	zero_reg,
4680	adjusted_reg,
4681	.{ .shifted_immediate = .{ .immediate = neg_imm_lsr_12, .lsl = .@"12" } },
4682	)) else {
4683	const imm_ra = try isel.allocIntReg();
4684	defer isel.freeReg(imm_ra);
4685	const imm_reg = switch (cond_int_info.bits) {
4686	else => unreachable,
4687	1...32 => imm_ra.w(),
4688	33...64 => imm_ra.x(),
4689	};
4690	try isel.emit(.subs(zero_reg, adjusted_reg, .{ .register = imm_reg }));
4691	try isel.movImmediate(imm_reg, @bitCast(delta_int));
4692	}
4693	}
4694
4695	switch (low_int) {
4696	0 => {},
4697	else => {
4698	if (std.math.cast(u12, low_int)) |pos_imm| try isel.emit(.sub(
4699	adjusted_reg,
4700	cond_reg,
4701	.{ .immediate = pos_imm },
4702	)) else if (std.math.cast(u12, -low_int)) |neg_imm| try isel.emit(.add(
4703	adjusted_reg,
4704	cond_reg,
4705	.{ .immediate = neg_imm },
4706	)) else if (if (@as(i12, @truncate(low_int)) == 0)
4707	std.math.cast(u12, low_int >> 12)
4708	else
4709	null) |pos_imm_lsr_12| try isel.emit(.sub(
4710	adjusted_reg,
4711	cond_reg,
4712	.{ .shifted_immediate = .{ .immediate = pos_imm_lsr_12, .lsl = .@"12" } },
4713	)) else if (if (@as(i12, @truncate(-low_int)) == 0)
4714	std.math.cast(u12, -low_int >> 12)
4715	else
4716	null) |neg_imm_lsr_12| try isel.emit(.add(
4717	adjusted_reg,
4718	cond_reg,
4719	.{ .shifted_immediate = .{ .immediate = neg_imm_lsr_12, .lsl = .@"12" } },
4720	)) else {
4721	const imm_ra = try isel.allocIntReg();
4722	defer isel.freeReg(imm_ra);
4723	const imm_reg = switch (cond_int_info.bits) {
4724	else => unreachable,
4725	1...32 => imm_ra.w(),
4726	33...64 => imm_ra.x(),
4727	};
4728	try isel.emit(.sub(adjusted_reg, cond_reg, .{ .register = imm_reg }));
4729	try isel.movImmediate(imm_reg, @bitCast(low_int));
4730	}
4731	},
4732	}
4733	}
4734	var case_item_index = case.items.len;
4735	while (case_item_index > 0) {
4736	case_item_index -= 1;
4737
4738	const item_val: Constant = .fromInterned(case.items[case_item_index].toInterned().?);
4739	var item_bigint_space: Constant.BigIntSpace = undefined;
4740	const item_bigint = item_val.toBigInt(&item_bigint_space, zcu);
4741	const item_int: i64 = if (item_bigint.positive) @bitCast(
4742	item_bigint.toInt(u64) catch
4743	return isel.fail("too big case item: {f}", .{isel.fmtConstant(item_val)}),
4744	) else item_bigint.toInt(i64) catch
4745	return isel.fail("too big case item: {f}", .{isel.fmtConstant(item_val)});
4746
4747	if (case_item_index > 0) {
4748	if (std.math.cast(u5, item_int)) |pos_imm| try isel.emit(.ccmp(
4749	cond_reg,
4750	.{ .immediate = pos_imm },
4751	.{ .n = false, .z = true, .c = false, .v = false },
4752	.ne,
4753	)) else if (std.math.cast(u5, -item_int)) |neg_imm| try isel.emit(.ccmn(
4754	cond_reg,
4755	.{ .immediate = neg_imm },
4756	.{ .n = false, .z = true, .c = false, .v = false },
4757	.ne,
4758	)) else {
4759	const imm_ra = try isel.allocIntReg();
4760	defer isel.freeReg(imm_ra);
4761	const imm_reg = switch (cond_int_info.bits) {
4762	else => unreachable,
4763	1...32 => imm_ra.w(),
4764	33...64 => imm_ra.x(),
4765	};
4766	try isel.emit(.ccmp(
4767	cond_reg,
4768	.{ .register = imm_reg },
4769	.{ .n = false, .z = true, .c = false, .v = false },
4770	.ne,
4771	));
4772	try isel.movImmediate(imm_reg, @bitCast(item_int));
4773	}
4774	} else {
4775	if (std.math.cast(u12, item_int)) |pos_imm| try isel.emit(.subs(
4776	zero_reg,
4777	cond_reg,
4778	.{ .immediate = pos_imm },
4779	)) else if (std.math.cast(u12, -item_int)) |neg_imm| try isel.emit(.adds(
4780	zero_reg,
4781	cond_reg,
4782	.{ .immediate = neg_imm },
4783	)) else if (if (@as(i12, @truncate(item_int)) == 0)
4784	std.math.cast(u12, item_int >> 12)
4785	else
4786	null) |pos_imm_lsr_12| try isel.emit(.subs(
4787	zero_reg,
4788	cond_reg,
4789	.{ .shifted_immediate = .{ .immediate = pos_imm_lsr_12, .lsl = .@"12" } },
4790	)) else if (if (@as(i12, @truncate(-item_int)) == 0)
4791	std.math.cast(u12, -item_int >> 12)
4792	else
4793	null) |neg_imm_lsr_12| try isel.emit(.adds(
4794	zero_reg,
4795	cond_reg,
4796	.{ .shifted_immediate = .{ .immediate = neg_imm_lsr_12, .lsl = .@"12" } },
4797	)) else {
4798	const imm_ra = try isel.allocIntReg();
4799	defer isel.freeReg(imm_ra);
4800	const imm_reg = switch (cond_int_info.bits) {
4801	else => unreachable,
4802	1...32 => imm_ra.w(),
4803	33...64 => imm_ra.x(),
4804	};
4805	try isel.emit(.subs(zero_reg, cond_reg, .{ .register = imm_reg }));
4806	try isel.movImmediate(imm_reg, @bitCast(item_int));
4807	}
4808	}
4809	}
4810	}
4811	if (cond_mat) |mat| try mat.finish(isel);
4812	if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
4813	},
4814	.@"try", .try_cold => {
4815	const unwrapped_try = isel.air.unwrapTry(air.inst_index);
4816	const error_union_ty = isel.air.typeOf(unwrapped_try.error_union, &zcu.intern_pool);
4817	const error_union_info = ip.indexToKey(error_union_ty.toIntern()).error_union_type;
4818	const payload_ty: ZigType = .fromInterned(error_union_info.payload_type);
4819
4820	const error_union_vi = try isel.use(unwrapped_try.error_union);
4821	if (isel.live_values.fetchRemove(air.inst_index)) |payload_vi| {
4822	defer payload_vi.value.deref(isel);
4823
4824	var payload_part_it = error_union_vi.field(
4825	error_union_ty,
4826	codegen.errUnionPayloadOffset(payload_ty, zcu),
4827	payload_vi.value.size(isel),
4828	);
4829	const payload_part_vi = try payload_part_it.only(isel);
4830	try payload_vi.value.copy(isel, payload_ty, payload_part_vi.?);
4831	}
4832
4833	const cont_label = isel.instructions.items.len;
4834	const cont_live_registers = isel.live_registers;
4835	try isel.body(unwrapped_try.else_body);
4836	try isel.merge(&cont_live_registers, .{});
4837
4838	var error_set_part_it = error_union_vi.field(
4839	error_union_ty,
4840	codegen.errUnionErrorOffset(payload_ty, zcu),
4841	ZigType.fromInterned(error_union_info.error_set_type).abiSize(zcu),
4842	);
4843	const error_set_part_vi = try error_set_part_it.only(isel);
4844	const error_set_part_mat = try error_set_part_vi.?.matReg(isel);
4845	try isel.emit(.cbz(
4846	error_set_part_mat.ra.w(),
4847	@intCast((isel.instructions.items.len + 1 - cont_label) << 2),
4848	));
4849	try error_set_part_mat.finish(isel);
4850
4851	if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
4852	},
4853	.try_ptr, .try_ptr_cold => {
4854	const unwrapped_try = isel.air.unwrapTryPtr(air.inst_index);
4855	const error_union_ty = isel.air.typeOf(unwrapped_try.error_union_ptr, ip).childType(zcu);
4856	const error_union_info = ip.indexToKey(error_union_ty.toIntern()).error_union_type;
4857	const payload_ty: ZigType = .fromInterned(error_union_info.payload_type);
4858
4859	const error_union_ptr_vi = try isel.use(unwrapped_try.error_union_ptr);
4860	const error_union_ptr_mat = try error_union_ptr_vi.matReg(isel);
4861	if (isel.live_values.fetchRemove(air.inst_index)) |payload_ptr_vi| unused: {
4862	defer payload_ptr_vi.value.deref(isel);
4863	switch (codegen.errUnionPayloadOffset(unwrapped_try.error_union_payload_ptr_ty.childType(zcu), zcu)) {
4864	0 => try payload_ptr_vi.value.move(isel, unwrapped_try.error_union_ptr),
4865	else => |payload_offset| {
4866	const payload_ptr_ra = try payload_ptr_vi.value.defReg(isel) orelse break :unused;
4867	const lo12: u12 = @truncate(payload_offset >> 0);
4868	const hi12: u12 = @intCast(payload_offset >> 12);
4869	if (hi12 > 0) try isel.emit(.add(
4870	payload_ptr_ra.x(),
4871	if (lo12 > 0) payload_ptr_ra.x() else error_union_ptr_mat.ra.x(),
4872	.{ .shifted_immediate = .{ .immediate = hi12, .lsl = .@"12" } },
4873	));
4874	if (lo12 > 0) try isel.emit(.add(payload_ptr_ra.x(), error_union_ptr_mat.ra.x(), .{ .immediate = lo12 }));
4875	},
4876	}
4877	}
4878
4879	const cont_label = isel.instructions.items.len;
4880	const cont_live_registers = isel.live_registers;
4881	try isel.body(unwrapped_try.else_body);
4882	try isel.merge(&cont_live_registers, .{});
4883
4884	const error_set_ra = try isel.allocIntReg();
4885	defer isel.freeReg(error_set_ra);
4886	try isel.loadReg(
4887	error_set_ra,
4888	ZigType.fromInterned(error_union_info.error_set_type).abiSize(zcu),
4889	.unsigned,
4890	error_union_ptr_mat.ra,
4891	codegen.errUnionErrorOffset(payload_ty, zcu),
4892	);
4893	try error_union_ptr_mat.finish(isel);
4894	try isel.emit(.cbz(
4895	error_set_ra.w(),
4896	@intCast((isel.instructions.items.len + 1 - cont_label) << 2),
4897	));
4898
4899	if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
4900	},
4901	.dbg_stmt => if (air.next()) |next_air_tag| continue :air_tag next_air_tag,
4902	.dbg_empty_stmt => {
4903	try isel.emit(.nop());
4904	if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
4905	},
4906	.dbg_inline_block => {
4907	const dbg_block = isel.air.unwrapDbgBlock(air.inst_index);
4908	try isel.block(air.inst_index, dbg_block.ty, dbg_block.body);
4909	if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
4910	},
4911	.dbg_var_ptr, .dbg_var_val, .dbg_arg_inline => {
4912	if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
4913	},
4914	.is_null, .is_non_null => |air_tag| {
4915	if (isel.live_values.fetchRemove(air.inst_index)) |is_vi| unused: {
4916	defer is_vi.value.deref(isel);
4917	const is_ra = try is_vi.value.defReg(isel) orelse break :unused;
4918
4919	const un_op = air.data(air.inst_index).un_op;
4920	const opt_ty = isel.air.typeOf(un_op, ip);
4921	const payload_ty = opt_ty.optionalChild(zcu);
4922	const payload_size = payload_ty.abiSize(zcu);
4923	const has_value_offset, const has_value_size = if (!opt_ty.optionalReprIsPayload(zcu))
4924	.{ payload_size, 1 }
4925	else if (payload_ty.isSlice(zcu))
4926	.{ 0, 8 }
4927	else
4928	.{ 0, payload_size };
4929
4930	try isel.emit(.csinc(is_ra.w(), .wzr, .wzr, .invert(switch (air_tag) {
4931	else => unreachable,
4932	.is_null => .eq,
4933	.is_non_null => .ne,
4934	})));
4935	const opt_vi = try isel.use(un_op);
4936	var has_value_part_it = opt_vi.field(opt_ty, has_value_offset, has_value_size);
4937	const has_value_part_vi = try has_value_part_it.only(isel);
4938	const has_value_part_mat = try has_value_part_vi.?.matReg(isel);
4939	try isel.emit(switch (has_value_size) {
4940	else => unreachable,
4941	1...4 => .subs(.wzr, has_value_part_mat.ra.w(), .{ .immediate = 0 }),
4942	5...8 => .subs(.xzr, has_value_part_mat.ra.x(), .{ .immediate = 0 }),
4943	});
4944	try has_value_part_mat.finish(isel);
4945	}
4946	if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
4947	},
4948	.is_err, .is_non_err => |air_tag| {
4949	if (isel.live_values.fetchRemove(air.inst_index)) |is_vi| unused: {
4950	defer is_vi.value.deref(isel);
4951	const is_ra = try is_vi.value.defReg(isel) orelse break :unused;
4952
4953	const un_op = air.data(air.inst_index).un_op;
4954	const error_union_ty = isel.air.typeOf(un_op, ip);
4955	const error_union_info = ip.indexToKey(error_union_ty.toIntern()).error_union_type;
4956	const error_set_ty: ZigType = .fromInterned(error_union_info.error_set_type);
4957	const payload_ty: ZigType = .fromInterned(error_union_info.payload_type);
4958	const error_set_offset = codegen.errUnionErrorOffset(payload_ty, zcu);
4959	const error_set_size = error_set_ty.abiSize(zcu);
4960
4961	try isel.emit(.csinc(is_ra.w(), .wzr, .wzr, .invert(switch (air_tag) {
4962	else => unreachable,
4963	.is_err => .ne,
4964	.is_non_err => .eq,
4965	})));
4966	const error_union_vi = try isel.use(un_op);
4967	var error_set_part_it = error_union_vi.field(error_union_ty, error_set_offset, error_set_size);
4968	const error_set_part_vi = try error_set_part_it.only(isel);
4969	const error_set_part_mat = try error_set_part_vi.?.matReg(isel);
4970	try isel.emit(.ands(.wzr, error_set_part_mat.ra.w(), .{ .immediate = .{
4971	.N = .word,
4972	.immr = 0,
4973	.imms = @intCast(8 * error_set_size - 1),
4974	} }));
4975	try error_set_part_mat.finish(isel);
4976	}
4977	if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
4978	},
4979	.load => {
4980	const ty_op = air.data(air.inst_index).ty_op;
4981	const ptr_ty = isel.air.typeOf(ty_op.operand, ip);
4982	const ptr_info = ptr_ty.ptrInfo(zcu);
4983	if (ptr_info.packed_offset.host_size > 0) return isel.fail("packed load", .{});
4984
4985	if (ptr_info.flags.is_volatile) _ = try isel.use(air.inst_index.toRef());
4986	if (isel.live_values.fetchRemove(air.inst_index)) |dst_vi| unused: {
4987	defer dst_vi.value.deref(isel);
4988	const size = dst_vi.value.size(isel);
4989	if (size <= Value.max_parts and ip.zigTypeTag(ptr_info.child) != .@"union") {
4990	const ptr_vi = try isel.use(ty_op.operand);
4991	const ptr_mat = try ptr_vi.matReg(isel);
4992	_ = try dst_vi.value.load(isel, ty_op.ty, ptr_mat.ra, .{
4993	.@"volatile" = ptr_info.flags.is_volatile,
4994	});
4995	try ptr_mat.finish(isel);
4996	} else {
4997	try dst_vi.value.defAddr(isel, .fromInterned(ptr_info.child), .{}) orelse break :unused;
4998
4999	try call.prepareReturn(isel);
5000	try call.finishReturn(isel);
5001
5002	try call.prepareCallee(isel);
5003	try isel.global_relocs.append(gpa, .{
5004	.name = "memcpy",
5005	.reloc = .{ .label = @intCast(isel.instructions.items.len) },
5006	});
5007	try isel.emit(.bl(0));
5008	try call.finishCallee(isel);
5009
5010	try call.prepareParams(isel);
5011	const ptr_vi = try isel.use(ty_op.operand);
5012	try isel.movImmediate(.x2, size);
5013	try call.paramLiveOut(isel, ptr_vi, .r1);
5014	try call.paramAddress(isel, dst_vi.value, .r0);
5015	try call.finishParams(isel);
5016	}
5017	}
5018
5019	if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
5020	},
5021	.ret, .ret_safe => {
5022	assert(isel.blocks.keys()[0] == Block.main);
5023	try isel.blocks.values()[0].branch(isel);
5024	if (isel.live_values.get(Block.main)) |ret_vi| {
5025	const un_op = air.data(air.inst_index).un_op;
5026	const src_vi = try isel.use(un_op);
5027	switch (ret_vi.parent(isel)) {
5028	.unallocated, .stack_slot => if (ret_vi.hint(isel)) |ret_ra| {
5029	try src_vi.liveOut(isel, ret_ra);
5030	} else {
5031	var ret_part_it = ret_vi.parts(isel);
5032	var src_part_it = src_vi.parts(isel);
5033	if (src_part_it.only()) |_| {
5034	try isel.values.ensureUnusedCapacity(gpa, ret_part_it.remaining);
5035	src_vi.setParts(isel, ret_part_it.remaining);
5036	while (ret_part_it.next()) |ret_part_vi| {
5037	const src_part_vi = src_vi.addPart(
5038	isel,
5039	ret_part_vi.get(isel).offset_from_parent,
5040	ret_part_vi.size(isel),
5041	);
5042	switch (ret_part_vi.signedness(isel)) {
5043	.signed => src_part_vi.setSignedness(isel, .signed),
5044	.unsigned => {},
5045	}
5046	if (ret_part_vi.isVector(isel)) src_part_vi.setIsVector(isel);
5047	}
5048	ret_part_it = ret_vi.parts(isel);
5049	src_part_it = src_vi.parts(isel);
5050	}
5051	while (ret_part_it.next()) |ret_part_vi| {
5052	const src_part_vi = src_part_it.next().?;
5053	assert(ret_part_vi.get(isel).offset_from_parent == src_part_vi.get(isel).offset_from_parent);
5054	assert(ret_part_vi.size(isel) == src_part_vi.size(isel));
5055	try src_part_vi.liveOut(isel, ret_part_vi.hint(isel).?);
5056	}
5057	},
5058	.value, .constant => unreachable,
5059	.address => |address_vi| {
5060	const ptr_mat = try address_vi.matReg(isel);
5061	try src_vi.store(isel, isel.air.typeOf(un_op, ip), ptr_mat.ra, .{});
5062	try ptr_mat.finish(isel);
5063	},
5064	}
5065	}
5066	if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
5067	},
5068	.ret_load => {
5069	const un_op = air.data(air.inst_index).un_op;
5070	const ptr_ty = isel.air.typeOf(un_op, ip);
5071	const ptr_info = ptr_ty.ptrInfo(zcu);
5072	if (ptr_info.packed_offset.host_size > 0) return isel.fail("packed load", .{});
5073
5074	assert(isel.blocks.keys()[0] == Block.main);
5075	try isel.blocks.values()[0].branch(isel);
5076	if (isel.live_values.get(Block.main)) |ret_vi| switch (ret_vi.parent(isel)) {
5077	.unallocated, .stack_slot => {
5078	var ret_part_it: Value.PartIterator = if (ret_vi.hint(isel)) |_| .initOne(ret_vi) else ret_vi.parts(isel);
5079	while (ret_part_it.next()) |ret_part_vi| try ret_part_vi.liveOut(isel, ret_part_vi.hint(isel).?);
5080	const ptr_vi = try isel.use(un_op);
5081	const ptr_mat = try ptr_vi.matReg(isel);
5082	_ = try ret_vi.load(isel, .fromInterned(ptr_info.child), ptr_mat.ra, .{});
5083	try ptr_mat.finish(isel);
5084	},
5085	.value, .constant => unreachable,
5086	.address => {},
5087	};
5088	if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
5089	},
5090	.store, .store_safe, .atomic_store_unordered => {
5091	const bin_op = air.data(air.inst_index).bin_op;
5092	const ptr_ty = isel.air.typeOf(bin_op.lhs, ip);
5093	const ptr_info = ptr_ty.ptrInfo(zcu);
5094	if (ptr_info.packed_offset.host_size > 0) return isel.fail("packed store", .{});
5095	if (bin_op.rhs.toInterned()) |rhs_val| if (ip.isUndef(rhs_val))
5096	break :air_tag if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
5097
5098	const src_vi = try isel.use(bin_op.rhs);
5099	const size = src_vi.size(isel);
5100	if (ZigType.fromInterned(ptr_info.child).zigTypeTag(zcu) != .@"union") switch (size) {
5101	0 => unreachable,
5102	1...Value.max_parts => {
5103	const ptr_vi = try isel.use(bin_op.lhs);
5104	const ptr_mat = try ptr_vi.matReg(isel);
5105	try src_vi.store(isel, isel.air.typeOf(bin_op.rhs, ip), ptr_mat.ra, .{
5106	.@"volatile" = ptr_info.flags.is_volatile,
5107	});
5108	try ptr_mat.finish(isel);
5109
5110	break :air_tag if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
5111	},
5112	else => {},
5113	};
5114	try call.prepareReturn(isel);
5115	try call.finishReturn(isel);
5116
5117	try call.prepareCallee(isel);
5118	try isel.global_relocs.append(gpa, .{
5119	.name = "memcpy",
5120	.reloc = .{ .label = @intCast(isel.instructions.items.len) },
5121	});
5122	try isel.emit(.bl(0));
5123	try call.finishCallee(isel);
5124
5125	try call.prepareParams(isel);
5126	const ptr_vi = try isel.use(bin_op.lhs);
5127	try isel.movImmediate(.x2, size);
5128	try call.paramAddress(isel, src_vi, .r1);
5129	try call.paramLiveOut(isel, ptr_vi, .r0);
5130	try call.finishParams(isel);
5131
5132	if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
5133	},
5134	.unreach => if (air.next()) |next_air_tag| continue :air_tag next_air_tag,
5135	.fptrunc, .fpext => {
5136	if (isel.live_values.fetchRemove(air.inst_index)) |dst_vi| unused: {
5137	defer dst_vi.value.deref(isel);
5138
5139	const ty_op = air.data(air.inst_index).ty_op;
5140	const dst_ty = ty_op.ty;
5141	const dst_bits = dst_ty.floatBits(isel.target);
5142	const src_ty = isel.air.typeOf(ty_op.operand, ip);
5143	const src_bits = src_ty.floatBits(isel.target);
5144	assert(dst_bits != src_bits);
5145	switch (@max(dst_bits, src_bits)) {
5146	else => unreachable,
5147	16, 32, 64 => {
5148	const dst_ra = try dst_vi.value.defReg(isel) orelse break :unused;
5149	const src_vi = try isel.use(ty_op.operand);
5150	const src_mat = try src_vi.matReg(isel);
5151	try isel.emit(.fcvt(switch (dst_bits) {
5152	else => unreachable,
5153	16 => dst_ra.h(),
5154	32 => dst_ra.s(),
5155	64 => dst_ra.d(),
5156	}, switch (src_bits) {
5157	else => unreachable,
5158	16 => src_mat.ra.h(),
5159	32 => src_mat.ra.s(),
5160	64 => src_mat.ra.d(),
5161	}));
5162	try src_mat.finish(isel);
5163	},
5164	80, 128 => {
5165	try call.prepareReturn(isel);
5166	switch (dst_bits) {
5167	else => unreachable,
5168	16, 32, 64, 128 => try call.returnLiveIn(isel, dst_vi.value, .v0),
5169	80 => {
5170	var dst_hi16_it = dst_vi.value.field(dst_ty, 8, 8);
5171	const dst_hi16_vi = try dst_hi16_it.only(isel);
5172	try call.returnLiveIn(isel, dst_hi16_vi.?, .r1);
5173	var dst_lo64_it = dst_vi.value.field(dst_ty, 0, 8);
5174	const dst_lo64_vi = try dst_lo64_it.only(isel);
5175	try call.returnLiveIn(isel, dst_lo64_vi.?, .r0);
5176	},
5177	}
5178	try call.finishReturn(isel);
5179
5180	try call.prepareCallee(isel);
5181	try isel.global_relocs.append(gpa, .{
5182	.name = switch (dst_bits) {
5183	else => unreachable,
5184	16 => switch (src_bits) {
5185	else => unreachable,
5186	32 => "__truncsfhf2",
5187	64 => "__truncdfhf2",
5188	80 => "__truncxfhf2",
5189	128 => "__trunctfhf2",
5190	},
5191	32 => switch (src_bits) {
5192	else => unreachable,
5193	16 => "__extendhfsf2",
5194	64 => "__truncdfsf2",
5195	80 => "__truncxfsf2",
5196	128 => "__trunctfsf2",
5197	},
5198	64 => switch (src_bits) {
5199	else => unreachable,
5200	16 => "__extendhfdf2",
5201	32 => "__extendsfdf2",
5202	80 => "__truncxfdf2",
5203	128 => "__trunctfdf2",
5204	},
5205	80 => switch (src_bits) {
5206	else => unreachable,
5207	16 => "__extendhfxf2",
5208	32 => "__extendsfxf2",
5209	64 => "__extenddfxf2",
5210	128 => "__trunctfxf2",
5211	},
5212	128 => switch (src_bits) {
5213	else => unreachable,
5214	16 => "__extendhftf2",
5215	32 => "__extendsftf2",
5216	64 => "__extenddftf2",
5217	80 => "__extendxftf2",
5218	},
5219	},
5220	.reloc = .{ .label = @intCast(isel.instructions.items.len) },
5221	});
5222	try isel.emit(.bl(0));
5223	try call.finishCallee(isel);
5224
5225	try call.prepareParams(isel);
5226	const src_vi = try isel.use(ty_op.operand);
5227	switch (src_bits) {
5228	else => unreachable,
5229	16, 32, 64, 128 => try call.paramLiveOut(isel, src_vi, .v0),
5230	80 => {
5231	var src_hi16_it = src_vi.field(src_ty, 8, 8);
5232	const src_hi16_vi = try src_hi16_it.only(isel);
5233	try call.paramLiveOut(isel, src_hi16_vi.?, .r1);
5234	var src_lo64_it = src_vi.field(src_ty, 0, 8);
5235	const src_lo64_vi = try src_lo64_it.only(isel);
5236	try call.paramLiveOut(isel, src_lo64_vi.?, .r0);
5237	},
5238	}
5239	try call.finishParams(isel);
5240	},
5241	}
5242	}
5243	if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
5244	},
5245	.int_cast => |air_tag| {
5246	if (isel.live_values.fetchRemove(air.inst_index)) |dst_vi| unused: {
5247	defer dst_vi.value.deref(isel);
5248
5249	const ty_op = air.data(air.inst_index).ty_op;
5250	const dst_ty = ty_op.ty;
5251	const dst_int_info = dst_ty.intInfo(zcu);
5252	const src_ty = isel.air.typeOf(ty_op.operand, ip);
5253	const src_int_info = src_ty.intInfo(zcu);
5254	const can_be_negative = dst_int_info.signedness == .signed and
5255	src_int_info.signedness == .signed;
5256	if ((dst_int_info.bits <= 8 and src_int_info.bits <= 8) or
5257	(dst_int_info.bits > 8 and dst_int_info.bits <= 16 and
5258	src_int_info.bits > 8 and src_int_info.bits <= 16) or
5259	(dst_int_info.bits > 16 and dst_int_info.bits <= 32 and
5260	src_int_info.bits > 16 and src_int_info.bits <= 32) or
5261	(dst_int_info.bits > 32 and dst_int_info.bits <= 64 and
5262	src_int_info.bits > 32 and src_int_info.bits <= 64) or
5263	(dst_int_info.bits > 64 and src_int_info.bits > 64 and
5264	(dst_int_info.bits - 1) / 128 == (src_int_info.bits - 1) / 128))
5265	{
5266	try dst_vi.value.move(isel, ty_op.operand);
5267	} else if (dst_int_info.bits <= 32 and src_int_info.bits <= 64) {
5268	const dst_ra = try dst_vi.value.defReg(isel) orelse break :unused;
5269	const src_vi = try isel.use(ty_op.operand);
5270	const src_mat = try src_vi.matReg(isel);
5271	try isel.emit(.orr(dst_ra.w(), .wzr, .{ .register = src_mat.ra.w() }));
5272	try src_mat.finish(isel);
5273	} else if (dst_int_info.bits <= 64 and src_int_info.bits <= 32) {
5274	const dst_ra = try dst_vi.value.defReg(isel) orelse break :unused;
5275	const src_vi = try isel.use(ty_op.operand);
5276	const src_mat = try src_vi.matReg(isel);
5277	try isel.emit(if (can_be_negative) .sbfm(dst_ra.x(), src_mat.ra.x(), .{
5278	.N = .doubleword,
5279	.immr = 0,
5280	.imms = @intCast(src_int_info.bits - 1),
5281	}) else .orr(dst_ra.w(), .wzr, .{ .register = src_mat.ra.w() }));
5282	try src_mat.finish(isel);
5283	} else if (dst_int_info.bits <= 32 and src_int_info.bits <= 128) {
5284	assert(src_int_info.bits > 64);
5285	const dst_ra = try dst_vi.value.defReg(isel) orelse break :unused;
5286	const src_vi = try isel.use(ty_op.operand);
5287
5288	var src_lo64_it = src_vi.field(src_ty, 0, 8);
5289	const src_lo64_vi = try src_lo64_it.only(isel);
5290	const src_lo64_mat = try src_lo64_vi.?.matReg(isel);
5291	try isel.emit(.orr(dst_ra.w(), .wzr, .{ .register = src_lo64_mat.ra.w() }));
5292	try src_lo64_mat.finish(isel);
5293	} else if (dst_int_info.bits <= 64 and src_int_info.bits <= 128) {
5294	assert(dst_int_info.bits > 32 and src_int_info.bits > 64);
5295	const src_vi = try isel.use(ty_op.operand);
5296
5297	var src_lo64_it = src_vi.field(src_ty, 0, 8);
5298	const src_lo64_vi = try src_lo64_it.only(isel);
5299	try dst_vi.value.copy(isel, dst_ty, src_lo64_vi.?);
5300	} else if (dst_int_info.bits <= 128 and src_int_info.bits <= 64) {
5301	assert(dst_int_info.bits > 64);
5302	const src_vi = try isel.use(ty_op.operand);
5303
5304	var dst_lo64_it = dst_vi.value.field(dst_ty, 0, 8);
5305	const dst_lo64_vi = try dst_lo64_it.only(isel);
5306	if (src_int_info.bits <= 32) unused_lo64: {
5307	const dst_lo64_ra = try dst_lo64_vi.?.defReg(isel) orelse break :unused_lo64;
5308	const src_mat = try src_vi.matReg(isel);
5309	try isel.emit(if (can_be_negative) .sbfm(dst_lo64_ra.x(), src_mat.ra.x(), .{
5310	.N = .doubleword,
5311	.immr = 0,
5312	.imms = @intCast(src_int_info.bits - 1),
5313	}) else .orr(dst_lo64_ra.w(), .wzr, .{ .register = src_mat.ra.w() }));
5314	try src_mat.finish(isel);
5315	} else try dst_lo64_vi.?.copy(isel, src_ty, src_vi);
5316
5317	var dst_hi64_it = dst_vi.value.field(dst_ty, 8, 8);
5318	const dst_hi64_vi = try dst_hi64_it.only(isel);
5319	const dst_hi64_ra = try dst_hi64_vi.?.defReg(isel);
5320	if (dst_hi64_ra) |dst_ra| switch (can_be_negative) {
5321	false => try isel.emit(.orr(dst_ra.x(), .xzr, .{ .register = .xzr })),
5322	true => {
5323	const src_mat = try src_vi.matReg(isel);
5324	try isel.emit(.sbfm(dst_ra.x(), src_mat.ra.x(), .{
5325	.N = .doubleword,
5326	.immr = @intCast(src_int_info.bits - 1),
5327	.imms = @intCast(src_int_info.bits - 1),
5328	}));
5329	try src_mat.finish(isel);
5330	},
5331	};
5332	} else return isel.fail("too big {t} {f} {f}", .{ air_tag, isel.fmtType(dst_ty), isel.fmtType(src_ty) });
5333	}
5334	if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
5335	},
5336	.int_cast_safe => |air_tag| {
5337	if (isel.live_values.fetchRemove(air.inst_index)) |dst_vi| unused: {
5338	defer dst_vi.value.deref(isel);
5339
5340	const ty_op = air.data(air.inst_index).ty_op;
5341	const dst_ty = ty_op.ty;
5342	const dst_int_info = dst_ty.intInfo(zcu);
5343	const src_ty = isel.air.typeOf(ty_op.operand, ip);
5344	const src_int_info = src_ty.intInfo(zcu);
5345	const can_be_negative = dst_int_info.signedness == .signed and
5346	src_int_info.signedness == .signed;
5347	const panic_id: Zcu.SimplePanicId = panic_id: switch (dst_ty.zigTypeTag(zcu)) {
5348	else => unreachable,
5349	.int => .integer_out_of_bounds,
5350	.@"enum" => {
5351	if (!dst_ty.isNonexhaustiveEnum(zcu)) {
5352	return isel.fail("bad {t} {f} {f}", .{ air_tag, isel.fmtType(dst_ty), isel.fmtType(src_ty) });
5353	}
5354	break :panic_id .invalid_enum_value;
5355	},
5356	};
5357	if (dst_ty.toIntern() == src_ty.toIntern()) {
5358	try dst_vi.value.move(isel, ty_op.operand);
5359	} else if (dst_int_info.bits <= 64 and src_int_info.bits <= 64) {
5360	const dst_ra = try dst_vi.value.defReg(isel) orelse break :unused;
5361	const src_vi = try isel.use(ty_op.operand);
5362	const dst_active_bits = dst_int_info.bits - @intFromBool(dst_int_info.signedness == .signed);
5363	const src_active_bits = src_int_info.bits - @intFromBool(src_int_info.signedness == .signed);
5364	if ((dst_int_info.signedness != .unsigned or src_int_info.signedness != .signed) and dst_active_bits >= src_active_bits) {
5365	const src_mat = try src_vi.matReg(isel);
5366	try isel.emit(if (can_be_negative and dst_active_bits > 32 and src_active_bits <= 32)
5367	.sbfm(dst_ra.x(), src_mat.ra.x(), .{
5368	.N = .doubleword,
5369	.immr = 0,
5370	.imms = @intCast(src_int_info.bits - 1),
5371	})
5372	else switch (src_int_info.bits) {
5373	else => unreachable,
5374	1...32 => .orr(dst_ra.w(), .wzr, .{ .register = src_mat.ra.w() }),
5375	33...64 => .orr(dst_ra.x(), .xzr, .{ .register = src_mat.ra.x() }),
5376	});
5377	try src_mat.finish(isel);
5378	} else {
5379	const skip_label = isel.instructions.items.len;
5380	try isel.emitPanic(panic_id);
5381	try isel.emit(.@"b."(
5382	.eq,
5383	@intCast((isel.instructions.items.len + 1 - skip_label) << 2),
5384	));
5385	if (can_be_negative) {
5386	const src_mat = src_mat: {
5387	const dst_lock = isel.lockReg(dst_ra);
5388	defer dst_lock.unlock(isel);
5389	break :src_mat try src_vi.matReg(isel);
5390	};
5391	try isel.emit(switch (src_int_info.bits) {
5392	else => unreachable,
5393	1...32 => .subs(.wzr, dst_ra.w(), .{ .register = src_mat.ra.w() }),
5394	33...64 => .subs(.xzr, dst_ra.x(), .{ .register = src_mat.ra.x() }),
5395	});
5396	try isel.emit(switch (@max(dst_int_info.bits, src_int_info.bits)) {
5397	else => unreachable,
5398	1...32 => .sbfm(dst_ra.w(), src_mat.ra.w(), .{
5399	.N = .word,
5400	.immr = 0,
5401	.imms = @intCast(dst_int_info.bits - 1),
5402	}),
5403	33...64 => .sbfm(dst_ra.x(), src_mat.ra.x(), .{
5404	.N = .doubleword,
5405	.immr = 0,
5406	.imms = @intCast(dst_int_info.bits - 1),
5407	}),
5408	});
5409	try src_mat.finish(isel);
5410	} else {
5411	const src_mat = try src_vi.matReg(isel);
5412	try isel.emit(switch (@min(dst_int_info.bits, src_int_info.bits)) {
5413	else => unreachable,
5414	1...32 => .orr(dst_ra.w(), .wzr, .{ .register = src_mat.ra.w() }),
5415	33...64 => .orr(dst_ra.x(), .xzr, .{ .register = src_mat.ra.x() }),
5416	});
5417	const active_bits = @min(dst_active_bits, src_active_bits);
5418	try isel.emit(switch (src_int_info.bits) {
5419	else => unreachable,
5420	1...32 => .ands(.wzr, src_mat.ra.w(), .{ .immediate = .{
5421	.N = .word,
5422	.immr = @intCast(32 - active_bits),
5423	.imms = @intCast(32 - active_bits - 1),
5424	} }),
5425	33...64 => .ands(.xzr, src_mat.ra.x(), .{ .immediate = .{
5426	.N = .doubleword,
5427	.immr = @intCast(64 - active_bits),
5428	.imms = @intCast(64 - active_bits - 1),
5429	} }),
5430	});
5431	try src_mat.finish(isel);
5432	}
5433	}
5434	} else return isel.fail("too big {t} {f} {f}", .{ air_tag, isel.fmtType(dst_ty), isel.fmtType(src_ty) });
5435	}
5436	if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
5437	},
5438	.trunc => |air_tag| {
5439	if (isel.live_values.fetchRemove(air.inst_index)) |dst_vi| unused: {
5440	defer dst_vi.value.deref(isel);
5441
5442	const ty_op = air.data(air.inst_index).ty_op;
5443	const dst_ty = ty_op.ty;
5444	const src_ty = isel.air.typeOf(ty_op.operand, ip);
5445	if (!dst_ty.isAbiInt(zcu) or !src_ty.isAbiInt(zcu)) return isel.fail("bad {t} {f} {f}", .{ air_tag, isel.fmtType(dst_ty), isel.fmtType(src_ty) });
5446	const dst_int_info = dst_ty.intInfo(zcu);
5447	switch (dst_int_info.bits) {
5448	0 => unreachable,
5449	1...64 => |dst_bits| {
5450	const dst_ra = try dst_vi.value.defReg(isel) orelse break :unused;
5451	const src_vi = try isel.use(ty_op.operand);
5452	var src_part_it = src_vi.field(src_ty, 0, @min(src_vi.size(isel), 8));
5453	const src_part_vi = try src_part_it.only(isel);
5454	const src_part_mat = try src_part_vi.?.matReg(isel);
5455	try isel.emit(switch (dst_bits) {
5456	else => unreachable,
5457	1...31 => |bits| switch (dst_int_info.signedness) {
5458	.signed => .sbfm(dst_ra.w(), src_part_mat.ra.w(), .{
5459	.N = .word,
5460	.immr = 0,
5461	.imms = @intCast(bits - 1),
5462	}),
5463	.unsigned => .ubfm(dst_ra.w(), src_part_mat.ra.w(), .{
5464	.N = .word,
5465	.immr = 0,
5466	.imms = @intCast(bits - 1),
5467	}),
5468	},
5469	32 => .orr(dst_ra.w(), .wzr, .{ .register = src_part_mat.ra.w() }),
5470	33...63 => |bits| switch (dst_int_info.signedness) {
5471	.signed => .sbfm(dst_ra.x(), src_part_mat.ra.x(), .{
5472	.N = .doubleword,
5473	.immr = 0,
5474	.imms = @intCast(bits - 1),
5475	}),
5476	.unsigned => .ubfm(dst_ra.x(), src_part_mat.ra.x(), .{
5477	.N = .doubleword,
5478	.immr = 0,
5479	.imms = @intCast(bits - 1),
5480	}),
5481	},
5482	64 => .orr(dst_ra.x(), .xzr, .{ .register = src_part_mat.ra.x() }),
5483	});
5484	try src_part_mat.finish(isel);
5485	},
5486	65...128 => |dst_bits| switch (src_ty.intInfo(zcu).bits) {
5487	0 => unreachable,
5488	65...128 => {
5489	const src_vi = try isel.use(ty_op.operand);
5490	var dst_hi64_it = dst_vi.value.field(dst_ty, 8, 8);
5491	const dst_hi64_vi = try dst_hi64_it.only(isel);
5492	if (try dst_hi64_vi.?.defReg(isel)) |dst_hi64_ra| {
5493	var src_hi64_it = src_vi.field(src_ty, 8, 8);
5494	const src_hi64_vi = try src_hi64_it.only(isel);
5495	const src_hi64_mat = try src_hi64_vi.?.matReg(isel);
5496	try isel.emit(switch (dst_int_info.signedness) {
5497	.signed => .sbfm(dst_hi64_ra.x(), src_hi64_mat.ra.x(), .{
5498	.N = .doubleword,
5499	.immr = 0,
5500	.imms = @intCast(dst_bits - 64 - 1),
5501	}),
5502	.unsigned => .ubfm(dst_hi64_ra.x(), src_hi64_mat.ra.x(), .{
5503	.N = .doubleword,
5504	.immr = 0,
5505	.imms = @intCast(dst_bits - 64 - 1),
5506	}),
5507	});
5508	try src_hi64_mat.finish(isel);
5509	}
5510	var dst_lo64_it = dst_vi.value.field(dst_ty, 0, 8);
5511	const dst_lo64_vi = try dst_lo64_it.only(isel);
5512	if (try dst_lo64_vi.?.defReg(isel)) |dst_lo64_ra| {
5513	var src_lo64_it = src_vi.field(src_ty, 0, 8);
5514	const src_lo64_vi = try src_lo64_it.only(isel);
5515	try src_lo64_vi.?.liveOut(isel, dst_lo64_ra);
5516	}
5517	},
5518	else => return isel.fail("too big {t} {f} {f}", .{ air_tag, isel.fmtType(dst_ty), isel.fmtType(src_ty) }),
5519	},
5520	else => return isel.fail("too big {t} {f} {f}", .{ air_tag, isel.fmtType(dst_ty), isel.fmtType(src_ty) }),
5521	}
5522	}
5523	if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
5524	},
5525	.optional_payload => {
5526	if (isel.live_values.fetchRemove(air.inst_index)) |payload_vi| unused: {
5527	defer payload_vi.value.deref(isel);
5528
5529	const ty_op = air.data(air.inst_index).ty_op;
5530	const opt_ty = isel.air.typeOf(ty_op.operand, ip);
5531	if (opt_ty.optionalReprIsPayload(zcu)) {
5532	try payload_vi.value.move(isel, ty_op.operand);
5533	break :unused;
5534	}
5535
5536	const opt_vi = try isel.use(ty_op.operand);
5537	var payload_part_it = opt_vi.field(opt_ty, 0, payload_vi.value.size(isel));
5538	const payload_part_vi = try payload_part_it.only(isel);
5539	try payload_vi.value.copy(isel, ty_op.ty, payload_part_vi.?);
5540	}
5541	if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
5542	},
5543	.optional_payload_ptr => {
5544	if (isel.live_values.fetchRemove(air.inst_index)) |payload_ptr_vi| {
5545	defer payload_ptr_vi.value.deref(isel);
5546	const ty_op = air.data(air.inst_index).ty_op;
5547	try payload_ptr_vi.value.move(isel, ty_op.operand);
5548	}
5549	if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
5550	},
5551	.optional_payload_ptr_set => {
5552	if (isel.live_values.fetchRemove(air.inst_index)) |payload_ptr_vi| {
5553	defer payload_ptr_vi.value.deref(isel);
5554	const ty_op = air.data(air.inst_index).ty_op;
5555	const opt_ty = isel.air.typeOf(ty_op.operand, ip).childType(zcu);
5556	if (!opt_ty.optionalReprIsPayload(zcu)) {
5557	const opt_ptr_vi = try isel.use(ty_op.operand);
5558	const opt_ptr_mat = try opt_ptr_vi.matReg(isel);
5559	const has_value_ra = try isel.allocIntReg();
5560	defer isel.freeReg(has_value_ra);
5561	try isel.storeReg(
5562	has_value_ra,
5563	1,
5564	opt_ptr_mat.ra,
5565	opt_ty.optionalChild(zcu).abiSize(zcu),
5566	);
5567	try opt_ptr_mat.finish(isel);
5568	try isel.emit(.movz(has_value_ra.w(), 1, .{ .lsl = .@"0" }));
5569	}
5570	try payload_ptr_vi.value.move(isel, ty_op.operand);
5571	}
5572	if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
5573	},
5574	.wrap_optional => {
5575	if (isel.live_values.fetchRemove(air.inst_index)) |opt_vi| unused: {
5576	defer opt_vi.value.deref(isel);
5577
5578	const ty_op = air.data(air.inst_index).ty_op;
5579	if (ty_op.ty.optionalReprIsPayload(zcu)) {
5580	try opt_vi.value.move(isel, ty_op.operand);
5581	break :unused;
5582	}
5583
5584	const payload_size = isel.air.typeOf(ty_op.operand, ip).abiSize(zcu);
5585	var payload_part_it = opt_vi.value.field(ty_op.ty, 0, payload_size);
5586	const payload_part_vi = try payload_part_it.only(isel);
5587	try payload_part_vi.?.move(isel, ty_op.operand);
5588	var has_value_part_it = opt_vi.value.field(ty_op.ty, payload_size, 1);
5589	const has_value_part_vi = try has_value_part_it.only(isel);
5590	const has_value_part_ra = try has_value_part_vi.?.defReg(isel) orelse break :unused;
5591	try isel.emit(.movz(has_value_part_ra.w(), 1, .{ .lsl = .@"0" }));
5592	}
5593	if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
5594	},
5595	.unwrap_errunion_payload => {
5596	if (isel.live_values.fetchRemove(air.inst_index)) |payload_vi| {
5597	defer payload_vi.value.deref(isel);
5598
5599	const ty_op = air.data(air.inst_index).ty_op;
5600	const error_union_ty = isel.air.typeOf(ty_op.operand, ip);
5601
5602	const error_union_vi = try isel.use(ty_op.operand);
5603	var payload_part_it = error_union_vi.field(
5604	error_union_ty,
5605	codegen.errUnionPayloadOffset(ty_op.ty, zcu),
5606	payload_vi.value.size(isel),
5607	);
5608	const payload_part_vi = try payload_part_it.only(isel);
5609	try payload_vi.value.copy(isel, ty_op.ty, payload_part_vi.?);
5610	}
5611	if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
5612	},
5613	.unwrap_errunion_err => {
5614	if (isel.live_values.fetchRemove(air.inst_index)) |error_set_vi| {
5615	defer error_set_vi.value.deref(isel);
5616
5617	const ty_op = air.data(air.inst_index).ty_op;
5618	const error_union_ty = isel.air.typeOf(ty_op.operand, ip);
5619
5620	const error_union_vi = try isel.use(ty_op.operand);
5621	var error_set_part_it = error_union_vi.field(
5622	error_union_ty,
5623	codegen.errUnionErrorOffset(error_union_ty.errorUnionPayload(zcu), zcu),
5624	error_set_vi.value.size(isel),
5625	);
5626	const error_set_part_vi = try error_set_part_it.only(isel);
5627	try error_set_vi.value.copy(isel, ty_op.ty, error_set_part_vi.?);
5628	}
5629	if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
5630	},
5631	.unwrap_errunion_payload_ptr => {
5632	if (isel.live_values.fetchRemove(air.inst_index)) |payload_ptr_vi| unused: {
5633	defer payload_ptr_vi.value.deref(isel);
5634	const ty_op = air.data(air.inst_index).ty_op;
5635	switch (codegen.errUnionPayloadOffset(ty_op.ty.childType(zcu), zcu)) {
5636	0 => try payload_ptr_vi.value.move(isel, ty_op.operand),
5637	else => |payload_offset| {
5638	const payload_ptr_ra = try payload_ptr_vi.value.defReg(isel) orelse break :unused;
5639	const error_union_ptr_vi = try isel.use(ty_op.operand);
5640	const error_union_ptr_mat = try error_union_ptr_vi.matReg(isel);
5641	const lo12: u12 = @truncate(payload_offset >> 0);
5642	const hi12: u12 = @intCast(payload_offset >> 12);
5643	if (hi12 > 0) try isel.emit(.add(
5644	payload_ptr_ra.x(),
5645	if (lo12 > 0) payload_ptr_ra.x() else error_union_ptr_mat.ra.x(),
5646	.{ .shifted_immediate = .{ .immediate = hi12, .lsl = .@"12" } },
5647	));
5648	if (lo12 > 0) try isel.emit(.add(payload_ptr_ra.x(), error_union_ptr_mat.ra.x(), .{ .immediate = lo12 }));
5649	try error_union_ptr_mat.finish(isel);
5650	},
5651	}
5652	}
5653	if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
5654	},
5655	.unwrap_errunion_err_ptr => {
5656	if (isel.live_values.fetchRemove(air.inst_index)) |error_vi| {
5657	defer error_vi.value.deref(isel);
5658	const ty_op = air.data(air.inst_index).ty_op;
5659	const error_union_ptr_ty = isel.air.typeOf(ty_op.operand, ip);
5660	const error_union_ptr_info = error_union_ptr_ty.ptrInfo(zcu);
5661	const error_union_ptr_vi = try isel.use(ty_op.operand);
5662	const error_union_ptr_mat = try error_union_ptr_vi.matReg(isel);
5663	_ = try error_vi.value.load(isel, ty_op.ty, error_union_ptr_mat.ra, .{
5664	.offset = codegen.errUnionErrorOffset(
5665	ZigType.fromInterned(error_union_ptr_info.child).errorUnionPayload(zcu),
5666	zcu,
5667	),
5668	.@"volatile" = error_union_ptr_info.flags.is_volatile,
5669	});
5670	try error_union_ptr_mat.finish(isel);
5671	}
5672	if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
5673	},
5674	.errunion_payload_ptr_set => {
5675	if (isel.live_values.fetchRemove(air.inst_index)) |payload_ptr_vi| unused: {
5676	defer payload_ptr_vi.value.deref(isel);
5677	const ty_op = air.data(air.inst_index).ty_op;
5678	const payload_ty = ty_op.ty.childType(zcu);
5679	const error_union_ty = isel.air.typeOf(ty_op.operand, ip).childType(zcu);
5680	const error_set_size = error_union_ty.errorUnionSet(zcu).abiSize(zcu);
5681	const error_union_ptr_vi = try isel.use(ty_op.operand);
5682	const error_union_ptr_mat = try error_union_ptr_vi.matReg(isel);
5683	if (error_set_size > 0) try isel.storeReg(
5684	.zr,
5685	error_set_size,
5686	error_union_ptr_mat.ra,
5687	codegen.errUnionErrorOffset(payload_ty, zcu),
5688	);
5689	switch (codegen.errUnionPayloadOffset(payload_ty, zcu)) {
5690	0 => {
5691	try error_union_ptr_mat.finish(isel);
5692	try payload_ptr_vi.value.move(isel, ty_op.operand);
5693	},
5694	else => |payload_offset| {
5695	const payload_ptr_ra = try payload_ptr_vi.value.defReg(isel) orelse break :unused;
5696	const lo12: u12 = @truncate(payload_offset >> 0);
5697	const hi12: u12 = @intCast(payload_offset >> 12);
5698	if (hi12 > 0) try isel.emit(.add(
5699	payload_ptr_ra.x(),
5700	if (lo12 > 0) payload_ptr_ra.x() else error_union_ptr_mat.ra.x(),
5701	.{ .shifted_immediate = .{ .immediate = hi12, .lsl = .@"12" } },
5702	));
5703	if (lo12 > 0) try isel.emit(.add(payload_ptr_ra.x(), error_union_ptr_mat.ra.x(), .{ .immediate = lo12 }));
5704	try error_union_ptr_mat.finish(isel);
5705	},
5706	}
5707	}
5708	if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
5709	},
5710	.wrap_errunion_payload => {
5711	if (isel.live_values.fetchRemove(air.inst_index)) |error_union_vi| {
5712	defer error_union_vi.value.deref(isel);
5713
5714	const ty_op = air.data(air.inst_index).ty_op;
5715	const error_union_ty = ty_op.ty;
5716	const error_union_info = ip.indexToKey(error_union_ty.toIntern()).error_union_type;
5717	const error_set_ty: ZigType = .fromInterned(error_union_info.error_set_type);
5718	const payload_ty: ZigType = .fromInterned(error_union_info.payload_type);
5719	const error_set_offset = codegen.errUnionErrorOffset(payload_ty, zcu);
5720	const payload_offset = codegen.errUnionPayloadOffset(payload_ty, zcu);
5721	const error_set_size = error_set_ty.abiSize(zcu);
5722	const payload_size = payload_ty.abiSize(zcu);
5723
5724	var payload_part_it = error_union_vi.value.field(error_union_ty, payload_offset, payload_size);
5725	const payload_part_vi = try payload_part_it.only(isel);
5726	try payload_part_vi.?.move(isel, ty_op.operand);
5727	var error_set_part_it = error_union_vi.value.field(error_union_ty, error_set_offset, error_set_size);
5728	const error_set_part_vi = try error_set_part_it.only(isel);
5729	if (try error_set_part_vi.?.defReg(isel)) |error_set_part_ra| try isel.emit(switch (error_set_size) {
5730	else => unreachable,
5731	1...4 => .orr(error_set_part_ra.w(), .wzr, .{ .register = .wzr }),
5732	5...8 => .orr(error_set_part_ra.x(), .xzr, .{ .register = .xzr }),
5733	});
5734	}
5735	if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
5736	},
5737	.wrap_errunion_err => {
5738	if (isel.live_values.fetchRemove(air.inst_index)) |error_union_vi| {
5739	defer error_union_vi.value.deref(isel);
5740
5741	const ty_op = air.data(air.inst_index).ty_op;
5742	const error_union_ty = ty_op.ty;
5743	const error_union_info = ip.indexToKey(error_union_ty.toIntern()).error_union_type;
5744	const error_set_ty: ZigType = .fromInterned(error_union_info.error_set_type);
5745	const payload_ty: ZigType = .fromInterned(error_union_info.payload_type);
5746	const error_set_offset = codegen.errUnionErrorOffset(payload_ty, zcu);
5747	const payload_offset = codegen.errUnionPayloadOffset(payload_ty, zcu);
5748	const error_set_size = error_set_ty.abiSize(zcu);
5749	const payload_size = payload_ty.abiSize(zcu);
5750
5751	var error_set_part_it = error_union_vi.value.field(error_union_ty, error_set_offset, error_set_size);
5752	const error_set_part_vi = try error_set_part_it.only(isel);
5753	try error_set_part_vi.?.move(isel, ty_op.operand);
5754	if (payload_size > 0) {
5755	var payload_part_it = error_union_vi.value.field(error_union_ty, payload_offset, payload_size);
5756	const payload_part_vi = try payload_part_it.only(isel);
5757	try payload_part_vi.?.defUndef(isel, payload_ty, .{});
5758	}
5759	}
5760	if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
5761	},
5762	.struct_field_ptr => {
5763	if (isel.live_values.fetchRemove(air.inst_index)) |dst_vi| unused: {
5764	defer dst_vi.value.deref(isel);
5765	const ty_pl = air.data(air.inst_index).ty_pl;
5766	const extra = isel.air.extraData(Air.StructField, ty_pl.payload).data;
5767	switch (codegen.fieldOffset(
5768	isel.air.typeOf(extra.struct_operand, ip),
5769	ty_pl.ty,
5770	extra.field_index,
5771	zcu,
5772	)) {
5773	0 => try dst_vi.value.move(isel, extra.struct_operand),
5774	else => |field_offset| {
5775	const dst_ra = try dst_vi.value.defReg(isel) orelse break :unused;
5776	const src_vi = try isel.use(extra.struct_operand);
5777	const src_mat = try src_vi.matReg(isel);
5778	const lo12: u12 = @truncate(field_offset >> 0);
5779	const hi12: u12 = @intCast(field_offset >> 12);
5780	if (hi12 > 0) try isel.emit(.add(
5781	dst_ra.x(),
5782	if (lo12 > 0) dst_ra.x() else src_mat.ra.x(),
5783	.{ .shifted_immediate = .{ .immediate = hi12, .lsl = .@"12" } },
5784	));
5785	if (lo12 > 0) try isel.emit(.add(dst_ra.x(), src_mat.ra.x(), .{ .immediate = lo12 }));
5786	try src_mat.finish(isel);
5787	},
5788	}
5789	}
5790	if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
5791	},
5792	.struct_field_ptr_index_0,
5793	.struct_field_ptr_index_1,
5794	.struct_field_ptr_index_2,
5795	.struct_field_ptr_index_3,
5796	=> |air_tag| {
5797	if (isel.live_values.fetchRemove(air.inst_index)) |dst_vi| unused: {
5798	defer dst_vi.value.deref(isel);
5799	const ty_op = air.data(air.inst_index).ty_op;
5800	switch (codegen.fieldOffset(
5801	isel.air.typeOf(ty_op.operand, ip),
5802	ty_op.ty,
5803	switch (air_tag) {
5804	else => unreachable,
5805	.struct_field_ptr_index_0 => 0,
5806	.struct_field_ptr_index_1 => 1,
5807	.struct_field_ptr_index_2 => 2,
5808	.struct_field_ptr_index_3 => 3,
5809	},
5810	zcu,
5811	)) {
5812	0 => try dst_vi.value.move(isel, ty_op.operand),
5813	else => |field_offset| {
5814	const dst_ra = try dst_vi.value.defReg(isel) orelse break :unused;
5815	const src_vi = try isel.use(ty_op.operand);
5816	const src_mat = try src_vi.matReg(isel);
5817	const lo12: u12 = @truncate(field_offset >> 0);
5818	const hi12: u12 = @intCast(field_offset >> 12);
5819	if (hi12 > 0) try isel.emit(.add(
5820	dst_ra.x(),
5821	if (lo12 > 0) dst_ra.x() else src_mat.ra.x(),
5822	.{ .shifted_immediate = .{ .immediate = hi12, .lsl = .@"12" } },
5823	));
5824	if (lo12 > 0) try isel.emit(.add(dst_ra.x(), src_mat.ra.x(), .{ .immediate = lo12 }));
5825	try src_mat.finish(isel);
5826	},
5827	}
5828	}
5829	if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
5830	},
5831	.agg_field_val => {
5832	if (isel.live_values.fetchRemove(air.inst_index)) |field_vi| unused: {
5833	defer field_vi.value.deref(isel);
5834
5835	const ty_pl = air.data(air.inst_index).ty_pl;
5836	const extra = isel.air.extraData(Air.StructField, ty_pl.payload).data;
5837	const agg_ty = isel.air.typeOf(extra.struct_operand, ip);
5838	const field_ty = ty_pl.ty;
5839	const field_bit_offset, const field_bit_size, const is_packed = switch (agg_ty.containerLayout(zcu)) {
5840	.auto, .@"extern" => .{
5841	8 * agg_ty.structFieldOffset(extra.field_index, zcu),
5842	8 * field_ty.abiSize(zcu),
5843	false,
5844	},
5845	.@"packed" => .{
5846	if (zcu.typeToPackedStruct(agg_ty)) |loaded_struct|
5847	zcu.structPackedFieldBitOffset(loaded_struct, extra.field_index)
5848	else
5849	0,
5850	field_ty.bitSize(zcu),
5851	true,
5852	},
5853	};
5854	if (is_packed) return isel.fail("packed field of {f}", .{
5855	isel.fmtType(agg_ty),
5856	});
5857
5858	const agg_vi = try isel.use(extra.struct_operand);
5859	switch (agg_ty.zigTypeTag(zcu)) {
5860	else => unreachable,
5861	.@"struct" => {
5862	var agg_part_it = agg_vi.field(agg_ty, @divExact(field_bit_offset, 8), @divExact(field_bit_size, 8));
5863	while (try agg_part_it.next(isel)) |agg_part| {
5864	var field_part_it = field_vi.value.field(ty_pl.ty, agg_part.offset, agg_part.vi.size(isel));
5865	const field_part_vi = try field_part_it.only(isel);
5866	if (field_part_vi.? == agg_part.vi) continue;
5867	var field_subpart_it = field_part_vi.?.parts(isel);
5868	const field_part_offset = if (field_subpart_it.only()) |field_subpart_vi|
5869	field_subpart_vi.get(isel).offset_from_parent
5870	else
5871	0;
5872	while (field_subpart_it.next()) |field_subpart_vi| {
5873	const field_subpart_ra = try field_subpart_vi.defReg(isel) orelse continue;
5874	const field_subpart_offset, const field_subpart_size = field_subpart_vi.position(isel);
5875	var agg_subpart_it = agg_part.vi.field(
5876	field_ty,
5877	agg_part.offset + field_subpart_offset - field_part_offset,
5878	field_subpart_size,
5879	);
5880	const agg_subpart_vi = try agg_subpart_it.only(isel);
5881	try agg_subpart_vi.?.liveOut(isel, field_subpart_ra);
5882	}
5883	}
5884	},
5885	.@"union" => {
5886	try field_vi.value.defAddr(isel, field_ty, .{}) orelse break :unused;
5887
5888	try call.prepareReturn(isel);
5889	try call.finishReturn(isel);
5890
5891	try call.prepareCallee(isel);
5892	try isel.global_relocs.append(gpa, .{
5893	.name = "memcpy",
5894	.reloc = .{ .label = @intCast(isel.instructions.items.len) },
5895	});
5896	try isel.emit(.bl(0));
5897	try call.finishCallee(isel);
5898
5899	try call.prepareParams(isel);
5900	const union_layout = agg_ty.unionGetLayout(zcu);
5901	var payload_it = agg_vi.field(agg_ty, union_layout.payloadOffset(), union_layout.payload_size);
5902	const payload_vi = try payload_it.only(isel);
5903	try isel.movImmediate(.x2, field_vi.value.size(isel));
5904	try call.paramAddress(isel, payload_vi.?, .r1);
5905	try call.paramAddress(isel, field_vi.value, .r0);
5906	try call.finishParams(isel);
5907	},
5908	}
5909	}
5910	if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
5911	},
5912	.set_union_tag => {
5913	const bin_op = air.data(air.inst_index).bin_op;
5914	const union_ty = isel.air.typeOf(bin_op.lhs, ip).childType(zcu);
5915	const union_layout = union_ty.unionGetLayout(zcu);
5916	const tag_vi = try isel.use(bin_op.rhs);
5917	const union_ptr_vi = try isel.use(bin_op.lhs);
5918	const union_ptr_mat = try union_ptr_vi.matReg(isel);
5919	try tag_vi.store(isel, isel.air.typeOf(bin_op.rhs, ip), union_ptr_mat.ra, .{
5920	.offset = union_layout.tagOffset(),
5921	});
5922	try union_ptr_mat.finish(isel);
5923	if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
5924	},
5925	.get_union_tag => {
5926	if (isel.live_values.fetchRemove(air.inst_index)) |tag_vi| {
5927	defer tag_vi.value.deref(isel);
5928	const ty_op = air.data(air.inst_index).ty_op;
5929	const union_ty = isel.air.typeOf(ty_op.operand, ip);
5930	const union_layout = union_ty.unionGetLayout(zcu);
5931	const union_vi = try isel.use(ty_op.operand);
5932	var tag_part_it = union_vi.field(union_ty, union_layout.tagOffset(), union_layout.tag_size);
5933	const tag_part_vi = try tag_part_it.only(isel);
5934	try tag_vi.value.copy(isel, ty_op.ty, tag_part_vi.?);
5935	}
5936	if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
5937	},
5938	.slice => {
5939	if (isel.live_values.fetchRemove(air.inst_index)) |slice_vi| {
5940	defer slice_vi.value.deref(isel);
5941	const ty_pl = air.data(air.inst_index).ty_pl;
5942	const bin_op = isel.air.extraData(Air.Bin, ty_pl.payload).data;
5943	var ptr_part_it = slice_vi.value.field(ty_pl.ty, 0, 8);
5944	const ptr_part_vi = try ptr_part_it.only(isel);
5945	try ptr_part_vi.?.move(isel, bin_op.lhs);
5946	var len_part_it = slice_vi.value.field(ty_pl.ty, 8, 8);
5947	const len_part_vi = try len_part_it.only(isel);
5948	try len_part_vi.?.move(isel, bin_op.rhs);
5949	}
5950	if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
5951	},
5952	.slice_len => {
5953	if (isel.live_values.fetchRemove(air.inst_index)) |len_vi| {
5954	defer len_vi.value.deref(isel);
5955	const ty_op = air.data(air.inst_index).ty_op;
5956	const slice_vi = try isel.use(ty_op.operand);
5957	var len_part_it = slice_vi.field(isel.air.typeOf(ty_op.operand, ip), 8, 8);
5958	const len_part_vi = try len_part_it.only(isel);
5959	try len_vi.value.copy(isel, ty_op.ty, len_part_vi.?);
5960	}
5961	if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
5962	},
5963	.slice_ptr => {
5964	if (isel.live_values.fetchRemove(air.inst_index)) |ptr_vi| {
5965	defer ptr_vi.value.deref(isel);
5966	const ty_op = air.data(air.inst_index).ty_op;
5967	const slice_vi = try isel.use(ty_op.operand);
5968	var ptr_part_it = slice_vi.field(isel.air.typeOf(ty_op.operand, ip), 0, 8);
5969	const ptr_part_vi = try ptr_part_it.only(isel);
5970	try ptr_vi.value.copy(isel, ty_op.ty, ptr_part_vi.?);
5971	}
5972	if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
5973	},
5974	.ptr_slice_len_ptr => {
5975	if (isel.live_values.fetchRemove(air.inst_index)) |dst_vi| unused: {
5976	defer dst_vi.value.deref(isel);
5977	const ty_op = air.data(air.inst_index).ty_op;
5978	const dst_ra = try dst_vi.value.defReg(isel) orelse break :unused;
5979	const src_vi = try isel.use(ty_op.operand);
5980	const src_mat = try src_vi.matReg(isel);
5981	try isel.emit(.add(dst_ra.x(), src_mat.ra.x(), .{ .immediate = 8 }));
5982	try src_mat.finish(isel);
5983	}
5984	if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
5985	},
5986	.ptr_slice_ptr_ptr => {
5987	if (isel.live_values.fetchRemove(air.inst_index)) |dst_vi| {
5988	defer dst_vi.value.deref(isel);
5989	const ty_op = air.data(air.inst_index).ty_op;
5990	try dst_vi.value.move(isel, ty_op.operand);
5991	}
5992	if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
5993	},
5994	.array_elem_val => {
5995	if (isel.live_values.fetchRemove(air.inst_index)) |elem_vi| unused: {
5996	defer elem_vi.value.deref(isel);
5997
5998	const bin_op = air.data(air.inst_index).bin_op;
5999	const array_ty = isel.air.typeOf(bin_op.lhs, ip);
6000	const elem_ty = array_ty.childType(zcu);
6001	const elem_size = elem_ty.abiSize(zcu);
6002	if (elem_size <= 16 and array_ty.arrayLenIncludingSentinel(zcu) <= Value.max_parts) if (bin_op.rhs.toInterned()) |index_val| {
6003	const elem_offset = elem_size * Constant.fromInterned(index_val).toUnsignedInt(zcu);
6004	const array_vi = try isel.use(bin_op.lhs);
6005	var elem_part_it = array_vi.field(array_ty, elem_offset, elem_size);
6006	const elem_part_vi = try elem_part_it.only(isel);
6007	try elem_vi.value.copy(isel, elem_ty, elem_part_vi.?);
6008	break :unused;
6009	};
6010	switch (elem_size) {
6011	0 => unreachable,
6012	1, 2, 4, 8 => {
6013	const elem_ra = try elem_vi.value.defReg(isel) orelse break :unused;
6014	const array_ptr_ra = try isel.allocIntReg();
6015	defer isel.freeReg(array_ptr_ra);
6016	const index_vi = try isel.use(bin_op.rhs);
6017	const index_mat = try index_vi.matReg(isel);
6018	try isel.emit(switch (elem_size) {
6019	else => unreachable,
6020	1 => if (elem_vi.value.isVector(isel)) .ldr(elem_ra.b(), .{ .extended_register = .{
6021	.base = array_ptr_ra.x(),
6022	.index = index_mat.ra.x(),
6023	.extend = .{ .lsl = 0 },
6024	} }) else switch (elem_vi.value.signedness(isel)) {
6025	.signed => .ldrsb(elem_ra.w(), .{ .extended_register = .{
6026	.base = array_ptr_ra.x(),
6027	.index = index_mat.ra.x(),
6028	.extend = .{ .lsl = 0 },
6029	} }),
6030	.unsigned => .ldrb(elem_ra.w(), .{ .extended_register = .{
6031	.base = array_ptr_ra.x(),
6032	.index = index_mat.ra.x(),
6033	.extend = .{ .lsl = 0 },
6034	} }),
6035	},
6036	2 => if (elem_vi.value.isVector(isel)) .ldr(elem_ra.h(), .{ .extended_register = .{
6037	.base = array_ptr_ra.x(),
6038	.index = index_mat.ra.x(),
6039	.extend = .{ .lsl = 1 },
6040	} }) else switch (elem_vi.value.signedness(isel)) {
6041	.signed => .ldrsh(elem_ra.w(), .{ .extended_register = .{
6042	.base = array_ptr_ra.x(),
6043	.index = index_mat.ra.x(),
6044	.extend = .{ .lsl = 1 },
6045	} }),
6046	.unsigned => .ldrh(elem_ra.w(), .{ .extended_register = .{
6047	.base = array_ptr_ra.x(),
6048	.index = index_mat.ra.x(),
6049	.extend = .{ .lsl = 1 },
6050	} }),
6051	},
6052	4 => .ldr(if (elem_vi.value.isVector(isel)) elem_ra.s() else elem_ra.w(), .{ .extended_register = .{
6053	.base = array_ptr_ra.x(),
6054	.index = index_mat.ra.x(),
6055	.extend = .{ .lsl = 2 },
6056	} }),
6057	8 => .ldr(if (elem_vi.value.isVector(isel)) elem_ra.d() else elem_ra.x(), .{ .extended_register = .{
6058	.base = array_ptr_ra.x(),
6059	.index = index_mat.ra.x(),
6060	.extend = .{ .lsl = 3 },
6061	} }),
6062	16 => .ldr(elem_ra.q(), .{ .extended_register = .{
6063	.base = array_ptr_ra.x(),
6064	.index = index_mat.ra.x(),
6065	.extend = .{ .lsl = 4 },
6066	} }),
6067	});
6068	try index_mat.finish(isel);
6069	const array_vi = try isel.use(bin_op.lhs);
6070	try array_vi.address(isel, 0, array_ptr_ra);
6071	},
6072	else => {
6073	const ptr_ra = try isel.allocIntReg();
6074	defer isel.freeReg(ptr_ra);
6075	if (!try elem_vi.value.load(isel, elem_ty, ptr_ra, .{})) break :unused;
6076	const index_vi = try isel.use(bin_op.rhs);
6077	try isel.elemPtr(ptr_ra, ptr_ra, .add, elem_size, index_vi);
6078	const array_vi = try isel.use(bin_op.lhs);
6079	try array_vi.address(isel, 0, ptr_ra);
6080	},
6081	}
6082	}
6083	if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
6084	},
6085	.slice_elem_val => {
6086	if (isel.live_values.fetchRemove(air.inst_index)) |elem_vi| unused: {
6087	defer elem_vi.value.deref(isel);
6088
6089	const bin_op = air.data(air.inst_index).bin_op;
6090	const slice_ty = isel.air.typeOf(bin_op.lhs, ip);
6091	const ptr_info = slice_ty.ptrInfo(zcu);
6092	const elem_size = elem_vi.value.size(isel);
6093	const elem_is_vector = elem_vi.value.isVector(isel);
6094	if (switch (elem_size) {
6095	0 => unreachable,
6096	1, 2, 4, 8 => true,
6097	16 => elem_is_vector,
6098	else => false,
6099	}) {
6100	const elem_ra = try elem_vi.value.defReg(isel) orelse break :unused;
6101	const slice_vi = try isel.use(bin_op.lhs);
6102	const index_vi = try isel.use(bin_op.rhs);
6103	var ptr_part_it = slice_vi.field(slice_ty, 0, 8);
6104	const ptr_part_vi = try ptr_part_it.only(isel);
6105	const base_mat = try ptr_part_vi.?.matReg(isel);
6106	const index_mat = try index_vi.matReg(isel);
6107	try isel.emit(switch (elem_size) {
6108	else => unreachable,
6109	1 => if (elem_is_vector) .ldr(elem_ra.b(), .{ .extended_register = .{
6110	.base = base_mat.ra.x(),
6111	.index = index_mat.ra.x(),
6112	.extend = .{ .lsl = 0 },
6113	} }) else switch (elem_vi.value.signedness(isel)) {
6114	.signed => .ldrsb(elem_ra.w(), .{ .extended_register = .{
6115	.base = base_mat.ra.x(),
6116	.index = index_mat.ra.x(),
6117	.extend = .{ .lsl = 0 },
6118	} }),
6119	.unsigned => .ldrb(elem_ra.w(), .{ .extended_register = .{
6120	.base = base_mat.ra.x(),
6121	.index = index_mat.ra.x(),
6122	.extend = .{ .lsl = 0 },
6123	} }),
6124	},
6125	2 => if (elem_is_vector) .ldr(elem_ra.h(), .{ .extended_register = .{
6126	.base = base_mat.ra.x(),
6127	.index = index_mat.ra.x(),
6128	.extend = .{ .lsl = 1 },
6129	} }) else switch (elem_vi.value.signedness(isel)) {
6130	.signed => .ldrsh(elem_ra.w(), .{ .extended_register = .{
6131	.base = base_mat.ra.x(),
6132	.index = index_mat.ra.x(),
6133	.extend = .{ .lsl = 1 },
6134	} }),
6135	.unsigned => .ldrh(elem_ra.w(), .{ .extended_register = .{
6136	.base = base_mat.ra.x(),
6137	.index = index_mat.ra.x(),
6138	.extend = .{ .lsl = 1 },
6139	} }),
6140	},
6141	4 => .ldr(if (elem_is_vector) elem_ra.s() else elem_ra.w(), .{ .extended_register = .{
6142	.base = base_mat.ra.x(),
6143	.index = index_mat.ra.x(),
6144	.extend = .{ .lsl = 2 },
6145	} }),
6146	8 => .ldr(if (elem_is_vector) elem_ra.d() else elem_ra.x(), .{ .extended_register = .{
6147	.base = base_mat.ra.x(),
6148	.index = index_mat.ra.x(),
6149	.extend = .{ .lsl = 3 },
6150	} }),
6151	16 => if (elem_is_vector) .ldr(elem_ra.q(), .{ .extended_register = .{
6152	.base = base_mat.ra.x(),
6153	.index = index_mat.ra.x(),
6154	.extend = .{ .lsl = 4 },
6155	} }) else unreachable,
6156	});
6157	try index_mat.finish(isel);
6158	try base_mat.finish(isel);
6159	} else {
6160	const elem_ptr_ra = try isel.allocIntReg();
6161	defer isel.freeReg(elem_ptr_ra);
6162	if (!try elem_vi.value.load(isel, slice_ty.childType(zcu), elem_ptr_ra, .{
6163	.@"volatile" = ptr_info.flags.is_volatile,
6164	})) break :unused;
6165	const slice_vi = try isel.use(bin_op.lhs);
6166	var ptr_part_it = slice_vi.field(slice_ty, 0, 8);
6167	const ptr_part_vi = try ptr_part_it.only(isel);
6168	const ptr_part_mat = try ptr_part_vi.?.matReg(isel);
6169	const index_vi = try isel.use(bin_op.rhs);
6170	try isel.elemPtr(elem_ptr_ra, ptr_part_mat.ra, .add, elem_size, index_vi);
6171	try ptr_part_mat.finish(isel);
6172	}
6173	}
6174	if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
6175	},
6176	.slice_elem_ptr => {
6177	if (isel.live_values.fetchRemove(air.inst_index)) |elem_ptr_vi| unused: {
6178	defer elem_ptr_vi.value.deref(isel);
6179	const elem_ptr_ra = try elem_ptr_vi.value.defReg(isel) orelse break :unused;
6180
6181	const ty_pl = air.data(air.inst_index).ty_pl;
6182	const bin_op = isel.air.extraData(Air.Bin, ty_pl.payload).data;
6183	const elem_size = ty_pl.ty.childType(zcu).abiSize(zcu);
6184
6185	const slice_vi = try isel.use(bin_op.lhs);
6186	var ptr_part_it = slice_vi.field(isel.air.typeOf(bin_op.lhs, ip), 0, 8);
6187	const ptr_part_vi = try ptr_part_it.only(isel);
6188	const ptr_part_mat = try ptr_part_vi.?.matReg(isel);
6189	const index_vi = try isel.use(bin_op.rhs);
6190	try isel.elemPtr(elem_ptr_ra, ptr_part_mat.ra, .add, elem_size, index_vi);
6191	try ptr_part_mat.finish(isel);
6192	}
6193	if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
6194	},
6195	.ptr_elem_val => {
6196	if (isel.live_values.fetchRemove(air.inst_index)) |elem_vi| unused: {
6197	defer elem_vi.value.deref(isel);
6198
6199	const bin_op = air.data(air.inst_index).bin_op;
6200	const ptr_ty = isel.air.typeOf(bin_op.lhs, ip);
6201	const ptr_info = ptr_ty.ptrInfo(zcu);
6202	const elem_size = elem_vi.value.size(isel);
6203	const elem_is_vector = elem_vi.value.isVector(isel);
6204	if (switch (elem_size) {
6205	0 => unreachable,
6206	1, 2, 4, 8 => true,
6207	16 => elem_is_vector,
6208	else => false,
6209	}) {
6210	const elem_ra = try elem_vi.value.defReg(isel) orelse break :unused;
6211	const base_vi = try isel.use(bin_op.lhs);
6212	const index_vi = try isel.use(bin_op.rhs);
6213	const base_mat = try base_vi.matReg(isel);
6214	const index_mat = try index_vi.matReg(isel);
6215	try isel.emit(switch (elem_size) {
6216	else => unreachable,
6217	1 => if (elem_is_vector) .ldr(elem_ra.b(), .{ .extended_register = .{
6218	.base = base_mat.ra.x(),
6219	.index = index_mat.ra.x(),
6220	.extend = .{ .lsl = 0 },
6221	} }) else switch (elem_vi.value.signedness(isel)) {
6222	.signed => .ldrsb(elem_ra.w(), .{ .extended_register = .{
6223	.base = base_mat.ra.x(),
6224	.index = index_mat.ra.x(),
6225	.extend = .{ .lsl = 0 },
6226	} }),
6227	.unsigned => .ldrb(elem_ra.w(), .{ .extended_register = .{
6228	.base = base_mat.ra.x(),
6229	.index = index_mat.ra.x(),
6230	.extend = .{ .lsl = 0 },
6231	} }),
6232	},
6233	2 => if (elem_is_vector) .ldr(elem_ra.h(), .{ .extended_register = .{
6234	.base = base_mat.ra.x(),
6235	.index = index_mat.ra.x(),
6236	.extend = .{ .lsl = 1 },
6237	} }) else switch (elem_vi.value.signedness(isel)) {
6238	.signed => .ldrsh(elem_ra.w(), .{ .extended_register = .{
6239	.base = base_mat.ra.x(),
6240	.index = index_mat.ra.x(),
6241	.extend = .{ .lsl = 1 },
6242	} }),
6243	.unsigned => .ldrh(elem_ra.w(), .{ .extended_register = .{
6244	.base = base_mat.ra.x(),
6245	.index = index_mat.ra.x(),
6246	.extend = .{ .lsl = 1 },
6247	} }),
6248	},
6249	4 => .ldr(if (elem_is_vector) elem_ra.s() else elem_ra.w(), .{ .extended_register = .{
6250	.base = base_mat.ra.x(),
6251	.index = index_mat.ra.x(),
6252	.extend = .{ .lsl = 2 },
6253	} }),
6254	8 => .ldr(if (elem_is_vector) elem_ra.d() else elem_ra.x(), .{ .extended_register = .{
6255	.base = base_mat.ra.x(),
6256	.index = index_mat.ra.x(),
6257	.extend = .{ .lsl = 3 },
6258	} }),
6259	16 => if (elem_is_vector) .ldr(elem_ra.q(), .{ .extended_register = .{
6260	.base = base_mat.ra.x(),
6261	.index = index_mat.ra.x(),
6262	.extend = .{ .lsl = 4 },
6263	} }) else unreachable,
6264	});
6265	try index_mat.finish(isel);
6266	try base_mat.finish(isel);
6267	} else {
6268	const elem_ptr_ra = try isel.allocIntReg();
6269	defer isel.freeReg(elem_ptr_ra);
6270	if (!try elem_vi.value.load(isel, ptr_ty.childType(zcu), elem_ptr_ra, .{
6271	.@"volatile" = ptr_info.flags.is_volatile,
6272	})) break :unused;
6273	const base_vi = try isel.use(bin_op.lhs);
6274	const base_mat = try base_vi.matReg(isel);
6275	const index_vi = try isel.use(bin_op.rhs);
6276	try isel.elemPtr(elem_ptr_ra, base_mat.ra, .add, elem_size, index_vi);
6277	try base_mat.finish(isel);
6278	}
6279	}
6280	if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
6281	},
6282	.ptr_elem_ptr => {
6283	if (isel.live_values.fetchRemove(air.inst_index)) |elem_ptr_vi| unused: {
6284	defer elem_ptr_vi.value.deref(isel);
6285	const elem_ptr_ra = try elem_ptr_vi.value.defReg(isel) orelse break :unused;
6286
6287	const ty_pl = air.data(air.inst_index).ty_pl;
6288	const bin_op = isel.air.extraData(Air.Bin, ty_pl.payload).data;
6289	const elem_size = ty_pl.ty.childType(zcu).abiSize(zcu);
6290
6291	const base_vi = try isel.use(bin_op.lhs);
6292	const base_mat = try base_vi.matReg(isel);
6293	const index_vi = try isel.use(bin_op.rhs);
6294	try isel.elemPtr(elem_ptr_ra, base_mat.ra, .add, elem_size, index_vi);
6295	try base_mat.finish(isel);
6296	}
6297	if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
6298	},
6299	.array_to_vector => unreachable, // legalize .expand_array_to_vector
6300	.array_to_slice => {
6301	if (isel.live_values.fetchRemove(air.inst_index)) |slice_vi| {
6302	defer slice_vi.value.deref(isel);
6303	const ty_op = air.data(air.inst_index).ty_op;
6304	var ptr_part_it = slice_vi.value.field(ty_op.ty, 0, 8);
6305	const ptr_part_vi = try ptr_part_it.only(isel);
6306	try ptr_part_vi.?.move(isel, ty_op.operand);
6307	var len_part_it = slice_vi.value.field(ty_op.ty, 8, 8);
6308	const len_part_vi = try len_part_it.only(isel);
6309	if (try len_part_vi.?.defReg(isel)) |len_ra| try isel.movImmediate(
6310	len_ra.x(),
6311	isel.air.typeOf(ty_op.operand, ip).childType(zcu).arrayLen(zcu),
6312	);
6313	}
6314	if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
6315	},
6316	.int_from_float, .int_from_float_optimized => |air_tag| {
6317	if (isel.live_values.fetchRemove(air.inst_index)) |dst_vi| unused: {
6318	defer dst_vi.value.deref(isel);
6319
6320	const ty_op = air.data(air.inst_index).ty_op;
6321	const dst_ty = ty_op.ty;
6322	const src_ty = isel.air.typeOf(ty_op.operand, ip);
6323	if (!dst_ty.isAbiInt(zcu)) return isel.fail("bad {t} {f} {f}", .{ air_tag, isel.fmtType(dst_ty), isel.fmtType(src_ty) });
6324	const dst_int_info = dst_ty.intInfo(zcu);
6325	const src_bits = src_ty.floatBits(isel.target);
6326	switch (@max(dst_int_info.bits, src_bits)) {
6327	0 => unreachable,
6328	1...64 => {
6329	const dst_ra = try dst_vi.value.defReg(isel) orelse break :unused;
6330	const need_fcvt = switch (src_bits) {
6331	else => unreachable,
6332	16 => !isel.target.cpu.has(.aarch64, .fullfp16),
6333	32, 64 => false,
6334	};
6335	const src_vi = try isel.use(ty_op.operand);
6336	const src_mat = try src_vi.matReg(isel);
6337	const src_ra = if (need_fcvt) try isel.allocVecReg() else src_mat.ra;
6338	defer if (need_fcvt) isel.freeReg(src_ra);
6339	const dst_reg = switch (dst_int_info.bits) {
6340	else => unreachable,
6341	1...32 => dst_ra.w(),
6342	33...64 => dst_ra.x(),
6343	};
6344	const src_reg = switch (src_bits) {
6345	else => unreachable,
6346	16 => if (need_fcvt) src_ra.s() else src_ra.h(),
6347	32 => src_ra.s(),
6348	64 => src_ra.d(),
6349	};
6350	try isel.emit(switch (dst_int_info.signedness) {
6351	.signed => .fcvtzs(dst_reg, src_reg),
6352	.unsigned => .fcvtzu(dst_reg, src_reg),
6353	});
6354	if (need_fcvt) try isel.emit(.fcvt(src_reg, src_mat.ra.h()));
6355	try src_mat.finish(isel);
6356	},
6357	65...128 => {
6358	try call.prepareReturn(isel);
6359	switch (dst_int_info.bits) {
6360	else => unreachable,
6361	1...64 => try call.returnLiveIn(isel, dst_vi.value, .r0),
6362	65...128 => {
6363	var dst_hi64_it = dst_vi.value.field(dst_ty, 8, 8);
6364	const dst_hi64_vi = try dst_hi64_it.only(isel);
6365	try call.returnLiveIn(isel, dst_hi64_vi.?, .r1);
6366	var dst_lo64_it = dst_vi.value.field(dst_ty, 0, 8);
6367	const dst_lo64_vi = try dst_lo64_it.only(isel);
6368	try call.returnLiveIn(isel, dst_lo64_vi.?, .r0);
6369	},
6370	}
6371	try call.finishReturn(isel);
6372
6373	try call.prepareCallee(isel);
6374	try isel.global_relocs.append(gpa, .{
6375	.name = switch (dst_int_info.bits) {
6376	else => unreachable,
6377	1...32 => switch (dst_int_info.signedness) {
6378	.signed => switch (src_bits) {
6379	else => unreachable,
6380	16 => "__fixhfsi",
6381	32 => "__fixsfsi",
6382	64 => "__fixdfsi",
6383	80 => "__fixxfsi",
6384	128 => "__fixtfsi",
6385	},
6386	.unsigned => switch (src_bits) {
6387	else => unreachable,
6388	16 => "__fixunshfsi",
6389	32 => "__fixunssfsi",
6390	64 => "__fixunsdfsi",
6391	80 => "__fixunsxfsi",
6392	128 => "__fixunstfsi",
6393	},
6394	},
6395	33...64 => switch (dst_int_info.signedness) {
6396	.signed => switch (src_bits) {
6397	else => unreachable,
6398	16 => "__fixhfdi",
6399	32 => "__fixsfdi",
6400	64 => "__fixdfdi",
6401	80 => "__fixxfdi",
6402	128 => "__fixtfdi",
6403	},
6404	.unsigned => switch (src_bits) {
6405	else => unreachable,
6406	16 => "__fixunshfdi",
6407	32 => "__fixunssfdi",
6408	64 => "__fixunsdfdi",
6409	80 => "__fixunsxfdi",
6410	128 => "__fixunstfdi",
6411	},
6412	},
6413	65...128 => switch (dst_int_info.signedness) {
6414	.signed => switch (src_bits) {
6415	else => unreachable,
6416	16 => "__fixhfti",
6417	32 => "__fixsfti",
6418	64 => "__fixdfti",
6419	80 => "__fixxfti",
6420	128 => "__fixtfti",
6421	},
6422	.unsigned => switch (src_bits) {
6423	else => unreachable,
6424	16 => "__fixunshfti",
6425	32 => "__fixunssfti",
6426	64 => "__fixunsdfti",
6427	80 => "__fixunsxfti",
6428	128 => "__fixunstfti",
6429	},
6430	},
6431	},
6432	.reloc = .{ .label = @intCast(isel.instructions.items.len) },
6433	});
6434	try isel.emit(.bl(0));
6435	try call.finishCallee(isel);
6436
6437	try call.prepareParams(isel);
6438	const src_vi = try isel.use(ty_op.operand);
6439	switch (src_bits) {
6440	else => unreachable,
6441	16, 32, 64, 128 => try call.paramLiveOut(isel, src_vi, .v0),
6442	80 => {
6443	var src_hi16_it = src_vi.field(src_ty, 8, 8);
6444	const src_hi16_vi = try src_hi16_it.only(isel);
6445	try call.paramLiveOut(isel, src_hi16_vi.?, .r1);
6446	var src_lo64_it = src_vi.field(src_ty, 0, 8);
6447	const src_lo64_vi = try src_lo64_it.only(isel);
6448	try call.paramLiveOut(isel, src_lo64_vi.?, .r0);
6449	},
6450	}
6451	try call.finishParams(isel);
6452	},
6453	else => return isel.fail("too big {t} {f} {f}", .{ air_tag, isel.fmtType(dst_ty), isel.fmtType(src_ty) }),
6454	}
6455	}
6456	if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
6457	},
6458	.float_from_int => |air_tag| {
6459	if (isel.live_values.fetchRemove(air.inst_index)) |dst_vi| unused: {
6460	defer dst_vi.value.deref(isel);
6461
6462	const ty_op = air.data(air.inst_index).ty_op;
6463	const dst_ty = ty_op.ty;
6464	const src_ty = isel.air.typeOf(ty_op.operand, ip);
6465	const dst_bits = dst_ty.floatBits(isel.target);
6466	if (!src_ty.isAbiInt(zcu)) return isel.fail("bad {t} {f} {f}", .{ air_tag, isel.fmtType(dst_ty), isel.fmtType(src_ty) });
6467	const src_int_info = src_ty.intInfo(zcu);
6468	switch (@max(dst_bits, src_int_info.bits)) {
6469	0 => unreachable,
6470	1...64 => {
6471	const dst_ra = try dst_vi.value.defReg(isel) orelse break :unused;
6472	const need_fcvt = switch (dst_bits) {
6473	else => unreachable,
6474	16 => !isel.target.cpu.has(.aarch64, .fullfp16),
6475	32, 64 => false,
6476	};
6477	if (need_fcvt) try isel.emit(.fcvt(dst_ra.h(), dst_ra.s()));
6478	const src_vi = try isel.use(ty_op.operand);
6479	const src_mat = try src_vi.matReg(isel);
6480	const dst_reg = switch (dst_bits) {
6481	else => unreachable,
6482	16 => if (need_fcvt) dst_ra.s() else dst_ra.h(),
6483	32 => dst_ra.s(),
6484	64 => dst_ra.d(),
6485	};
6486	const src_reg = switch (src_int_info.bits) {
6487	else => unreachable,
6488	1...32 => src_mat.ra.w(),
6489	33...64 => src_mat.ra.x(),
6490	};
6491	try isel.emit(switch (src_int_info.signedness) {
6492	.signed => .scvtf(dst_reg, src_reg),
6493	.unsigned => .ucvtf(dst_reg, src_reg),
6494	});
6495	try src_mat.finish(isel);
6496	},
6497	65...128 => {
6498	try call.prepareReturn(isel);
6499	switch (dst_bits) {
6500	else => unreachable,
6501	16, 32, 64, 128 => try call.returnLiveIn(isel, dst_vi.value, .v0),
6502	80 => {
6503	var dst_hi16_it = dst_vi.value.field(dst_ty, 8, 8);
6504	const dst_hi16_vi = try dst_hi16_it.only(isel);
6505	try call.returnLiveIn(isel, dst_hi16_vi.?, .r1);
6506	var dst_lo64_it = dst_vi.value.field(dst_ty, 0, 8);
6507	const dst_lo64_vi = try dst_lo64_it.only(isel);
6508	try call.returnLiveIn(isel, dst_lo64_vi.?, .r0);
6509	},
6510	}
6511	try call.finishReturn(isel);
6512
6513	try call.prepareCallee(isel);
6514	try isel.global_relocs.append(gpa, .{
6515	.name = switch (src_int_info.bits) {
6516	else => unreachable,
6517	1...32 => switch (src_int_info.signedness) {
6518	.signed => switch (dst_bits) {
6519	else => unreachable,
6520	16 => "__floatsihf",
6521	32 => "__floatsisf",
6522	64 => "__floatsidf",
6523	80 => "__floatsixf",
6524	128 => "__floatsitf",
6525	},
6526	.unsigned => switch (dst_bits) {
6527	else => unreachable,
6528	16 => "__floatunsihf",
6529	32 => "__floatunsisf",
6530	64 => "__floatunsidf",
6531	80 => "__floatunsixf",
6532	128 => "__floatunsitf",
6533	},
6534	},
6535	33...64 => switch (src_int_info.signedness) {
6536	.signed => switch (dst_bits) {
6537	else => unreachable,
6538	16 => "__floatdihf",
6539	32 => "__floatdisf",
6540	64 => "__floatdidf",
6541	80 => "__floatdixf",
6542	128 => "__floatditf",
6543	},
6544	.unsigned => switch (dst_bits) {
6545	else => unreachable,
6546	16 => "__floatundihf",
6547	32 => "__floatundisf",
6548	64 => "__floatundidf",
6549	80 => "__floatundixf",
6550	128 => "__floatunditf",
6551	},
6552	},
6553	65...128 => switch (src_int_info.signedness) {
6554	.signed => switch (dst_bits) {
6555	else => unreachable,
6556	16 => "__floattihf",
6557	32 => "__floattisf",
6558	64 => "__floattidf",
6559	80 => "__floattixf",
6560	128 => "__floattitf",
6561	},
6562	.unsigned => switch (dst_bits) {
6563	else => unreachable,
6564	16 => "__floatuntihf",
6565	32 => "__floatuntisf",
6566	64 => "__floatuntidf",
6567	80 => "__floatuntixf",
6568	128 => "__floatuntitf",
6569	},
6570	},
6571	},
6572	.reloc = .{ .label = @intCast(isel.instructions.items.len) },
6573	});
6574	try isel.emit(.bl(0));
6575	try call.finishCallee(isel);
6576
6577	try call.prepareParams(isel);
6578	const src_vi = try isel.use(ty_op.operand);
6579	switch (src_int_info.bits) {
6580	else => unreachable,
6581	1...64 => try call.paramLiveOut(isel, src_vi, .r0),
6582	65...128 => {
6583	var src_hi64_it = src_vi.field(src_ty, 8, 8);
6584	const src_hi64_vi = try src_hi64_it.only(isel);
6585	try call.paramLiveOut(isel, src_hi64_vi.?, .r1);
6586	var src_lo64_it = src_vi.field(src_ty, 0, 8);
6587	const src_lo64_vi = try src_lo64_it.only(isel);
6588	try call.paramLiveOut(isel, src_lo64_vi.?, .r0);
6589	},
6590	}
6591	try call.finishParams(isel);
6592	},
6593	else => return isel.fail("too big {t} {f} {f}", .{ air_tag, isel.fmtType(dst_ty), isel.fmtType(src_ty) }),
6594	}
6595	}
6596	if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
6597	},
6598	.memset, .memset_safe => |air_tag| {
6599	const bin_op = air.data(air.inst_index).bin_op;
6600	const dst_ty = isel.air.typeOf(bin_op.lhs, ip);
6601	const dst_info = dst_ty.ptrInfo(zcu);
6602	const fill_byte: union(enum) { constant: u8, value: Air.Inst.Ref } = fill_byte: {
6603	if (bin_op.rhs.toInterned()) |fill_val| {
6604	if (ip.isUndef(fill_val)) switch (air_tag) {
6605	else => unreachable,
6606	.memset => break :air_tag if (air.next()) |next_air_tag| continue :air_tag next_air_tag,
6607	.memset_safe => break :fill_byte .{ .constant = 0xaa },
6608	};
6609	if (try isel.hasRepeatedByteRepr(.fromInterned(fill_val))) |fill_byte|
6610	break :fill_byte .{ .constant = fill_byte };
6611	}
6612	switch (dst_ty.indexableElem(zcu).abiSize(zcu)) {
6613	0 => unreachable,
6614	1 => break :fill_byte .{ .value = bin_op.rhs },
6615	2, 4, 8 => |size| {
6616	const dst_vi = try isel.use(bin_op.lhs);
6617	const ptr_ra = try isel.allocIntReg();
6618	const fill_vi = try isel.use(bin_op.rhs);
6619	const fill_mat = try fill_vi.matReg(isel);
6620	const len_mat: Value.Materialize = len_mat: switch (dst_info.flags.size) {
6621	.one => .{ .vi = undefined, .ra = try isel.allocIntReg() },
6622	.many => unreachable,
6623	.slice => {
6624	var dst_len_it = dst_vi.field(dst_ty, 8, 8);
6625	const dst_len_vi = try dst_len_it.only(isel);
6626	break :len_mat try dst_len_vi.?.matReg(isel);
6627	},
6628	.c => unreachable,
6629	};
6630
6631	const skip_label = isel.instructions.items.len;
6632	_ = try isel.instructions.addOne(gpa);
6633	try isel.emit(.sub(len_mat.ra.x(), len_mat.ra.x(), .{ .immediate = 1 }));
6634	try isel.emit(switch (size) {
6635	else => unreachable,
6636	2 => .strh(fill_mat.ra.w(), .{ .post_index = .{ .base = ptr_ra.x(), .index = 2 } }),
6637	4 => .str(fill_mat.ra.w(), .{ .post_index = .{ .base = ptr_ra.x(), .index = 4 } }),
6638	8 => .str(fill_mat.ra.x(), .{ .post_index = .{ .base = ptr_ra.x(), .index = 8 } }),
6639	});
6640	isel.instructions.items[skip_label] = .cbnz(
6641	len_mat.ra.x(),
6642	-@as(i21, @intCast((isel.instructions.items.len - 1 - skip_label) << 2)),
6643	);
6644	switch (dst_info.flags.size) {
6645	.one => {
6646	const len_imm = ZigType.fromInterned(dst_info.child).arrayLen(zcu);
6647	assert(len_imm > 0);
6648	try isel.movImmediate(len_mat.ra.x(), len_imm);
6649	isel.freeReg(len_mat.ra);
6650	try fill_mat.finish(isel);
6651	isel.freeReg(ptr_ra);
6652	try dst_vi.liveOut(isel, ptr_ra);
6653	},
6654	.many => unreachable,
6655	.slice => {
6656	try isel.emit(.cbz(
6657	len_mat.ra.x(),
6658	@intCast((isel.instructions.items.len + 1 - skip_label) << 2),
6659	));
6660	try len_mat.finish(isel);
6661	try fill_mat.finish(isel);
6662	isel.freeReg(ptr_ra);
6663	var dst_ptr_it = dst_vi.field(dst_ty, 0, 8);
6664	const dst_ptr_vi = try dst_ptr_it.only(isel);
6665	try dst_ptr_vi.?.liveOut(isel, ptr_ra);
6666	},
6667	.c => unreachable,
6668	}
6669
6670	break :air_tag if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
6671	},
6672	else => return isel.fail("too big {t} {f}", .{ air_tag, isel.fmtType(dst_ty) }),
6673	}
6674	};
6675
6676	try call.prepareReturn(isel);
6677	try call.finishReturn(isel);
6678
6679	try call.prepareCallee(isel);
6680	try isel.global_relocs.append(gpa, .{
6681	.name = "memset",
6682	.reloc = .{ .label = @intCast(isel.instructions.items.len) },
6683	});
6684	try isel.emit(.bl(0));
6685	try call.finishCallee(isel);
6686
6687	try call.prepareParams(isel);
6688	const dst_vi = try isel.use(bin_op.lhs);
6689	switch (dst_info.flags.size) {
6690	.one => {
6691	try isel.movImmediate(.x2, ZigType.fromInterned(dst_info.child).abiSize(zcu));
6692	switch (fill_byte) {
6693	.constant => |byte| try isel.movImmediate(.w1, byte),
6694	.value => |byte| try call.paramLiveOut(isel, try isel.use(byte), .r1),
6695	}
6696	try call.paramLiveOut(isel, dst_vi, .r0);
6697	},
6698	.many => unreachable,
6699	.slice => {
6700	var dst_ptr_it = dst_vi.field(dst_ty, 0, 8);
6701	const dst_ptr_vi = try dst_ptr_it.only(isel);
6702	var dst_len_it = dst_vi.field(dst_ty, 8, 8);
6703	const dst_len_vi = try dst_len_it.only(isel);
6704	try isel.elemPtr(.r2, .zr, .add, ZigType.fromInterned(dst_info.child).abiSize(zcu), dst_len_vi.?);
6705	switch (fill_byte) {
6706	.constant => |byte| try isel.movImmediate(.w1, byte),
6707	.value => |byte| try call.paramLiveOut(isel, try isel.use(byte), .r1),
6708	}
6709	try call.paramLiveOut(isel, dst_ptr_vi.?, .r0);
6710	},
6711	.c => unreachable,
6712	}
6713	try call.finishParams(isel);
6714
6715	if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
6716	},
6717	.memcpy, .memmove => |air_tag| {
6718	const bin_op = air.data(air.inst_index).bin_op;
6719	const dst_ty = isel.air.typeOf(bin_op.lhs, ip);
6720	const dst_info = dst_ty.ptrInfo(zcu);
6721
6722	try call.prepareReturn(isel);
6723	try call.finishReturn(isel);
6724
6725	try call.prepareCallee(isel);
6726	try isel.global_relocs.append(gpa, .{
6727	.name = @tagName(air_tag),
6728	.reloc = .{ .label = @intCast(isel.instructions.items.len) },
6729	});
6730	try isel.emit(.bl(0));
6731	try call.finishCallee(isel);
6732
6733	try call.prepareParams(isel);
6734	switch (dst_info.flags.size) {
6735	.one => {
6736	const dst_vi = try isel.use(bin_op.lhs);
6737	const src_vi = try isel.use(bin_op.rhs);
6738	try isel.movImmediate(.x2, ZigType.fromInterned(dst_info.child).abiSize(zcu));
6739	try call.paramLiveOut(isel, src_vi, .r1);
6740	try call.paramLiveOut(isel, dst_vi, .r0);
6741	},
6742	.many => unreachable,
6743	.slice => {
6744	const dst_vi = try isel.use(bin_op.lhs);
6745	var dst_ptr_it = dst_vi.field(dst_ty, 0, 8);
6746	const dst_ptr_vi = try dst_ptr_it.only(isel);
6747	var dst_len_it = dst_vi.field(dst_ty, 8, 8);
6748	const dst_len_vi = try dst_len_it.only(isel);
6749	const src_vi = try isel.use(bin_op.rhs);
6750	try isel.elemPtr(.r2, .zr, .add, ZigType.fromInterned(dst_info.child).abiSize(zcu), dst_len_vi.?);
6751	try call.paramLiveOut(isel, src_vi, .r1);
6752	try call.paramLiveOut(isel, dst_ptr_vi.?, .r0);
6753	},
6754	.c => unreachable,
6755	}
6756	try call.finishParams(isel);
6757
6758	if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
6759	},
6760	.atomic_load => {
6761	const atomic_load = air.data(air.inst_index).atomic_load;
6762	const ptr_ty = isel.air.typeOf(atomic_load.ptr, ip);
6763	const ptr_info = ptr_ty.ptrInfo(zcu);
6764	if (atomic_load.order != .unordered) return isel.fail("ordered atomic load", .{});
6765	if (ptr_info.packed_offset.host_size > 0) return isel.fail("packed atomic load", .{});
6766
6767	if (isel.live_values.fetchRemove(air.inst_index)) |dst_vi| {
6768	defer dst_vi.value.deref(isel);
6769	var ptr_mat: ?Value.Materialize = null;
6770	var dst_part_it = dst_vi.value.parts(isel);
6771	while (dst_part_it.next()) |dst_part_vi| {
6772	const dst_ra = try dst_part_vi.defReg(isel) orelse continue;
6773	if (ptr_mat == null) {
6774	const ptr_vi = try isel.use(atomic_load.ptr);
6775	ptr_mat = try ptr_vi.matReg(isel);
6776	}
6777	try isel.emit(switch (dst_part_vi.size(isel)) {
6778	else => |size| return isel.fail("bad atomic load size of {d} from {f}", .{
6779	size, isel.fmtType(ptr_ty),
6780	}),
6781	1 => switch (dst_part_vi.signedness(isel)) {
6782	.signed => .ldrsb(dst_ra.w(), .{ .unsigned_offset = .{
6783	.base = ptr_mat.?.ra.x(),
6784	.offset = @intCast(dst_part_vi.get(isel).offset_from_parent),
6785	} }),
6786	.unsigned => .ldrb(dst_ra.w(), .{ .unsigned_offset = .{
6787	.base = ptr_mat.?.ra.x(),
6788	.offset = @intCast(dst_part_vi.get(isel).offset_from_parent),
6789	} }),
6790	},
6791	2 => switch (dst_part_vi.signedness(isel)) {
6792	.signed => .ldrsh(dst_ra.w(), .{ .unsigned_offset = .{
6793	.base = ptr_mat.?.ra.x(),
6794	.offset = @intCast(dst_part_vi.get(isel).offset_from_parent),
6795	} }),
6796	.unsigned => .ldrh(dst_ra.w(), .{ .unsigned_offset = .{
6797	.base = ptr_mat.?.ra.x(),
6798	.offset = @intCast(dst_part_vi.get(isel).offset_from_parent),
6799	} }),
6800	},
6801	4 => .ldr(dst_ra.w(), .{ .unsigned_offset = .{
6802	.base = ptr_mat.?.ra.x(),
6803	.offset = @intCast(dst_part_vi.get(isel).offset_from_parent),
6804	} }),
6805	8 => .ldr(dst_ra.x(), .{ .unsigned_offset = .{
6806	.base = ptr_mat.?.ra.x(),
6807	.offset = @intCast(dst_part_vi.get(isel).offset_from_parent),
6808	} }),
6809	});
6810	}
6811	if (ptr_mat) |mat| try mat.finish(isel);
6812	} else if (ptr_info.flags.is_volatile) return isel.fail("volatile atomic load", .{});
6813
6814	if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
6815	},
6816	.error_name => {
6817	if (isel.live_values.fetchRemove(air.inst_index)) |name_vi| unused: {
6818	defer name_vi.value.deref(isel);
6819	var ptr_part_it = name_vi.value.field(.slice_const_u8_sentinel_0, 0, 8);
6820	const ptr_part_vi = try ptr_part_it.only(isel);
6821	const ptr_part_ra = try ptr_part_vi.?.defReg(isel);
6822	var len_part_it = name_vi.value.field(.slice_const_u8_sentinel_0, 8, 8);
6823	const len_part_vi = try len_part_it.only(isel);
6824	const len_part_ra = try len_part_vi.?.defReg(isel);
6825	if (ptr_part_ra == null and len_part_ra == null) break :unused;
6826
6827	const un_op = air.data(air.inst_index).un_op;
6828	const error_vi = try isel.use(un_op);
6829	const error_mat = try error_vi.matReg(isel);
6830	const ptr_ra = try isel.allocIntReg();
6831	defer isel.freeReg(ptr_ra);
6832	const start_ra, const end_ra = range_ras: {
6833	const name_lock: RegLock = if (len_part_ra != null) if (ptr_part_ra) |name_ptr_ra|
6834	isel.tryLockReg(name_ptr_ra)
6835	else
6836	.empty else .empty;
6837	defer name_lock.unlock(isel);
6838	break :range_ras .{ try isel.allocIntReg(), try isel.allocIntReg() };
6839	};
6840	defer {
6841	isel.freeReg(start_ra);
6842	isel.freeReg(end_ra);
6843	}
6844	if (len_part_ra) |name_len_ra| try isel.emit(.sub(
6845	name_len_ra.w(),
6846	end_ra.w(),
6847	.{ .register = start_ra.w() },
6848	));
6849	if (ptr_part_ra) |name_ptr_ra| try isel.emit(.add(
6850	name_ptr_ra.x(),
6851	ptr_ra.x(),
6852	.{ .extended_register = .{
6853	.register = start_ra.w(),
6854	.extend = .{ .uxtw = 0 },
6855	} },
6856	));
6857	if (len_part_ra) |_| try isel.emit(.sub(end_ra.w(), end_ra.w(), .{ .immediate = 1 }));
6858	try isel.emit(.ldp(start_ra.w(), end_ra.w(), .{ .base = start_ra.x() }));
6859	try isel.emit(.add(start_ra.x(), ptr_ra.x(), .{ .extended_register = .{
6860	.register = error_mat.ra.w(),
6861	.extend = switch (zcu.errorSetBits()) {
6862	else => unreachable,
6863	1...8 => .{ .uxtb = 2 },
6864	9...16 => .{ .uxth = 2 },
6865	17...32 => .{ .uxtw = 2 },
6866	},
6867	} }));
6868	try isel.lazy_relocs.append(gpa, .{
6869	.symbol = .{ .kind = .const_data, .ty = .anyerror_type },
6870	.reloc = .{ .label = @intCast(isel.instructions.items.len) },
6871	});
6872	try isel.emit(.add(ptr_ra.x(), ptr_ra.x(), .{ .immediate = 0 }));
6873	try isel.lazy_relocs.append(gpa, .{
6874	.symbol = .{ .kind = .const_data, .ty = .anyerror_type },
6875	.reloc = .{ .label = @intCast(isel.instructions.items.len) },
6876	});
6877	try isel.emit(.adrp(ptr_ra.x(), 0));
6878	try error_mat.finish(isel);
6879	}
6880	if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
6881	},
6882	.aggregate_init => {
6883	if (isel.live_values.fetchRemove(air.inst_index)) |agg_vi| {
6884	defer agg_vi.value.deref(isel);
6885
6886	const ty_pl = air.data(air.inst_index).ty_pl;
6887	const agg_ty = ty_pl.ty;
6888	switch (ip.indexToKey(agg_ty.toIntern())) {
6889	.array_type => |array_type| {
6890	const elems: []const Air.Inst.Ref =
6891	@ptrCast(isel.air.extra.items[ty_pl.payload..][0..@intCast(array_type.len)]);
6892	var elem_offset: u64 = 0;
6893	const elem_size = ZigType.fromInterned(array_type.child).abiSize(zcu);
6894	for (elems) |elem| {
6895	var agg_part_it = agg_vi.value.field(agg_ty, elem_offset, elem_size);
6896	const agg_part_vi = try agg_part_it.only(isel);
6897	try agg_part_vi.?.move(isel, elem);
6898	elem_offset += elem_size;
6899	}
6900	switch (array_type.sentinel) {
6901	.none => {},
6902	else => |sentinel| {
6903	var agg_part_it = agg_vi.value.field(agg_ty, elem_offset, elem_size);
6904	const agg_part_vi = try agg_part_it.only(isel);
6905	try agg_part_vi.?.move(isel, .fromIntern(sentinel));
6906	},
6907	}
6908	},
6909	.struct_type => {
6910	const loaded_struct = ip.loadStructType(agg_ty.toIntern());
6911	const elems: []const Air.Inst.Ref =
6912	@ptrCast(isel.air.extra.items[ty_pl.payload..][0..loaded_struct.field_types.len]);
6913	var field_offset: u64 = 0;
6914	var field_it = loaded_struct.iterateRuntimeOrder(ip);
6915	while (field_it.next()) |field_index| {
6916	const field_ty: ZigType = .fromInterned(loaded_struct.field_types.get(ip)[field_index]);
6917	field_offset = loaded_struct.field_offsets.get(ip)[field_index];
6918	const field_size = field_ty.abiSize(zcu);
6919	if (field_size == 0) continue;
6920	var agg_part_it = agg_vi.value.field(agg_ty, field_offset, field_size);
6921	const agg_part_vi = try agg_part_it.only(isel);
6922	try agg_part_vi.?.move(isel, elems[field_index]);
6923	field_offset += field_size;
6924	}
6925	assert(loaded_struct.alignment.forward(field_offset) == agg_vi.value.size(isel));
6926	},
6927	.tuple_type => |tuple_type| {
6928	const elems: []const Air.Inst.Ref =
6929	@ptrCast(isel.air.extra.items[ty_pl.payload..][0..tuple_type.types.len]);
6930	var tuple_align: InternPool.Alignment = .@"1";
6931	var field_offset: u64 = 0;
6932	for (
6933	tuple_type.types.get(ip),
6934	tuple_type.values.get(ip),
6935	elems,
6936	) |field_ty_index, field_val, elem| {
6937	if (field_val != .none) continue;
6938	const field_ty: ZigType = .fromInterned(field_ty_index);
6939	const field_align = field_ty.abiAlignment(zcu);
6940	tuple_align = tuple_align.maxStrict(field_align);
6941	field_offset = field_align.forward(field_offset);
6942	const field_size = field_ty.abiSize(zcu);
6943	if (field_size == 0) continue;
6944	var agg_part_it = agg_vi.value.field(agg_ty, field_offset, field_size);
6945	const agg_part_vi = try agg_part_it.only(isel);
6946	try agg_part_vi.?.move(isel, elem);
6947	field_offset += field_size;
6948	}
6949	assert(tuple_align.forward(field_offset) == agg_vi.value.size(isel));
6950	},
6951	else => return isel.fail("aggregate init {f}", .{isel.fmtType(agg_ty)}),
6952	}
6953	}
6954	if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
6955	},
6956	.union_init => |air_tag| {
6957	if (isel.live_values.fetchRemove(air.inst_index)) |union_vi| unused: {
6958	defer union_vi.value.deref(isel);
6959
6960	const ty_pl = air.data(air.inst_index).ty_pl;
6961	const extra = isel.air.extraData(Air.UnionInit, ty_pl.payload).data;
6962	const union_ty = ty_pl.ty;
6963	const loaded_union = ip.loadUnionType(union_ty.toIntern());
6964	const union_layout = ZigType.getUnionLayout(loaded_union, zcu);
6965
6966	if (union_layout.tag_size > 0) unused_tag: {
6967	const loaded_tag = ip.loadEnumType(loaded_union.enum_tag_type);
6968	var tag_it = union_vi.value.field(union_ty, union_layout.tagOffset(), union_layout.tag_size);
6969	const tag_vi = try tag_it.only(isel);
6970	const tag_ra = try tag_vi.?.defReg(isel) orelse break :unused_tag;
6971	switch (union_layout.tag_size) {
6972	0 => unreachable,
6973	1...4 => try isel.movImmediate(tag_ra.w(), @as(u32, switch (loaded_tag.field_values.len) {
6974	0 => extra.field_index,
6975	else => switch (ip.indexToKey(loaded_tag.field_values.get(ip)[extra.field_index]).int.storage) {
6976	.u64 => |imm| @intCast(imm),
6977	.i64 => |imm| @bitCast(@as(i32, @intCast(imm))),
6978	else => unreachable,
6979	},
6980	})),
6981	5...8 => try isel.movImmediate(tag_ra.x(), switch (loaded_tag.field_values.len) {
6982	0 => extra.field_index,
6983	else => switch (ip.indexToKey(loaded_tag.field_values.get(ip)[extra.field_index]).int.storage) {
6984	.u64 => |imm| imm,
6985	.i64 => |imm| @bitCast(imm),
6986	else => unreachable,
6987	},
6988	}),
6989	else => return isel.fail("too big {t} {f}", .{ air_tag, isel.fmtType(union_ty) }),
6990	}
6991	}
6992	var payload_it = union_vi.value.field(union_ty, union_layout.payloadOffset(), union_layout.payload_size);
6993	const payload_vi = try payload_it.only(isel);
6994	try payload_vi.?.defAddr(isel, union_ty, .{ .root_vi = union_vi.value }) orelse break :unused;
6995
6996	try call.prepareReturn(isel);
6997	try call.finishReturn(isel);
6998
6999	try call.prepareCallee(isel);
7000	try isel.global_relocs.append(gpa, .{
7001	.name = "memcpy",
7002	.reloc = .{ .label = @intCast(isel.instructions.items.len) },
7003	});
7004	try isel.emit(.bl(0));
7005	try call.finishCallee(isel);
7006
7007	try call.prepareParams(isel);
7008	const init_vi = try isel.use(extra.init);
7009	try isel.movImmediate(.x2, init_vi.size(isel));
7010	try call.paramAddress(isel, init_vi, .r1);
7011	try call.paramAddress(isel, payload_vi.?, .r0);
7012	try call.finishParams(isel);
7013	}
7014	if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
7015	},
7016	.prefetch => {
7017	const prefetch = air.data(air.inst_index).prefetch;
7018	if (!(prefetch.rw == .write and prefetch.cache == .instruction)) {
7019	const maybe_slice_ty = isel.air.typeOf(prefetch.ptr, ip);
7020	const maybe_slice_vi = try isel.use(prefetch.ptr);
7021	const ptr_vi = if (maybe_slice_ty.isSlice(zcu)) ptr_vi: {
7022	var ptr_part_it = maybe_slice_vi.field(maybe_slice_ty, 0, 8);
7023	const ptr_part_vi = try ptr_part_it.only(isel);
7024	break :ptr_vi ptr_part_vi.?;
7025	} else maybe_slice_vi;
7026	const ptr_mat = try ptr_vi.matReg(isel);
7027	try isel.emit(.prfm(.{
7028	.policy = switch (prefetch.locality) {
7029	1, 2, 3 => .keep,
7030	0 => .strm,
7031	},
7032	.target = switch (prefetch.locality) {
7033	0, 3 => .l1,
7034	2 => .l2,
7035	1 => .l3,
7036	},
7037	.type = switch (prefetch.rw) {
7038	.read => switch (prefetch.cache) {
7039	.data => .pld,
7040	.instruction => .pli,
7041	},
7042	.write => switch (prefetch.cache) {
7043	.data => .pst,
7044	.instruction => unreachable,
7045	},
7046	},
7047	}, .{ .base = ptr_mat.ra.x() }));
7048	try ptr_mat.finish(isel);
7049	}
7050	if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
7051	},
7052	.mul_add => {
7053	if (isel.live_values.fetchRemove(air.inst_index)) |res_vi| unused: {
7054	defer res_vi.value.deref(isel);
7055
7056	const pl_op = air.data(air.inst_index).pl_op;
7057	const bin_op = isel.air.extraData(Air.Bin, pl_op.payload).data;
7058	const ty = isel.air.typeOf(pl_op.operand, ip);
7059	switch (ty.floatBits(isel.target)) {
7060	else => unreachable,
7061	16, 32, 64 => |bits| {
7062	const res_ra = try res_vi.value.defReg(isel) orelse break :unused;
7063	const need_fcvt = switch (bits) {
7064	else => unreachable,
7065	16 => !isel.target.cpu.has(.aarch64, .fullfp16),
7066	32, 64 => false,
7067	};
7068	if (need_fcvt) try isel.emit(.fcvt(res_ra.h(), res_ra.s()));
7069	const lhs_vi = try isel.use(bin_op.lhs);
7070	const rhs_vi = try isel.use(bin_op.rhs);
7071	const addend_vi = try isel.use(pl_op.operand);
7072	const lhs_mat = try lhs_vi.matReg(isel);
7073	const rhs_mat = try rhs_vi.matReg(isel);
7074	const addend_mat = try addend_vi.matReg(isel);
7075	const lhs_ra = if (need_fcvt) try isel.allocVecReg() else lhs_mat.ra;
7076	defer if (need_fcvt) isel.freeReg(lhs_ra);
7077	const rhs_ra = if (need_fcvt) try isel.allocVecReg() else rhs_mat.ra;
7078	defer if (need_fcvt) isel.freeReg(rhs_ra);
7079	const addend_ra = if (need_fcvt) try isel.allocVecReg() else addend_mat.ra;
7080	defer if (need_fcvt) isel.freeReg(addend_ra);
7081	try isel.emit(bits: switch (bits) {
7082	else => unreachable,
7083	16 => if (need_fcvt)
7084	continue :bits 32
7085	else
7086	.fmadd(res_ra.h(), lhs_ra.h(), rhs_ra.h(), addend_ra.h()),
7087	32 => .fmadd(res_ra.s(), lhs_ra.s(), rhs_ra.s(), addend_ra.s()),
7088	64 => .fmadd(res_ra.d(), lhs_ra.d(), rhs_ra.d(), addend_ra.d()),
7089	});
7090	if (need_fcvt) {
7091	try isel.emit(.fcvt(addend_ra.s(), addend_mat.ra.h()));
7092	try isel.emit(.fcvt(rhs_ra.s(), rhs_mat.ra.h()));
7093	try isel.emit(.fcvt(lhs_ra.s(), lhs_mat.ra.h()));
7094	}
7095	try addend_mat.finish(isel);
7096	try rhs_mat.finish(isel);
7097	try lhs_mat.finish(isel);
7098	},
7099	80, 128 => |bits| {
7100	try call.prepareReturn(isel);
7101	switch (bits) {
7102	else => unreachable,
7103	16, 32, 64, 128 => try call.returnLiveIn(isel, res_vi.value, .v0),
7104	80 => {
7105	var res_hi16_it = res_vi.value.field(ty, 8, 8);
7106	const res_hi16_vi = try res_hi16_it.only(isel);
7107	try call.returnLiveIn(isel, res_hi16_vi.?, .r1);
7108	var res_lo64_it = res_vi.value.field(ty, 0, 8);
7109	const res_lo64_vi = try res_lo64_it.only(isel);
7110	try call.returnLiveIn(isel, res_lo64_vi.?, .r0);
7111	},
7112	}
7113	try call.finishReturn(isel);
7114
7115	try call.prepareCallee(isel);
7116	try isel.global_relocs.append(gpa, .{
7117	.name = switch (bits) {
7118	else => unreachable,
7119	16 => "__fmah",
7120	32 => "fmaf",
7121	64 => "fma",
7122	80 => "__fmax",
7123	128 => "fmaf128",
7124	},
7125	.reloc = .{ .label = @intCast(isel.instructions.items.len) },
7126	});
7127	try isel.emit(.bl(0));
7128	try call.finishCallee(isel);
7129
7130	try call.prepareParams(isel);
7131	const lhs_vi = try isel.use(bin_op.lhs);
7132	const rhs_vi = try isel.use(bin_op.rhs);
7133	const addend_vi = try isel.use(pl_op.operand);
7134	switch (bits) {
7135	else => unreachable,
7136	16, 32, 64, 128 => {
7137	try call.paramLiveOut(isel, addend_vi, .v2);
7138	try call.paramLiveOut(isel, rhs_vi, .v1);
7139	try call.paramLiveOut(isel, lhs_vi, .v0);
7140	},
7141	80 => {
7142	var addend_hi16_it = addend_vi.field(ty, 8, 8);
7143	const addend_hi16_vi = try addend_hi16_it.only(isel);
7144	try call.paramLiveOut(isel, addend_hi16_vi.?, .r5);
7145	var addend_lo64_it = addend_vi.field(ty, 0, 8);
7146	const addend_lo64_vi = try addend_lo64_it.only(isel);
7147	try call.paramLiveOut(isel, addend_lo64_vi.?, .r4);
7148	var rhs_hi16_it = rhs_vi.field(ty, 8, 8);
7149	const rhs_hi16_vi = try rhs_hi16_it.only(isel);
7150	try call.paramLiveOut(isel, rhs_hi16_vi.?, .r3);
7151	var rhs_lo64_it = rhs_vi.field(ty, 0, 8);
7152	const rhs_lo64_vi = try rhs_lo64_it.only(isel);
7153	try call.paramLiveOut(isel, rhs_lo64_vi.?, .r2);
7154	var lhs_hi16_it = lhs_vi.field(ty, 8, 8);
7155	const lhs_hi16_vi = try lhs_hi16_it.only(isel);
7156	try call.paramLiveOut(isel, lhs_hi16_vi.?, .r1);
7157	var lhs_lo64_it = lhs_vi.field(ty, 0, 8);
7158	const lhs_lo64_vi = try lhs_lo64_it.only(isel);
7159	try call.paramLiveOut(isel, lhs_lo64_vi.?, .r0);
7160	},
7161	}
7162	try call.finishParams(isel);
7163	},
7164	}
7165	}
7166	if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
7167	},
7168	.field_parent_ptr => {
7169	if (isel.live_values.fetchRemove(air.inst_index)) |dst_vi| unused: {
7170	defer dst_vi.value.deref(isel);
7171	const ty_pl = air.data(air.inst_index).ty_pl;
7172	const extra = isel.air.extraData(Air.FieldParentPtr, ty_pl.payload).data;
7173	switch (codegen.fieldOffset(
7174	ty_pl.ty,
7175	isel.air.typeOf(extra.field_ptr, ip),
7176	extra.field_index,
7177	zcu,
7178	)) {
7179	0 => try dst_vi.value.move(isel, extra.field_ptr),
7180	else => |field_offset| {
7181	const dst_ra = try dst_vi.value.defReg(isel) orelse break :unused;
7182	const src_vi = try isel.use(extra.field_ptr);
7183	const src_mat = try src_vi.matReg(isel);
7184	const lo12: u12 = @truncate(field_offset >> 0);
7185	const hi12: u12 = @intCast(field_offset >> 12);
7186	if (hi12 > 0) try isel.emit(.sub(
7187	dst_ra.x(),
7188	if (lo12 > 0) dst_ra.x() else src_mat.ra.x(),
7189	.{ .shifted_immediate = .{ .immediate = hi12, .lsl = .@"12" } },
7190	));
7191	if (lo12 > 0) try isel.emit(.sub(dst_ra.x(), src_mat.ra.x(), .{ .immediate = lo12 }));
7192	try src_mat.finish(isel);
7193	},
7194	}
7195	}
7196	if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
7197	},
7198	.wasm_memory_size, .wasm_memory_grow => unreachable,
7199	.cmp_lte_errors_len => {
7200	if (isel.live_values.fetchRemove(air.inst_index)) |is_vi| unused: {
7201	defer is_vi.value.deref(isel);
7202	const is_ra = try is_vi.value.defReg(isel) orelse break :unused;
7203	try isel.emit(.csinc(is_ra.w(), .wzr, .wzr, .invert(.ls)));
7204
7205	const un_op = air.data(air.inst_index).un_op;
7206	const error_vi = try isel.use(un_op);
7207	const error_mat = try error_vi.matReg(isel);
7208	const ptr_ra = try isel.allocIntReg();
7209	defer isel.freeReg(ptr_ra);
7210	try isel.emit(.subs(.wzr, error_mat.ra.w(), .{ .register = ptr_ra.w() }));
7211	try isel.lazy_relocs.append(gpa, .{
7212	.symbol = .{ .kind = .const_data, .ty = .anyerror_type },
7213	.reloc = .{ .label = @intCast(isel.instructions.items.len) },
7214	});
7215	try isel.emit(.ldr(ptr_ra.w(), .{ .base = ptr_ra.x() }));
7216	try isel.lazy_relocs.append(gpa, .{
7217	.symbol = .{ .kind = .const_data, .ty = .anyerror_type },
7218	.reloc = .{ .label = @intCast(isel.instructions.items.len) },
7219	});
7220	try isel.emit(.adrp(ptr_ra.x(), 0));
7221	try error_mat.finish(isel);
7222	}
7223	if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
7224	},
7225	.runtime_nav_ptr => {
7226	if (isel.live_values.fetchRemove(air.inst_index)) |ptr_vi| unused: {
7227	defer ptr_vi.value.deref(isel);
7228	const ptr_ra = try ptr_vi.value.defReg(isel) orelse break :unused;
7229
7230	const ty_nav = air.data(air.inst_index).ty_nav;
7231	if (ZigType.fromInterned(ip.getNav(ty_nav.nav).resolved.?.type).isRuntimeFnOrHasRuntimeBits(zcu)) switch (true) {
7232	false => {
7233	try isel.nav_relocs.append(gpa, .{
7234	.nav = ty_nav.nav,
7235	.reloc = .{ .label = @intCast(isel.instructions.items.len) },
7236	});
7237	try isel.emit(.adr(ptr_ra.x(), 0));
7238	},
7239	true => {
7240	try isel.nav_relocs.append(gpa, .{
7241	.nav = ty_nav.nav,
7242	.reloc = .{ .label = @intCast(isel.instructions.items.len) },
7243	});
7244	if (ip.getNav(ty_nav.nav).getExtern(ip)) |_|
7245	try isel.emit(.ldr(ptr_ra.x(), .{ .unsigned_offset = .{ .base = ptr_ra.x(), .offset = 0 } }))
7246	else
7247	try isel.emit(.add(ptr_ra.x(), ptr_ra.x(), .{ .immediate = 0 }));
7248	try isel.nav_relocs.append(gpa, .{
7249	.nav = ty_nav.nav,
7250	.reloc = .{ .label = @intCast(isel.instructions.items.len) },
7251	});
7252	try isel.emit(.adrp(ptr_ra.x(), 0));
7253	},
7254	} else try isel.movImmediate(ptr_ra.x(), zcu.navAlignment(ty_nav.nav).forward(0xaaaaaaaaaaaaaaaa));
7255	}
7256	if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
7257	},
7258	.c_va_arg => {
7259	const maybe_arg_vi = isel.live_values.fetchRemove(air.inst_index);
7260	defer if (maybe_arg_vi) |arg_vi| arg_vi.value.deref(isel);
7261	const ty_op = air.data(air.inst_index).ty_op;
7262	const ty = ty_op.ty;
7263	var param_it: CallAbiIterator = .init;
7264	const param_vi = try param_it.param(isel, ty);
7265	defer param_vi.?.deref(isel);
7266	const passed_vi = switch (param_vi.?.parent(isel)) {
7267	.unallocated => param_vi.?,
7268	.stack_slot, .value, .constant => unreachable,
7269	.address => |address_vi| address_vi,
7270	};
7271	const passed_size: u5 = @intCast(passed_vi.alignment(isel).forward(passed_vi.size(isel)));
7272	const passed_is_vector = passed_vi.isVector(isel);
7273
7274	const va_list_ptr_vi = try isel.use(ty_op.operand);
7275	const va_list_ptr_mat = try va_list_ptr_vi.matReg(isel);
7276	const offs_ra = try isel.allocIntReg();
7277	defer isel.freeReg(offs_ra);
7278	const stack_ra = try isel.allocIntReg();
7279	defer isel.freeReg(stack_ra);
7280
7281	var part_vis: [2]Value.Index = undefined;
7282	var arg_part_ras: [2]?Register.Alias = @splat(null);
7283	const parts_len = parts_len: {
7284	var parts_len: u2 = 0;
7285	var part_it = passed_vi.parts(isel);
7286	while (part_it.next()) |part_vi| : (parts_len += 1) {
7287	part_vis[parts_len] = part_vi;
7288	const arg_vi = maybe_arg_vi orelse continue;
7289	const part_offset, const part_size = part_vi.position(isel);
7290	var arg_part_it = arg_vi.value.field(ty, part_offset, part_size);
7291	const arg_part_vi = try arg_part_it.only(isel);
7292	arg_part_ras[parts_len] = try arg_part_vi.?.defReg(isel);
7293	}
7294	break :parts_len parts_len;
7295	};
7296
7297	const done_label = isel.instructions.items.len;
7298	try isel.emit(.str(stack_ra.x(), .{ .unsigned_offset = .{
7299	.base = va_list_ptr_mat.ra.x(),
7300	.offset = 0,
7301	} }));
7302	try isel.emit(switch (parts_len) {
7303	else => unreachable,
7304	1 => if (arg_part_ras[0]) |arg_part_ra| switch (part_vis[0].size(isel)) {
7305	else => unreachable,
7306	1 => if (arg_part_ra.isVector()) .ldr(arg_part_ra.b(), .{ .post_index = .{
7307	.base = stack_ra.x(),
7308	.index = passed_size,
7309	} }) else switch (part_vis[0].signedness(isel)) {
7310	.signed => .ldrsb(arg_part_ra.w(), .{ .post_index = .{
7311	.base = stack_ra.x(),
7312	.index = passed_size,
7313	} }),
7314	.unsigned => .ldrb(arg_part_ra.w(), .{ .post_index = .{
7315	.base = stack_ra.x(),
7316	.index = passed_size,
7317	} }),
7318	},
7319	2 => if (arg_part_ra.isVector()) .ldr(arg_part_ra.h(), .{ .post_index = .{
7320	.base = stack_ra.x(),
7321	.index = passed_size,
7322	} }) else switch (part_vis[0].signedness(isel)) {
7323	.signed => .ldrsh(arg_part_ra.w(), .{ .post_index = .{
7324	.base = stack_ra.x(),
7325	.index = passed_size,
7326	} }),
7327	.unsigned => .ldrh(arg_part_ra.w(), .{ .post_index = .{
7328	.base = stack_ra.x(),
7329	.index = passed_size,
7330	} }),
7331	},
7332	4 => .ldr(if (arg_part_ra.isVector()) arg_part_ra.s() else arg_part_ra.w(), .{ .post_index = .{
7333	.base = stack_ra.x(),
7334	.index = passed_size,
7335	} }),
7336	8 => .ldr(if (arg_part_ra.isVector()) arg_part_ra.d() else arg_part_ra.x(), .{ .post_index = .{
7337	.base = stack_ra.x(),
7338	.index = passed_size,
7339	} }),
7340	16 => .ldr(arg_part_ra.q(), .{ .post_index = .{
7341	.base = stack_ra.x(),
7342	.index = passed_size,
7343	} }),
7344	} else .add(stack_ra.x(), stack_ra.x(), .{ .immediate = passed_size }),
7345	2 => if (arg_part_ras[0] != null or arg_part_ras[1] != null) .ldp(
7346	@as(Register.Alias, arg_part_ras[0] orelse .zr).x(),
7347	@as(Register.Alias, arg_part_ras[1] orelse .zr).x(),
7348	.{ .post_index = .{
7349	.base = stack_ra.x(),
7350	.index = passed_size,
7351	} },
7352	) else .add(stack_ra.x(), stack_ra.x(), .{ .immediate = passed_size }),
7353	});
7354	try isel.emit(.ldr(stack_ra.x(), .{ .unsigned_offset = .{
7355	.base = va_list_ptr_mat.ra.x(),
7356	.offset = 0,
7357	} }));
7358	switch (isel.va_list) {
7359	.other => {},
7360	.sysv => {
7361	const stack_label = isel.instructions.items.len;
7362	try isel.emit(.b(
7363	@intCast((isel.instructions.items.len + 1 - done_label) << 2),
7364	));
7365	switch (parts_len) {
7366	else => unreachable,
7367	1 => if (arg_part_ras[0]) |arg_part_ra| try isel.emit(switch (part_vis[0].size(isel)) {
7368	else => unreachable,
7369	1 => if (arg_part_ra.isVector()) .ldr(arg_part_ra.b(), .{ .extended_register = .{
7370	.base = stack_ra.x(),
7371	.index = offs_ra.w(),
7372	.extend = .{ .sxtw = 0 },
7373	} }) else switch (part_vis[0].signedness(isel)) {
7374	.signed => .ldrsb(arg_part_ra.w(), .{ .extended_register = .{
7375	.base = stack_ra.x(),
7376	.index = offs_ra.w(),
7377	.extend = .{ .sxtw = 0 },
7378	} }),
7379	.unsigned => .ldrb(arg_part_ra.w(), .{ .extended_register = .{
7380	.base = stack_ra.x(),
7381	.index = offs_ra.w(),
7382	.extend = .{ .sxtw = 0 },
7383	} }),
7384	},
7385	2 => if (arg_part_ra.isVector()) .ldr(arg_part_ra.h(), .{ .extended_register = .{
7386	.base = stack_ra.x(),
7387	.index = offs_ra.w(),
7388	.extend = .{ .sxtw = 0 },
7389	} }) else switch (part_vis[0].signedness(isel)) {
7390	.signed => .ldrsh(arg_part_ra.w(), .{ .extended_register = .{
7391	.base = stack_ra.x(),
7392	.index = offs_ra.w(),
7393	.extend = .{ .sxtw = 0 },
7394	} }),
7395	.unsigned => .ldrh(arg_part_ra.w(), .{ .extended_register = .{
7396	.base = stack_ra.x(),
7397	.index = offs_ra.w(),
7398	.extend = .{ .sxtw = 0 },
7399	} }),
7400	},
7401	4 => .ldr(if (arg_part_ra.isVector()) arg_part_ra.s() else arg_part_ra.w(), .{ .extended_register = .{
7402	.base = stack_ra.x(),
7403	.index = offs_ra.w(),
7404	.extend = .{ .sxtw = 0 },
7405	} }),
7406	8 => .ldr(if (arg_part_ra.isVector()) arg_part_ra.d() else arg_part_ra.x(), .{ .extended_register = .{
7407	.base = stack_ra.x(),
7408	.index = offs_ra.w(),
7409	.extend = .{ .sxtw = 0 },
7410	} }),
7411	16 => .ldr(arg_part_ra.q(), .{ .extended_register = .{
7412	.base = stack_ra.x(),
7413	.index = offs_ra.w(),
7414	.extend = .{ .sxtw = 0 },
7415	} }),
7416	}),
7417	2 => if (arg_part_ras[0] != null or arg_part_ras[1] != null) {
7418	try isel.emit(.ldp(
7419	@as(Register.Alias, arg_part_ras[0] orelse .zr).x(),
7420	@as(Register.Alias, arg_part_ras[1] orelse .zr).x(),
7421	.{ .base = stack_ra.x() },
7422	));
7423	try isel.emit(.add(stack_ra.x(), stack_ra.x(), .{ .extended_register = .{
7424	.register = offs_ra.w(),
7425	.extend = .{ .sxtw = 0 },
7426	} }));
7427	},
7428	}
7429	try isel.emit(.ldr(stack_ra.x(), .{ .unsigned_offset = .{
7430	.base = va_list_ptr_mat.ra.x(),
7431	.offset = if (passed_is_vector) 16 else 8,
7432	} }));
7433	try isel.emit(.@"b."(
7434	.gt,
7435	@intCast((isel.instructions.items.len + 1 - stack_label) << 2),
7436	));
7437	try isel.emit(.str(stack_ra.w(), .{ .unsigned_offset = .{
7438	.base = va_list_ptr_mat.ra.x(),
7439	.offset = if (passed_is_vector) 28 else 24,
7440	} }));
7441	try isel.emit(.adds(stack_ra.w(), offs_ra.w(), .{ .immediate = passed_size }));
7442	try isel.emit(.tbz(
7443	offs_ra.w(),
7444	31,
7445	@intCast((isel.instructions.items.len + 1 - stack_label) << 2),
7446	));
7447	try isel.emit(.ldr(offs_ra.w(), .{ .unsigned_offset = .{
7448	.base = va_list_ptr_mat.ra.x(),
7449	.offset = if (passed_is_vector) 28 else 24,
7450	} }));
7451	},
7452	}
7453	try va_list_ptr_mat.finish(isel);
7454	if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
7455	},
7456	.c_va_copy => {
7457	if (isel.live_values.fetchRemove(air.inst_index)) |va_list_vi| {
7458	defer va_list_vi.value.deref(isel);
7459	const ty_op = air.data(air.inst_index).ty_op;
7460	const va_list_ptr_vi = try isel.use(ty_op.operand);
7461	const va_list_ptr_mat = try va_list_ptr_vi.matReg(isel);
7462	_ = try va_list_vi.value.load(isel, ty_op.ty, va_list_ptr_mat.ra, .{});
7463	try va_list_ptr_mat.finish(isel);
7464	}
7465	if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
7466	},
7467	.c_va_end => if (air.next()) |next_air_tag| continue :air_tag next_air_tag,
7468	.c_va_start => {
7469	if (isel.live_values.fetchRemove(air.inst_index)) |va_list_vi| {
7470	defer va_list_vi.value.deref(isel);
7471	const ty = air.data(air.inst_index).ty;
7472	switch (isel.va_list) {
7473	.other => |va_list| if (try va_list_vi.value.defReg(isel)) |va_list_ra| try isel.emit(.add(
7474	va_list_ra.x(),
7475	va_list.base.x(),
7476	.{ .immediate = @intCast(va_list.offset) },
7477	)),
7478	.sysv => |va_list| {
7479	var vr_offs_it = va_list_vi.value.field(ty, 28, 4);
7480	const vr_offs_vi = try vr_offs_it.only(isel);
7481	if (try vr_offs_vi.?.defReg(isel)) |vr_offs_ra| try isel.movImmediate(
7482	vr_offs_ra.w(),
7483	@as(u32, @bitCast(va_list.__vr_offs)),
7484	);
7485	var gr_offs_it = va_list_vi.value.field(ty, 24, 4);
7486	const gr_offs_vi = try gr_offs_it.only(isel);
7487	if (try gr_offs_vi.?.defReg(isel)) |gr_offs_ra| try isel.movImmediate(
7488	gr_offs_ra.w(),
7489	@as(u32, @bitCast(va_list.__gr_offs)),
7490	);
7491	var vr_top_it = va_list_vi.value.field(ty, 16, 8);
7492	const vr_top_vi = try vr_top_it.only(isel);
7493	if (try vr_top_vi.?.defReg(isel)) |vr_top_ra| try isel.emit(.add(
7494	vr_top_ra.x(),
7495	va_list.__vr_top.base.x(),
7496	.{ .immediate = @intCast(va_list.__vr_top.offset) },
7497	));
7498	var gr_top_it = va_list_vi.value.field(ty, 8, 8);
7499	const gr_top_vi = try gr_top_it.only(isel);
7500	if (try gr_top_vi.?.defReg(isel)) |gr_top_ra| try isel.emit(.add(
7501	gr_top_ra.x(),
7502	va_list.__gr_top.base.x(),
7503	.{ .immediate = @intCast(va_list.__gr_top.offset) },
7504	));
7505	var stack_it = va_list_vi.value.field(ty, 0, 8);
7506	const stack_vi = try stack_it.only(isel);
7507	if (try stack_vi.?.defReg(isel)) |stack_ra| try isel.emit(.add(
7508	stack_ra.x(),
7509	va_list.__stack.base.x(),
7510	.{ .immediate = @intCast(va_list.__stack.offset) },
7511	));
7512	},
7513	}
7514	}
7515	if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
7516	},
7517	.work_item_id, .work_group_size, .work_group_id, .spirv_runtime_array_len => unreachable,
7518	}
7519	assert(air.body_index == 0);
7520	}
7521
7522	pub fn verify(isel: *Select, check_values: bool) void {
7523	if (!std.debug.runtime_safety) return;
7524	assert(isel.blocks.count() == 1 and isel.blocks.keys()[0] == Select.Block.main);
7525	assert(isel.active_loops.items.len == 0);
7526	assert(isel.dom_start == 0 and isel.dom_len == 0);
7527	var live_reg_it = isel.live_registers.iterator();
7528	while (live_reg_it.next()) |live_reg_entry| switch (live_reg_entry.value.*) {
7529	_ => {
7530	isel.dumpValues(.all);
7531	unreachable;
7532	},
7533	.allocating, .free => {},
7534	};
7535	if (check_values) for (isel.values.items) |value| if (value.refs != 0) {
7536	isel.dumpValues(.only_referenced);
7537	unreachable;
7538	};
7539	}
7540
7541	/// Stack Frame Layout
7542	/// +-+-----------------------------------+
7543	/// |R| allocated stack |
7544	/// +-+-----------------------------------+
7545	/// |S| caller frame record | +---------------+
7546	/// +-+-----------------------------------+ <-| entry/exit FP |
7547	/// |R| caller frame | +---------------+
7548	/// +-+-----------------------------------+
7549	/// |R| variable incoming stack arguments | +---------------+
7550	/// +-+-----------------------------------+ <-| __stack |
7551	/// |S| named incoming stack arguments | +---------------+
7552	/// +-+-----------------------------------+ <-| entry/exit SP |
7553	/// |S| incoming gr arguments | | __gr_top |
7554	/// +-+-----------------------------------+ +---------------+
7555	/// |S| alignment gap |
7556	/// +-+-----------------------------------+
7557	/// |S| frame record | +----------+
7558	/// +-+-----------------------------------+ <-| FP |
7559	/// |S| incoming vr arguments | | __vr_top |
7560	/// +-+-----------------------------------+ +----------+
7561	/// |L| alignment gap |
7562	/// +-+-----------------------------------+
7563	/// |L| callee saved vr area |
7564	/// +-+-----------------------------------+
7565	/// |L| callee saved gr area | +----------------------+
7566	/// +-+-----------------------------------+ <-| prologue/epilogue SP |
7567	/// |R| realignment gap | +----------------------+
7568	/// +-+-----------------------------------+
7569	/// |L| locals |
7570	/// +-+-----------------------------------+
7571	/// |S| outgoing stack arguments | +----+
7572	/// +-+-----------------------------------+ <-| SP |
7573	/// |R| unallocated stack | +----+
7574	/// +-+-----------------------------------+
7575	/// [S] Size computed by `analyze`, can be used by the body.
7576	/// [L] Size computed by `layout`, can be used by the prologue/epilogue.
7577	/// [R] Size unknown until runtime, can vary from one call to the next.
7578	///
7579	/// Constraints that led to this layout:
7580	/// * FP to __stack/__gr_top/__vr_top must only pass through [S]
7581	/// * SP to outgoing stack arguments/locals must only pass through [S]
7582	/// * entry/exit SP to prologue/epilogue SP must only pass through [S/L]
7583	/// * all save areas must be at a positive offset from prologue/epilogue SP
7584	/// * the entry/exit SP to prologue/epilogue SP distance must
7585	/// - be a multiple of 16 due to hardware restrictions on the value of SP
7586	/// - conform to the limit from the first matching condition in the
7587	/// following list due to instruction encoding limitations
7588	/// 1. callee saved gr count >= 2: multiple of 8 of at most 504 bytes
7589	/// 2. callee saved vr count >= 2: multiple of 8 of at most 504 bytes
7590	/// 3. callee saved gr count >= 1: at most 255 bytes
7591	/// 4. callee saved vr count >= 1: at most 255 bytes
7592	/// 5. variable incoming vr argument count >= 2: multiple of 16 of at most 1008 bytes
7593	/// 6. variable incoming vr argument count >= 1: at most 255 bytes
7594	/// 7. have frame record: multiple of 8 of at most 504 bytes
7595	pub fn layout(
7596	isel: *Select,
7597	incoming: CallAbiIterator,
7598	is_sysv_var_args: bool,
7599	saved_gra_len: u7,
7600	saved_vra_len: u7,
7601	mod: *const Module,
7602	) !usize {
7603	const zcu = isel.pt.zcu;
7604	const ip = &zcu.intern_pool;
7605	const nav = ip.getNav(isel.nav_index);
7606	wip_mir_log.debug("{f}<body>:\n", .{nav.fqn.fmt(ip)});
7607
7608	const stack_size: u24 = @intCast(InternPool.Alignment.@"16".forward(isel.stack_size));
7609
7610	var saves_buf: [10 + 8 + 8 + 2 + 8]struct {
7611	class: enum { integer, vector },
7612	needs_restore: bool,
7613	register: Register,
7614	offset: u10,
7615	size: u5,
7616	} = undefined;
7617	const saves, const saves_size, const frame_record_offset = saves: {
7618	var saves_len: usize = 0;
7619	var saves_size: u10 = 0;
7620	var save_ra: Register.Alias = undefined;
7621
7622	// callee saved gr area
7623	save_ra = .r19;
7624	while (save_ra != .r29) : (save_ra = @fromBackingInt(@intCast(@backingInt(save_ra) + 1))) {
7625	if (!isel.saved_registers.contains(save_ra)) continue;
7626	saves_size = std.mem.alignForward(u10, saves_size, 8);
7627	saves_buf[saves_len] = .{
7628	.class = .integer,
7629	.needs_restore = true,
7630	.register = save_ra.x(),
7631	.offset = saves_size,
7632	.size = 8,
7633	};
7634	saves_len += 1;
7635	saves_size += 8;
7636	}
7637	var deferred_gr = if (saves_size == 8 or (saves_size % 16 != 0 and saved_gra_len % 2 != 0)) gr: {
7638	saves_len -= 1;
7639	saves_size -= 8;
7640	break :gr saves_buf[saves_len].register;
7641	} else null;
7642	defer assert(deferred_gr == null);
7643
7644	// callee saved vr area
7645	save_ra = .v8;
7646	while (save_ra != .v16) : (save_ra = @fromBackingInt(@intCast(@backingInt(save_ra) + 1))) {
7647	if (!isel.saved_registers.contains(save_ra)) continue;
7648	saves_size = std.mem.alignForward(u10, saves_size, 8);
7649	saves_buf[saves_len] = .{
7650	.class = .vector,
7651	.needs_restore = true,
7652	.register = save_ra.d(),
7653	.offset = saves_size,
7654	.size = 8,
7655	};
7656	saves_len += 1;
7657	saves_size += 8;
7658	}
7659	if (deferred_gr != null and saved_gra_len % 2 == 0) {
7660	saves_size = std.mem.alignForward(u10, saves_size, 8);
7661	saves_buf[saves_len] = .{
7662	.class = .integer,
7663	.needs_restore = true,
7664	.register = deferred_gr.?,
7665	.offset = saves_size,
7666	.size = 8,
7667	};
7668	saves_len += 1;
7669	saves_size += 8;
7670	deferred_gr = null;
7671	}
7672	if (saves_size % 16 != 0 and saved_vra_len % 2 != 0) {
7673	const prev_save = &saves_buf[saves_len - 1];
7674	switch (prev_save.class) {
7675	.integer => {},
7676	.vector => {
7677	prev_save.register = prev_save.register.alias.q();
7678	prev_save.size = 16;
7679	saves_size += 8;
7680	},
7681	}
7682	}
7683
7684	// incoming vr arguments
7685	save_ra = if (mod.strip) incoming.nsrn else CallAbiIterator.nsrn_start;
7686	while (save_ra != if (is_sysv_var_args) CallAbiIterator.nsrn_end else incoming.nsrn) : (save_ra = @fromBackingInt(@intCast(@backingInt(save_ra) + 1))) {
7687	saves_size = std.mem.alignForward(u10, saves_size, 16);
7688	saves_buf[saves_len] = .{
7689	.class = .vector,
7690	.needs_restore = false,
7691	.register = save_ra.q(),
7692	.offset = saves_size,
7693	.size = 16,
7694	};
7695	saves_len += 1;
7696	saves_size += 16;
7697	}
7698
7699	// frame record
7700	saves_size = std.mem.alignForward(u10, saves_size, 16);
7701	const frame_record_offset = saves_size;
7702	saves_buf[saves_len] = .{
7703	.class = .integer,
7704	.needs_restore = true,
7705	.register = .fp,
7706	.offset = saves_size,
7707	.size = 8,
7708	};
7709	saves_len += 1;
7710	saves_size += 8;
7711
7712	saves_size = std.mem.alignForward(u10, saves_size, 8);
7713	saves_buf[saves_len] = .{
7714	.class = .integer,
7715	.needs_restore = true,
7716	.register = .lr,
7717	.offset = saves_size,
7718	.size = 8,
7719	};
7720	saves_len += 1;
7721	saves_size += 8;
7722
7723	// incoming gr arguments
7724	if (deferred_gr) |gr| {
7725	saves_size = std.mem.alignForward(u10, saves_size, 8);
7726	saves_buf[saves_len] = .{
7727	.class = .integer,
7728	.needs_restore = true,
7729	.register = gr,
7730	.offset = saves_size,
7731	.size = 8,
7732	};
7733	saves_len += 1;
7734	saves_size += 8;
7735	deferred_gr = null;
7736	} else switch (@as(u1, @truncate(saved_gra_len))) {
7737	0 => {},
7738	1 => saves_size += 8,
7739	}
7740	save_ra = if (mod.strip) incoming.ngrn else CallAbiIterator.ngrn_start;
7741	while (save_ra != if (is_sysv_var_args) CallAbiIterator.ngrn_end else incoming.ngrn) : (save_ra = @fromBackingInt(@intCast(@backingInt(save_ra) + 1))) {
7742	saves_size = std.mem.alignForward(u10, saves_size, 8);
7743	saves_buf[saves_len] = .{
7744	.class = .integer,
7745	.needs_restore = false,
7746	.register = save_ra.x(),
7747	.offset = saves_size,
7748	.size = 8,
7749	};
7750	saves_len += 1;
7751	saves_size += 8;
7752	}
7753
7754	assert(InternPool.Alignment.@"16".check(saves_size));
7755	break :saves .{ saves_buf[0..saves_len], saves_size, frame_record_offset };
7756	};
7757
7758	{
7759	wip_mir_log.debug("{f}<prologue>:", .{nav.fqn.fmt(ip)});
7760	var save_index: usize = 0;
7761	while (save_index < saves.len) if (save_index + 2 <= saves.len and
7762	saves[save_index + 0].class == saves[save_index + 1].class and
7763	saves[save_index + 0].size == saves[save_index + 1].size and
7764	saves[save_index + 0].offset + saves[save_index + 0].size == saves[save_index + 1].offset)
7765	{
7766	try isel.emit(.stp(
7767	saves[save_index + 0].register,
7768	saves[save_index + 1].register,
7769	switch (saves[save_index + 0].offset) {
7770	0 => .{ .pre_index = .{
7771	.base = .sp,
7772	.index = @intCast(-@as(i11, saves_size)),
7773	} },
7774	else => |offset| .{ .signed_offset = .{
7775	.base = .sp,
7776	.offset = @intCast(offset),
7777	} },
7778	},
7779	));
7780	save_index += 2;
7781	} else {
7782	try isel.emit(.str(
7783	saves[save_index].register,
7784	switch (saves[save_index].offset) {
7785	0 => .{ .pre_index = .{
7786	.base = .sp,
7787	.index = @intCast(-@as(i11, saves_size)),
7788	} },
7789	else => |offset| .{ .unsigned_offset = .{
7790	.base = .sp,
7791	.offset = @intCast(offset),
7792	} },
7793	},
7794	));
7795	save_index += 1;
7796	};
7797
7798	try isel.emit(.add(.fp, .sp, .{ .immediate = frame_record_offset }));
7799	const scratch_reg: Register = if (isel.stack_align == .@"16")
7800	.sp
7801	else if (stack_size == 0 and frame_record_offset == 0)
7802	.fp
7803	else
7804	.ip0;
7805	const stack_size_lo: u12 = @truncate(stack_size >> 0);
7806	const stack_size_hi: u12 = @truncate(stack_size >> 12);
7807	if (mod.stack_check) {
7808	if (stack_size_hi > 2) {
7809	try isel.movImmediate(.ip1, stack_size_hi);
7810	const loop_label = isel.instructions.items.len;
7811	try isel.emit(.sub(.sp, .sp, .{
7812	.shifted_immediate = .{ .immediate = 1, .lsl = .@"12" },
7813	}));
7814	try isel.emit(.sub(.ip1, .ip1, .{ .immediate = 1 }));
7815	try isel.emit(.ldr(.xzr, .{ .base = .sp }));
7816	try isel.emit(.cbnz(.ip1, -@as(i21, @intCast(
7817	(isel.instructions.items.len - loop_label) << 2,
7818	))));
7819	} else for (0..stack_size_hi) |_| {
7820	try isel.emit(.sub(.sp, .sp, .{
7821	.shifted_immediate = .{ .immediate = 1, .lsl = .@"12" },
7822	}));
7823	try isel.emit(.ldr(.xzr, .{ .base = .sp }));
7824	}
7825	if (stack_size_lo > 0) try isel.emit(.sub(
7826	scratch_reg,
7827	.sp,
7828	.{ .immediate = stack_size_lo },
7829	)) else if (scratch_reg.alias == Register.Alias.ip0)
7830	try isel.emit(.add(scratch_reg, .sp, .{ .immediate = 0 }));
7831	} else {
7832	if (stack_size_hi > 0) try isel.emit(.sub(scratch_reg, .sp, .{
7833	.shifted_immediate = .{ .immediate = stack_size_hi, .lsl = .@"12" },
7834	}));
7835	if (stack_size_lo > 0) try isel.emit(.sub(
7836	scratch_reg,
7837	if (stack_size_hi > 0) scratch_reg else .sp,
7838	.{ .immediate = stack_size_lo },
7839	)) else if (scratch_reg.alias == Register.Alias.ip0 and stack_size_hi == 0)
7840	try isel.emit(.add(scratch_reg, .sp, .{ .immediate = 0 }));
7841	}
7842	if (isel.stack_align != .@"16") try isel.emit(.@"and"(.sp, scratch_reg, .{ .immediate = .{
7843	.N = .doubleword,
7844	.immr = -%isel.stack_align.toLog2Units(),
7845	.imms = ~isel.stack_align.toLog2Units(),
7846	} }));
7847	wip_mir_log.debug("", .{});
7848	}
7849
7850	const epilogue = isel.instructions.items.len;
7851	if (isel.returns) {
7852	try isel.emit(.ret(.lr));
7853	var save_index: usize = 0;
7854	var first_offset: ?u10 = null;
7855	while (save_index < saves.len) {
7856	if (save_index + 2 <= saves.len and saves[save_index + 1].needs_restore and
7857	saves[save_index + 0].class == saves[save_index + 1].class and
7858	saves[save_index + 0].size == saves[save_index + 1].size and
7859	saves[save_index + 0].offset + saves[save_index + 0].size == saves[save_index + 1].offset)
7860	{
7861	try isel.emit(.ldp(
7862	saves[save_index + 0].register,
7863	saves[save_index + 1].register,
7864	if (first_offset) |offset| .{ .signed_offset = .{
7865	.base = .sp,
7866	.offset = @intCast(saves[save_index + 0].offset - offset),
7867	} } else form: {
7868	first_offset = @intCast(saves[save_index + 0].offset);
7869	break :form .{ .post_index = .{
7870	.base = .sp,
7871	.index = @intCast(saves_size - first_offset.?),
7872	} };
7873	},
7874	));
7875	save_index += 2;
7876	} else if (saves[save_index].needs_restore) {
7877	try isel.emit(.ldr(
7878	saves[save_index].register,
7879	if (first_offset) |offset| .{ .unsigned_offset = .{
7880	.base = .sp,
7881	.offset = saves[save_index + 0].offset - offset,
7882	} } else form: {
7883	const offset = saves[save_index + 0].offset;
7884	first_offset = offset;
7885	break :form .{ .post_index = .{
7886	.base = .sp,
7887	.index = @intCast(saves_size - offset),
7888	} };
7889	},
7890	));
7891	save_index += 1;
7892	} else save_index += 1;
7893	}
7894	const offset = stack_size + first_offset.?;
7895	const offset_lo: u12 = @truncate(offset >> 0);
7896	const offset_hi: u12 = @truncate(offset >> 12);
7897	if (isel.stack_align != .@"16" or (offset_lo > 0 and offset_hi > 0)) {
7898	const fp_offset = @as(i11, first_offset.?) - frame_record_offset;
7899	try isel.emit(if (fp_offset >= 0)
7900	.add(.sp, .fp, .{ .immediate = @intCast(fp_offset) })
7901	else
7902	.sub(.sp, .fp, .{ .immediate = @intCast(-fp_offset) }));
7903	} else {
7904	if (offset_hi > 0) try isel.emit(.add(.sp, .sp, .{
7905	.shifted_immediate = .{ .immediate = offset_hi, .lsl = .@"12" },
7906	}));
7907	if (offset_lo > 0) try isel.emit(.add(.sp, .sp, .{
7908	.immediate = offset_lo,
7909	}));
7910	}
7911	wip_mir_log.debug("{f}<epilogue>:\n", .{nav.fqn.fmt(ip)});
7912	}
7913	return epilogue;
7914	}
7915
7916	fn fmtDom(isel: *Select, inst: Air.Inst.Index, start: u32, len: u32) struct {
7917	isel: *Select,
7918	inst: Air.Inst.Index,
7919	start: u32,
7920	len: u32,
7921	pub fn format(data: @This(), writer: *std.Io.Writer) std.Io.Writer.Error!void {
7922	try writer.print("%{d} -> {{", .{@backingInt(data.inst)});
7923	var first = true;
7924	for (data.isel.blocks.keys()[0..data.len], 0..) |block_inst_index, dom_index| {
7925	if (@as(u1, @truncate(data.isel.dom.items[
7926	data.start + dom_index / @bitSizeOf(DomInt)
7927	] >> @truncate(dom_index))) == 0) continue;
7928	if (first) {
7929	first = false;
7930	} else {
7931	try writer.writeByte(',');
7932	}
7933	switch (block_inst_index) {
7934	Block.main => try writer.writeAll(" %main"),
7935	else => try writer.print(" %{d}", .{@backingInt(block_inst_index)}),
7936	}
7937	}
7938	if (!first) try writer.writeByte(' ');
7939	try writer.writeByte('}');
7940	}
7941	} {
7942	return .{ .isel = isel, .inst = inst, .start = start, .len = len };
7943	}
7944
7945	fn fmtLoopLive(isel: *Select, loop_inst: Air.Inst.Index) struct {
7946	isel: *Select,
7947	inst: Air.Inst.Index,
7948	pub fn format(data: @This(), writer: *std.Io.Writer) std.Io.Writer.Error!void {
7949	const loops = data.isel.loops.values();
7950	const loop_index = data.isel.loops.getIndex(data.inst).?;
7951	const live_insts =
7952	data.isel.loop_live.list.items[loops[loop_index].live..loops[loop_index + 1].live];
7953
7954	try writer.print("%{d} <- {{", .{@backingInt(data.inst)});
7955	var first = true;
7956	for (live_insts) |live_inst| {
7957	if (first) {
7958	first = false;
7959	} else {
7960	try writer.writeByte(',');
7961	}
7962	try writer.print(" %{d}", .{@backingInt(live_inst)});
7963	}
7964	if (!first) try writer.writeByte(' ');
7965	try writer.writeByte('}');
7966	}
7967	} {
7968	return .{ .isel = isel, .inst = loop_inst };
7969	}
7970
7971	fn fmtType(isel: *Select, ty: ZigType) ZigType.Formatter {
7972	return ty.fmt(isel.pt);
7973	}
7974
7975	fn fmtConstant(isel: *Select, constant: Constant) @typeInfo(@TypeOf(Constant.fmtValue)).@"fn".return_type.? {
7976	return constant.fmtValue(isel.pt);
7977	}
7978
7979	fn block(
7980	isel: *Select,
7981	air_inst_index: Air.Inst.Index,
7982	res_ty: ZigType,
7983	air_body: []const Air.Inst.Index,
7984	) !void {
7985	if (res_ty.toIntern() != .noreturn_type) {
7986	isel.blocks.putAssumeCapacityNoClobber(air_inst_index, .{
7987	.live_registers = isel.live_registers,
7988	.target_label = @intCast(isel.instructions.items.len),
7989	});
7990	}
7991	try isel.body(air_body);
7992	if (res_ty.toIntern() != .noreturn_type) {
7993	const block_entry = isel.blocks.pop().?;
7994	assert(block_entry.key == air_inst_index);
7995	if (isel.live_values.fetchRemove(air_inst_index)) |result_vi| result_vi.value.deref(isel);
7996	}
7997	}
7998
7999	fn emit(isel: *Select, instruction: codegen.aarch64.encoding.Instruction) !void {
8000	wip_mir_log.debug(" | {f}", .{instruction});
8001	try isel.instructions.append(isel.pt.zcu.gpa, instruction);
8002	}
8003
8004	fn emitPanic(isel: *Select, panic_id: Zcu.SimplePanicId) !void {
8005	const zcu = isel.pt.zcu;
8006	try isel.nav_relocs.append(zcu.gpa, .{
8007	.nav = switch (zcu.intern_pool.indexToKey(zcu.std_lang_decl_values.get(panic_id.toStdLangDecl()))) {
8008	else => unreachable,
8009	inline .@"extern", .func => |func| func.owner_nav,
8010	},
8011	.reloc = .{ .label = @intCast(isel.instructions.items.len) },
8012	});
8013	try isel.emit(.bl(0));
8014	}
8015
8016	fn emitLiteral(isel: *Select, bytes: []const u8) !void {
8017	const words: []align(1) const u32 = @ptrCast(bytes);
8018	const literals = try isel.literals.addManyAsSlice(isel.pt.zcu.gpa, words.len);
8019	switch (isel.target.cpu.arch.endian()) {
8020	.little => @memcpy(literals, words),
8021	.big => for (words, 0..) |word, word_index| {
8022	literals[literals.len - 1 - word_index] = @byteSwap(word);
8023	},
8024	}
8025	}
8026
8027	fn fail(isel: *Select, comptime format: []const u8, args: anytype) codegen.Error {
8028	@branchHint(.cold);
8029	return isel.pt.zcu.codegenFail(isel.nav_index, format, args);
8030	}
8031
8032	/// dst = src
8033	fn movImmediate(isel: *Select, dst_reg: Register, src_imm: u64) !void {
8034	const sf = dst_reg.format.general;
8035	if (src_imm == 0) {
8036	const zr: Register = switch (sf) {
8037	.word => .wzr,
8038	.doubleword => .xzr,
8039	};
8040	return isel.emit(.orr(dst_reg, zr, .{ .register = zr }));
8041	}
8042
8043	const Part = u16;
8044	const min_part: Part = std.math.minInt(Part);
8045	const max_part: Part = std.math.maxInt(Part);
8046
8047	const parts: [4]Part = @bitCast(switch (sf) {
8048	.word => @as(u32, @intCast(src_imm)),
8049	.doubleword => @as(u64, @intCast(src_imm)),
8050	});
8051	const width: u7 = switch (sf) {
8052	.word => 32,
8053	.doubleword => 64,
8054	};
8055	const parts_len: u3 = @intCast(@divExact(width, @bitSizeOf(Part)));
8056	var equal_min_count: u3 = 0;
8057	var equal_max_count: u3 = 0;
8058	for (parts[0..parts_len]) |part| {
8059	equal_min_count += @intFromBool(part == min_part);
8060	equal_max_count += @intFromBool(part == max_part);
8061	}
8062
8063	const equal_fill_count, const fill_part: Part = if (equal_min_count >= equal_max_count)
8064	.{ equal_min_count, min_part }
8065	else
8066	.{ equal_max_count, max_part };
8067	var remaining_parts = @max(parts_len - equal_fill_count, 1);
8068
8069	if (remaining_parts > 1) {
8070	var elem_width: u8 = 2;
8071	while (elem_width <= width) : (elem_width <<= 1) {
8072	const emask = @as(u64, std.math.maxInt(u64)) >> @intCast(64 - elem_width);
8073	const rmask = @divExact(@as(u64, switch (sf) {
8074	.word => std.math.maxInt(u32),
8075	.doubleword => std.math.maxInt(u64),
8076	}), emask);
8077	const elem = src_imm & emask;
8078	if (src_imm != elem * rmask) continue;
8079	const imask: u64 = @bitCast(@as(i64, @bitCast(elem << 63)) >> 63);
8080	const lsb0 = elem ^ (imask & emask);
8081	const lsb1 = (lsb0 - 1) | lsb0;
8082	if ((lsb1 +% 1) & lsb1 == 0) {
8083	const lo: u6 = @intCast(@ctz(lsb0));
8084	const hi: u6 = @intCast(@clz(lsb0) - (64 - elem_width));
8085	const mid: u6 = @intCast(elem_width - lo - hi);
8086	const smask: u6 = @truncate(imask);
8087	const mid_masked = mid & ~smask;
8088	return isel.emit(.orr(
8089	dst_reg,
8090	switch (sf) {
8091	.word => .wzr,
8092	.doubleword => .xzr,
8093	},
8094	.{ .immediate = .{
8095	.N = @fromBackingInt(@intCast(elem_width >> 6)),
8096	.immr = hi + mid_masked,
8097	.imms = ((((lo + hi) & smask) | mid_masked) - 1) | -%@as(u6, @truncate(elem_width)) << 1,
8098	} },
8099	));
8100	}
8101	}
8102	}
8103
8104	var part_index = parts_len;
8105	while (part_index > 0) {
8106	part_index -= 1;
8107	if (part_index >= remaining_parts and parts[part_index] == fill_part) continue;
8108	remaining_parts -= 1;
8109	try isel.emit(if (remaining_parts > 0) .movk(
8110	dst_reg,
8111	parts[part_index],
8112	.{ .lsl = @fromBackingInt(@intCast(part_index)) },
8113	) else switch (fill_part) {
8114	else => unreachable,
8115	min_part => .movz(
8116	dst_reg,
8117	parts[part_index],
8118	.{ .lsl = @fromBackingInt(@intCast(part_index)) },
8119	),
8120	max_part => .movn(
8121	dst_reg,
8122	~parts[part_index],
8123	.{ .lsl = @fromBackingInt(@intCast(part_index)) },
8124	),
8125	});
8126	}
8127	assert(remaining_parts == 0);
8128	}
8129
8130	/// elem_ptr = base +- elem_size * index
8131	/// elem_ptr, base, and index may alias
8132	fn elemPtr(
8133	isel: *Select,
8134	elem_ptr_ra: Register.Alias,
8135	base_ra: Register.Alias,
8136	op: codegen.aarch64.encoding.Instruction.AddSubtractOp,
8137	elem_size: u64,
8138	index_vi: Value.Index,
8139	) !void {
8140	const index_mat = try index_vi.matReg(isel);
8141	switch (@popCount(elem_size)) {
8142	0 => unreachable,
8143	1 => try isel.emit(switch (op) {
8144	.add => switch (base_ra) {
8145	else => .add(elem_ptr_ra.x(), base_ra.x(), .{ .shifted_register = .{
8146	.register = index_mat.ra.x(),
8147	.shift = .{ .lsl = @intCast(@ctz(elem_size)) },
8148	} }),
8149	.zr => switch (@ctz(elem_size)) {
8150	0 => .orr(elem_ptr_ra.x(), .xzr, .{ .register = index_mat.ra.x() }),
8151	else => |shift| .ubfm(elem_ptr_ra.x(), index_mat.ra.x(), .{
8152	.N = .doubleword,
8153	.immr = @intCast(64 - shift),
8154	.imms = @intCast(63 - shift),
8155	}),
8156	},
8157	},
8158	.sub => .sub(elem_ptr_ra.x(), base_ra.x(), .{ .shifted_register = .{
8159	.register = index_mat.ra.x(),
8160	.shift = .{ .lsl = @intCast(@ctz(elem_size)) },
8161	} }),
8162	}),
8163	2 => {
8164	const shift: u6 = @intCast(@ctz(elem_size));
8165	const temp_ra, const free_temp_ra = temp_ra: switch (op) {
8166	.add => switch (base_ra) {
8167	else => {
8168	const temp_ra = try isel.allocIntReg();
8169	errdefer isel.freeReg(temp_ra);
8170	try isel.emit(.add(elem_ptr_ra.x(), base_ra.x(), .{ .shifted_register = .{
8171	.register = temp_ra.x(),
8172	.shift = .{ .lsl = shift },
8173	} }));
8174	break :temp_ra .{ temp_ra, true };
8175	},
8176	.zr => {
8177	if (shift > 0) try isel.emit(.ubfm(elem_ptr_ra.x(), elem_ptr_ra.x(), .{
8178	.N = .doubleword,
8179	.immr = -%shift,
8180	.imms = ~shift,
8181	}));
8182	break :temp_ra .{ elem_ptr_ra, false };
8183	},
8184	},
8185	.sub => {
8186	const temp_ra = try isel.allocIntReg();
8187	errdefer isel.freeReg(temp_ra);
8188	try isel.emit(.sub(elem_ptr_ra.x(), base_ra.x(), .{ .shifted_register = .{
8189	.register = temp_ra.x(),
8190	.shift = .{ .lsl = shift },
8191	} }));
8192	break :temp_ra .{ temp_ra, true };
8193	},
8194	};
8195	defer if (free_temp_ra) isel.freeReg(temp_ra);
8196	try isel.emit(.add(temp_ra.x(), index_mat.ra.x(), .{ .shifted_register = .{
8197	.register = index_mat.ra.x(),
8198	.shift = .{ .lsl = @intCast(63 - @clz(elem_size) - shift) },
8199	} }));
8200	},
8201	else => {
8202	const elem_size_lsb1 = (elem_size - 1) | elem_size;
8203	if ((elem_size_lsb1 +% 1) & elem_size_lsb1 == 0) {
8204	const shift: u6 = @intCast(@ctz(elem_size));
8205	const temp_ra = temp_ra: switch (op) {
8206	.add => {
8207	const temp_ra = try isel.allocIntReg();
8208	errdefer isel.freeReg(temp_ra);
8209	try isel.emit(.sub(elem_ptr_ra.x(), base_ra.x(), .{ .shifted_register = .{
8210	.register = temp_ra.x(),
8211	.shift = .{ .lsl = shift },
8212	} }));
8213	break :temp_ra temp_ra;
8214	},
8215	.sub => switch (base_ra) {
8216	else => {
8217	const temp_ra = try isel.allocIntReg();
8218	errdefer isel.freeReg(temp_ra);
8219	try isel.emit(.add(elem_ptr_ra.x(), base_ra.x(), .{ .shifted_register = .{
8220	.register = temp_ra.x(),
8221	.shift = .{ .lsl = shift },
8222	} }));
8223	break :temp_ra temp_ra;
8224	},
8225	.zr => {
8226	if (shift > 0) try isel.emit(.ubfm(elem_ptr_ra.x(), elem_ptr_ra.x(), .{
8227	.N = .doubleword,
8228	.immr = -%shift,
8229	.imms = ~shift,
8230	}));
8231	break :temp_ra elem_ptr_ra;
8232	},
8233	},
8234	};
8235	defer if (temp_ra != elem_ptr_ra) isel.freeReg(temp_ra);
8236	try isel.emit(.sub(temp_ra.x(), index_mat.ra.x(), .{ .shifted_register = .{
8237	.register = index_mat.ra.x(),
8238	.shift = .{ .lsl = @intCast(64 - @clz(elem_size) - shift) },
8239	} }));
8240	} else {
8241	try isel.emit(switch (op) {
8242	.add => .madd(elem_ptr_ra.x(), index_mat.ra.x(), elem_ptr_ra.x(), base_ra.x()),
8243	.sub => .msub(elem_ptr_ra.x(), index_mat.ra.x(), elem_ptr_ra.x(), base_ra.x()),
8244	});
8245	try isel.movImmediate(elem_ptr_ra.x(), elem_size);
8246	}
8247	},
8248	}
8249	try index_mat.finish(isel);
8250	}
8251
8252	fn clzLimb(
8253	isel: *Select,
8254	res_ra: Register.Alias,
8255	src_int_info: std.lang.Type.Int,
8256	src_ra: Register.Alias,
8257	) !void {
8258	switch (src_int_info.bits) {
8259	else => unreachable,
8260	1...31 => |bits| {
8261	try isel.emit(.sub(res_ra.w(), res_ra.w(), .{
8262	.immediate = @intCast(32 - bits),
8263	}));
8264	switch (src_int_info.signedness) {
8265	.signed => {
8266	try isel.emit(.clz(res_ra.w(), res_ra.w()));
8267	try isel.emit(.ubfm(res_ra.w(), src_ra.w(), .{
8268	.N = .word,
8269	.immr = 0,
8270	.imms = @intCast(bits - 1),
8271	}));
8272	},
8273	.unsigned => try isel.emit(.clz(res_ra.w(), src_ra.w())),
8274	}
8275	},
8276	32 => try isel.emit(.clz(res_ra.w(), src_ra.w())),
8277	33...63 => |bits| {
8278	try isel.emit(.sub(res_ra.w(), res_ra.w(), .{
8279	.immediate = @intCast(64 - bits),
8280	}));
8281	switch (src_int_info.signedness) {
8282	.signed => {
8283	try isel.emit(.clz(res_ra.x(), res_ra.x()));
8284	try isel.emit(.ubfm(res_ra.x(), src_ra.x(), .{
8285	.N = .doubleword,
8286	.immr = 0,
8287	.imms = @intCast(bits - 1),
8288	}));
8289	},
8290	.unsigned => try isel.emit(.clz(res_ra.x(), src_ra.x())),
8291	}
8292	},
8293	64 => try isel.emit(.clz(res_ra.x(), src_ra.x())),
8294	}
8295	}
8296
8297	fn ctzLimb(
8298	isel: *Select,
8299	res_ra: Register.Alias,
8300	src_int_info: std.lang.Type.Int,
8301	src_ra: Register.Alias,
8302	) !void {
8303	switch (src_int_info.bits) {
8304	else => unreachable,
8305	1...31 => |bits| {
8306	try isel.emit(.clz(res_ra.w(), res_ra.w()));
8307	try isel.emit(.rbit(res_ra.w(), res_ra.w()));
8308	try isel.emit(.orr(res_ra.w(), src_ra.w(), .{ .immediate = .{
8309	.N = .word,
8310	.immr = @intCast(32 - bits),
8311	.imms = @intCast(32 - bits - 1),
8312	} }));
8313	},
8314	32 => {
8315	try isel.emit(.clz(res_ra.w(), res_ra.w()));
8316	try isel.emit(.rbit(res_ra.w(), src_ra.w()));
8317	},
8318	33...63 => |bits| {
8319	try isel.emit(.clz(res_ra.x(), res_ra.x()));
8320	try isel.emit(.rbit(res_ra.x(), res_ra.x()));
8321	try isel.emit(.orr(res_ra.x(), src_ra.x(), .{ .immediate = .{
8322	.N = .doubleword,
8323	.immr = @intCast(64 - bits),
8324	.imms = @intCast(64 - bits - 1),
8325	} }));
8326	},
8327	64 => {
8328	try isel.emit(.clz(res_ra.x(), res_ra.x()));
8329	try isel.emit(.rbit(res_ra.x(), src_ra.x()));
8330	},
8331	}
8332	}
8333
8334	fn cmp(
8335	isel: *Select,
8336	res_ra: Register.Alias,
8337	ty: ZigType,
8338	orig_lhs_vi: Value.Index,
8339	op: std.math.CompareOperator,
8340	orig_rhs_vi: Value.Index,
8341	) !struct { cset_label: usize } {
8342	var lhs_vi = orig_lhs_vi;
8343	var rhs_vi = orig_rhs_vi;
8344	if (!ty.isRuntimeFloat()) {
8345	const int_info: std.lang.Type.Int = if (ty.toIntern() == .bool_type)
8346	.{ .signedness = .unsigned, .bits = 1 }
8347	else if (ty.isAbiInt(isel.pt.zcu))
8348	ty.intInfo(isel.pt.zcu)
8349	else if (ty.isPtrAtRuntime(isel.pt.zcu))
8350	.{ .signedness = .unsigned, .bits = 64 }
8351	else
8352	return isel.fail("bad cmp_{t} {f}", .{ op, isel.fmtType(ty) });
8353	if (int_info.bits > 256) return isel.fail("too big cmp_{t} {f}", .{ op, isel.fmtType(ty) });
8354	try isel.emit(.csinc(res_ra.w(), .wzr, .wzr, .invert(cond: switch (op) {
8355	.lt => switch (int_info.signedness) {
8356	.signed => .lt,
8357	.unsigned => .lo,
8358	},
8359	.lte => switch (int_info.bits) {
8360	else => unreachable,
8361	1...64 => switch (int_info.signedness) {
8362	.signed => .le,
8363	.unsigned => .ls,
8364	},
8365	65...128 => {
8366	std.mem.swap(Value.Index, &lhs_vi, &rhs_vi);
8367	continue :cond .gte;
8368	},
8369	},
8370	.eq => .eq,
8371	.gte => switch (int_info.signedness) {
8372	.signed => .ge,
8373	.unsigned => .hs,
8374	},
8375	.gt => switch (int_info.bits) {
8376	else => unreachable,
8377	1...64 => switch (int_info.signedness) {
8378	.signed => .gt,
8379	.unsigned => .hi,
8380	},
8381	65...128 => {
8382	std.mem.swap(Value.Index, &lhs_vi, &rhs_vi);
8383	continue :cond .lt;
8384	},
8385	},
8386	.neq => .ne,
8387	})));
8388	const cset_label = isel.instructions.items.len;
8389
8390	var part_offset = lhs_vi.size(isel);
8391	while (part_offset > 0) {
8392	const part_size = @min(part_offset, 8);
8393	part_offset -= part_size;
8394	var lhs_part_it = lhs_vi.field(ty, part_offset, part_size);
8395	const lhs_part_vi = try lhs_part_it.only(isel);
8396	const lhs_part_mat = try lhs_part_vi.?.matReg(isel);
8397	var rhs_part_it = rhs_vi.field(ty, part_offset, part_size);
8398	const rhs_part_vi = try rhs_part_it.only(isel);
8399	const rhs_part_mat = try rhs_part_vi.?.matReg(isel);
8400	try isel.emit(switch (part_size) {
8401	else => unreachable,
8402	1...4 => switch (part_offset) {
8403	0 => .subs(.wzr, lhs_part_mat.ra.w(), .{ .register = rhs_part_mat.ra.w() }),
8404	else => switch (op) {
8405	.lt, .lte, .gte, .gt => .sbcs(
8406	.wzr,
8407	lhs_part_mat.ra.w(),
8408	rhs_part_mat.ra.w(),
8409	),
8410	.eq, .neq => .ccmp(
8411	lhs_part_mat.ra.w(),
8412	.{ .register = rhs_part_mat.ra.w() },
8413	.{ .n = false, .z = false, .c = false, .v = false },
8414	.eq,
8415	),
8416	},
8417	},
8418	5...8 => switch (part_offset) {
8419	0 => .subs(.xzr, lhs_part_mat.ra.x(), .{ .register = rhs_part_mat.ra.x() }),
8420	else => switch (op) {
8421	.lt, .lte, .gte, .gt => .sbcs(
8422	.xzr,
8423	lhs_part_mat.ra.x(),
8424	rhs_part_mat.ra.x(),
8425	),
8426	.eq, .neq => .ccmp(
8427	lhs_part_mat.ra.x(),
8428	.{ .register = rhs_part_mat.ra.x() },
8429	.{ .n = false, .z = false, .c = false, .v = false },
8430	.eq,
8431	),
8432	},
8433	},
8434	});
8435	try rhs_part_mat.finish(isel);
8436	try lhs_part_mat.finish(isel);
8437	}
8438	return .{ .cset_label = cset_label };
8439	}
8440	switch (ty.floatBits(isel.target)) {
8441	else => unreachable,
8442	16, 32, 64 => |bits| {
8443	const need_fcvt = switch (bits) {
8444	else => unreachable,
8445	16 => !isel.target.cpu.has(.aarch64, .fullfp16),
8446	32, 64 => false,
8447	};
8448	try isel.emit(.csinc(res_ra.w(), .wzr, .wzr, .invert(switch (op) {
8449	.lt => .lo,
8450	.lte => .ls,
8451	.eq => .eq,
8452	.gte => .ge,
8453	.gt => .gt,
8454	.neq => .ne,
8455	})));
8456	const cset_label = isel.instructions.items.len;
8457
8458	const lhs_mat = try lhs_vi.matReg(isel);
8459	const rhs_mat = try rhs_vi.matReg(isel);
8460	const lhs_ra = if (need_fcvt) try isel.allocVecReg() else lhs_mat.ra;
8461	defer if (need_fcvt) isel.freeReg(lhs_ra);
8462	const rhs_ra = if (need_fcvt) try isel.allocVecReg() else rhs_mat.ra;
8463	defer if (need_fcvt) isel.freeReg(rhs_ra);
8464	try isel.emit(bits: switch (bits) {
8465	else => unreachable,
8466	16 => if (need_fcvt)
8467	continue :bits 32
8468	else
8469	.fcmp(lhs_ra.h(), .{ .register = rhs_ra.h() }),
8470	32 => .fcmp(lhs_ra.s(), .{ .register = rhs_ra.s() }),
8471	64 => .fcmp(lhs_ra.d(), .{ .register = rhs_ra.d() }),
8472	});
8473	if (need_fcvt) {
8474	try isel.emit(.fcvt(rhs_ra.s(), rhs_mat.ra.h()));
8475	try isel.emit(.fcvt(lhs_ra.s(), lhs_mat.ra.h()));
8476	}
8477	try rhs_mat.finish(isel);
8478	try lhs_mat.finish(isel);
8479	return .{ .cset_label = cset_label };
8480	},
8481	80, 128 => |bits| {
8482	try call.prepareReturn(isel);
8483	try call.returnFill(isel, .r0);
8484	try isel.emit(.csinc(res_ra.w(), .wzr, .wzr, .invert(cond: switch (op) {
8485	.lt => .lt,
8486	.lte => .le,
8487	.eq => .eq,
8488	.gte => {
8489	std.mem.swap(Value.Index, &lhs_vi, &rhs_vi);
8490	continue :cond .lte;
8491	},
8492	.gt => {
8493	std.mem.swap(Value.Index, &lhs_vi, &rhs_vi);
8494	continue :cond .lt;
8495	},
8496	.neq => .ne,
8497	})));
8498	const cset_label = isel.instructions.items.len;
8499	try isel.emit(.subs(.wzr, .w0, .{ .immediate = 0 }));
8500	try call.finishReturn(isel);
8501
8502	try call.prepareCallee(isel);
8503	try isel.global_relocs.append(isel.pt.zcu.gpa, .{
8504	.name = switch (bits) {
8505	else => unreachable,
8506	16 => "__cmphf2",
8507	32 => "__cmpsf2",
8508	64 => "__cmpdf2",
8509	80 => "__cmpxf2",
8510	128 => "__cmptf2",
8511	},
8512	.reloc = .{ .label = @intCast(isel.instructions.items.len) },
8513	});
8514	try isel.emit(.bl(0));
8515	try call.finishCallee(isel);
8516
8517	try call.prepareParams(isel);
8518	switch (bits) {
8519	else => unreachable,
8520	16, 32, 64, 128 => {
8521	try call.paramLiveOut(isel, rhs_vi, .v1);
8522	try call.paramLiveOut(isel, lhs_vi, .v0);
8523	},
8524	80 => {
8525	var rhs_hi16_it = rhs_vi.field(ty, 8, 8);
8526	const rhs_hi16_vi = try rhs_hi16_it.only(isel);
8527	try call.paramLiveOut(isel, rhs_hi16_vi.?, .r3);
8528	var rhs_lo64_it = rhs_vi.field(ty, 0, 8);
8529	const rhs_lo64_vi = try rhs_lo64_it.only(isel);
8530	try call.paramLiveOut(isel, rhs_lo64_vi.?, .r2);
8531	var lhs_hi16_it = lhs_vi.field(ty, 8, 8);
8532	const lhs_hi16_vi = try lhs_hi16_it.only(isel);
8533	try call.paramLiveOut(isel, lhs_hi16_vi.?, .r1);
8534	var lhs_lo64_it = lhs_vi.field(ty, 0, 8);
8535	const lhs_lo64_vi = try lhs_lo64_it.only(isel);
8536	try call.paramLiveOut(isel, lhs_lo64_vi.?, .r0);
8537	},
8538	}
8539	try call.finishParams(isel);
8540	return .{ .cset_label = cset_label };
8541	},
8542	}
8543	}
8544
8545	fn loadReg(
8546	isel: *Select,
8547	ra: Register.Alias,
8548	size: u64,
8549	signedness: std.lang.Signedness,
8550	base_ra: Register.Alias,
8551	offset: i65,
8552	) !void {
8553	switch (size) {
8554	0 => unreachable,
8555	1 => {
8556	if (std.math.cast(u12, offset)) |unsigned_offset| return isel.emit(if (ra.isVector()) .ldr(
8557	ra.b(),
8558	.{ .unsigned_offset = .{
8559	.base = base_ra.x(),
8560	.offset = unsigned_offset,
8561	} },
8562	) else switch (signedness) {
8563	.signed => .ldrsb(ra.w(), .{ .unsigned_offset = .{
8564	.base = base_ra.x(),
8565	.offset = unsigned_offset,
8566	} }),
8567	.unsigned => .ldrb(ra.w(), .{ .unsigned_offset = .{
8568	.base = base_ra.x(),
8569	.offset = unsigned_offset,
8570	} }),
8571	});
8572	if (std.math.cast(i9, offset)) |signed_offset| return isel.emit(if (ra.isVector())
8573	.ldur(ra.b(), base_ra.x(), signed_offset)
8574	else switch (signedness) {
8575	.signed => .ldursb(ra.w(), base_ra.x(), signed_offset),
8576	.unsigned => .ldurb(ra.w(), base_ra.x(), signed_offset),
8577	});
8578	},
8579	2 => {
8580	if (std.math.cast(u13, offset)) |unsigned_offset| if (unsigned_offset % 2 == 0)
8581	return isel.emit(if (ra.isVector()) .ldr(
8582	ra.h(),
8583	.{ .unsigned_offset = .{
8584	.base = base_ra.x(),
8585	.offset = unsigned_offset,
8586	} },
8587	) else switch (signedness) {
8588	.signed => .ldrsh(
8589	ra.w(),
8590	.{ .unsigned_offset = .{
8591	.base = base_ra.x(),
8592	.offset = unsigned_offset,
8593	} },
8594	),
8595	.unsigned => .ldrh(
8596	ra.w(),
8597	.{ .unsigned_offset = .{
8598	.base = base_ra.x(),
8599	.offset = unsigned_offset,
8600	} },
8601	),
8602	});
8603	if (std.math.cast(i9, offset)) |signed_offset| return isel.emit(if (ra.isVector())
8604	.ldur(ra.h(), base_ra.x(), signed_offset)
8605	else switch (signedness) {
8606	.signed => .ldursh(ra.w(), base_ra.x(), signed_offset),
8607	.unsigned => .ldurh(ra.w(), base_ra.x(), signed_offset),
8608	});
8609	},
8610	3 => {
8611	const lo16_ra = try isel.allocIntReg();
8612	defer isel.freeReg(lo16_ra);
8613	try isel.emit(.orr(ra.w(), lo16_ra.w(), .{ .shifted_register = .{
8614	.register = ra.w(),
8615	.shift = .{ .lsl = 16 },
8616	} }));
8617	try isel.loadReg(ra, 1, signedness, base_ra, offset + 2);
8618	return isel.loadReg(lo16_ra, 2, .unsigned, base_ra, offset);
8619	},
8620	4 => {
8621	if (std.math.cast(u14, offset)) |unsigned_offset| if (unsigned_offset % 4 == 0) return isel.emit(.ldr(
8622	if (ra.isVector()) ra.s() else ra.w(),
8623	.{ .unsigned_offset = .{
8624	.base = base_ra.x(),
8625	.offset = unsigned_offset,
8626	} },
8627	));
8628	if (std.math.cast(i9, offset)) |signed_offset| return isel.emit(.ldur(
8629	if (ra.isVector()) ra.s() else ra.w(),
8630	base_ra.x(),
8631	signed_offset,
8632	));
8633	},
8634	5, 6 => {
8635	const lo32_ra = try isel.allocIntReg();
8636	defer isel.freeReg(lo32_ra);
8637	try isel.emit(.orr(ra.x(), lo32_ra.x(), .{ .shifted_register = .{
8638	.register = ra.x(),
8639	.shift = .{ .lsl = 32 },
8640	} }));
8641	try isel.loadReg(ra, size - 4, signedness, base_ra, offset + 4);
8642	return isel.loadReg(lo32_ra, 4, .unsigned, base_ra, offset);
8643	},
8644	7 => {
8645	const lo32_ra = try isel.allocIntReg();
8646	defer isel.freeReg(lo32_ra);
8647	const lo48_ra = try isel.allocIntReg();
8648	defer isel.freeReg(lo48_ra);
8649	try isel.emit(.orr(ra.x(), lo48_ra.x(), .{ .shifted_register = .{
8650	.register = ra.x(),
8651	.shift = .{ .lsl = 32 + 16 },
8652	} }));
8653	try isel.loadReg(ra, 1, signedness, base_ra, offset + 4 + 2);
8654	try isel.emit(.orr(lo48_ra.x(), lo32_ra.x(), .{ .shifted_register = .{
8655	.register = lo48_ra.x(),
8656	.shift = .{ .lsl = 32 },
8657	} }));
8658	try isel.loadReg(lo48_ra, 2, .unsigned, base_ra, offset + 4);
8659	return isel.loadReg(lo32_ra, 4, .unsigned, base_ra, offset);
8660	},
8661	8 => {
8662	if (std.math.cast(u15, offset)) |unsigned_offset| if (unsigned_offset % 8 == 0) return isel.emit(.ldr(
8663	if (ra.isVector()) ra.d() else ra.x(),
8664	.{ .unsigned_offset = .{
8665	.base = base_ra.x(),
8666	.offset = unsigned_offset,
8667	} },
8668	));
8669	if (std.math.cast(i9, offset)) |signed_offset| return isel.emit(.ldur(
8670	if (ra.isVector()) ra.d() else ra.x(),
8671	base_ra.x(),
8672	signed_offset,
8673	));
8674	},
8675	16 => {
8676	if (std.math.cast(u16, offset)) |unsigned_offset| if (unsigned_offset % 16 == 0) return isel.emit(.ldr(
8677	ra.q(),
8678	.{ .unsigned_offset = .{
8679	.base = base_ra.x(),
8680	.offset = unsigned_offset,
8681	} },
8682	));
8683	if (std.math.cast(i9, offset)) |signed_offset| return isel.emit(.ldur(ra.q(), base_ra.x(), signed_offset));
8684	},
8685	else => return isel.fail("bad load size: {d}", .{size}),
8686	}
8687	const ptr_ra = try isel.allocIntReg();
8688	defer isel.freeReg(ptr_ra);
8689	try isel.loadReg(ra, size, signedness, ptr_ra, 0);
8690	if (std.math.cast(u24, offset)) |pos_offset| {
8691	const lo12: u12 = @truncate(pos_offset >> 0);
8692	const hi12: u12 = @intCast(pos_offset >> 12);
8693	if (hi12 > 0) try isel.emit(.add(
8694	ptr_ra.x(),
8695	if (lo12 > 0) ptr_ra.x() else base_ra.x(),
8696	.{ .shifted_immediate = .{ .immediate = hi12, .lsl = .@"12" } },
8697	));
8698	if (lo12 > 0 or hi12 == 0) try isel.emit(.add(ptr_ra.x(), base_ra.x(), .{ .immediate = lo12 }));
8699	} else if (std.math.cast(u24, -offset)) |neg_offset| {
8700	const lo12: u12 = @truncate(neg_offset >> 0);
8701	const hi12: u12 = @intCast(neg_offset >> 12);
8702	if (hi12 > 0) try isel.emit(.sub(
8703	ptr_ra.x(),
8704	if (lo12 > 0) ptr_ra.x() else base_ra.x(),
8705	.{ .shifted_immediate = .{ .immediate = hi12, .lsl = .@"12" } },
8706	));
8707	if (lo12 > 0 or hi12 == 0) try isel.emit(.sub(ptr_ra.x(), base_ra.x(), .{ .immediate = lo12 }));
8708	} else {
8709	try isel.emit(.add(ptr_ra.x(), base_ra.x(), .{ .register = ptr_ra.x() }));
8710	try isel.movImmediate(ptr_ra.x(), @truncate(@as(u65, @bitCast(offset))));
8711	}
8712	}
8713
8714	fn storeReg(
8715	isel: *Select,
8716	ra: Register.Alias,
8717	size: u64,
8718	base_ra: Register.Alias,
8719	offset: i65,
8720	) !void {
8721	switch (size) {
8722	0 => unreachable,
8723	1 => {
8724	if (std.math.cast(u12, offset)) |unsigned_offset| return isel.emit(if (ra.isVector()) .str(
8725	ra.b(),
8726	.{ .unsigned_offset = .{
8727	.base = base_ra.x(),
8728	.offset = unsigned_offset,
8729	} },
8730	) else .strb(
8731	ra.w(),
8732	.{ .unsigned_offset = .{
8733	.base = base_ra.x(),
8734	.offset = unsigned_offset,
8735	} },
8736	));
8737	if (std.math.cast(i9, offset)) |signed_offset| return isel.emit(if (ra.isVector())
8738	.stur(ra.b(), base_ra.x(), signed_offset)
8739	else
8740	.sturb(ra.w(), base_ra.x(), signed_offset));
8741	},
8742	2 => {
8743	if (std.math.cast(u13, offset)) |unsigned_offset| if (unsigned_offset % 2 == 0)
8744	return isel.emit(if (ra.isVector()) .str(
8745	ra.h(),
8746	.{ .unsigned_offset = .{
8747	.base = base_ra.x(),
8748	.offset = unsigned_offset,
8749	} },
8750	) else .strh(
8751	ra.w(),
8752	.{ .unsigned_offset = .{
8753	.base = base_ra.x(),
8754	.offset = unsigned_offset,
8755	} },
8756	));
8757	if (std.math.cast(i9, offset)) |signed_offset| return isel.emit(if (ra.isVector())
8758	.stur(ra.h(), base_ra.x(), signed_offset)
8759	else
8760	.sturh(ra.w(), base_ra.x(), signed_offset));
8761	},
8762	3 => {
8763	const hi8_ra = try isel.allocIntReg();
8764	defer isel.freeReg(hi8_ra);
8765	try isel.storeReg(hi8_ra, 1, base_ra, offset + 2);
8766	try isel.storeReg(ra, 2, base_ra, offset);
8767	return isel.emit(.ubfm(hi8_ra.w(), ra.w(), .{
8768	.N = .word,
8769	.immr = 16,
8770	.imms = 16 + 8 - 1,
8771	}));
8772	},
8773	4 => {
8774	if (std.math.cast(u14, offset)) |unsigned_offset| if (unsigned_offset % 4 == 0) return isel.emit(.str(
8775	if (ra.isVector()) ra.s() else ra.w(),
8776	.{ .unsigned_offset = .{
8777	.base = base_ra.x(),
8778	.offset = unsigned_offset,
8779	} },
8780	));
8781	if (std.math.cast(i9, offset)) |signed_offset| return isel.emit(.stur(
8782	if (ra.isVector()) ra.s() else ra.w(),
8783	base_ra.x(),
8784	signed_offset,
8785	));
8786	},
8787	5 => {
8788	const hi8_ra = try isel.allocIntReg();
8789	defer isel.freeReg(hi8_ra);
8790	try isel.storeReg(hi8_ra, 1, base_ra, offset + 4);
8791	try isel.storeReg(ra, 4, base_ra, offset);
8792	return isel.emit(.ubfm(hi8_ra.x(), ra.x(), .{
8793	.N = .doubleword,
8794	.immr = 32,
8795	.imms = 32 + 8 - 1,
8796	}));
8797	},
8798	6 => {
8799	const hi16_ra = try isel.allocIntReg();
8800	defer isel.freeReg(hi16_ra);
8801	try isel.storeReg(hi16_ra, 2, base_ra, offset + 4);
8802	try isel.storeReg(ra, 4, base_ra, offset);
8803	return isel.emit(.ubfm(hi16_ra.x(), ra.x(), .{
8804	.N = .doubleword,
8805	.immr = 32,
8806	.imms = 32 + 16 - 1,
8807	}));
8808	},
8809	7 => {
8810	const hi16_ra = try isel.allocIntReg();
8811	defer isel.freeReg(hi16_ra);
8812	const hi8_ra = try isel.allocIntReg();
8813	defer isel.freeReg(hi8_ra);
8814	try isel.storeReg(hi8_ra, 1, base_ra, offset + 6);
8815	try isel.storeReg(hi16_ra, 2, base_ra, offset + 4);
8816	try isel.storeReg(ra, 4, base_ra, offset);
8817	try isel.emit(.ubfm(hi8_ra.x(), ra.x(), .{
8818	.N = .doubleword,
8819	.immr = 32 + 16,
8820	.imms = 32 + 16 + 8 - 1,
8821	}));
8822	return isel.emit(.ubfm(hi16_ra.x(), ra.x(), .{
8823	.N = .doubleword,
8824	.immr = 32,
8825	.imms = 32 + 16 - 1,
8826	}));
8827	},
8828	8 => {
8829	if (std.math.cast(u15, offset)) |unsigned_offset| if (unsigned_offset % 8 == 0) return isel.emit(.str(
8830	if (ra.isVector()) ra.d() else ra.x(),
8831	.{ .unsigned_offset = .{
8832	.base = base_ra.x(),
8833	.offset = unsigned_offset,
8834	} },
8835	));
8836	if (std.math.cast(i9, offset)) |signed_offset| return isel.emit(.stur(
8837	if (ra.isVector()) ra.d() else ra.x(),
8838	base_ra.x(),
8839	signed_offset,
8840	));
8841	},
8842	16 => {
8843	if (std.math.cast(u16, offset)) |unsigned_offset| if (unsigned_offset % 16 == 0) return isel.emit(.str(
8844	ra.q(),
8845	.{ .unsigned_offset = .{
8846	.base = base_ra.x(),
8847	.offset = unsigned_offset,
8848	} },
8849	));
8850	if (std.math.cast(i9, offset)) |signed_offset| return isel.emit(.stur(ra.q(), base_ra.x(), signed_offset));
8851	},
8852	else => return isel.fail("bad store size: {d}", .{size}),
8853	}
8854	const ptr_ra = try isel.allocIntReg();
8855	defer isel.freeReg(ptr_ra);
8856	try isel.storeReg(ra, size, ptr_ra, 0);
8857	if (std.math.cast(u24, offset)) |pos_offset| {
8858	const lo12: u12 = @truncate(pos_offset >> 0);
8859	const hi12: u12 = @intCast(pos_offset >> 12);
8860	if (hi12 > 0) try isel.emit(.add(
8861	ptr_ra.x(),
8862	if (lo12 > 0) ptr_ra.x() else base_ra.x(),
8863	.{ .shifted_immediate = .{ .immediate = hi12, .lsl = .@"12" } },
8864	));
8865	if (lo12 > 0 or hi12 == 0) try isel.emit(.add(ptr_ra.x(), base_ra.x(), .{ .immediate = lo12 }));
8866	} else if (std.math.cast(u24, -offset)) |neg_offset| {
8867	const lo12: u12 = @truncate(neg_offset >> 0);
8868	const hi12: u12 = @intCast(neg_offset >> 12);
8869	if (hi12 > 0) try isel.emit(.sub(
8870	ptr_ra.x(),
8871	if (lo12 > 0) ptr_ra.x() else base_ra.x(),
8872	.{ .shifted_immediate = .{ .immediate = hi12, .lsl = .@"12" } },
8873	));
8874	if (lo12 > 0 or hi12 == 0) try isel.emit(.sub(ptr_ra.x(), base_ra.x(), .{ .immediate = lo12 }));
8875	} else {
8876	try isel.emit(.add(ptr_ra.x(), base_ra.x(), .{ .register = ptr_ra.x() }));
8877	try isel.movImmediate(ptr_ra.x(), @truncate(@as(u65, @bitCast(offset))));
8878	}
8879	}
8880
8881	const DomInt = u8;
8882
8883	pub const Value = struct {
8884	refs: u32,
8885	flags: Flags,
8886	offset_from_parent: u64,
8887	parent_payload: Parent.Payload,
8888	location_payload: Location.Payload,
8889	parts: Value.Index,
8890
8891	/// Must be at least 16 to compute call abi.
8892	/// Must be at least 16, the largest hardware alignment.
8893	pub const max_parts = 16;
8894	pub const PartsLen = std.math.IntFittingRange(0, Value.max_parts);
8895
8896	comptime {
8897	if (!std.debug.runtime_safety) assert(@sizeOf(Value) == 32);
8898	}
8899
8900	pub const Flags = packed struct(u32) {
8901	alignment: InternPool.Alignment,
8902	parent_tag: Parent.Tag,
8903	location_tag: Location.Tag,
8904	parts_len_minus_one: std.math.IntFittingRange(0, Value.max_parts - 1),
8905	unused: u18 = 0,
8906	};
8907
8908	pub const Parent = union(enum(u3)) {
8909	unallocated: void,
8910	stack_slot: Indirect,
8911	address: Value.Index,
8912	value: Value.Index,
8913	constant: Constant,
8914
8915	pub const Tag = @typeInfo(Parent).@"union".tag_type.?;
8916	pub const Payload = Payload: {
8917	const info = @typeInfo(Parent).@"union";
8918	break :Payload @Union(.auto, null, info.field_names, info.field_types[0..], &@splat(.{}));
8919	};
8920	};
8921
8922	pub const Location = union(enum(u1)) {
8923	large: struct {
8924	size: u64,
8925	},
8926	small: struct {
8927	size: u5,
8928	signedness: std.lang.Signedness,
8929	is_vector: bool,
8930	hint: Register.Alias,
8931	register: Register.Alias,
8932	},
8933
8934	pub const Tag = @typeInfo(Location).@"union".tag_type.?;
8935	pub const Payload = Payload: {
8936	const info = @typeInfo(Location).@"union";
8937	break :Payload @Union(.auto, null, info.field_names, info.field_types[0..], &@splat(.{}));
8938	};
8939	};
8940
8941	pub const Indirect = packed struct(u32) {
8942	base: Register.Alias,
8943	offset: i25,
8944
8945	pub fn withOffset(ind: Indirect, offset: i25) Indirect {
8946	return .{
8947	.base = ind.base,
8948	.offset = ind.offset + offset,
8949	};
8950	}
8951	};
8952
8953	pub const Index = enum(u32) {
8954	allocating = std.math.maxInt(u32) - 1,
8955	free = std.math.maxInt(u32) - 0,
8956	_,
8957
8958	fn get(vi: Value.Index, isel: *Select) *Value {
8959	return &isel.values.items[@backingInt(vi)];
8960	}
8961
8962	fn setAlignment(vi: Value.Index, isel: *Select, new_alignment: InternPool.Alignment) void {
8963	vi.get(isel).flags.alignment = new_alignment;
8964	}
8965
8966	pub fn alignment(vi: Value.Index, isel: *Select) InternPool.Alignment {
8967	return vi.get(isel).flags.alignment;
8968	}
8969
8970	pub fn setParent(vi: Value.Index, isel: *Select, new_parent: Parent) void {
8971	const value = vi.get(isel);
8972	assert(value.flags.parent_tag == .unallocated);
8973	value.flags.parent_tag = new_parent;
8974	value.parent_payload = switch (new_parent) {
8975	.unallocated => unreachable,
8976	inline else => |payload, tag| @unionInit(Parent.Payload, @tagName(tag), payload),
8977	};
8978	if (value.refs > 0) switch (new_parent) {
8979	.unallocated => unreachable,
8980	.stack_slot, .constant => {},
8981	.address, .value => |parent_vi| _ = parent_vi.ref(isel),
8982	};
8983	}
8984
8985	pub fn changeStackSlot(vi: Value.Index, isel: *Select, new_stack_slot: Indirect) void {
8986	const value = vi.get(isel);
8987	assert(value.flags.parent_tag == .stack_slot);
8988	value.flags.parent_tag = .unallocated;
8989	vi.setParent(isel, .{ .stack_slot = new_stack_slot });
8990	}
8991
8992	pub fn parent(vi: Value.Index, isel: *Select) Parent {
8993	const value = vi.get(isel);
8994	return switch (value.flags.parent_tag) {
8995	inline else => |tag| @unionInit(
8996	Parent,
8997	@tagName(tag),
8998	@field(value.parent_payload, @tagName(tag)),
8999	),
9000	};
9001	}
9002
9003	pub fn valueParent(initial_vi: Value.Index, isel: *Select) struct { u64, Value.Index } {
9004	var offset: u64 = 0;
9005	var vi = initial_vi;
9006	parent: switch (vi.parent(isel)) {
9007	else => return .{ offset, vi },
9008	.value => |parent_vi| {
9009	offset += vi.position(isel)[0];
9010	vi = parent_vi;
9011	continue :parent parent_vi.parent(isel);
9012	},
9013	}
9014	}
9015
9016	pub fn location(vi: Value.Index, isel: *Select) Location {
9017	const value = vi.get(isel);
9018	return switch (value.flags.location_tag) {
9019	inline else => |tag| @unionInit(
9020	Location,
9021	@tagName(tag),
9022	@field(value.location_payload, @tagName(tag)),
9023	),
9024	};
9025	}
9026
9027	pub fn position(vi: Value.Index, isel: *Select) struct { u64, u64 } {
9028	return .{ vi.get(isel).offset_from_parent, vi.size(isel) };
9029	}
9030
9031	pub fn size(vi: Value.Index, isel: *Select) u64 {
9032	return switch (vi.location(isel)) {
9033	inline else => |loc| loc.size,
9034	};
9035	}
9036
9037	fn setHint(vi: Value.Index, isel: *Select, new_hint: Register.Alias) void {
9038	vi.get(isel).location_payload.small.hint = new_hint;
9039	}
9040
9041	pub fn hint(vi: Value.Index, isel: *Select) ?Register.Alias {
9042	return switch (vi.location(isel)) {
9043	.large => null,
9044	.small => |loc| switch (loc.hint) {
9045	.zr => null,
9046	else => |hint_reg| hint_reg,
9047	},
9048	};
9049	}
9050
9051	fn setSignedness(vi: Value.Index, isel: *Select, new_signedness: std.lang.Signedness) void {
9052	const value = vi.get(isel);
9053	assert(value.location_payload.small.size <= 2);
9054	value.location_payload.small.signedness = new_signedness;
9055	}
9056
9057	pub fn signedness(vi: Value.Index, isel: *Select) std.lang.Signedness {
9058	const value = vi.get(isel);
9059	return switch (value.flags.location_tag) {
9060	.large => .unsigned,
9061	.small => value.location_payload.small.signedness,
9062	};
9063	}
9064
9065	fn setIsVector(vi: Value.Index, isel: *Select) void {
9066	const is_vector = &vi.get(isel).location_payload.small.is_vector;
9067	assert(!is_vector.*);
9068	is_vector.* = true;
9069	}
9070
9071	pub fn isVector(vi: Value.Index, isel: *Select) bool {
9072	const value = vi.get(isel);
9073	return switch (value.flags.location_tag) {
9074	.large => false,
9075	.small => value.location_payload.small.is_vector,
9076	};
9077	}
9078
9079	pub fn register(vi: Value.Index, isel: *Select) ?Register.Alias {
9080	return switch (vi.location(isel)) {
9081	.large => null,
9082	.small => |loc| switch (loc.register) {
9083	.zr => null,
9084	else => |reg| reg,
9085	},
9086	};
9087	}
9088
9089	pub fn isUsed(vi: Value.Index, isel: *Select) bool {
9090	return vi.valueParent(isel)[1].parent(isel) != .unallocated or vi.hasRegisterRecursive(isel);
9091	}
9092
9093	fn hasRegisterRecursive(vi: Value.Index, isel: *Select) bool {
9094	if (vi.register(isel)) |_| return true;
9095	var part_it = vi.parts(isel);
9096	if (part_it.only() == null) while (part_it.next()) |part_vi| if (part_vi.hasRegisterRecursive(isel)) return true;
9097	return false;
9098	}
9099
9100	fn setParts(vi: Value.Index, isel: *Select, parts_len: Value.PartsLen) void {
9101	assert(parts_len > 1);
9102	const value = vi.get(isel);
9103	assert(value.flags.parts_len_minus_one == 0);
9104	value.parts = @fromBackingInt(@intCast(isel.values.items.len));
9105	value.flags.parts_len_minus_one = @intCast(parts_len - 1);
9106	}
9107
9108	fn addPart(vi: Value.Index, isel: *Select, part_offset: u64, part_size: u64) Value.Index {
9109	const part_vi = isel.initValueAdvanced(vi.alignment(isel), part_offset, part_size);
9110	tracking_log.debug("${d} <- ${d}[{d}]", .{
9111	@backingInt(part_vi),
9112	@backingInt(vi),
9113	part_offset,
9114	});
9115	part_vi.setParent(isel, .{ .value = vi });
9116	return part_vi;
9117	}
9118
9119	pub fn parts(vi: Value.Index, isel: *Select) Value.PartIterator {
9120	const value = vi.get(isel);
9121	return switch (value.flags.parts_len_minus_one) {
9122	0 => .initOne(vi),
9123	else => |parts_len_minus_one| .{
9124	.vi = value.parts,
9125	.remaining = @as(Value.PartsLen, parts_len_minus_one) + 1,
9126	},
9127	};
9128	}
9129
9130	fn containingParts(vi: Value.Index, isel: *Select, part_offset: u64, part_size: u64) Value.PartIterator {
9131	const start_vi = vi.partAtOffset(isel, part_offset);
9132	const start_offset, const start_size = start_vi.position(isel);
9133	if (part_offset >= start_offset and part_size <= start_size) return .initOne(start_vi);
9134	const end_vi = vi.partAtOffset(isel, part_size - 1 + part_offset);
9135	return .{
9136	.vi = start_vi,
9137	.remaining = @intCast(@backingInt(end_vi) - @backingInt(start_vi) + 1),
9138	};
9139	}
9140	comptime {
9141	_ = containingParts;
9142	}
9143
9144	fn partAtOffset(vi: Value.Index, isel: *Select, offset: u64) Value.Index {
9145	const SearchPartIndex = std.math.IntFittingRange(0, Value.max_parts * 2 - 1);
9146	const value = vi.get(isel);
9147	var last: SearchPartIndex = value.flags.parts_len_minus_one;
9148	if (last == 0) return vi;
9149	var first: SearchPartIndex = 0;
9150	last += 1;
9151	while (true) {
9152	const mid = (first + last) / 2;
9153	const mid_vi: Value.Index = @fromBackingInt(@intCast(@backingInt(value.parts) + mid));
9154	if (mid == first) return mid_vi;
9155	if (offset < mid_vi.get(isel).offset_from_parent) last = mid else first = mid;
9156	}
9157	}
9158
9159	fn field(
9160	vi: Value.Index,
9161	ty: ZigType,
9162	field_offset: u64,
9163	field_size: u64,
9164	) Value.FieldPartIterator {
9165	assert(field_size > 0);
9166	return .{
9167	.vi = vi,
9168	.ty = ty,
9169	.field_offset = field_offset,
9170	.field_size = field_size,
9171	.next_offset = 0,
9172	};
9173	}
9174
9175	fn ref(initial_vi: Value.Index, isel: *Select) Value.Index {
9176	var vi = initial_vi;
9177	while (true) {
9178	const refs = &vi.get(isel).refs;
9179	refs.* += 1;
9180	if (refs.* > 1) return initial_vi;
9181	switch (vi.parent(isel)) {
9182	.unallocated, .stack_slot, .constant => {},
9183	.address, .value => |parent_vi| {
9184	vi = parent_vi;
9185	continue;
9186	},
9187	}
9188	return initial_vi;
9189	}
9190	}
9191
9192	pub fn deref(initial_vi: Value.Index, isel: *Select) void {
9193	var vi = initial_vi;
9194	while (true) {
9195	const refs = &vi.get(isel).refs;
9196	refs.* -= 1;
9197	if (refs.* > 0) return;
9198	switch (vi.parent(isel)) {
9199	.unallocated, .constant => {},
9200	.stack_slot => {
9201	// reuse stack slot
9202	},
9203	.address, .value => |parent_vi| {
9204	vi = parent_vi;
9205	continue;
9206	},
9207	}
9208	return;
9209	}
9210	}
9211
9212	fn move(dst_vi: Value.Index, isel: *Select, src_ref: Air.Inst.Ref) !void {
9213	try dst_vi.copy(
9214	isel,
9215	isel.air.typeOf(src_ref, &isel.pt.zcu.intern_pool),
9216	try isel.use(src_ref),
9217	);
9218	}
9219
9220	fn copy(dst_vi: Value.Index, isel: *Select, ty: ZigType, src_vi: Value.Index) !void {
9221	try dst_vi.copyAdvanced(isel, src_vi, .{
9222	.ty = ty,
9223	.dst_vi = dst_vi,
9224	.dst_offset = 0,
9225	.src_vi = src_vi,
9226	.src_offset = 0,
9227	});
9228	}
9229
9230	fn copyAdvanced(dst_vi: Value.Index, isel: *Select, src_vi: Value.Index, root: struct {
9231	ty: ZigType,
9232	dst_vi: Value.Index,
9233	dst_offset: u64,
9234	src_vi: Value.Index,
9235	src_offset: u64,
9236	}) !void {
9237	if (dst_vi == src_vi) return;
9238	var dst_part_it = dst_vi.parts(isel);
9239	if (dst_part_it.only()) |dst_part_vi| {
9240	var src_part_it = src_vi.parts(isel);
9241	if (src_part_it.only()) |src_part_vi| only: {
9242	const src_part_size = src_part_vi.size(isel);
9243	if (src_part_size > @as(@TypeOf(src_part_size), if (src_part_vi.isVector(isel)) 16 else 8)) {
9244	var subpart_it = root.src_vi.field(root.ty, root.src_offset, src_part_size - 1);
9245	_ = try subpart_it.next(isel);
9246	src_part_it = src_vi.parts(isel);
9247	assert(src_part_it.only() == null);
9248	break :only;
9249	}
9250	return src_part_vi.liveOut(isel, try dst_part_vi.defReg(isel) orelse return);
9251	}
9252	while (src_part_it.next()) |src_part_vi| {
9253	const src_part_offset, const src_part_size = src_part_vi.position(isel);
9254	var dst_field_it = root.dst_vi.field(root.ty, root.dst_offset + src_part_offset, src_part_size);
9255	const dst_field_vi = try dst_field_it.only(isel);
9256	try dst_field_vi.?.copyAdvanced(isel, src_part_vi, .{
9257	.ty = root.ty,
9258	.dst_vi = root.dst_vi,
9259	.dst_offset = root.dst_offset + src_part_offset,
9260	.src_vi = root.src_vi,
9261	.src_offset = root.src_offset + src_part_offset,
9262	});
9263	}
9264	} else while (dst_part_it.next()) |dst_part_vi| {
9265	const dst_part_offset, const dst_part_size = dst_part_vi.position(isel);
9266	var src_field_it = root.src_vi.field(root.ty, root.src_offset + dst_part_offset, dst_part_size);
9267	const src_part_vi = try src_field_it.only(isel);
9268	try dst_part_vi.copyAdvanced(isel, src_part_vi.?, .{
9269	.ty = root.ty,
9270	.dst_vi = root.dst_vi,
9271	.dst_offset = root.dst_offset + dst_part_offset,
9272	.src_vi = root.src_vi,
9273	.src_offset = root.src_offset + dst_part_offset,
9274	});
9275	}
9276	}
9277
9278	const AddOrSubtractOptions = struct {
9279	overflow: Overflow,
9280
9281	const Overflow = union(enum) {
9282	@"unreachable",
9283	panic: Zcu.SimplePanicId,
9284	wrap,
9285	ra: Register.Alias,
9286
9287	fn defCond(overflow: Overflow, isel: *Select, cond: codegen.aarch64.encoding.ConditionCode) !void {
9288	switch (overflow) {
9289	.@"unreachable" => unreachable,
9290	.panic => |panic_id| {
9291	const skip_label = isel.instructions.items.len;
9292	try isel.emitPanic(panic_id);
9293	try isel.emit(.@"b."(
9294	cond.invert(),
9295	@intCast((isel.instructions.items.len + 1 - skip_label) << 2),
9296	));
9297	},
9298	.wrap => {},
9299	.ra => |overflow_ra| try isel.emit(.csinc(overflow_ra.w(), .wzr, .wzr, cond.invert())),
9300	}
9301	}
9302	};
9303	};
9304	fn addOrSubtract(
9305	res_vi: Value.Index,
9306	isel: *Select,
9307	ty: ZigType,
9308	lhs_vi: Value.Index,
9309	op: codegen.aarch64.encoding.Instruction.AddSubtractOp,
9310	rhs_vi: Value.Index,
9311	opts: AddOrSubtractOptions,
9312	) !void {
9313	const zcu = isel.pt.zcu;
9314	if (!ty.isAbiInt(zcu)) return isel.fail("bad {t} {f}", .{ op, isel.fmtType(ty) });
9315	const int_info = ty.intInfo(zcu);
9316	if (int_info.bits > 128) return isel.fail("too big {t} {f}", .{ op, isel.fmtType(ty) });
9317	var part_offset = res_vi.size(isel);
9318	var need_wrap = switch (opts.overflow) {
9319	.@"unreachable" => false,
9320	.panic, .wrap, .ra => true,
9321	};
9322	var need_carry = switch (opts.overflow) {
9323	.@"unreachable", .wrap => false,
9324	.panic, .ra => true,
9325	};
9326	while (part_offset > 0) : (need_wrap = false) {
9327	const part_size = @min(part_offset, 8);
9328	part_offset -= part_size;
9329	var wrapped_res_part_it = res_vi.field(ty, part_offset, part_size);
9330	const wrapped_res_part_vi = try wrapped_res_part_it.only(isel);
9331	const wrapped_res_part_ra = wrapped_res_part_ra: {
9332	const overflow_ra_lock: RegLock = switch (opts.overflow) {
9333	.ra => |ra| isel.lockReg(ra),
9334	else => .empty,
9335	};
9336	defer overflow_ra_lock.unlock(isel);
9337	break :wrapped_res_part_ra try wrapped_res_part_vi.?.defReg(isel) orelse if (need_carry) .zr else continue;
9338	};
9339	const unwrapped_res_part_ra = unwrapped_res_part_ra: {
9340	if (!need_wrap) break :unwrapped_res_part_ra wrapped_res_part_ra;
9341	if (int_info.bits % 32 == 0) {
9342	try opts.overflow.defCond(isel, switch (int_info.signedness) {
9343	.signed => .vs,
9344	.unsigned => switch (op) {
9345	.add => .cs,
9346	.sub => .cc,
9347	},
9348	});
9349	break :unwrapped_res_part_ra wrapped_res_part_ra;
9350	}
9351	need_carry = false;
9352	const wrapped_part_ra, const unwrapped_part_ra = part_ra: switch (opts.overflow) {
9353	.@"unreachable" => unreachable,
9354	.panic, .ra => switch (int_info.signedness) {
9355	.signed => {
9356	try opts.overflow.defCond(isel, .ne);
9357	const wrapped_part_ra = switch (wrapped_res_part_ra) {
9358	else => |res_part_ra| res_part_ra,
9359	.zr => try isel.allocIntReg(),
9360	};
9361	errdefer if (wrapped_part_ra != wrapped_res_part_ra) isel.freeReg(wrapped_part_ra);
9362	const unwrapped_part_ra = unwrapped_part_ra: {
9363	const wrapped_res_part_lock: RegLock = switch (wrapped_res_part_ra) {
9364	else => |res_part_ra| isel.lockReg(res_part_ra),
9365	.zr => .empty,
9366	};
9367	defer wrapped_res_part_lock.unlock(isel);
9368	break :unwrapped_part_ra try isel.allocIntReg();
9369	};
9370	errdefer isel.freeReg(unwrapped_part_ra);
9371	switch (part_size) {
9372	else => unreachable,
9373	1...4 => try isel.emit(.subs(.wzr, wrapped_part_ra.w(), .{ .register = unwrapped_part_ra.w() })),
9374	5...8 => try isel.emit(.subs(.xzr, wrapped_part_ra.x(), .{ .register = unwrapped_part_ra.x() })),
9375	}
9376	break :part_ra .{ wrapped_part_ra, unwrapped_part_ra };
9377	},
9378	.unsigned => {
9379	const unwrapped_part_ra = unwrapped_part_ra: {
9380	const wrapped_res_part_lock: RegLock = switch (wrapped_res_part_ra) {
9381	else => |res_part_ra| isel.lockReg(res_part_ra),
9382	.zr => .empty,
9383	};
9384	defer wrapped_res_part_lock.unlock(isel);
9385	break :unwrapped_part_ra try isel.allocIntReg();
9386	};
9387	errdefer isel.freeReg(unwrapped_part_ra);
9388	const bit: u6 = @truncate(int_info.bits);
9389	switch (opts.overflow) {
9390	.@"unreachable", .wrap => unreachable,
9391	.panic => |panic_id| {
9392	const skip_label = isel.instructions.items.len;
9393	try isel.emitPanic(panic_id);
9394	try isel.emit(.tbz(
9395	switch (bit) {
9396	0, 32 => unreachable,
9397	1...31 => unwrapped_part_ra.w(),
9398	33...63 => unwrapped_part_ra.x(),
9399	},
9400	bit,
9401	@intCast((isel.instructions.items.len + 1 - skip_label) << 2),
9402	));
9403	},
9404	.ra => |overflow_ra| try isel.emit(switch (bit) {
9405	0, 32 => unreachable,
9406	1...31 => .ubfm(overflow_ra.w(), unwrapped_part_ra.w(), .{
9407	.N = .word,
9408	.immr = bit,
9409	.imms = bit,
9410	}),
9411	33...63 => .ubfm(overflow_ra.x(), unwrapped_part_ra.x(), .{
9412	.N = .doubleword,
9413	.immr = bit,
9414	.imms = bit,
9415	}),
9416	}),
9417	}
9418	break :part_ra .{ wrapped_res_part_ra, unwrapped_part_ra };
9419	},
9420	},
9421	.wrap => .{ wrapped_res_part_ra, wrapped_res_part_ra },
9422	};
9423	defer if (wrapped_part_ra != wrapped_res_part_ra) isel.freeReg(wrapped_part_ra);
9424	errdefer if (unwrapped_part_ra != wrapped_res_part_ra) isel.freeReg(unwrapped_part_ra);
9425	if (wrapped_part_ra != .zr) try isel.emit(switch (part_size) {
9426	else => unreachable,
9427	1...4 => switch (int_info.signedness) {
9428	.signed => .sbfm(wrapped_part_ra.w(), unwrapped_part_ra.w(), .{
9429	.N = .word,
9430	.immr = 0,
9431	.imms = @truncate(int_info.bits - 1),
9432	}),
9433	.unsigned => .ubfm(wrapped_part_ra.w(), unwrapped_part_ra.w(), .{
9434	.N = .word,
9435	.immr = 0,
9436	.imms = @truncate(int_info.bits - 1),
9437	}),
9438	},
9439	5...8 => switch (int_info.signedness) {
9440	.signed => .sbfm(wrapped_part_ra.x(), unwrapped_part_ra.x(), .{
9441	.N = .doubleword,
9442	.immr = 0,
9443	.imms = @truncate(int_info.bits - 1),
9444	}),
9445	.unsigned => .ubfm(wrapped_part_ra.x(), unwrapped_part_ra.x(), .{
9446	.N = .doubleword,
9447	.immr = 0,
9448	.imms = @truncate(int_info.bits - 1),
9449	}),
9450	},
9451	});
9452	break :unwrapped_res_part_ra unwrapped_part_ra;
9453	};
9454	defer if (unwrapped_res_part_ra != wrapped_res_part_ra) isel.freeReg(unwrapped_res_part_ra);
9455	var lhs_part_it = lhs_vi.field(ty, part_offset, part_size);
9456	const lhs_part_vi = try lhs_part_it.only(isel);
9457	const lhs_part_mat = try lhs_part_vi.?.matReg(isel);
9458	var rhs_part_it = rhs_vi.field(ty, part_offset, part_size);
9459	const rhs_part_vi = try rhs_part_it.only(isel);
9460	const rhs_part_mat = try rhs_part_vi.?.matReg(isel);
9461	try isel.emit(switch (part_size) {
9462	else => unreachable,
9463	1...4 => switch (op) {
9464	.add => switch (part_offset) {
9465	0 => switch (need_carry) {
9466	false => .add(unwrapped_res_part_ra.w(), lhs_part_mat.ra.w(), .{ .register = rhs_part_mat.ra.w() }),
9467	true => .adds(unwrapped_res_part_ra.w(), lhs_part_mat.ra.w(), .{ .register = rhs_part_mat.ra.w() }),
9468	},
9469	else => switch (need_carry) {
9470	false => .adc(unwrapped_res_part_ra.w(), lhs_part_mat.ra.w(), rhs_part_mat.ra.w()),
9471	true => .adcs(unwrapped_res_part_ra.w(), lhs_part_mat.ra.w(), rhs_part_mat.ra.w()),
9472	},
9473	},
9474	.sub => switch (part_offset) {
9475	0 => switch (need_carry) {
9476	false => .sub(unwrapped_res_part_ra.w(), lhs_part_mat.ra.w(), .{ .register = rhs_part_mat.ra.w() }),
9477	true => .subs(unwrapped_res_part_ra.w(), lhs_part_mat.ra.w(), .{ .register = rhs_part_mat.ra.w() }),
9478	},
9479	else => switch (need_carry) {
9480	false => .sbc(unwrapped_res_part_ra.w(), lhs_part_mat.ra.w(), rhs_part_mat.ra.w()),
9481	true => .sbcs(unwrapped_res_part_ra.w(), lhs_part_mat.ra.w(), rhs_part_mat.ra.w()),
9482	},
9483	},
9484	},
9485	5...8 => switch (op) {
9486	.add => switch (part_offset) {
9487	0 => switch (need_carry) {
9488	false => .add(unwrapped_res_part_ra.x(), lhs_part_mat.ra.x(), .{ .register = rhs_part_mat.ra.x() }),
9489	true => .adds(unwrapped_res_part_ra.x(), lhs_part_mat.ra.x(), .{ .register = rhs_part_mat.ra.x() }),
9490	},
9491	else => switch (need_carry) {
9492	false => .adc(unwrapped_res_part_ra.x(), lhs_part_mat.ra.x(), rhs_part_mat.ra.x()),
9493	true => .adcs(unwrapped_res_part_ra.x(), lhs_part_mat.ra.x(), rhs_part_mat.ra.x()),
9494	},
9495	},
9496	.sub => switch (part_offset) {
9497	0 => switch (need_carry) {
9498	false => .sub(unwrapped_res_part_ra.x(), lhs_part_mat.ra.x(), .{ .register = rhs_part_mat.ra.x() }),
9499	true => .subs(unwrapped_res_part_ra.x(), lhs_part_mat.ra.x(), .{ .register = rhs_part_mat.ra.x() }),
9500	},
9501	else => switch (need_carry) {
9502	false => .sbc(unwrapped_res_part_ra.x(), lhs_part_mat.ra.x(), rhs_part_mat.ra.x()),
9503	true => .sbcs(unwrapped_res_part_ra.x(), lhs_part_mat.ra.x(), rhs_part_mat.ra.x()),
9504	},
9505	},
9506	},
9507	});
9508	try rhs_part_mat.finish(isel);
9509	try lhs_part_mat.finish(isel);
9510	need_carry = true;
9511	}
9512	}
9513
9514	const MemoryAccessOptions = struct {
9515	root_vi: Value.Index = .free,
9516	offset: u64 = 0,
9517	@"volatile": bool = false,
9518	split: bool = true,
9519	wrap: ?std.lang.Type.Int = null,
9520	expected_live_registers: *const LiveRegisters = &.initFill(.free),
9521	};
9522
9523	fn load(
9524	vi: Value.Index,
9525	isel: *Select,
9526	root_ty: ZigType,
9527	base_ra: Register.Alias,
9528	opts: MemoryAccessOptions,
9529	) !bool {
9530	const root_vi = switch (opts.root_vi) {
9531	_ => |root_vi| root_vi,
9532	.allocating => unreachable,
9533	.free => vi,
9534	};
9535	var part_it = vi.parts(isel);
9536	if (part_it.only()) |part_vi| only: {
9537	const part_size = part_vi.size(isel);
9538	const part_is_vector = part_vi.isVector(isel);
9539	if (part_size > @as(@TypeOf(part_size), if (part_is_vector) 16 else 8)) {
9540	if (!opts.split) return false;
9541	var subpart_it = root_vi.field(root_ty, opts.offset, part_size - 1);
9542	_ = try subpart_it.next(isel);
9543	part_it = vi.parts(isel);
9544	assert(part_it.only() == null);
9545	break :only;
9546	}
9547	const part_ra = if (try part_vi.defReg(isel)) |part_ra|
9548	part_ra
9549	else if (opts.@"volatile")
9550	.zr
9551	else
9552	return false;
9553	const part_lock: RegLock = switch (part_ra) {
9554	else => isel.lockReg(part_ra),
9555	.zr => .empty,
9556	};
9557	defer switch (opts.expected_live_registers.get(part_ra)) {
9558	_ => {},
9559	.allocating => unreachable,
9560	.free => part_lock.unlock(isel),
9561	};
9562	if (opts.wrap) |int_info| switch (int_info.bits) {
9563	else => unreachable,
9564	1...7, 9...15, 17...31 => |bits| try isel.emit(switch (int_info.signedness) {
9565	.signed => .sbfm(part_ra.w(), part_ra.w(), .{
9566	.N = .word,
9567	.immr = 0,
9568	.imms = @intCast(bits - 1),
9569	}),
9570	.unsigned => .ubfm(part_ra.w(), part_ra.w(), .{
9571	.N = .word,
9572	.immr = 0,
9573	.imms = @intCast(bits - 1),
9574	}),
9575	}),
9576	8, 16, 32 => {},
9577	33...63 => |bits| try isel.emit(switch (int_info.signedness) {
9578	.signed => .sbfm(part_ra.x(), part_ra.x(), .{
9579	.N = .doubleword,
9580	.immr = 0,
9581	.imms = @intCast(bits - 1),
9582	}),
9583	.unsigned => .ubfm(part_ra.x(), part_ra.x(), .{
9584	.N = .doubleword,
9585	.immr = 0,
9586	.imms = @intCast(bits - 1),
9587	}),
9588	}),
9589	64 => {},
9590	};
9591	try isel.loadReg(part_ra, part_size, part_vi.signedness(isel), base_ra, opts.offset);
9592	return true;
9593	}
9594	var used = false;
9595	while (part_it.next()) |part_vi| used |= try part_vi.load(isel, root_ty, base_ra, .{
9596	.root_vi = root_vi,
9597	.offset = opts.offset + part_vi.get(isel).offset_from_parent,
9598	.@"volatile" = opts.@"volatile",
9599	.split = opts.split,
9600	.wrap = switch (part_it.remaining) {
9601	else => null,
9602	0 => if (opts.wrap) |wrap| .{
9603	.signedness = wrap.signedness,
9604	.bits = @intCast(wrap.bits - 8 * part_vi.position(isel)[0]),
9605	} else null,
9606	},
9607	.expected_live_registers = opts.expected_live_registers,
9608	});
9609	return used;
9610	}
9611
9612	fn store(
9613	vi: Value.Index,
9614	isel: *Select,
9615	root_ty: ZigType,
9616	base_ra: Register.Alias,
9617	opts: MemoryAccessOptions,
9618	) !void {
9619	const root_vi = switch (opts.root_vi) {
9620	_ => |root_vi| root_vi,
9621	.allocating => unreachable,
9622	.free => vi,
9623	};
9624	var part_it = vi.parts(isel);
9625	if (part_it.only()) |part_vi| only: {
9626	const part_size = part_vi.size(isel);
9627	const part_is_vector = part_vi.isVector(isel);
9628	if (part_size > @as(@TypeOf(part_size), if (part_is_vector) 16 else 8)) {
9629	if (!opts.split) return;
9630	var subpart_it = root_vi.field(root_ty, opts.offset, part_size - 1);
9631	_ = try subpart_it.next(isel);
9632	part_it = vi.parts(isel);
9633	assert(part_it.only() == null);
9634	break :only;
9635	}
9636	const part_mat = try part_vi.matReg(isel);
9637	try isel.storeReg(part_mat.ra, part_size, base_ra, opts.offset);
9638	return part_mat.finish(isel);
9639	}
9640	while (part_it.next()) |part_vi| try part_vi.store(isel, root_ty, base_ra, .{
9641	.root_vi = root_vi,
9642	.offset = opts.offset + part_vi.get(isel).offset_from_parent,
9643	.@"volatile" = opts.@"volatile",
9644	.split = opts.split,
9645	.wrap = switch (part_it.remaining) {
9646	else => null,
9647	0 => if (opts.wrap) |wrap| .{
9648	.signedness = wrap.signedness,
9649	.bits = @intCast(wrap.bits - 8 * part_vi.position(isel)[0]),
9650	} else null,
9651	},
9652	.expected_live_registers = opts.expected_live_registers,
9653	});
9654	}
9655
9656	fn mat(vi: Value.Index, isel: *Select) !void {
9657	if (false) {
9658	var part_it: Value.PartIterator = if (vi.size(isel) > 8) vi.parts(isel) else .initOne(vi);
9659	if (part_it.only()) |part_vi| only: {
9660	const mat_ra = mat_ra: {
9661	if (part_vi.register(isel)) |mat_ra| {
9662	part_vi.get(isel).location_payload.small.register = .zr;
9663	const live_vi = isel.live_registers.getPtr(mat_ra);
9664	assert(live_vi.* == part_vi);
9665	live_vi.* = .allocating;
9666	break :mat_ra mat_ra;
9667	}
9668	if (part_vi.hint(isel)) |hint_ra| {
9669	const live_vi = isel.live_registers.getPtr(hint_ra);
9670	if (live_vi.* == .free) {
9671	live_vi.* = .allocating;
9672	isel.saved_registers.insert(hint_ra);
9673	break :mat_ra hint_ra;
9674	}
9675	}
9676	const part_size = part_vi.size(isel);
9677	const part_is_vector = part_vi.isVector(isel);
9678	if (part_size <= @as(@TypeOf(part_size), if (part_is_vector) 16 else 8))
9679	switch (if (part_is_vector) isel.tryAllocVecReg() else isel.tryAllocIntReg()) {
9680	.allocated => |ra| break :mat_ra ra,
9681	.fill_candidate, .out_of_registers => {},
9682	};
9683	_, const parent_vi = vi.valueParent(isel);
9684	switch (parent_vi.parent(isel)) {
9685	.unallocated => parent_vi.setParent(isel, .{ .stack_slot = parent_vi.allocStackSlot(isel) }),
9686	else => {},
9687	}
9688	break :only;
9689	};
9690	assert(isel.live_registers.get(mat_ra) == .allocating);
9691	try Value.Materialize.finish(.{ .vi = part_vi, .ra = mat_ra }, isel);
9692	} else while (part_it.next()) |part_vi| try part_vi.mat(isel);
9693	} else {
9694	_, const parent_vi = vi.valueParent(isel);
9695	switch (parent_vi.parent(isel)) {
9696	.unallocated => parent_vi.setParent(isel, .{ .stack_slot = parent_vi.allocStackSlot(isel) }),
9697	else => {},
9698	}
9699	}
9700	}
9701
9702	fn matReg(vi: Value.Index, isel: *Select) !Value.Materialize {
9703	const mat_ra = mat_ra: {
9704	if (vi.register(isel)) |mat_ra| {
9705	vi.get(isel).location_payload.small.register = .zr;
9706	const live_vi = isel.live_registers.getPtr(mat_ra);
9707	assert(live_vi.* == vi);
9708	live_vi.* = .allocating;
9709	break :mat_ra mat_ra;
9710	}
9711	if (vi.hint(isel)) |hint_ra| {
9712	const live_vi = isel.live_registers.getPtr(hint_ra);
9713	if (live_vi.* == .free) {
9714	live_vi.* = .allocating;
9715	isel.saved_registers.insert(hint_ra);
9716	break :mat_ra hint_ra;
9717	}
9718	}
9719	break :mat_ra if (vi.isVector(isel)) try isel.allocVecReg() else try isel.allocIntReg();
9720	};
9721	assert(isel.live_registers.get(mat_ra) == .allocating);
9722	return .{ .vi = vi, .ra = mat_ra };
9723	}
9724
9725	fn defAddr(
9726	def_vi: Value.Index,
9727	isel: *Select,
9728	root_ty: ZigType,
9729	opts: struct {
9730	root_vi: Value.Index = .free,
9731	wrap: ?std.lang.Type.Int = null,
9732	expected_live_registers: *const LiveRegisters = &.initFill(.free),
9733	},
9734	) !?void {
9735	if (!def_vi.isUsed(isel)) return null;
9736	const offset_from_parent: i65, const parent_vi = def_vi.valueParent(isel);
9737	const stack_slot, const allocated = switch (parent_vi.parent(isel)) {
9738	.unallocated => .{ parent_vi.allocStackSlot(isel), true },
9739	.stack_slot => |stack_slot| .{ stack_slot, false },
9740	else => unreachable,
9741	};
9742	_ = try def_vi.load(isel, root_ty, stack_slot.base, .{
9743	.root_vi = opts.root_vi,
9744	.offset = @intCast(stack_slot.offset + offset_from_parent),
9745	.split = false,
9746	.wrap = opts.wrap,
9747	.expected_live_registers = opts.expected_live_registers,
9748	});
9749	if (allocated) parent_vi.setParent(isel, .{ .stack_slot = stack_slot });
9750	}
9751
9752	fn defReg(def_vi: Value.Index, isel: *Select) !?Register.Alias {
9753	var vi = def_vi;
9754	var offset: i65 = 0;
9755	var def_ra: ?Register.Alias = null;
9756	while (true) {
9757	if (vi.register(isel)) |ra| {
9758	vi.get(isel).location_payload.small.register = .zr;
9759	const live_vi = isel.live_registers.getPtr(ra);
9760	assert(live_vi.* == vi);
9761	if (def_ra == null and vi != def_vi) {
9762	var part_it = vi.parts(isel);
9763	assert(part_it.only() == null);
9764
9765	const first_part_vi = part_it.next().?;
9766	const first_part_value = first_part_vi.get(isel);
9767	assert(first_part_value.offset_from_parent == 0);
9768	first_part_value.location_payload.small.register = ra;
9769	live_vi.* = first_part_vi;
9770
9771	const vi_size = vi.size(isel);
9772	while (part_it.next()) |part_vi| {
9773	const part_offset, const part_size = part_vi.position(isel);
9774	const part_mat = try part_vi.matReg(isel);
9775	try isel.emit(if (part_vi.isVector(isel)) emit: {
9776	assert(part_offset == 0 and part_size == vi_size);
9777	break :emit switch (vi_size) {
9778	else => unreachable,
9779	2 => if (isel.target.cpu.has(.aarch64, .fullfp16))
9780	.fmov(ra.h(), .{ .register = part_mat.ra.h() })
9781	else
9782	.dup(ra.h(), part_mat.ra.@"h[]"(0)),
9783	4 => .fmov(ra.s(), .{ .register = part_mat.ra.s() }),
9784	8 => .fmov(ra.d(), .{ .register = part_mat.ra.d() }),
9785	16 => .orr(ra.@"16b"(), part_mat.ra.@"16b"(), .{ .register = part_mat.ra.@"16b"() }),
9786	};
9787	} else switch (vi_size) {
9788	else => unreachable,
9789	1...4 => .bfm(ra.w(), part_mat.ra.w(), .{
9790	.N = .word,
9791	.immr = @as(u5, @truncate(32 - 8 * part_offset)),
9792	.imms = @intCast(8 * part_size - 1),
9793	}),
9794	5...8 => .bfm(ra.x(), part_mat.ra.x(), .{
9795	.N = .doubleword,
9796	.immr = @as(u6, @truncate(64 - 8 * part_offset)),
9797	.imms = @intCast(8 * part_size - 1),
9798	}),
9799	});
9800	try part_mat.finish(isel);
9801	}
9802	vi = def_vi;
9803	offset = 0;
9804	continue;
9805	}
9806	live_vi.* = .free;
9807	def_ra = ra;
9808	}
9809	offset += vi.get(isel).offset_from_parent;
9810	switch (vi.parent(isel)) {
9811	else => unreachable,
9812	.unallocated => return def_ra,
9813	.stack_slot => |stack_slot| {
9814	offset += stack_slot.offset;
9815	const def_is_vector = def_vi.isVector(isel);
9816	const ra = def_ra orelse if (def_is_vector) try isel.allocVecReg() else try isel.allocIntReg();
9817	defer if (def_ra == null) isel.freeReg(ra);
9818	try isel.storeReg(ra, def_vi.size(isel), stack_slot.base, offset);
9819	return ra;
9820	},
9821	.value => |parent_vi| vi = parent_vi,
9822	}
9823	}
9824	}
9825
9826	pub fn defUndef(def_vi: Value.Index, isel: *Select, root_ty: ZigType, opts: struct {
9827	root_vi: Value.Index = .free,
9828	offset: u64 = 0,
9829	split: bool = true,
9830	}) !void {
9831	const root_vi = switch (opts.root_vi) {
9832	_ => |root_vi| root_vi,
9833	.allocating => unreachable,
9834	.free => def_vi,
9835	};
9836	var part_it = def_vi.parts(isel);
9837	if (part_it.only()) |part_vi| only: {
9838	const part_size = part_vi.size(isel);
9839	const part_is_vector = part_vi.isVector(isel);
9840	if (part_size > @as(@TypeOf(part_size), if (part_is_vector) 16 else 8)) {
9841	if (!opts.split) return;
9842	var subpart_it = root_vi.field(root_ty, opts.offset, part_size - 1);
9843	_ = try subpart_it.next(isel);
9844	part_it = def_vi.parts(isel);
9845	assert(part_it.only() == null);
9846	break :only;
9847	}
9848	return if (try part_vi.defReg(isel)) |part_ra| try isel.emit(if (part_is_vector)
9849	.movi(switch (part_size) {
9850	else => unreachable,
9851	1...8 => part_ra.@"8b"(),
9852	9...16 => part_ra.@"16b"(),
9853	}, 0xaa, .{ .lsl = 0 })
9854	else switch (part_size) {
9855	else => unreachable,
9856	1...4 => .orr(part_ra.w(), .wzr, .{ .immediate = .{
9857	.N = .word,
9858	.immr = 0b000001,
9859	.imms = 0b111100,
9860	} }),
9861	5...8 => .orr(part_ra.x(), .xzr, .{ .immediate = .{
9862	.N = .word,
9863	.immr = 0b000001,
9864	.imms = 0b111100,
9865	} }),
9866	});
9867	}
9868	while (part_it.next()) |part_vi| try part_vi.defUndef(isel, root_ty, .{
9869	.root_vi = root_vi,
9870	});
9871	}
9872
9873	pub fn liveIn(
9874	vi: Value.Index,
9875	isel: *Select,
9876	src_ra: Register.Alias,
9877	expected_live_registers: *const LiveRegisters,
9878	) !void {
9879	const src_live_vi = isel.live_registers.getPtr(src_ra);
9880	if (vi.register(isel)) |dst_ra| {
9881	const dst_live_vi = isel.live_registers.getPtr(dst_ra);
9882	assert(dst_live_vi.* == vi);
9883	if (dst_ra == src_ra) {
9884	src_live_vi.* = .allocating;
9885	return;
9886	}
9887	dst_live_vi.* = .allocating;
9888	if (try isel.fill(src_ra)) {
9889	assert(src_live_vi.* == .free);
9890	src_live_vi.* = .allocating;
9891	}
9892	assert(src_live_vi.* == .allocating);
9893	try isel.emit(switch (dst_ra.isVector()) {
9894	false => switch (src_ra.isVector()) {
9895	false => switch (vi.size(isel)) {
9896	else => unreachable,
9897	1...4 => .orr(dst_ra.w(), .wzr, .{ .register = src_ra.w() }),
9898	5...8 => .orr(dst_ra.x(), .xzr, .{ .register = src_ra.x() }),
9899	},
9900	true => switch (vi.size(isel)) {
9901	else => unreachable,
9902	2 => if (isel.target.cpu.has(.aarch64, .fullfp16))
9903	.fmov(dst_ra.w(), .{ .register = src_ra.h() })
9904	else
9905	.umov(dst_ra.w(), src_ra.@"h[]"(0)),
9906	4 => .fmov(dst_ra.w(), .{ .register = src_ra.s() }),
9907	8 => .fmov(dst_ra.x(), .{ .register = src_ra.d() }),
9908	},
9909	},
9910	true => switch (src_ra.isVector()) {
9911	false => size: switch (vi.size(isel)) {
9912	else => unreachable,
9913	2 => if (isel.target.cpu.has(.aarch64, .fullfp16))
9914	.fmov(dst_ra.h(), .{ .register = src_ra.w() })
9915	else
9916	continue :size 4,
9917	4 => .fmov(dst_ra.s(), .{ .register = src_ra.w() }),
9918	8 => .fmov(dst_ra.d(), .{ .register = src_ra.x() }),
9919	},
9920	true => switch (vi.size(isel)) {
9921	else => unreachable,
9922	2 => if (isel.target.cpu.has(.aarch64, .fullfp16))
9923	.fmov(dst_ra.h(), .{ .register = src_ra.h() })
9924	else
9925	.dup(dst_ra.h(), src_ra.@"h[]"(0)),
9926	4 => .fmov(dst_ra.s(), .{ .register = src_ra.s() }),
9927	8 => .fmov(dst_ra.d(), .{ .register = src_ra.d() }),
9928	16 => .orr(dst_ra.@"16b"(), src_ra.@"16b"(), .{ .register = src_ra.@"16b"() }),
9929	},
9930	},
9931	});
9932	assert(dst_live_vi.* == .allocating);
9933	dst_live_vi.* = switch (expected_live_registers.get(dst_ra)) {
9934	_ => .allocating,
9935	.allocating => .allocating,
9936	.free => .free,
9937	};
9938	} else if (try isel.fill(src_ra)) {
9939	assert(src_live_vi.* == .free);
9940	src_live_vi.* = .allocating;
9941	}
9942	assert(src_live_vi.* == .allocating);
9943	vi.get(isel).location_payload.small.register = src_ra;
9944	}
9945
9946	pub fn defLiveIn(
9947	vi: Value.Index,
9948	isel: *Select,
9949	src_ra: Register.Alias,
9950	expected_live_registers: *const LiveRegisters,
9951	) !void {
9952	try vi.liveIn(isel, src_ra, expected_live_registers);
9953	const offset_from_parent, const parent_vi = vi.valueParent(isel);
9954	switch (parent_vi.parent(isel)) {
9955	.unallocated => {},
9956	.stack_slot => |stack_slot| if (stack_slot.base != Register.Alias.fp) try isel.storeReg(
9957	src_ra,
9958	vi.size(isel),
9959	stack_slot.base,
9960	@as(i65, stack_slot.offset) + offset_from_parent,
9961	),
9962	else => unreachable,
9963	}
9964	try vi.spillReg(isel, src_ra, 0, expected_live_registers);
9965	}
9966
9967	fn spillReg(
9968	vi: Value.Index,
9969	isel: *Select,
9970	src_ra: Register.Alias,
9971	start_offset: u64,
9972	expected_live_registers: *const LiveRegisters,
9973	) !void {
9974	assert(isel.live_registers.get(src_ra) == .allocating);
9975	var part_it = vi.parts(isel);
9976	if (part_it.only()) |part_vi| {
9977	const dst_ra = part_vi.register(isel) orelse return;
9978	if (dst_ra == src_ra) return;
9979	const part_size = part_vi.size(isel);
9980	const part_ra = if (part_vi.isVector(isel)) try isel.allocIntReg() else dst_ra;
9981	defer if (part_ra != dst_ra) isel.freeReg(part_ra);
9982	if (part_ra != dst_ra) try isel.emit(part_size: switch (part_size) {
9983	else => unreachable,
9984	2 => if (isel.target.cpu.has(.aarch64, .fullfp16))
9985	.fmov(dst_ra.h(), .{ .register = part_ra.w() })
9986	else
9987	continue :part_size 4,
9988	4 => .fmov(dst_ra.s(), .{ .register = part_ra.w() }),
9989	8 => .fmov(dst_ra.d(), .{ .register = part_ra.x() }),
9990	});
9991	try isel.emit(switch (start_offset + part_size) {
9992	else => unreachable,
9993	1...4 => |end_offset| switch (part_vi.signedness(isel)) {
9994	.signed => .sbfm(part_ra.w(), src_ra.w(), .{
9995	.N = .word,
9996	.immr = @intCast(8 * start_offset),
9997	.imms = @intCast(8 * end_offset - 1),
9998	}),
9999	.unsigned => .ubfm(part_ra.w(), src_ra.w(), .{
10000	.N = .word,
10001	.immr = @intCast(8 * start_offset),
10002	.imms = @intCast(8 * end_offset - 1),
10003	}),
10004	},
10005	5...8 => |end_offset| switch (part_vi.signedness(isel)) {
10006	.signed => .sbfm(part_ra.x(), src_ra.x(), .{
10007	.N = .doubleword,
10008	.immr = @intCast(8 * start_offset),
10009	.imms = @intCast(8 * end_offset - 1),
10010	}),
10011	.unsigned => .ubfm(part_ra.x(), src_ra.x(), .{
10012	.N = .doubleword,
10013	.immr = @intCast(8 * start_offset),
10014	.imms = @intCast(8 * end_offset - 1),
10015	}),
10016	},
10017	});
10018	const value_ra = &part_vi.get(isel).location_payload.small.register;
10019	assert(value_ra.* == dst_ra);
10020	value_ra.* = .zr;
10021	const dst_live_vi = isel.live_registers.getPtr(dst_ra);
10022	assert(dst_live_vi.* == part_vi);
10023	dst_live_vi.* = switch (expected_live_registers.get(dst_ra)) {
10024	_ => .allocating,
10025	.allocating => unreachable,
10026	.free => .free,
10027	};
10028	} else while (part_it.next()) |part_vi| try part_vi.spillReg(
10029	isel,
10030	src_ra,
10031	start_offset + part_vi.get(isel).offset_from_parent,
10032	expected_live_registers,
10033	);
10034	}
10035
10036	fn liveOut(vi: Value.Index, isel: *Select, ra: Register.Alias) !void {
10037	assert(try isel.fill(ra));
10038	const live_vi = isel.live_registers.getPtr(ra);
10039	assert(live_vi.* == .free);
10040	live_vi.* = .allocating;
10041	try Value.Materialize.finish(.{ .vi = vi, .ra = ra }, isel);
10042	}
10043
10044	fn allocStackSlot(vi: Value.Index, isel: *Select) Value.Indirect {
10045	const offset = vi.alignment(isel).forward(isel.stack_size);
10046	isel.stack_size = @intCast(offset + vi.size(isel));
10047	tracking_log.debug("${d} -> [sp, #0x{x}]", .{ @backingInt(vi), @abs(offset) });
10048	return .{
10049	.base = .sp,
10050	.offset = @intCast(offset),
10051	};
10052	}
10053
10054	fn address(initial_vi: Value.Index, isel: *Select, initial_offset: u64, ptr_ra: Register.Alias) !void {
10055	var vi = initial_vi;
10056	var offset: i65 = vi.get(isel).offset_from_parent + initial_offset;
10057	parent: switch (vi.parent(isel)) {
10058	.unallocated => {
10059	const stack_slot = vi.allocStackSlot(isel);
10060	vi.setParent(isel, .{ .stack_slot = stack_slot });
10061	continue :parent .{ .stack_slot = stack_slot };
10062	},
10063	.stack_slot => |stack_slot| {
10064	offset += stack_slot.offset;
10065	const lo12: u12 = @truncate(@abs(offset) >> 0);
10066	const hi12: u12 = @intCast(@abs(offset) >> 12);
10067	if (hi12 > 0) try isel.emit(if (offset >= 0) .add(
10068	ptr_ra.x(),
10069	if (lo12 > 0) ptr_ra.x() else stack_slot.base.x(),
10070	.{ .shifted_immediate = .{ .immediate = hi12, .lsl = .@"12" } },
10071	) else .sub(
10072	ptr_ra.x(),
10073	if (lo12 > 0) ptr_ra.x() else stack_slot.base.x(),
10074	.{ .shifted_immediate = .{ .immediate = hi12, .lsl = .@"12" } },
10075	));
10076	if (lo12 > 0 or hi12 == 0) try isel.emit(if (offset >= 0) .add(
10077	ptr_ra.x(),
10078	stack_slot.base.x(),
10079	.{ .immediate = lo12 },
10080	) else .sub(
10081	ptr_ra.x(),
10082	stack_slot.base.x(),
10083	.{ .immediate = lo12 },
10084	));
10085	},
10086	.address => |address_vi| try address_vi.liveOut(isel, ptr_ra),
10087	.value => |parent_vi| {
10088	vi = parent_vi;
10089	offset += vi.get(isel).offset_from_parent;
10090	continue :parent vi.parent(isel);
10091	},
10092	.constant => |constant| {
10093	const pt = isel.pt;
10094	const zcu = pt.zcu;
10095	switch (true) {
10096	false => {
10097	try isel.uav_relocs.append(zcu.gpa, .{
10098	.uav = .{
10099	.val = constant.toIntern(),
10100	.orig_ty = (try pt.singleConstPtrType(constant.typeOf(zcu))).toIntern(),
10101	},
10102	.reloc = .{
10103	.label = @intCast(isel.instructions.items.len),
10104	.addend = @intCast(offset),
10105	},
10106	});
10107	try isel.emit(.adr(ptr_ra.x(), 0));
10108	},
10109	true => {
10110	try isel.uav_relocs.append(zcu.gpa, .{
10111	.uav = .{
10112	.val = constant.toIntern(),
10113	.orig_ty = (try pt.singleConstPtrType(constant.typeOf(zcu))).toIntern(),
10114	},
10115	.reloc = .{
10116	.label = @intCast(isel.instructions.items.len),
10117	.addend = @intCast(offset),
10118	},
10119	});
10120	try isel.emit(.add(ptr_ra.x(), ptr_ra.x(), .{ .immediate = 0 }));
10121	try isel.uav_relocs.append(zcu.gpa, .{
10122	.uav = .{
10123	.val = constant.toIntern(),
10124	.orig_ty = (try pt.singleConstPtrType(constant.typeOf(zcu))).toIntern(),
10125	},
10126	.reloc = .{
10127	.label = @intCast(isel.instructions.items.len),
10128	.addend = @intCast(offset),
10129	},
10130	});
10131	try isel.emit(.adrp(ptr_ra.x(), 0));
10132	},
10133	}
10134	},
10135	}
10136	}
10137	};
10138
10139	pub const PartIterator = struct {
10140	vi: Value.Index,
10141	remaining: Value.PartsLen,
10142
10143	fn initOne(vi: Value.Index) PartIterator {
10144	return .{ .vi = vi, .remaining = 1 };
10145	}
10146
10147	pub fn next(it: *PartIterator) ?Value.Index {
10148	if (it.remaining == 0) return null;
10149	it.remaining -= 1;
10150	defer it.vi = @fromBackingInt(@intCast(@backingInt(it.vi) + 1));
10151	return it.vi;
10152	}
10153
10154	pub fn peek(it: PartIterator) ?Value.Index {
10155	var it_mut = it;
10156	return it_mut.next();
10157	}
10158
10159	pub fn only(it: PartIterator) ?Value.Index {
10160	return if (it.remaining == 1) it.vi else null;
10161	}
10162	};
10163
10164	const FieldPartIterator = struct {
10165	vi: Value.Index,
10166	ty: ZigType,
10167	field_offset: u64,
10168	field_size: u64,
10169	next_offset: u64,
10170
10171	fn next(it: *FieldPartIterator, isel: *Select) !?struct { offset: u64, vi: Value.Index } {
10172	const next_offset = it.next_offset;
10173	const next_part_size = it.field_size - next_offset;
10174	if (next_part_size == 0) return null;
10175	var next_part_offset = it.field_offset + next_offset;
10176
10177	const zcu = isel.pt.zcu;
10178	const ip = &zcu.intern_pool;
10179	var vi = it.vi;
10180	var ty = it.ty;
10181	var ty_size = vi.size(isel);
10182	assert(ty_size == ty.abiSize(zcu));
10183	var offset: u64 = 0;
10184	var size = ty_size;
10185	assert(next_part_offset + next_part_size <= size);
10186	while (next_part_offset > 0 or next_part_size < size) {
10187	const part_vi = vi.partAtOffset(isel, next_part_offset);
10188	if (part_vi != vi) {
10189	vi = part_vi;
10190	const part_offset, size = part_vi.position(isel);
10191	assert(part_offset <= next_part_offset and part_offset + size > next_part_offset);
10192	offset += part_offset;
10193	next_part_offset -= part_offset;
10194	continue;
10195	}
10196	try isel.values.ensureUnusedCapacity(zcu.gpa, Value.max_parts);
10197	type_key: switch (ip.indexToKey(ty.toIntern())) {
10198	else => return isel.fail("Value.FieldPartIterator.next({f})", .{isel.fmtType(ty)}),
10199	.int_type => |int_type| switch (int_type.bits) {
10200	0 => unreachable,
10201	1...64 => unreachable,
10202	65...256 => |bits| if (offset == 0 and size == ty_size) {
10203	const parts_len = @divCeil(bits, 64);
10204	vi.setParts(isel, @intCast(parts_len));
10205	for (0..parts_len) |part_index| _ = vi.addPart(isel, 8 * part_index, 8);
10206	},
10207	else => return isel.fail("Value.FieldPartIterator.next({f})", .{isel.fmtType(ty)}),
10208	},
10209	.ptr_type => |ptr_type| switch (ptr_type.flags.size) {
10210	.one, .many, .c => unreachable,
10211	.slice => if (offset == 0 and size == ty_size) {
10212	vi.setParts(isel, 2);
10213	_ = vi.addPart(isel, 0, 8);
10214	_ = vi.addPart(isel, 8, 8);
10215	} else unreachable,
10216	},
10217	.opt_type => |child_type| if (ty.optionalReprIsPayload(zcu)) continue :type_key ip.indexToKey(child_type) else {
10218	const child_ty: ZigType = .fromInterned(child_type);
10219	const child_size = child_ty.abiSize(zcu);
10220	if (offset == 0 and size == child_size) {
10221	ty = child_ty;
10222	ty_size = child_size;
10223	continue :type_key ip.indexToKey(child_type);
10224	}
10225	switch (child_size) {
10226	0...8, 16 => if (offset == 0 and size == ty_size) {
10227	vi.setParts(isel, 2);
10228	_ = vi.addPart(isel, 0, child_size);
10229	_ = vi.addPart(isel, child_size, 1);
10230	} else unreachable,
10231	9...15 => if (offset == 0 and size == ty_size) {
10232	vi.setParts(isel, 2);
10233	_ = vi.addPart(isel, 0, 8);
10234	_ = vi.addPart(isel, 8, ty_size - 8);
10235	} else if (offset == 8 and size == ty_size - 8) {
10236	vi.setParts(isel, 2);
10237	_ = vi.addPart(isel, 0, child_size - 8);
10238	_ = vi.addPart(isel, child_size - 8, 1);
10239	} else unreachable,
10240	else => return isel.fail("Value.FieldPartIterator.next({f})", .{isel.fmtType(ty)}),
10241	}
10242	},
10243	.array_type => |array_type| {
10244	const min_part_log2_stride: u5 = if (size > 16) 4 else if (size > 8) 3 else 0;
10245	const array_len = array_type.lenIncludingSentinel();
10246	if (array_len > Value.max_parts and
10247	(@divCeil(size, @as(u64, 1) << min_part_log2_stride)) > Value.max_parts)
10248	return isel.fail("Value.FieldPartIterator.next({f})", .{isel.fmtType(ty)});
10249	const alignment = vi.alignment(isel);
10250	const Part = struct { offset: u64, size: u64 };
10251	var parts: [Value.max_parts]Part = undefined;
10252	var parts_len: Value.PartsLen = 0;
10253	const elem_ty: ZigType = .fromInterned(array_type.child);
10254	const elem_size = elem_ty.abiSize(zcu);
10255	const elem_signedness = if (ty.isAbiInt(zcu)) elem_signedness: {
10256	const elem_int_info = elem_ty.intInfo(zcu);
10257	break :elem_signedness if (elem_int_info.bits <= 16) elem_int_info.signedness else null;
10258	} else null;
10259	const elem_is_vector = elem_size <= 16 and
10260	CallAbiIterator.homogeneousAggregateBaseType(zcu, elem_ty.toIntern()) != null;
10261	var elem_end: u64 = 0;
10262	for (0..@intCast(array_len)) |_| {
10263	const elem_begin = elem_end;
10264	if (elem_begin >= offset + size) break;
10265	elem_end = elem_begin + elem_size;
10266	if (elem_end <= offset) continue;
10267	if (offset >= elem_begin and offset + size <= elem_begin + elem_size) {
10268	ty = elem_ty;
10269	ty_size = elem_size;
10270	offset -= elem_begin;
10271	continue :type_key ip.indexToKey(elem_ty.toIntern());
10272	}
10273	if (parts_len > 0) combine: {
10274	const prev_part = &parts[parts_len - 1];
10275	const combined_size = elem_end - prev_part.offset;
10276	if (combined_size > @as(u64, 1) << @min(
10277	min_part_log2_stride,
10278	alignment.toLog2Units(),
10279	@ctz(prev_part.offset),
10280	)) break :combine;
10281	prev_part.size = combined_size;
10282	continue;
10283	}
10284	parts[parts_len] = .{ .offset = elem_begin, .size = elem_size };
10285	parts_len += 1;
10286	}
10287	vi.setParts(isel, parts_len);
10288	for (parts[0..parts_len]) |part| {
10289	const subpart_vi = vi.addPart(isel, part.offset - offset, part.size);
10290	if (elem_signedness) |signedness| subpart_vi.setSignedness(isel, signedness);
10291	if (elem_is_vector) subpart_vi.setIsVector(isel);
10292	}
10293	},
10294	.anyframe_type => unreachable,
10295	.error_union_type => |error_union_type| {
10296	const min_part_log2_stride: u5 = if (size > 16) 4 else if (size > 8) 3 else 0;
10297	if ((@divCeil(size, @as(u64, 1) << min_part_log2_stride)) > Value.max_parts)
10298	return isel.fail("Value.FieldPartIterator.next({f})", .{isel.fmtType(ty)});
10299	const alignment = vi.alignment(isel);
10300	const payload_ty: ZigType = .fromInterned(error_union_type.payload_type);
10301	const error_set_offset = codegen.errUnionErrorOffset(payload_ty, zcu);
10302	const payload_offset = codegen.errUnionPayloadOffset(payload_ty, zcu);
10303	const Part = struct { offset: u64, size: u64, signedness: ?std.lang.Signedness, is_vector: bool };
10304	var parts: [2]Part = undefined;
10305	var parts_len: Value.PartsLen = 0;
10306	var field_end: u64 = 0;
10307	for (0..2) |field_index| {
10308	const field_ty: ZigType, const field_begin = switch (@as(enum { error_set, payload }, switch (field_index) {
10309	0 => if (error_set_offset < payload_offset) .error_set else .payload,
10310	1 => if (error_set_offset < payload_offset) .payload else .error_set,
10311	else => unreachable,
10312	})) {
10313	.error_set => .{ .fromInterned(error_union_type.error_set_type), error_set_offset },
10314	.payload => .{ payload_ty, payload_offset },
10315	};
10316	if (field_begin >= offset + size) break;
10317	const field_size = field_ty.abiSize(zcu);
10318	if (field_size == 0) continue;
10319	field_end = field_begin + field_size;
10320	if (field_end <= offset) continue;
10321	if (offset >= field_begin and offset + size <= field_begin + field_size) {
10322	ty = field_ty;
10323	ty_size = field_size;
10324	offset -= field_begin;
10325	continue :type_key ip.indexToKey(field_ty.toIntern());
10326	}
10327	const field_signedness = if (field_ty.isAbiInt(zcu)) field_signedness: {
10328	const field_int_info = field_ty.intInfo(zcu);
10329	break :field_signedness if (field_int_info.bits <= 16) field_int_info.signedness else null;
10330	} else null;
10331	const field_is_vector = field_size <= 16 and
10332	CallAbiIterator.homogeneousAggregateBaseType(zcu, field_ty.toIntern()) != null;
10333	if (parts_len > 0) combine: {
10334	const prev_part = &parts[parts_len - 1];
10335	const combined_size = field_end - prev_part.offset;
10336	if (combined_size > @as(u64, 1) << @min(
10337	min_part_log2_stride,
10338	alignment.toLog2Units(),
10339	@ctz(prev_part.offset),
10340	)) break :combine;
10341	prev_part.size = combined_size;
10342	prev_part.signedness = null;
10343	prev_part.is_vector &= field_is_vector;
10344	continue;
10345	}
10346	parts[parts_len] = .{
10347	.offset = field_begin,
10348	.size = field_size,
10349	.signedness = field_signedness,
10350	.is_vector = field_is_vector,
10351	};
10352	parts_len += 1;
10353	}
10354	vi.setParts(isel, parts_len);
10355	for (parts[0..parts_len]) |part| {
10356	const subpart_vi = vi.addPart(isel, part.offset - offset, part.size);
10357	if (part.signedness) |signedness| subpart_vi.setSignedness(isel, signedness);
10358	if (part.is_vector) subpart_vi.setIsVector(isel);
10359	}
10360	},
10361	.simple_type => |simple_type| switch (simple_type) {
10362	.f16, .f32, .f64, .f128, .c_longdouble => return isel.fail("Value.FieldPartIterator.next({f})", .{isel.fmtType(ty)}),
10363	.f80 => continue :type_key .{ .int_type = .{ .signedness = .unsigned, .bits = 80 } },
10364	.usize,
10365	.isize,
10366	.c_char,
10367	.c_short,
10368	.c_ushort,
10369	.c_int,
10370	.c_uint,
10371	.c_long,
10372	.c_ulong,
10373	.c_longlong,
10374	.c_ulonglong,
10375	=> continue :type_key .{ .int_type = ty.intInfo(zcu) },
10376	.anyopaque,
10377	.void,
10378	.type,
10379	.comptime_int,
10380	.comptime_float,
10381	.noreturn,
10382	.null,
10383	.undefined,
10384	.enum_literal,
10385	.adhoc_inferred_error_set,
10386	.generic_poison,
10387	=> unreachable,
10388	.bool => continue :type_key .{ .int_type = .{ .signedness = .unsigned, .bits = 1 } },
10389	.anyerror => continue :type_key .{ .int_type = .{
10390	.signedness = .unsigned,
10391	.bits = zcu.errorSetBits(),
10392	} },
10393	},
10394	.struct_type => {
10395	const loaded_struct = ip.loadStructType(ty.toIntern());
10396	switch (loaded_struct.layout) {
10397	.auto, .@"extern" => {},
10398	.@"packed" => continue :type_key .{
10399	.int_type = ip.indexToKey(loaded_struct.packed_backing_int_type).int_type,
10400	},
10401	}
10402	const min_part_log2_stride: u5 = if (size > 16) 4 else if (size > 8) 3 else 0;
10403	if (loaded_struct.field_types.len > Value.max_parts and
10404	(@divCeil(size, @as(u64, 1) << min_part_log2_stride)) > Value.max_parts)
10405	return isel.fail("Value.FieldPartIterator.next({f})", .{isel.fmtType(ty)});
10406	const alignment = vi.alignment(isel);
10407	const Part = struct { offset: u64, size: u64, signedness: ?std.lang.Signedness, is_vector: bool };
10408	var parts: [Value.max_parts]Part = undefined;
10409	var parts_len: Value.PartsLen = 0;
10410	var field_end: u64 = 0;
10411	var field_it = loaded_struct.iterateRuntimeOrder(ip);
10412	while (field_it.next()) |field_index| {
10413	const field_ty: ZigType = .fromInterned(loaded_struct.field_types.get(ip)[field_index]);
10414	const field_begin = switch (loaded_struct.field_aligns.getOrNone(ip, field_index)) {
10415	.none => field_ty.abiAlignment(zcu),
10416	else => |field_align| field_align,
10417	}.forward(field_end);
10418	if (field_begin >= offset + size) break;
10419	const field_size = field_ty.abiSize(zcu);
10420	field_end = field_begin + field_size;
10421	if (field_end <= offset) continue;
10422	if (offset >= field_begin and offset + size <= field_begin + field_size) {
10423	ty = field_ty;
10424	ty_size = field_size;
10425	offset -= field_begin;
10426	continue :type_key ip.indexToKey(field_ty.toIntern());
10427	}
10428	const field_signedness = if (field_ty.isAbiInt(zcu)) field_signedness: {
10429	const field_int_info = field_ty.intInfo(zcu);
10430	break :field_signedness if (field_int_info.bits <= 16) field_int_info.signedness else null;
10431	} else null;
10432	const field_is_vector = field_size <= 16 and
10433	CallAbiIterator.homogeneousAggregateBaseType(zcu, field_ty.toIntern()) != null;
10434	if (parts_len > 0) combine: {
10435	const prev_part = &parts[parts_len - 1];
10436	const combined_size = field_end - prev_part.offset;
10437	if (combined_size > @as(u64, 1) << @min(
10438	min_part_log2_stride,
10439	alignment.toLog2Units(),
10440	@ctz(prev_part.offset),
10441	)) break :combine;
10442	prev_part.size = combined_size;
10443	prev_part.signedness = null;
10444	prev_part.is_vector &= field_is_vector;
10445	continue;
10446	}
10447	parts[parts_len] = .{
10448	.offset = field_begin,
10449	.size = field_size,
10450	.signedness = field_signedness,
10451	.is_vector = field_is_vector,
10452	};
10453	parts_len += 1;
10454	}
10455	vi.setParts(isel, parts_len);
10456	for (parts[0..parts_len]) |part| {
10457	const subpart_vi = vi.addPart(isel, part.offset - offset, part.size);
10458	if (part.signedness) |signedness| subpart_vi.setSignedness(isel, signedness);
10459	if (part.is_vector) subpart_vi.setIsVector(isel);
10460	}
10461	},
10462	.tuple_type => |tuple_type| {
10463	const min_part_log2_stride: u5 = if (size > 16) 4 else if (size > 8) 3 else 0;
10464	if (tuple_type.types.len > Value.max_parts and
10465	(@divCeil(size, @as(u64, 1) << min_part_log2_stride)) > Value.max_parts)
10466	return isel.fail("Value.FieldPartIterator.next({f})", .{isel.fmtType(ty)});
10467	const alignment = vi.alignment(isel);
10468	const Part = struct { offset: u64, size: u64, is_vector: bool };
10469	var parts: [Value.max_parts]Part = undefined;
10470	var parts_len: Value.PartsLen = 0;
10471	var field_end: u64 = 0;
10472	for (tuple_type.types.get(ip), tuple_type.values.get(ip)) |field_type, field_value| {
10473	if (field_value != .none) continue;
10474	const field_ty: ZigType = .fromInterned(field_type);
10475	const field_begin = field_ty.abiAlignment(zcu).forward(field_end);
10476	if (field_begin >= offset + size) break;
10477	const field_size = field_ty.abiSize(zcu);
10478	if (field_size == 0) continue;
10479	field_end = field_begin + field_size;
10480	if (field_end <= offset) continue;
10481	if (offset >= field_begin and offset + size <= field_begin + field_size) {
10482	ty = field_ty;
10483	ty_size = field_size;
10484	offset -= field_begin;
10485	continue :type_key ip.indexToKey(field_ty.toIntern());
10486	}
10487	const field_is_vector = field_size <= 16 and
10488	CallAbiIterator.homogeneousAggregateBaseType(zcu, field_ty.toIntern()) != null;
10489	if (parts_len > 0) combine: {
10490	const prev_part = &parts[parts_len - 1];
10491	const combined_size = field_end - prev_part.offset;
10492	if (combined_size > @as(u64, 1) << @min(
10493	min_part_log2_stride,
10494	alignment.toLog2Units(),
10495	@ctz(prev_part.offset),
10496	)) break :combine;
10497	prev_part.size = combined_size;
10498	prev_part.is_vector &= field_is_vector;
10499	continue;
10500	}
10501	parts[parts_len] = .{ .offset = field_begin, .size = field_size, .is_vector = field_is_vector };
10502	parts_len += 1;
10503	}
10504	vi.setParts(isel, parts_len);
10505	for (parts[0..parts_len]) |part| {
10506	const subpart_vi = vi.addPart(isel, part.offset - offset, part.size);
10507	if (part.is_vector) subpart_vi.setIsVector(isel);
10508	}
10509	},
10510	.union_type => {
10511	const loaded_union = ip.loadUnionType(ty.toIntern());
10512	switch (loaded_union.layout) {
10513	.auto, .@"extern" => {},
10514	.@"packed" => continue :type_key .{ .int_type = .{
10515	.signedness = .unsigned,
10516	.bits = @intCast(ty.bitSize(zcu)),
10517	} },
10518	}
10519	const min_part_log2_stride: u5 = if (size > 16) 4 else if (size > 8) 3 else 0;
10520	if ((@divCeil(size, @as(u64, 1) << min_part_log2_stride)) > Value.max_parts)
10521	return isel.fail("Value.FieldPartIterator.next({f})", .{isel.fmtType(ty)});
10522	const union_layout = ZigType.getUnionLayout(loaded_union, zcu);
10523	const alignment = vi.alignment(isel);
10524	const tag_offset = union_layout.tagOffset();
10525	const payload_offset = union_layout.payloadOffset();
10526	const Part = struct { offset: u64, size: u64, signedness: ?std.lang.Signedness };
10527	var parts: [2]Part = undefined;
10528	var parts_len: Value.PartsLen = 0;
10529	var field_end: u64 = 0;
10530	for (0..2) |field_index| {
10531	const field: enum { tag, payload } = switch (field_index) {
10532	0 => if (tag_offset < payload_offset) .tag else .payload,
10533	1 => if (tag_offset < payload_offset) .payload else .tag,
10534	else => unreachable,
10535	};
10536	const field_size, const field_begin = switch (field) {
10537	.tag => .{ union_layout.tag_size, tag_offset },
10538	.payload => .{ union_layout.payload_size, payload_offset },
10539	};
10540	if (field_begin >= offset + size) break;
10541	if (field_size == 0) continue;
10542	field_end = field_begin + field_size;
10543	if (field_end <= offset) continue;
10544	const field_signedness = field_signedness: switch (field) {
10545	.tag => {
10546	if (offset >= field_begin and offset + size <= field_begin + field_size) {
10547	ty = .fromInterned(loaded_union.enum_tag_type);
10548	ty_size = field_size;
10549	offset -= field_begin;
10550	continue :type_key ip.indexToKey(loaded_union.enum_tag_type);
10551	}
10552	const loaded_enum = ip.loadEnumType(loaded_union.enum_tag_type);
10553	break :field_signedness ip.indexToKey(loaded_enum.int_tag_type).int_type.signedness;
10554	},
10555	.payload => null,
10556	};
10557	if (parts_len > 0) combine: {
10558	const prev_part = &parts[parts_len - 1];
10559	const combined_size = field_end - prev_part.offset;
10560	if (combined_size > @as(u64, 1) << @min(
10561	min_part_log2_stride,
10562	alignment.toLog2Units(),
10563	@ctz(prev_part.offset),
10564	)) break :combine;
10565	prev_part.size = combined_size;
10566	prev_part.signedness = null;
10567	continue;
10568	}
10569	parts[parts_len] = .{
10570	.offset = field_begin,
10571	.size = field_size,
10572	.signedness = field_signedness,
10573	};
10574	parts_len += 1;
10575	}
10576	vi.setParts(isel, parts_len);
10577	for (parts[0..parts_len]) |part| {
10578	const subpart_vi = vi.addPart(isel, part.offset - offset, part.size);
10579	if (part.signedness) |signedness| subpart_vi.setSignedness(isel, signedness);
10580	}
10581	},
10582	.opaque_type, .func_type => continue :type_key .{ .simple_type = .anyopaque },
10583	.enum_type => continue :type_key ip.indexToKey(ip.loadEnumType(ty.toIntern()).int_tag_type),
10584	.error_set_type,
10585	.inferred_error_set_type,
10586	=> continue :type_key .{ .simple_type = .anyerror },
10587	.undef,
10588	.simple_value,
10589	.@"extern",
10590	.func,
10591	.int,
10592	.err,
10593	.error_union,
10594	.enum_literal,
10595	.enum_tag,
10596	.float,
10597	.ptr,
10598	.slice,
10599	.opt,
10600	.aggregate,
10601	.un,
10602	.memoized_call,
10603	=> unreachable, // values, not types
10604	}
10605	}
10606	it.next_offset = next_offset + size;
10607	return .{ .offset = next_part_offset - next_offset, .vi = vi };
10608	}
10609
10610	fn only(it: *FieldPartIterator, isel: *Select) !?Value.Index {
10611	const part = try it.next(isel);
10612	assert(part.?.offset == 0);
10613	return if (try it.next(isel)) |_| null else part.?.vi;
10614	}
10615	};
10616
10617	const Materialize = struct {
10618	vi: Value.Index,
10619	ra: Register.Alias,
10620
10621	fn finish(mat: Value.Materialize, isel: *Select) codegen.Error!void {
10622	const live_vi = isel.live_registers.getPtr(mat.ra);
10623	assert(live_vi.* == .allocating);
10624	var vi = mat.vi;
10625	var offset: u64 = 0;
10626	const size = mat.vi.size(isel);
10627	free: while (true) {
10628	if (vi.register(isel)) |ra| {
10629	if (ra != mat.ra) break :free try isel.emit(if (vi == mat.vi) if (mat.ra.isVector()) switch (size) {
10630	else => unreachable,
10631	2 => if (isel.target.cpu.has(.aarch64, .fullfp16))
10632	.fmov(mat.ra.h(), .{ .register = ra.h() })
10633	else
10634	.dup(mat.ra.h(), ra.@"h[]"(0)),
10635	4 => .fmov(mat.ra.s(), .{ .register = ra.s() }),
10636	8 => .fmov(mat.ra.d(), .{ .register = ra.d() }),
10637	16 => .orr(mat.ra.@"16b"(), ra.@"16b"(), .{ .register = ra.@"16b"() }),
10638	} else switch (size) {
10639	else => unreachable,
10640	1...4 => .orr(mat.ra.w(), .wzr, .{ .register = ra.w() }),
10641	5...8 => .orr(mat.ra.x(), .xzr, .{ .register = ra.x() }),
10642	} else switch (offset + size) {
10643	else => unreachable,
10644	1...4 => |end_offset| switch (mat.vi.signedness(isel)) {
10645	.signed => .sbfm(mat.ra.w(), ra.w(), .{
10646	.N = .word,
10647	.immr = @intCast(8 * offset),
10648	.imms = @intCast(8 * end_offset - 1),
10649	}),
10650	.unsigned => .ubfm(mat.ra.w(), ra.w(), .{
10651	.N = .word,
10652	.immr = @intCast(8 * offset),
10653	.imms = @intCast(8 * end_offset - 1),
10654	}),
10655	},
10656	5...8 => |end_offset| switch (mat.vi.signedness(isel)) {
10657	.signed => .sbfm(mat.ra.x(), ra.x(), .{
10658	.N = .doubleword,
10659	.immr = @intCast(8 * offset),
10660	.imms = @intCast(8 * end_offset - 1),
10661	}),
10662	.unsigned => .ubfm(mat.ra.x(), ra.x(), .{
10663	.N = .doubleword,
10664	.immr = @intCast(8 * offset),
10665	.imms = @intCast(8 * end_offset - 1),
10666	}),
10667	},
10668	});
10669	mat.vi.get(isel).location_payload.small.register = mat.ra;
10670	live_vi.* = mat.vi;
10671	return;
10672	}
10673	offset += vi.get(isel).offset_from_parent;
10674	switch (vi.parent(isel)) {
10675	.unallocated => {
10676	mat.vi.get(isel).location_payload.small.register = mat.ra;
10677	live_vi.* = mat.vi;
10678	return;
10679	},
10680	.stack_slot => |stack_slot| break :free try isel.loadReg(
10681	mat.ra,
10682	size,
10683	mat.vi.signedness(isel),
10684	stack_slot.base,
10685	@as(i65, stack_slot.offset) + offset,
10686	),
10687	.address => |base_vi| {
10688	const base_mat = try base_vi.matReg(isel);
10689	try isel.loadReg(mat.ra, size, mat.vi.signedness(isel), base_mat.ra, offset);
10690	break :free try base_mat.finish(isel);
10691	},
10692	.value => |parent_vi| vi = parent_vi,
10693	.constant => |initial_constant| {
10694	const zcu = isel.pt.zcu;
10695	const ip = &zcu.intern_pool;
10696	var constant = initial_constant.toIntern();
10697	var constant_key = ip.indexToKey(constant);
10698	while (true) {
10699	constant_key: switch (constant_key) {
10700	.int_type,
10701	.ptr_type,
10702	.array_type,
10703	.vector_type,
10704	.opt_type,
10705	.anyframe_type,
10706	.error_union_type,
10707	.simple_type,
10708	.struct_type,
10709	.tuple_type,
10710	.union_type,
10711	.opaque_type,
10712	.enum_type,
10713	.func_type,
10714	.error_set_type,
10715	.inferred_error_set_type,
10716
10717	.enum_literal,
10718	.memoized_call,
10719	=> unreachable, // not a runtime value
10720	.undef => break :free try isel.emit(if (mat.ra.isVector()) .movi(switch (size) {
10721	else => unreachable,
10722	1...8 => mat.ra.@"8b"(),
10723	9...16 => mat.ra.@"16b"(),
10724	}, 0xaa, .{ .lsl = 0 }) else switch (size) {
10725	else => unreachable,
10726	1...4 => .orr(mat.ra.w(), .wzr, .{ .immediate = .{
10727	.N = .word,
10728	.immr = 0b000001,
10729	.imms = 0b111100,
10730	} }),
10731	5...8 => .orr(mat.ra.x(), .xzr, .{ .immediate = .{
10732	.N = .word,
10733	.immr = 0b000001,
10734	.imms = 0b111100,
10735	} }),
10736	}),
10737	.simple_value => |simple_value| switch (simple_value) {
10738	.void, .null, .@"unreachable" => unreachable,
10739	.true => continue :constant_key .{ .int = .{
10740	.ty = .bool_type,
10741	.storage = .{ .u64 = 1 },
10742	} },
10743	.false => continue :constant_key .{ .int = .{
10744	.ty = .bool_type,
10745	.storage = .{ .u64 = 0 },
10746	} },
10747	},
10748	.int => |int| break :free switch (int.storage) {
10749	.u64 => |imm| try isel.movImmediate(switch (size) {
10750	else => unreachable,
10751	1...4 => mat.ra.w(),
10752	5...8 => mat.ra.x(),
10753	}, @bitCast(std.math.shr(u64, imm, 8 * offset))),
10754	.i64 => |imm| switch (size) {
10755	else => unreachable,
10756	1...4 => try isel.movImmediate(mat.ra.w(), @as(u32, @bitCast(@as(i32, @truncate(std.math.shr(i64, imm, 8 * offset)))))),
10757	5...8 => try isel.movImmediate(mat.ra.x(), @bitCast(std.math.shr(i64, imm, 8 * offset))),
10758	},
10759	.big_int => |big_int| {
10760	assert(size == 8);
10761	var imm: u64 = 0;
10762	const limb_bits = @bitSizeOf(std.math.big.Limb);
10763	const limbs = @divExact(64, limb_bits);
10764	var limb_index: usize = @intCast(@divExact(offset, @divExact(limb_bits, 8)) + limbs);
10765	for (0..limbs) |_| {
10766	limb_index -= 1;
10767	if (limb_index >= big_int.limbs.len) continue;
10768	if (limb_bits < 64) imm <<= limb_bits;
10769	imm |= big_int.limbs[limb_index];
10770	}
10771	if (!big_int.positive) {
10772	limb_index = @min(limb_index, big_int.limbs.len);
10773	imm = while (limb_index > 0) {
10774	limb_index -= 1;
10775	if (big_int.limbs[limb_index] != 0) break ~imm;
10776	} else -%imm;
10777	}
10778	try isel.movImmediate(mat.ra.x(), imm);
10779	},
10780	},
10781	.err => |err| continue :constant_key .{ .int = .{
10782	.ty = err.ty,
10783	.storage = .{ .u64 = ip.getErrorValueIfExists(err.name).? },
10784	} },
10785	.error_union => |error_union| {
10786	const error_union_type = ip.indexToKey(error_union.ty).error_union_type;
10787	const error_set_ty: ZigType = .fromInterned(error_union_type.error_set_type);
10788	const payload_ty: ZigType = .fromInterned(error_union_type.payload_type);
10789	const error_set_offset = codegen.errUnionErrorOffset(payload_ty, zcu);
10790	const error_set_size = error_set_ty.abiSize(zcu);
10791	if (offset >= error_set_offset and offset + size <= error_set_offset + error_set_size) {
10792	offset -= error_set_offset;
10793	continue :constant_key switch (error_union.val) {
10794	.err_name => |err_name| .{ .err = .{
10795	.ty = error_union_type.error_set_type,
10796	.name = err_name,
10797	} },
10798	.payload => .{ .int = .{
10799	.ty = error_union_type.error_set_type,
10800	.storage = .{ .u64 = 0 },
10801	} },
10802	};
10803	}
10804	const payload_offset = codegen.errUnionPayloadOffset(payload_ty, zcu);
10805	const payload_size = payload_ty.abiSize(zcu);
10806	if (offset >= payload_offset and offset + size <= payload_offset + payload_size) {
10807	offset -= payload_offset;
10808	switch (error_union.val) {
10809	.err_name => continue :constant_key .{ .undef = error_union_type.payload_type },
10810	.payload => |payload| {
10811	constant = payload;
10812	constant_key = ip.indexToKey(constant);
10813	continue :constant_key constant_key;
10814	},
10815	}
10816	}
10817	},
10818	.enum_tag => |enum_tag| continue :constant_key .{ .int = ip.indexToKey(enum_tag.int).int },
10819	.float => |float| storage: switch (float.storage) {
10820	.f16 => |imm| {
10821	if (!mat.ra.isVector()) continue :constant_key .{ .int = .{
10822	.ty = .u16_type,
10823	.storage = .{ .u64 = @as(u16, @bitCast(imm)) },
10824	} };
10825	const feat_fp16 = isel.target.cpu.has(.aarch64, .fullfp16);
10826	if (feat_fp16) {
10827	const Repr = std.math.FloatRepr(f16);
10828	const repr: Repr = @bitCast(imm);
10829	if (repr.mantissa & std.math.maxInt(Repr.Mantissa) >> 5 == 0 and switch (repr.exponent) {
10830	.denormal, .infinite => false,
10831	else => std.math.cast(i3, repr.exponent.unbias() - 1) != null,
10832	}) break :free try isel.emit(.fmov(mat.ra.h(), .{ .immediate = imm }));
10833	}
10834	const bits: u16 = @bitCast(imm);
10835	if (bits == 0) break :free try isel.emit(.movi(mat.ra.d(), 0b00000000, .replicate));
10836	if (bits & std.math.maxInt(u8) == 0) break :free try isel.emit(.movi(
10837	mat.ra.@"4h"(),
10838	@intCast(@shrExact(bits, 8)),
10839	.{ .lsl = 8 },
10840	));
10841	const temp_ra = try isel.allocIntReg();
10842	defer isel.freeReg(temp_ra);
10843	try isel.emit(.fmov(if (feat_fp16) mat.ra.h() else mat.ra.s(), .{ .register = temp_ra.w() }));
10844	break :free try isel.movImmediate(temp_ra.w(), bits);
10845	},
10846	.f32 => |imm| {
10847	if (!mat.ra.isVector()) continue :constant_key .{ .int = .{
10848	.ty = .u32_type,
10849	.storage = .{ .u64 = @as(u32, @bitCast(imm)) },
10850	} };
10851	const Repr = std.math.FloatRepr(f32);
10852	const repr: Repr = @bitCast(imm);
10853	if (repr.mantissa & std.math.maxInt(Repr.Mantissa) >> 5 == 0 and switch (repr.exponent) {
10854	.denormal, .infinite => false,
10855	else => std.math.cast(i3, repr.exponent.unbias() - 1) != null,
10856	}) break :free try isel.emit(.fmov(mat.ra.s(), .{ .immediate = @floatCast(imm) }));
10857	const bits: u32 = @bitCast(imm);
10858	if (bits == 0) break :free try isel.emit(.movi(mat.ra.d(), 0b00000000, .replicate));
10859	if (bits & std.math.maxInt(u24) == 0) break :free try isel.emit(.movi(
10860	mat.ra.@"2s"(),
10861	@intCast(@shrExact(bits, 24)),
10862	.{ .lsl = 24 },
10863	));
10864	const temp_ra = try isel.allocIntReg();
10865	defer isel.freeReg(temp_ra);
10866	try isel.emit(.fmov(mat.ra.s(), .{ .register = temp_ra.w() }));
10867	break :free try isel.movImmediate(temp_ra.w(), bits);
10868	},
10869	.f64 => |imm| {
10870	if (!mat.ra.isVector()) continue :constant_key .{ .int = .{
10871	.ty = .u64_type,
10872	.storage = .{ .u64 = @as(u64, @bitCast(imm)) },
10873	} };
10874	const Repr = std.math.FloatRepr(f64);
10875	const repr: Repr = @bitCast(imm);
10876	if (repr.mantissa & std.math.maxInt(Repr.Mantissa) >> 5 == 0 and switch (repr.exponent) {
10877	.denormal, .infinite => false,
10878	else => std.math.cast(i3, repr.exponent.unbias() - 1) != null,
10879	}) break :free try isel.emit(.fmov(mat.ra.d(), .{ .immediate = @floatCast(imm) }));
10880	const bits: u64 = @bitCast(imm);
10881	if (bits == 0) break :free try isel.emit(.movi(mat.ra.d(), 0b00000000, .replicate));
10882	const temp_ra = try isel.allocIntReg();
10883	defer isel.freeReg(temp_ra);
10884	try isel.emit(.fmov(mat.ra.d(), .{ .register = temp_ra.x() }));
10885	break :free try isel.movImmediate(temp_ra.x(), bits);
10886	},
10887	.f80 => |imm| break :free try isel.movImmediate(
10888	mat.ra.x(),
10889	@truncate(std.math.shr(u80, @bitCast(imm), 8 * offset)),
10890	),
10891	.f128 => |imm| switch (ZigType.fromInterned(float.ty).floatBits(isel.target)) {
10892	else => unreachable,
10893	16 => continue :storage .{ .f16 = @floatCast(imm) },
10894	32 => continue :storage .{ .f32 = @floatCast(imm) },
10895	64 => continue :storage .{ .f64 = @floatCast(imm) },
10896	128 => {
10897	const bits: u128 = @bitCast(imm);
10898	const hi64: u64 = @intCast(bits >> 64);
10899	const lo64: u64 = @truncate(bits >> 0);
10900	const temp_ra = try isel.allocIntReg();
10901	defer isel.freeReg(temp_ra);
10902	switch (hi64) {
10903	0 => {},
10904	else => {
10905	try isel.emit(.fmov(mat.ra.@"d[]"(1), .{ .register = temp_ra.x() }));
10906	try isel.movImmediate(temp_ra.x(), hi64);
10907	},
10908	}
10909	break :free switch (lo64) {
10910	0 => try isel.emit(.movi(switch (hi64) {
10911	else => mat.ra.d(),
10912	0 => mat.ra.@"2d"(),
10913	}, 0b00000000, .replicate)),
10914	else => {
10915	try isel.emit(.fmov(mat.ra.d(), .{ .register = temp_ra.x() }));
10916	try isel.movImmediate(temp_ra.x(), lo64);
10917	},
10918	};
10919	},
10920	},
10921	},
10922	.ptr => |ptr| {
10923	assert(offset == 0 and size == 8);
10924	break :free switch (ptr.base_addr) {
10925	.nav => |nav| if (ZigType.fromInterned(ip.getNav(nav).resolved.?.type).isRuntimeFnOrHasRuntimeBits(zcu)) switch (true) {
10926	false => {
10927	try isel.nav_relocs.append(zcu.gpa, .{
10928	.nav = nav,
10929	.reloc = .{
10930	.label = @intCast(isel.instructions.items.len),
10931	.addend = ptr.byte_offset,
10932	},
10933	});
10934	try isel.emit(.adr(mat.ra.x(), 0));
10935	},
10936	true => {
10937	try isel.nav_relocs.append(zcu.gpa, .{
10938	.nav = nav,
10939	.reloc = .{
10940	.label = @intCast(isel.instructions.items.len),
10941	.addend = ptr.byte_offset,
10942	},
10943	});
10944	if (ip.getNav(nav).getExtern(ip)) |_|
10945	try isel.emit(.ldr(mat.ra.x(), .{ .unsigned_offset = .{ .base = mat.ra.x(), .offset = 0 } }))
10946	else
10947	try isel.emit(.add(mat.ra.x(), mat.ra.x(), .{ .immediate = 0 }));
10948	try isel.nav_relocs.append(zcu.gpa, .{
10949	.nav = nav,
10950	.reloc = .{
10951	.label = @intCast(isel.instructions.items.len),
10952	.addend = ptr.byte_offset,
10953	},
10954	});
10955	try isel.emit(.adrp(mat.ra.x(), 0));
10956	},
10957	} else continue :constant_key .{ .int = .{
10958	.ty = .usize_type,
10959	.storage = .{ .u64 = zcu.navAlignment(nav).forward(0xaaaaaaaaaaaaaaaa) },
10960	} },
10961	.uav => |uav| if (ZigType.fromInterned(ip.typeOf(uav.val)).isRuntimeFnOrHasRuntimeBits(zcu)) switch (true) {
10962	false => {
10963	try isel.uav_relocs.append(zcu.gpa, .{
10964	.uav = uav,
10965	.reloc = .{
10966	.label = @intCast(isel.instructions.items.len),
10967	.addend = ptr.byte_offset,
10968	},
10969	});
10970	try isel.emit(.adr(mat.ra.x(), 0));
10971	},
10972	true => {
10973	try isel.uav_relocs.append(zcu.gpa, .{
10974	.uav = uav,
10975	.reloc = .{
10976	.label = @intCast(isel.instructions.items.len),
10977	.addend = ptr.byte_offset,
10978	},
10979	});
10980	try isel.emit(.add(mat.ra.x(), mat.ra.x(), .{ .immediate = 0 }));
10981	try isel.uav_relocs.append(zcu.gpa, .{
10982	.uav = uav,
10983	.reloc = .{
10984	.label = @intCast(isel.instructions.items.len),
10985	.addend = ptr.byte_offset,
10986	},
10987	});
10988	try isel.emit(.adrp(mat.ra.x(), 0));
10989	},
10990	} else continue :constant_key .{ .int = .{
10991	.ty = .usize_type,
10992	.storage = .{ .u64 = ZigType.fromInterned(uav.orig_ty).ptrAlignment(zcu).forward(0xaaaaaaaaaaaaaaaa) },
10993	} },
10994	.int => continue :constant_key .{ .int = .{
10995	.ty = .usize_type,
10996	.storage = .{ .u64 = ptr.byte_offset },
10997	} },
10998	.eu_payload => |base| {
10999	var base_ptr = ip.indexToKey(base).ptr;
11000	const eu_ty = ip.indexToKey(base_ptr.ty).ptr_type.child;
11001	const payload_ty = ip.indexToKey(eu_ty).error_union_type.payload_type;
11002	base_ptr.byte_offset += codegen.errUnionPayloadOffset(.fromInterned(payload_ty), zcu) + ptr.byte_offset;
11003	continue :constant_key .{ .ptr = base_ptr };
11004	},
11005	.opt_payload => |base| {
11006	var base_ptr = ip.indexToKey(base).ptr;
11007	base_ptr.byte_offset += ptr.byte_offset;
11008	continue :constant_key .{ .ptr = base_ptr };
11009	},
11010	.field => |field| {
11011	var base_ptr = ip.indexToKey(field.base).ptr;
11012	const agg_ty: ZigType = .fromInterned(ip.indexToKey(base_ptr.ty).ptr_type.child);
11013	base_ptr.byte_offset += agg_ty.structFieldOffset(@intCast(field.index), zcu) + ptr.byte_offset;
11014	continue :constant_key .{ .ptr = base_ptr };
11015	},
11016	.comptime_alloc, .comptime_field, .arr_elem => unreachable,
11017	};
11018	},
11019	.slice => |slice| switch (offset) {
11020	0 => continue :constant_key switch (ip.indexToKey(slice.ptr)) {
11021	else => unreachable,
11022	.undef => |undef| .{ .undef = undef },
11023	.ptr => |ptr| .{ .ptr = ptr },
11024	},
11025	else => {
11026	assert(offset == @divExact(isel.target.ptrBitWidth(), 8));
11027	offset = 0;
11028	continue :constant_key .{ .int = ip.indexToKey(slice.len).int };
11029	},
11030	},
11031	.opt => |opt| {
11032	const child_ty = ip.indexToKey(opt.ty).opt_type;
11033	const child_size = ZigType.fromInterned(child_ty).abiSize(zcu);
11034	if (offset == child_size and size == 1) {
11035	offset = 0;
11036	continue :constant_key .{ .simple_value = switch (opt.val) {
11037	.none => .false,
11038	else => .true,
11039	} };
11040	}
11041	const opt_ty: ZigType = .fromInterned(opt.ty);
11042	if (offset + size <= child_size) continue :constant_key switch (opt.val) {
11043	.none => if (opt_ty.optionalReprIsPayload(zcu)) .{ .int = .{
11044	.ty = opt.ty,
11045	.storage = .{ .u64 = 0 },
11046	} } else .{ .undef = child_ty },
11047	else => |child| {
11048	constant = child;
11049	constant_key = ip.indexToKey(constant);
11050	continue :constant_key constant_key;
11051	},
11052	};
11053	},
11054	.aggregate => |aggregate| switch (ip.indexToKey(aggregate.ty)) {
11055	else => unreachable,
11056	.array_type => |array_type| {
11057	const elem_size = ZigType.fromInterned(array_type.child).abiSize(zcu);
11058	const elem_offset = @mod(offset, elem_size);
11059	if (size <= elem_size - elem_offset) {
11060	defer offset = elem_offset;
11061	continue :constant_key switch (aggregate.storage) {
11062	.bytes => |bytes| .{ .int = .{ .ty = .u8_type, .storage = .{
11063	.u64 = bytes.toSlice(array_type.lenIncludingSentinel(), ip)[@intCast(@divFloor(offset, elem_size))],
11064	} } },
11065	.elems => |elems| {
11066	constant = elems[@intCast(@divFloor(offset, elem_size))];
11067	constant_key = ip.indexToKey(constant);
11068	continue :constant_key constant_key;
11069	},
11070	.repeated_elem => |repeated_elem| {
11071	constant = repeated_elem;
11072	constant_key = ip.indexToKey(constant);
11073	continue :constant_key constant_key;
11074	},
11075	};
11076	}
11077	},
11078	.vector_type => {},
11079	.struct_type => {
11080	const loaded_struct = ip.loadStructType(aggregate.ty);
11081	switch (loaded_struct.layout) {
11082	.auto => {
11083	var field_offset: u64 = 0;
11084	var field_it = loaded_struct.iterateRuntimeOrder(ip);
11085	while (field_it.next()) |field_index| {
11086	if (loaded_struct.field_is_comptime_bits.get(ip, field_index)) continue;
11087	const field_ty: ZigType = .fromInterned(loaded_struct.field_types.get(ip)[field_index]);
11088	field_offset = loaded_struct.field_offsets.get(ip)[field_index];
11089	const field_size = field_ty.abiSize(zcu);
11090	if (offset >= field_offset and offset + size <= field_offset + field_size) {
11091	offset -= field_offset;
11092	constant = switch (aggregate.storage) {
11093	.bytes => unreachable,
11094	.elems => |elems| elems[field_index],
11095	.repeated_elem => |repeated_elem| repeated_elem,
11096	};
11097	constant_key = ip.indexToKey(constant);
11098	continue :constant_key constant_key;
11099	}
11100	field_offset += field_size;
11101	}
11102	},
11103	.@"extern", .@"packed" => {},
11104	}
11105	},
11106	.tuple_type => |tuple_type| {
11107	var field_offset: u64 = 0;
11108	for (tuple_type.types.get(ip), tuple_type.values.get(ip), 0..) |field_type, field_value, field_index| {
11109	if (field_value != .none) continue;
11110	const field_ty: ZigType = .fromInterned(field_type);
11111	field_offset = field_ty.abiAlignment(zcu).forward(field_offset);
11112	const field_size = field_ty.abiSize(zcu);
11113	if (offset >= field_offset and offset + size <= field_offset + field_size) {
11114	offset -= field_offset;
11115	constant = switch (aggregate.storage) {
11116	.bytes => unreachable,
11117	.elems => |elems| elems[field_index],
11118	.repeated_elem => |repeated_elem| repeated_elem,
11119	};
11120	constant_key = ip.indexToKey(constant);
11121	continue :constant_key constant_key;
11122	}
11123	field_offset += field_size;
11124	}
11125	},
11126	},
11127	.un => |un| {
11128	const loaded_union = ip.loadUnionType(un.ty);
11129	const union_layout = ZigType.getUnionLayout(loaded_union, zcu);
11130	if (loaded_union.has_runtime_tag) {
11131	const tag_offset = union_layout.tagOffset();
11132	if (offset >= tag_offset and offset + size <= tag_offset + union_layout.tag_size) {
11133	offset -= tag_offset;
11134	continue :constant_key switch (ip.indexToKey(un.tag)) {
11135	else => unreachable,
11136	.int => |int| .{ .int = int },
11137	.enum_tag => |enum_tag| .{ .enum_tag = enum_tag },
11138	};
11139	}
11140	}
11141	const payload_offset = union_layout.payloadOffset();
11142	if (offset >= payload_offset and offset + size <= payload_offset + union_layout.payload_size) {
11143	offset -= payload_offset;
11144	constant = un.val;
11145	constant_key = ip.indexToKey(constant);
11146	continue :constant_key constant_key;
11147	}
11148	},
11149	else => {},
11150	}
11151	var buffer: [16]u8 = @splat(0);
11152	if (ZigType.fromInterned(constant_key.typeOf()).abiSize(zcu) <= buffer.len and
11153	try isel.writeToMemory(.fromInterned(constant), &buffer))
11154	{
11155	constant_key = if (mat.ra.isVector()) .{ .float = switch (size) {
11156	else => unreachable,
11157	2 => .{ .ty = .f16_type, .storage = .{ .f16 = @bitCast(std.mem.readInt(
11158	u16,
11159	buffer[@intCast(offset)..][0..2],
11160	isel.target.cpu.arch.endian(),
11161	)) } },
11162	4 => .{ .ty = .f32_type, .storage = .{ .f32 = @bitCast(std.mem.readInt(
11163	u32,
11164	buffer[@intCast(offset)..][0..4],
11165	isel.target.cpu.arch.endian(),
11166	)) } },
11167	8 => .{ .ty = .f64_type, .storage = .{ .f64 = @bitCast(std.mem.readInt(
11168	u64,
11169	buffer[@intCast(offset)..][0..8],
11170	isel.target.cpu.arch.endian(),
11171	)) } },
11172	16 => .{ .ty = .f128_type, .storage = .{ .f128 = @bitCast(std.mem.readInt(
11173	u128,
11174	buffer[@intCast(offset)..][0..16],
11175	isel.target.cpu.arch.endian(),
11176	)) } },
11177	} } else .{ .int = .{
11178	.ty = .u64_type,
11179	.storage = .{ .u64 = switch (size) {
11180	else => unreachable,
11181	inline 1...8 => |ct_size| std.mem.readInt(
11182	@Int(.unsigned, 8 * ct_size),
11183	buffer[@intCast(offset)..][0..ct_size],
11184	isel.target.cpu.arch.endian(),
11185	),
11186	} },
11187	} };
11188	offset = 0;
11189	continue;
11190	}
11191	return isel.fail("unsupported value <{f}, {f}>", .{
11192	isel.fmtType(.fromInterned(constant_key.typeOf())),
11193	isel.fmtConstant(.fromInterned(constant)),
11194	});
11195	}
11196	},
11197	}
11198	}
11199	live_vi.* = .free;
11200	}
11201	};
11202	};
11203	fn initValue(isel: *Select, ty: ZigType) Value.Index {
11204	const zcu = isel.pt.zcu;
11205	return isel.initValueAdvanced(ty.abiAlignment(zcu), 0, ty.abiSize(zcu));
11206	}
11207	fn initValueAdvanced(
11208	isel: *Select,
11209	parent_alignment: InternPool.Alignment,
11210	offset_from_parent: u64,
11211	size: u64,
11212	) Value.Index {
11213	defer isel.values.addOneAssumeCapacity().* = .{
11214	.refs = 0,
11215	.flags = .{
11216	.alignment = .fromLog2Units(@min(parent_alignment.toLog2Units(), @ctz(offset_from_parent))),
11217	.parent_tag = .unallocated,
11218	.location_tag = if (size > 16) .large else .small,
11219	.parts_len_minus_one = 0,
11220	},
11221	.offset_from_parent = offset_from_parent,
11222	.parent_payload = .{ .unallocated = {} },
11223	.location_payload = if (size > 16) .{ .large = .{
11224	.size = size,
11225	} } else .{ .small = .{
11226	.size = @intCast(size),
11227	.signedness = .unsigned,
11228	.is_vector = false,
11229	.hint = .zr,
11230	.register = .zr,
11231	} },
11232	.parts = undefined,
11233	};
11234	return @fromBackingInt(@intCast(isel.values.items.len));
11235	}
11236	const WhichValues = enum { only_referenced, all };
11237	pub fn dumpValues(isel: *Select, which: WhichValues) void {
11238	dumpValuesInner(isel, which) catch |err| @panic(@errorName(err));
11239	}
11240	fn dumpValuesInner(isel: *Select, which: WhichValues) !void {
11241	const zcu = isel.pt.zcu;
11242	const gpa = zcu.gpa;
11243	const ip = &zcu.intern_pool;
11244	const nav = ip.getNav(isel.nav_index);
11245
11246	const locked_stderr = std.debug.lockStderr(&.{});
11247	defer std.debug.unlockStderr();
11248	const stderr = &locked_stderr.file_writer.interface;
11249
11250	var reverse_live_values: std.array_hash_map.Auto(Value.Index, std.ArrayList(Air.Inst.Index)) = .empty;
11251	defer {
11252	for (reverse_live_values.values()) |*list| list.deinit(gpa);
11253	reverse_live_values.deinit(gpa);
11254	}
11255	{
11256	try reverse_live_values.ensureTotalCapacity(gpa, isel.live_values.count());
11257	var live_val_it = isel.live_values.iterator();
11258	while (live_val_it.next()) |live_val_entry| switch (live_val_entry.value_ptr.*) {
11259	_ => {
11260	const gop = reverse_live_values.getOrPutAssumeCapacity(live_val_entry.value_ptr.*);
11261	if (!gop.found_existing) gop.value_ptr.* = .empty;
11262	try gop.value_ptr.append(gpa, live_val_entry.key_ptr.*);
11263	},
11264	.allocating, .free => unreachable,
11265	};
11266	}
11267
11268	var reverse_live_registers: std.AutoHashMapUnmanaged(Value.Index, Register.Alias) = .empty;
11269	defer reverse_live_registers.deinit(gpa);
11270	{
11271	try reverse_live_registers.ensureTotalCapacity(gpa, @typeInfo(Register.Alias).@"enum".field_names.len);
11272	var live_reg_it = isel.live_registers.iterator();
11273	while (live_reg_it.next()) |live_reg_entry| switch (live_reg_entry.value.*) {
11274	_ => reverse_live_registers.putAssumeCapacityNoClobber(live_reg_entry.value.*, live_reg_entry.key),
11275	.allocating, .free => {},
11276	};
11277	}
11278
11279	var roots: std.array_hash_map.Auto(Value.Index, u32) = .empty;
11280	defer roots.deinit(gpa);
11281	{
11282	try roots.ensureTotalCapacity(gpa, isel.values.items.len);
11283	var vi: Value.Index = @fromBackingInt(@intCast(isel.values.items.len));
11284	while (@backingInt(vi) > 0) {
11285	vi = @fromBackingInt(@intCast(@backingInt(vi) - 1));
11286	if (which == .only_referenced and vi.get(isel).refs == 0) continue;
11287	while (true) switch (vi.parent(isel)) {
11288	.unallocated, .stack_slot, .constant => break,
11289	.value => |parent_vi| vi = parent_vi,
11290	.address => |address_vi| break roots.putAssumeCapacity(address_vi, 0),
11291	};
11292	roots.putAssumeCapacity(vi, 0);
11293	}
11294	}
11295
11296	try stderr.print("# Begin {s} Value Dump: {f}:\n", .{ @typeName(Select), nav.fqn.fmt(ip) });
11297	while (roots.pop()) |root_entry| {
11298	const vi = root_entry.key;
11299	const value = vi.get(isel);
11300	try stderr.splatByteAll(' ', 2 * (@as(usize, 1) + root_entry.value));
11301	try stderr.print("${d}", .{@backingInt(vi)});
11302	{
11303	var first = true;
11304	if (reverse_live_values.get(vi)) |aiis| for (aiis.items) |aii| {
11305	if (aii == Block.main) {
11306	try stderr.print("{s}%main", .{if (first) " <- " else ", "});
11307	} else {
11308	try stderr.print("{s}%{d}", .{ if (first) " <- " else ", ", @backingInt(aii) });
11309	}
11310	first = false;
11311	};
11312	if (reverse_live_registers.get(vi)) |ra| {
11313	try stderr.print("{s}{t}", .{ if (first) " <- " else ", ", ra });
11314	first = false;
11315	}
11316	}
11317	try stderr.writeByte(':');
11318	switch (value.flags.parent_tag) {
11319	.unallocated => if (value.offset_from_parent != 0) try stderr.print(" +0x{x}", .{value.offset_from_parent}),
11320	.stack_slot => {
11321	try stderr.print(" [{t}, #{s}0x{x}", .{
11322	value.parent_payload.stack_slot.base,
11323	if (value.parent_payload.stack_slot.offset < 0) "-" else "",
11324	@abs(value.parent_payload.stack_slot.offset),
11325	});
11326	if (value.offset_from_parent != 0) try stderr.print("+0x{x}", .{value.offset_from_parent});
11327	try stderr.writeByte(']');
11328	},
11329	.value => try stderr.print(" ${d}+0x{x}", .{ @backingInt(value.parent_payload.value), value.offset_from_parent }),
11330	.address => try stderr.print(" ${d}[0x{x}]", .{ @backingInt(value.parent_payload.address), value.offset_from_parent }),
11331	.constant => try stderr.print(" <{f}, {f}>", .{
11332	isel.fmtType(value.parent_payload.constant.typeOf(zcu)),
11333	isel.fmtConstant(value.parent_payload.constant),
11334	}),
11335	}
11336	try stderr.print(" align({t})", .{value.flags.alignment});
11337	switch (value.flags.location_tag) {
11338	.large => try stderr.print(" size=0x{x} large", .{value.location_payload.large.size}),
11339	.small => {
11340	const loc = value.location_payload.small;
11341	try stderr.print(" size=0x{x}", .{loc.size});
11342	switch (loc.signedness) {
11343	.unsigned => {},
11344	.signed => try stderr.writeAll(" signed"),
11345	}
11346	if (loc.hint != .zr) try stderr.print(" hint={t}", .{loc.hint});
11347	if (loc.register != .zr) try stderr.print(" loc={t}", .{loc.register});
11348	},
11349	}
11350	try stderr.print(" refs={d}\n", .{value.refs});
11351
11352	var part_index = value.flags.parts_len_minus_one;
11353	if (part_index > 0) while (true) : (part_index -= 1) {
11354	roots.putAssumeCapacityNoClobber(
11355	@fromBackingInt(@intCast(@backingInt(value.parts) + part_index)),
11356	root_entry.value + 1,
11357	);
11358	if (part_index == 0) break;
11359	};
11360	}
11361	try stderr.print("# End {s} Value Dump: {f}\n\n", .{ @typeName(Select), nav.fqn.fmt(ip) });
11362	}
11363
11364	fn hasRepeatedByteRepr(isel: *Select, constant: Constant) error{OutOfMemory}!?u8 {
11365	const zcu = isel.pt.zcu;
11366	const ty = constant.typeOf(zcu);
11367	const abi_size = std.math.cast(usize, ty.abiSize(zcu)) orelse return null;
11368	const byte_buffer = try zcu.gpa.alloc(u8, abi_size);
11369	defer zcu.gpa.free(byte_buffer);
11370	return if (try isel.writeToMemory(constant, byte_buffer) and
11371	std.mem.allEqual(u8, byte_buffer[1..], byte_buffer[0])) byte_buffer[0] else null;
11372	}
11373
11374	fn writeToMemory(isel: *Select, constant: Constant, buffer: []u8) error{OutOfMemory}!bool {
11375	const zcu = isel.pt.zcu;
11376	const ip = &zcu.intern_pool;
11377	if (try isel.writeKeyToMemory(ip.indexToKey(constant.toIntern()), buffer)) return true;
11378	constant.writeToMemory(zcu, buffer) catch |err| switch (err) {
11379	error.OutOfMemory => |e| return e,
11380	error.ReinterpretDeclRef, error.IllDefinedMemoryLayout => return false,
11381	};
11382	return true;
11383	}
11384	fn writeKeyToMemory(isel: *Select, constant_key: InternPool.Key, buffer: []u8) error{OutOfMemory}!bool {
11385	const zcu = isel.pt.zcu;
11386	const ip = &zcu.intern_pool;
11387	switch (constant_key) {
11388	.int_type,
11389	.ptr_type,
11390	.array_type,
11391	.vector_type,
11392	.opt_type,
11393	.anyframe_type,
11394	.error_union_type,
11395	.simple_type,
11396	.struct_type,
11397	.tuple_type,
11398	.union_type,
11399	.opaque_type,
11400	.enum_type,
11401	.func_type,
11402	.error_set_type,
11403	.inferred_error_set_type,
11404
11405	.enum_literal,
11406	.memoized_call,
11407	=> unreachable, // not a runtime value
11408	.err => |err| {
11409	const error_int = ip.getErrorValueIfExists(err.name).?;
11410	switch (buffer.len) {
11411	else => unreachable,
11412	inline 1...4 => |size| std.mem.writeInt(
11413	@Int(.unsigned, 8 * size),
11414	buffer[0..size],
11415	@intCast(error_int),
11416	isel.target.cpu.arch.endian(),
11417	),
11418	}
11419	},
11420	.error_union => |error_union| {
11421	const error_union_type = ip.indexToKey(error_union.ty).error_union_type;
11422	const error_set_ty: ZigType = .fromInterned(error_union_type.error_set_type);
11423	const payload_ty: ZigType = .fromInterned(error_union_type.payload_type);
11424	const error_set = buffer[@intCast(codegen.errUnionErrorOffset(payload_ty, zcu))..][0..@intCast(error_set_ty.abiSize(zcu))];
11425	switch (error_union.val) {
11426	.err_name => |err_name| if (!try isel.writeKeyToMemory(.{ .err = .{
11427	.ty = error_set_ty.toIntern(),
11428	.name = err_name,
11429	} }, error_set)) return false,
11430	.payload => |payload| {
11431	if (!try isel.writeToMemory(
11432	.fromInterned(payload),
11433	buffer[@intCast(codegen.errUnionPayloadOffset(payload_ty, zcu))..][0..@intCast(payload_ty.abiSize(zcu))],
11434	)) return false;
11435	@memset(error_set, 0);
11436	},
11437	}
11438	},
11439	.opt => |opt| {
11440	const child_size: usize = @intCast(ZigType.fromInterned(ip.indexToKey(opt.ty).opt_type).abiSize(zcu));
11441	switch (opt.val) {
11442	.none => if (!ZigType.fromInterned(opt.ty).optionalReprIsPayload(zcu)) {
11443	buffer[child_size] = @intFromBool(false);
11444	} else @memset(buffer[0..child_size], 0x00),
11445	else => |child_constant| {
11446	if (!try isel.writeToMemory(.fromInterned(child_constant), buffer[0..child_size])) return false;
11447	if (!ZigType.fromInterned(opt.ty).optionalReprIsPayload(zcu)) buffer[child_size] = @intFromBool(true);
11448	},
11449	}
11450	},
11451	.aggregate => |aggregate| switch (ip.indexToKey(aggregate.ty)) {
11452	else => unreachable,
11453	.array_type => |array_type| {
11454	var elem_offset: usize = 0;
11455	const elem_size: usize = @intCast(ZigType.fromInterned(array_type.child).abiSize(zcu));
11456	const len_including_sentinel: usize = @intCast(array_type.lenIncludingSentinel());
11457	switch (aggregate.storage) {
11458	.bytes => |bytes| @memcpy(buffer[0..len_including_sentinel], bytes.toSlice(len_including_sentinel, ip)),
11459	.elems => |elems| for (elems) |elem| {
11460	if (!try isel.writeToMemory(.fromInterned(elem), buffer[elem_offset..][0..elem_size])) return false;
11461	elem_offset += elem_size;
11462	},
11463	.repeated_elem => |repeated_elem| for (0..len_including_sentinel) |_| {
11464	if (!try isel.writeToMemory(.fromInterned(repeated_elem), buffer[elem_offset..][0..elem_size])) return false;
11465	elem_offset += elem_size;
11466	},
11467	}
11468	},
11469	.vector_type => return false,
11470	.struct_type => {
11471	const loaded_struct = ip.loadStructType(aggregate.ty);
11472	switch (loaded_struct.layout) {
11473	.auto => {
11474	var field_offset: u64 = 0;
11475	var field_it = loaded_struct.iterateRuntimeOrder(ip);
11476	while (field_it.next()) |field_index| {
11477	if (loaded_struct.field_is_comptime_bits.get(ip, field_index)) continue;
11478	const field_ty: ZigType = .fromInterned(loaded_struct.field_types.get(ip)[field_index]);
11479	field_offset = loaded_struct.field_offsets.get(ip)[field_index];
11480	const field_size = field_ty.abiSize(zcu);
11481	if (!try isel.writeToMemory(.fromInterned(switch (aggregate.storage) {
11482	.bytes => unreachable,
11483	.elems => |elems| elems[field_index],
11484	.repeated_elem => |repeated_elem| repeated_elem,
11485	}), buffer[@intCast(field_offset)..][0..@intCast(field_size)])) return false;
11486	field_offset += field_size;
11487	}
11488	},
11489	.@"extern", .@"packed" => return false,
11490	}
11491	},
11492	.tuple_type => |tuple_type| {
11493	var field_offset: u64 = 0;
11494	for (tuple_type.types.get(ip), tuple_type.values.get(ip), 0..) |field_type, field_value, field_index| {
11495	if (field_value != .none) continue;
11496	const field_ty: ZigType = .fromInterned(field_type);
11497	field_offset = field_ty.abiAlignment(zcu).forward(field_offset);
11498	const field_size = field_ty.abiSize(zcu);
11499	if (!try isel.writeToMemory(.fromInterned(switch (aggregate.storage) {
11500	.bytes => unreachable,
11501	.elems => |elems| elems[field_index],
11502	.repeated_elem => |repeated_elem| repeated_elem,
11503	}), buffer[@intCast(field_offset)..][0..@intCast(field_size)])) return false;
11504	field_offset += field_size;
11505	}
11506	},
11507	},
11508	else => return false,
11509	}
11510	return true;
11511	}
11512
11513	const TryAllocRegResult = union(enum) {
11514	allocated: Register.Alias,
11515	fill_candidate: Register.Alias,
11516	out_of_registers,
11517	};
11518
11519	fn tryAllocIntReg(isel: *Select) TryAllocRegResult {
11520	var failed_result: TryAllocRegResult = .out_of_registers;
11521	var ra: Register.Alias = .r0;
11522	while (true) : (ra = @fromBackingInt(@intCast(@backingInt(ra) + 1))) {
11523	if (ra == .r18) continue; // The Platform Register
11524	if (ra == Register.Alias.fp) continue;
11525	const live_vi = isel.live_registers.getPtr(ra);
11526	switch (live_vi.*) {
11527	_ => switch (failed_result) {
11528	.allocated => unreachable,
11529	.fill_candidate => {},
11530	.out_of_registers => failed_result = .{ .fill_candidate = ra },
11531	},
11532	.allocating => {},
11533	.free => {
11534	live_vi.* = .allocating;
11535	isel.saved_registers.insert(ra);
11536	return .{ .allocated = ra };
11537	},
11538	}
11539	if (ra == Register.Alias.lr) return failed_result;
11540	}
11541	}
11542
11543	fn allocIntReg(isel: *Select) !Register.Alias {
11544	switch (isel.tryAllocIntReg()) {
11545	.allocated => |ra| return ra,
11546	.fill_candidate => |ra| {
11547	assert(try isel.fillMemory(ra));
11548	const live_vi = isel.live_registers.getPtr(ra);
11549	assert(live_vi.* == .free);
11550	live_vi.* = .allocating;
11551	return ra;
11552	},
11553	.out_of_registers => return isel.fail("ran out of registers", .{}),
11554	}
11555	}
11556
11557	fn tryAllocVecReg(isel: *Select) TryAllocRegResult {
11558	var failed_result: TryAllocRegResult = .out_of_registers;
11559	var ra: Register.Alias = .v0;
11560	while (true) : (ra = @fromBackingInt(@intCast(@backingInt(ra) + 1))) {
11561	const live_vi = isel.live_registers.getPtr(ra);
11562	switch (live_vi.*) {
11563	_ => switch (failed_result) {
11564	.allocated => unreachable,
11565	.fill_candidate => {},
11566	.out_of_registers => failed_result = .{ .fill_candidate = ra },
11567	},
11568	.allocating => {},
11569	.free => {
11570	live_vi.* = .allocating;
11571	isel.saved_registers.insert(ra);
11572	return .{ .allocated = ra };
11573	},
11574	}
11575	if (ra == Register.Alias.v31) return failed_result;
11576	}
11577	}
11578
11579	fn allocVecReg(isel: *Select) !Register.Alias {
11580	switch (isel.tryAllocVecReg()) {
11581	.allocated => |ra| return ra,
11582	.fill_candidate => |ra| {
11583	assert(try isel.fillMemory(ra));
11584	return ra;
11585	},
11586	.out_of_registers => return isel.fail("ran out of registers", .{}),
11587	}
11588	}
11589
11590	const RegLock = struct {
11591	ra: Register.Alias,
11592	const empty: RegLock = .{ .ra = .zr };
11593	fn unlock(lock: RegLock, isel: *Select) void {
11594	switch (lock.ra) {
11595	else => |ra| isel.freeReg(ra),
11596	.zr => {},
11597	}
11598	}
11599	};
11600	fn lockReg(isel: *Select, ra: Register.Alias) RegLock {
11601	assert(ra != .zr);
11602	const live_vi = isel.live_registers.getPtr(ra);
11603	assert(live_vi.* == .free);
11604	live_vi.* = .allocating;
11605	return .{ .ra = ra };
11606	}
11607	fn tryLockReg(isel: *Select, ra: Register.Alias) RegLock {
11608	assert(ra != .zr);
11609	const live_vi = isel.live_registers.getPtr(ra);
11610	switch (live_vi.*) {
11611	_ => unreachable,
11612	.allocating => return .{ .ra = .zr },
11613	.free => {
11614	live_vi.* = .allocating;
11615	return .{ .ra = ra };
11616	},
11617	}
11618	}
11619
11620	fn freeReg(isel: *Select, ra: Register.Alias) void {
11621	assert(ra != .zr);
11622	const live_vi = isel.live_registers.getPtr(ra);
11623	assert(live_vi.* == .allocating);
11624	live_vi.* = .free;
11625	}
11626
11627	fn use(isel: *Select, air_ref: Air.Inst.Ref) !Value.Index {
11628	const zcu = isel.pt.zcu;
11629	const ip = &zcu.intern_pool;
11630	try isel.values.ensureUnusedCapacity(zcu.gpa, 1);
11631	const vi, const ty = if (air_ref.toIndex()) |air_inst_index| vi_ty: {
11632	const live_gop = try isel.live_values.getOrPut(zcu.gpa, air_inst_index);
11633	if (live_gop.found_existing) return live_gop.value_ptr.*;
11634	const ty = isel.air.typeOf(air_ref, ip);
11635	const vi = isel.initValue(ty);
11636	tracking_log.debug("${d} <- %{d}", .{
11637	@backingInt(vi),
11638	@backingInt(air_inst_index),
11639	});
11640	live_gop.value_ptr.* = vi.ref(isel);
11641	break :vi_ty .{ vi, ty };
11642	} else vi_ty: {
11643	const constant: Constant = .fromInterned(air_ref.toInterned().?);
11644	const ty = constant.typeOf(zcu);
11645	const vi = isel.initValue(ty);
11646	tracking_log.debug("${d} <- <{f}, {f}>", .{
11647	@backingInt(vi),
11648	isel.fmtType(ty),
11649	isel.fmtConstant(constant),
11650	});
11651	vi.setParent(isel, .{ .constant = constant });
11652	break :vi_ty .{ vi, ty };
11653	};
11654	if (ty.isAbiInt(zcu)) {
11655	const int_info = ty.intInfo(zcu);
11656	if (int_info.bits <= 16) vi.setSignedness(isel, int_info.signedness);
11657	} else if (vi.size(isel) <= 16 and
11658	CallAbiIterator.homogeneousAggregateBaseType(zcu, ty.toIntern()) != null) vi.setIsVector(isel);
11659	return vi;
11660	}
11661
11662	fn fill(isel: *Select, dst_ra: Register.Alias) codegen.Error!bool {
11663	switch (dst_ra) {
11664	else => {},
11665	Register.Alias.fp, .zr, .sp, .pc, .fpcr, .fpsr, .ffr => return false,
11666	}
11667	const dst_live_vi = isel.live_registers.getPtr(dst_ra);
11668	const dst_vi = switch (dst_live_vi.*) {
11669	_ => |dst_vi| dst_vi,
11670	.allocating => return false,
11671	.free => return true,
11672	};
11673	const src_ra = src_ra: {
11674	if (dst_vi.hint(isel)) |hint_ra| {
11675	assert(dst_live_vi.* == dst_vi);
11676	dst_live_vi.* = .allocating;
11677	defer dst_live_vi.* = dst_vi;
11678	if (try isel.fill(hint_ra)) {
11679	isel.saved_registers.insert(hint_ra);
11680	break :src_ra hint_ra;
11681	}
11682	}
11683	switch (if (dst_vi.isVector(isel)) isel.tryAllocVecReg() else isel.tryAllocIntReg()) {
11684	.allocated => |ra| break :src_ra ra,
11685	.fill_candidate, .out_of_registers => return isel.fillMemory(dst_ra),
11686	}
11687	};
11688	try dst_vi.liveIn(isel, src_ra, comptime &.initFill(.free));
11689	const src_live_vi = isel.live_registers.getPtr(src_ra);
11690	assert(src_live_vi.* == .allocating);
11691	src_live_vi.* = dst_vi;
11692	return true;
11693	}
11694
11695	fn fillMemory(isel: *Select, dst_ra: Register.Alias) codegen.Error!bool {
11696	const dst_live_vi = isel.live_registers.getPtr(dst_ra);
11697	const dst_vi = switch (dst_live_vi.*) {
11698	_ => |dst_vi| dst_vi,
11699	.allocating => return false,
11700	.free => return true,
11701	};
11702	const dst_vi_ra = &dst_vi.get(isel).location_payload.small.register;
11703	assert(dst_vi_ra.* == dst_ra);
11704	const base_ra = if (dst_ra.isVector()) try isel.allocIntReg() else dst_ra;
11705	defer if (base_ra != dst_ra) isel.freeReg(base_ra);
11706	try isel.emit(switch (dst_vi.size(isel)) {
11707	else => unreachable,
11708	1 => if (dst_ra.isVector())
11709	.ldr(dst_ra.b(), .{ .base = base_ra.x() })
11710	else switch (dst_vi.signedness(isel)) {
11711	.signed => .ldrsb(dst_ra.w(), .{ .base = base_ra.x() }),
11712	.unsigned => .ldrb(dst_ra.w(), .{ .base = base_ra.x() }),
11713	},
11714	2 => if (dst_ra.isVector())
11715	.ldr(dst_ra.h(), .{ .base = base_ra.x() })
11716	else switch (dst_vi.signedness(isel)) {
11717	.signed => .ldrsh(dst_ra.w(), .{ .base = base_ra.x() }),
11718	.unsigned => .ldrh(dst_ra.w(), .{ .base = base_ra.x() }),
11719	},
11720	4 => .ldr(if (dst_ra.isVector()) dst_ra.s() else dst_ra.w(), .{ .base = base_ra.x() }),
11721	8 => .ldr(if (dst_ra.isVector()) dst_ra.d() else dst_ra.x(), .{ .base = base_ra.x() }),
11722	16 => .ldr(dst_ra.q(), .{ .base = base_ra.x() }),
11723	});
11724	dst_vi_ra.* = .zr;
11725	try dst_vi.address(isel, 0, base_ra);
11726	dst_live_vi.* = .free;
11727	return true;
11728	}
11729
11730	/// Merges possibly differing value tracking into a consistent state.
11731	///
11732	/// At a conditional branch, if a value is expected in the same register on both
11733	/// paths, or only expected in a register on only one path, tracking is updated:
11734	///
11735	/// $0 -> r0 // final state is now consistent with both paths
11736	/// b.cond else
11737	/// then:
11738	/// $0 -> r0 // updated if not already consistent with else
11739	/// ...
11740	/// b end
11741	/// else:
11742	/// $0 -> r0
11743	/// ...
11744	/// end:
11745	///
11746	/// At a conditional branch, if a value is expected in different registers on
11747	/// each path, mov instructions are emitted:
11748	///
11749	/// $0 -> r0 // final state is now consistent with both paths
11750	/// b.cond else
11751	/// then:
11752	/// $0 -> r0 // updated to be consistent with else
11753	/// mov x1, x0 // emitted to merge the inconsistent states
11754	/// $0 -> r1
11755	/// ...
11756	/// b end
11757	/// else:
11758	/// $0 -> r0
11759	/// ...
11760	/// end:
11761	///
11762	/// At a loop, a value that is expected in a register at the repeats is updated:
11763	///
11764	/// $0 -> r0 // final state is now consistent with all paths
11765	/// loop:
11766	/// $0 -> r0 // updated to be consistent with the repeats
11767	/// ...
11768	/// $0 -> r0
11769	/// b.cond loop
11770	/// ...
11771	/// $0 -> r0
11772	/// b loop
11773	///
11774	/// At a loop, a value that is expected in a register at the top is filled:
11775	///
11776	/// $0 -> [sp, #A] // final state is now consistent with all paths
11777	/// loop:
11778	/// $0 -> [sp, #A] // updated to be consistent with the repeats
11779	/// ldr x0, [sp, #A] // emitted to merge the inconsistent states
11780	/// $0 -> r0
11781	/// ...
11782	/// $0 -> [sp, #A]
11783	/// b.cond loop
11784	/// ...
11785	/// $0 -> [sp, #A]
11786	/// b loop
11787	///
11788	/// At a loop, if a value that is expected in different registers on each path,
11789	/// mov instructions are emitted:
11790	///
11791	/// $0 -> r0 // final state is now consistent with all paths
11792	/// loop:
11793	/// $0 -> r0 // updated to be consistent with the repeats
11794	/// mov x1, x0 // emitted to merge the inconsistent states
11795	/// $0 -> r1
11796	/// ...
11797	/// $0 -> r0
11798	/// b.cond loop
11799	/// ...
11800	/// $0 -> r0
11801	/// b loop
11802	fn merge(
11803	isel: *Select,
11804	expected_live_registers: *const LiveRegisters,
11805	comptime opts: struct { fill_extra: bool = false },
11806	) !void {
11807	var live_reg_it = isel.live_registers.iterator();
11808	while (live_reg_it.next()) |live_reg_entry| {
11809	const ra = live_reg_entry.key;
11810	const actual_vi = live_reg_entry.value;
11811	const expected_vi = expected_live_registers.get(ra);
11812	switch (expected_vi) {
11813	else => switch (actual_vi.*) {
11814	_ => {},
11815	.allocating => unreachable,
11816	.free => actual_vi.* = .allocating,
11817	},
11818	.free => {},
11819	}
11820	}
11821	live_reg_it = isel.live_registers.iterator();
11822	while (live_reg_it.next()) |live_reg_entry| {
11823	const ra = live_reg_entry.key;
11824	const actual_vi = live_reg_entry.value;
11825	const expected_vi = expected_live_registers.get(ra);
11826	switch (expected_vi) {
11827	_ => {
11828	switch (actual_vi.*) {
11829	_ => _ = if (opts.fill_extra) {
11830	assert(try isel.fillMemory(ra));
11831	assert(actual_vi.* == .free);
11832	},
11833	.allocating => actual_vi.* = .free,
11834	.free => unreachable,
11835	}
11836	try expected_vi.liveIn(isel, ra, expected_live_registers);
11837	},
11838	.allocating => if (if (opts.fill_extra) try isel.fillMemory(ra) else try isel.fill(ra)) {
11839	assert(actual_vi.* == .free);
11840	actual_vi.* = .allocating;
11841	},
11842	.free => if (opts.fill_extra) assert(try isel.fillMemory(ra) and actual_vi.* == .free),
11843	}
11844	}
11845	live_reg_it = isel.live_registers.iterator();
11846	while (live_reg_it.next()) |live_reg_entry| {
11847	const ra = live_reg_entry.key;
11848	const actual_vi = live_reg_entry.value;
11849	const expected_vi = expected_live_registers.get(ra);
11850	switch (expected_vi) {
11851	_ => {
11852	assert(actual_vi.* == .allocating and expected_vi.register(isel) == ra);
11853	actual_vi.* = expected_vi;
11854	},
11855	.allocating => assert(actual_vi.* == .allocating),
11856	.free => if (opts.fill_extra) assert(actual_vi.* == .free),
11857	}
11858	}
11859	}
11860
11861	const call = struct {
11862	const param_reg: Value.Index = @fromBackingInt(@intCast(@backingInt(Value.Index.allocating) - 2));
11863	const callee_clobbered_reg: Value.Index = @fromBackingInt(@intCast(@backingInt(Value.Index.allocating) - 1));
11864	const caller_saved_regs: LiveRegisters = .init(.{
11865	.r0 = param_reg,
11866	.r1 = param_reg,
11867	.r2 = param_reg,
11868	.r3 = param_reg,
11869	.r4 = param_reg,
11870	.r5 = param_reg,
11871	.r6 = param_reg,
11872	.r7 = param_reg,
11873	.r8 = param_reg,
11874	.r9 = callee_clobbered_reg,
11875	.r10 = callee_clobbered_reg,
11876	.r11 = callee_clobbered_reg,
11877	.r12 = callee_clobbered_reg,
11878	.r13 = callee_clobbered_reg,
11879	.r14 = callee_clobbered_reg,
11880	.r15 = callee_clobbered_reg,
11881	.r16 = callee_clobbered_reg,
11882	.r17 = callee_clobbered_reg,
11883	.r18 = callee_clobbered_reg,
11884	.r19 = .free,
11885	.r20 = .free,
11886	.r21 = .free,
11887	.r22 = .free,
11888	.r23 = .free,
11889	.r24 = .free,
11890	.r25 = .free,
11891	.r26 = .free,
11892	.r27 = .free,
11893	.r28 = .free,
11894	.r29 = .free,
11895	.r30 = callee_clobbered_reg,
11896	.zr = .free,
11897	.sp = .free,
11898
11899	.pc = .free,
11900
11901	.v0 = param_reg,
11902	.v1 = param_reg,
11903	.v2 = param_reg,
11904	.v3 = param_reg,
11905	.v4 = param_reg,
11906	.v5 = param_reg,
11907	.v6 = param_reg,
11908	.v7 = param_reg,
11909	.v8 = .free,
11910	.v9 = .free,
11911	.v10 = .free,
11912	.v11 = .free,
11913	.v12 = .free,
11914	.v13 = .free,
11915	.v14 = .free,
11916	.v15 = .free,
11917	.v16 = callee_clobbered_reg,
11918	.v17 = callee_clobbered_reg,
11919	.v18 = callee_clobbered_reg,
11920	.v19 = callee_clobbered_reg,
11921	.v20 = callee_clobbered_reg,
11922	.v21 = callee_clobbered_reg,
11923	.v22 = callee_clobbered_reg,
11924	.v23 = callee_clobbered_reg,
11925	.v24 = callee_clobbered_reg,
11926	.v25 = callee_clobbered_reg,
11927	.v26 = callee_clobbered_reg,
11928	.v27 = callee_clobbered_reg,
11929	.v28 = callee_clobbered_reg,
11930	.v29 = callee_clobbered_reg,
11931	.v30 = callee_clobbered_reg,
11932	.v31 = callee_clobbered_reg,
11933
11934	.fpcr = .free,
11935	.fpsr = .free,
11936
11937	.p0 = callee_clobbered_reg,
11938	.p1 = callee_clobbered_reg,
11939	.p2 = callee_clobbered_reg,
11940	.p3 = callee_clobbered_reg,
11941	.p4 = callee_clobbered_reg,
11942	.p5 = callee_clobbered_reg,
11943	.p6 = callee_clobbered_reg,
11944	.p7 = callee_clobbered_reg,
11945	.p8 = callee_clobbered_reg,
11946	.p9 = callee_clobbered_reg,
11947	.p10 = callee_clobbered_reg,
11948	.p11 = callee_clobbered_reg,
11949	.p12 = callee_clobbered_reg,
11950	.p13 = callee_clobbered_reg,
11951	.p14 = callee_clobbered_reg,
11952	.p15 = callee_clobbered_reg,
11953
11954	.ffr = .free,
11955	});
11956	fn prepareReturn(isel: *Select) !void {
11957	var live_reg_it = isel.live_registers.iterator();
11958	while (live_reg_it.next()) |live_reg_entry| switch (caller_saved_regs.get(live_reg_entry.key)) {
11959	else => unreachable,
11960	param_reg, callee_clobbered_reg => switch (live_reg_entry.value.*) {
11961	_ => {},
11962	.allocating => unreachable,
11963	.free => live_reg_entry.value.* = .allocating,
11964	},
11965	.free => {},
11966	};
11967	}
11968	fn returnFill(isel: *Select, ra: Register.Alias) !void {
11969	const live_vi = isel.live_registers.getPtr(ra);
11970	if (try isel.fill(ra)) {
11971	assert(live_vi.* == .free);
11972	live_vi.* = .allocating;
11973	}
11974	assert(live_vi.* == .allocating);
11975	}
11976	fn returnLiveIn(isel: *Select, vi: Value.Index, ra: Register.Alias) !void {
11977	try vi.defLiveIn(isel, ra, &caller_saved_regs);
11978	}
11979	fn finishReturn(isel: *Select) !void {
11980	var live_reg_it = isel.live_registers.iterator();
11981	while (live_reg_it.next()) |live_reg_entry| {
11982	switch (live_reg_entry.value.*) {
11983	_ => |live_vi| switch (live_vi.size(isel)) {
11984	else => unreachable,
11985	1, 2, 4, 8 => {},
11986	16 => {
11987	assert(try isel.fillMemory(live_reg_entry.key));
11988	assert(live_reg_entry.value.* == .free);
11989	switch (caller_saved_regs.get(live_reg_entry.key)) {
11990	else => unreachable,
11991	param_reg, callee_clobbered_reg => live_reg_entry.value.* = .allocating,
11992	.free => {},
11993	}
11994	continue;
11995	},
11996	},
11997	.allocating, .free => {},
11998	}
11999	switch (caller_saved_regs.get(live_reg_entry.key)) {
12000	else => unreachable,
12001	param_reg, callee_clobbered_reg => switch (live_reg_entry.value.*) {
12002	_ => {
12003	assert(try isel.fill(live_reg_entry.key));
12004	assert(live_reg_entry.value.* == .free);
12005	live_reg_entry.value.* = .allocating;
12006	},
12007	.allocating => {},
12008	.free => unreachable,
12009	},
12010	.free => {},
12011	}
12012	}
12013	}
12014	fn prepareCallee(isel: *Select) !void {
12015	var live_reg_it = isel.live_registers.iterator();
12016	while (live_reg_it.next()) |live_reg_entry| switch (caller_saved_regs.get(live_reg_entry.key)) {
12017	else => unreachable,
12018	param_reg => assert(live_reg_entry.value.* == .allocating),
12019	callee_clobbered_reg => isel.freeReg(live_reg_entry.key),
12020	.free => {},
12021	};
12022	}
12023	fn finishCallee(_: *Select) !void {}
12024	fn prepareParams(_: *Select) !void {}
12025	fn paramLiveOut(isel: *Select, vi: Value.Index, ra: Register.Alias) !void {
12026	isel.freeReg(ra);
12027	try vi.liveOut(isel, ra);
12028	const live_vi = isel.live_registers.getPtr(ra);
12029	if (live_vi.* == .free) live_vi.* = .allocating;
12030	}
12031	fn paramAddress(isel: *Select, vi: Value.Index, ra: Register.Alias) !void {
12032	isel.freeReg(ra);
12033	try vi.address(isel, 0, ra);
12034	const live_vi = isel.live_registers.getPtr(ra);
12035	if (live_vi.* == .free) live_vi.* = .allocating;
12036	}
12037	fn finishParams(isel: *Select) !void {
12038	var live_reg_it = isel.live_registers.iterator();
12039	while (live_reg_it.next()) |live_reg_entry| switch (caller_saved_regs.get(live_reg_entry.key)) {
12040	else => unreachable,
12041	param_reg => switch (live_reg_entry.value.*) {
12042	_ => {},
12043	.allocating => live_reg_entry.value.* = .free,
12044	.free => unreachable,
12045	},
12046	callee_clobbered_reg, .free => {},
12047	};
12048	}
12049	};
12050
12051	pub const CallAbiIterator = struct {
12052	/// Next General-purpose Register Number
12053	ngrn: Register.Alias,
12054	/// Next SIMD and Floating-point Register Number
12055	nsrn: Register.Alias,
12056	/// next stacked argument address
12057	nsaa: u24,
12058
12059	pub const ngrn_start: Register.Alias = .r0;
12060	pub const ngrn_end: Register.Alias = .r8;
12061	pub const nsrn_start: Register.Alias = .v0;
12062	pub const nsrn_end: Register.Alias = .v8;
12063	pub const nsaa_start: u42 = 0;
12064
12065	pub const init: CallAbiIterator = .{
12066	// A.1
12067	.ngrn = ngrn_start,
12068	// A.2
12069	.nsrn = nsrn_start,
12070	// A.3
12071	.nsaa = nsaa_start,
12072	};
12073
12074	pub fn param(it: *CallAbiIterator, isel: *Select, ty: ZigType) !?Value.Index {
12075	const zcu = isel.pt.zcu;
12076	const ip = &zcu.intern_pool;
12077
12078	if (!ty.hasRuntimeBits(zcu)) return null;
12079	try isel.values.ensureUnusedCapacity(zcu.gpa, Value.max_parts);
12080	const wip_vi = isel.initValue(ty);
12081	type_key: switch (ip.indexToKey(ty.toIntern())) {
12082	else => return isel.fail("CallAbiIterator.param({f})", .{isel.fmtType(ty)}),
12083	.int_type => |int_type| switch (int_type.bits) {
12084	0 => unreachable,
12085	1...16 => {
12086	wip_vi.setSignedness(isel, int_type.signedness);
12087	// C.7
12088	it.integer(isel, wip_vi);
12089	},
12090	// C.7
12091	17...64 => it.integer(isel, wip_vi),
12092	// C.9
12093	65...128 => it.integers(isel, wip_vi, @splat(@divExact(wip_vi.size(isel), 2))),
12094	else => it.indirect(isel, wip_vi),
12095	},
12096	.array_type => switch (wip_vi.size(isel)) {
12097	0 => unreachable,
12098	1...8 => it.integer(isel, wip_vi),
12099	9...16 => |size| it.integers(isel, wip_vi, .{ 8, size - 8 }),
12100	else => it.indirect(isel, wip_vi),
12101	},
12102	.ptr_type => |ptr_type| switch (ptr_type.flags.size) {
12103	.one, .many, .c => continue :type_key .{ .int_type = .{
12104	.signedness = .unsigned,
12105	.bits = 64,
12106	} },
12107	.slice => it.integers(isel, wip_vi, @splat(8)),
12108	},
12109	.opt_type => |child_type| if (ty.optionalReprIsPayload(zcu))
12110	continue :type_key ip.indexToKey(child_type)
12111	else switch (ZigType.fromInterned(child_type).abiSize(zcu)) {
12112	0 => continue :type_key .{ .simple_type = .bool },
12113	1...7 => it.integer(isel, wip_vi),
12114	8...15 => |child_size| it.integers(isel, wip_vi, .{ 8, child_size - 7 }),
12115	else => return isel.fail("CallAbiIterator.param({f})", .{isel.fmtType(ty)}),
12116	},
12117	.anyframe_type => unreachable,
12118	.error_union_type => |error_union_type| switch (wip_vi.size(isel)) {
12119	0 => unreachable,
12120	1...8 => it.integer(isel, wip_vi),
12121	9...16 => {
12122	var sizes: [2]u64 = @splat(0);
12123	const payload_ty: ZigType = .fromInterned(error_union_type.payload_type);
12124	{
12125	const error_set_ty: ZigType = .fromInterned(error_union_type.error_set_type);
12126	const offset = codegen.errUnionErrorOffset(payload_ty, zcu);
12127	const end = offset % 8 + error_set_ty.abiSize(zcu);
12128	const part_index: usize = @intCast(offset / 8);
12129	sizes[part_index] = @max(sizes[part_index], @min(end, 8));
12130	if (end > 8) sizes[part_index + 1] = @max(sizes[part_index + 1], end - 8);
12131	}
12132	{
12133	const offset = codegen.errUnionPayloadOffset(payload_ty, zcu);
12134	const end = offset % 8 + payload_ty.abiSize(zcu);
12135	const part_index: usize = @intCast(offset / 8);
12136	sizes[part_index] = @max(sizes[part_index], @min(end, 8));
12137	if (end > 8) sizes[part_index + 1] = @max(sizes[part_index + 1], end - 8);
12138	}
12139	it.integers(isel, wip_vi, sizes);
12140	},
12141	else => it.indirect(isel, wip_vi),
12142	},
12143	.simple_type => |simple_type| switch (simple_type) {
12144	.f16, .f32, .f64, .f128, .c_longdouble => it.vector(isel, wip_vi),
12145	.f80 => continue :type_key .{ .int_type = .{ .signedness = .unsigned, .bits = 80 } },
12146	.usize,
12147	.isize,
12148	.c_char,
12149	.c_short,
12150	.c_ushort,
12151	.c_int,
12152	.c_uint,
12153	.c_long,
12154	.c_ulong,
12155	.c_longlong,
12156	.c_ulonglong,
12157	=> continue :type_key .{ .int_type = ty.intInfo(zcu) },
12158	// B.1
12159	.anyopaque => it.indirect(isel, wip_vi),
12160	.bool => continue :type_key .{ .int_type = .{ .signedness = .unsigned, .bits = 1 } },
12161	.anyerror => continue :type_key .{ .int_type = .{
12162	.signedness = .unsigned,
12163	.bits = zcu.errorSetBits(),
12164	} },
12165	.void,
12166	.type,
12167	.comptime_int,
12168	.comptime_float,
12169	.noreturn,
12170	.null,
12171	.undefined,
12172	.enum_literal,
12173	.adhoc_inferred_error_set,
12174	.generic_poison,
12175	=> unreachable,
12176	},
12177	.struct_type => {
12178	const loaded_struct = ip.loadStructType(ty.toIntern());
12179	switch (loaded_struct.layout) {
12180	.auto, .@"extern" => {},
12181	.@"packed" => continue :type_key ip.indexToKey(loaded_struct.packed_backing_int_type),
12182	}
12183	const size = wip_vi.size(isel);
12184	if (size <= 16 * 4) homogeneous_aggregate: {
12185	const fdt = homogeneousStructBaseType(zcu, &loaded_struct) orelse break :homogeneous_aggregate;
12186	const parts_len = @shrExact(size, fdt.log2Size());
12187	if (parts_len > 4) break :homogeneous_aggregate;
12188	it.vectors(isel, wip_vi, fdt, @intCast(parts_len));
12189	break :type_key;
12190	}
12191	switch (size) {
12192	0 => unreachable,
12193	1...8 => it.integer(isel, wip_vi),
12194	9...16 => {
12195	var part_offset: u64 = 0;
12196	var part_sizes: [2]u64 = undefined;
12197	var parts_len: Value.PartsLen = 0;
12198	var next_field_end: u64 = 0;
12199	var field_it = loaded_struct.iterateRuntimeOrder(ip);
12200	while (part_offset < size) {
12201	const field_end = next_field_end;
12202	const next_field_begin = if (field_it.next()) |field_index| next_field_begin: {
12203	const field_ty: ZigType = .fromInterned(loaded_struct.field_types.get(ip)[field_index]);
12204	const next_field_begin = switch (loaded_struct.field_aligns.getOrNone(ip, field_index)) {
12205	.none => field_ty.abiAlignment(zcu),
12206	else => |field_align| field_align,
12207	}.forward(field_end);
12208	next_field_end = next_field_begin + field_ty.abiSize(zcu);
12209	break :next_field_begin next_field_begin;
12210	} else std.mem.alignForward(u64, size, 8);
12211	while (next_field_begin - part_offset >= 8) {
12212	const part_size = field_end - part_offset;
12213	part_sizes[parts_len] = part_size;
12214	assert(part_offset + part_size <= size);
12215	parts_len += 1;
12216	part_offset = next_field_begin;
12217	}
12218	}
12219	assert(parts_len == part_sizes.len);
12220	it.integers(isel, wip_vi, part_sizes);
12221	},
12222	else => it.indirect(isel, wip_vi),
12223	}
12224	},
12225	.tuple_type => |tuple_type| {
12226	const size = wip_vi.size(isel);
12227	if (size <= 16 * 4) homogeneous_aggregate: {
12228	const fdt = homogeneousTupleBaseType(zcu, tuple_type) orelse break :homogeneous_aggregate;
12229	const parts_len = @shrExact(size, fdt.log2Size());
12230	if (parts_len > 4) break :homogeneous_aggregate;
12231	it.vectors(isel, wip_vi, fdt, @intCast(parts_len));
12232	break :type_key;
12233	}
12234	switch (size) {
12235	0 => unreachable,
12236	1...8 => it.integer(isel, wip_vi),
12237	9...16 => {
12238	var part_offset: u64 = 0;
12239	var part_sizes: [2]u64 = undefined;
12240	var parts_len: Value.PartsLen = 0;
12241	var next_field_end: u64 = 0;
12242	var field_index: usize = 0;
12243	while (part_offset < size) {
12244	const field_end = next_field_end;
12245	const next_field_begin = while (field_index < tuple_type.types.len) {
12246	defer field_index += 1;
12247	if (tuple_type.values.get(ip)[field_index] != .none) continue;
12248	const field_ty: ZigType = .fromInterned(tuple_type.types.get(ip)[field_index]);
12249	const next_field_begin = field_ty.abiAlignment(zcu).forward(field_end);
12250	next_field_end = next_field_begin + field_ty.abiSize(zcu);
12251	break next_field_begin;
12252	} else std.mem.alignForward(u64, size, 8);
12253	while (next_field_begin - part_offset >= 8) {
12254	const part_size = @min(field_end - part_offset, 8);
12255	part_sizes[parts_len] = part_size;
12256	assert(part_offset + part_size <= size);
12257	parts_len += 1;
12258	part_offset += part_size;
12259	if (part_offset >= field_end) part_offset = next_field_begin;
12260	}
12261	}
12262	assert(parts_len == part_sizes.len);
12263	it.integers(isel, wip_vi, part_sizes);
12264	},
12265	else => it.indirect(isel, wip_vi),
12266	}
12267	},
12268	.union_type => {
12269	const loaded_union = ip.loadUnionType(ty.toIntern());
12270	switch (loaded_union.layout) {
12271	.auto, .@"extern" => {},
12272	.@"packed" => continue :type_key .{ .int_type = .{
12273	.signedness = .unsigned,
12274	.bits = @intCast(ty.bitSize(zcu)),
12275	} },
12276	}
12277	switch (wip_vi.size(isel)) {
12278	0 => unreachable,
12279	1...8 => it.integer(isel, wip_vi),
12280	9...16 => {
12281	const union_layout = ZigType.getUnionLayout(loaded_union, zcu);
12282	var sizes: [2]u64 = @splat(0);
12283	{
12284	const offset = union_layout.tagOffset();
12285	const end = offset % 8 + union_layout.tag_size;
12286	const part_index: usize = @intCast(offset / 8);
12287	sizes[part_index] = @max(sizes[part_index], @min(end, 8));
12288	if (end > 8) sizes[part_index + 1] = @max(sizes[part_index + 1], end - 8);
12289	}
12290	{
12291	const offset = union_layout.payloadOffset();
12292	const end = offset % 8 + union_layout.payload_size;
12293	const part_index: usize = @intCast(offset / 8);
12294	sizes[part_index] = @max(sizes[part_index], @min(end, 8));
12295	if (end > 8) sizes[part_index + 1] = @max(sizes[part_index + 1], end - 8);
12296	}
12297	it.integers(isel, wip_vi, sizes);
12298	},
12299	else => it.indirect(isel, wip_vi),
12300	}
12301	},
12302	.opaque_type, .func_type => continue :type_key .{ .simple_type = .anyopaque },
12303	.enum_type => continue :type_key ip.indexToKey(ip.loadEnumType(ty.toIntern()).int_tag_type),
12304	.error_set_type,
12305	.inferred_error_set_type,
12306	=> continue :type_key .{ .simple_type = .anyerror },
12307	.undef,
12308	.simple_value,
12309	.@"extern",
12310	.func,
12311	.int,
12312	.err,
12313	.error_union,
12314	.enum_literal,
12315	.enum_tag,
12316	.float,
12317	.ptr,
12318	.slice,
12319	.opt,
12320	.aggregate,
12321	.un,
12322	.memoized_call,
12323	=> unreachable, // values, not types
12324	}
12325	return wip_vi.ref(isel);
12326	}
12327
12328	pub fn nonSysvVarArg(it: *CallAbiIterator, isel: *Select, ty: ZigType) !?Value.Index {
12329	const ngrn = it.ngrn;
12330	defer it.ngrn = ngrn;
12331	it.ngrn = ngrn_end;
12332	const nsrn = it.nsrn;
12333	defer it.nsrn = nsrn;
12334	it.nsrn = nsrn_end;
12335	return it.param(isel, ty);
12336	}
12337
12338	pub fn ret(it: *CallAbiIterator, isel: *Select, ty: ZigType) !?Value.Index {
12339	const wip_vi = try it.param(isel, ty) orelse return null;
12340	switch (wip_vi.parent(isel)) {
12341	.unallocated, .stack_slot => {},
12342	.value, .constant => unreachable,
12343	.address => |address_vi| {
12344	assert(address_vi.hint(isel) == ngrn_start);
12345	address_vi.setHint(isel, ngrn_end);
12346	},
12347	}
12348	return wip_vi;
12349	}
12350
12351	pub const FundamentalDataType = enum {
12352	half,
12353	single,
12354	double,
12355	quad,
12356	vector64,
12357	vector128,
12358	fn log2Size(fdt: FundamentalDataType) u3 {
12359	return switch (fdt) {
12360	.half => 1,
12361	.single => 2,
12362	.double, .vector64 => 3,
12363	.quad, .vector128 => 4,
12364	};
12365	}
12366	fn size(fdt: FundamentalDataType) u64 {
12367	return @as(u64, 1) << fdt.log2Size();
12368	}
12369	};
12370	fn homogeneousAggregateBaseType(zcu: *Zcu, initial_ty: InternPool.Index) ?FundamentalDataType {
12371	const ip = &zcu.intern_pool;
12372	var ty = initial_ty;
12373	return type_key: switch (ip.indexToKey(ty)) {
12374	else => null,
12375	.array_type => |array_type| {
12376	ty = array_type.child;
12377	continue :type_key ip.indexToKey(ty);
12378	},
12379	.vector_type => switch (ZigType.fromInterned(ty).abiSize(zcu)) {
12380	else => null,
12381	8 => .vector64,
12382	16 => .vector128,
12383	},
12384	.simple_type => |simple_type| switch (simple_type) {
12385	.f16 => .half,
12386	.f32 => .single,
12387	.f64 => .double,
12388	.f128 => .quad,
12389	.c_longdouble => switch (zcu.getTarget().cTypeBitSize(.longdouble).?) {
12390	else => unreachable,
12391	64 => .double,
12392	80 => null,
12393	128 => .quad,
12394	},
12395	else => null,
12396	},
12397	.struct_type => homogeneousStructBaseType(zcu, &ip.loadStructType(ty)),
12398	.tuple_type => |tuple_type| homogeneousTupleBaseType(zcu, tuple_type),
12399	};
12400	}
12401	fn homogeneousStructBaseType(zcu: *Zcu, loaded_struct: *const InternPool.LoadedStructType) ?FundamentalDataType {
12402	const ip = &zcu.intern_pool;
12403	var common_fdt: ?FundamentalDataType = null;
12404	for (0.., loaded_struct.field_types.get(ip)) |field_index, field_ty| {
12405	if (loaded_struct.field_is_comptime_bits.get(ip, field_index)) continue;
12406	if (loaded_struct.field_aligns.getOrNone(ip, field_index) != .none) return null;
12407	if (!ZigType.fromInterned(field_ty).hasRuntimeBits(zcu)) continue;
12408	const fdt = homogeneousAggregateBaseType(zcu, field_ty);
12409	if (common_fdt == null) common_fdt = fdt else if (fdt != common_fdt) return null;
12410	}
12411	return common_fdt;
12412	}
12413	fn homogeneousTupleBaseType(zcu: *Zcu, tuple_type: InternPool.Key.TupleType) ?FundamentalDataType {
12414	const ip = &zcu.intern_pool;
12415	var common_fdt: ?FundamentalDataType = null;
12416	for (tuple_type.values.get(ip), tuple_type.types.get(ip)) |field_val, field_ty| {
12417	if (field_val != .none) continue;
12418	const fdt = homogeneousAggregateBaseType(zcu, field_ty);
12419	if (common_fdt == null) common_fdt = fdt else if (fdt != common_fdt) return null;
12420	}
12421	return common_fdt;
12422	}
12423
12424	const Spec = struct {
12425	offset: u64,
12426	size: u64,
12427	};
12428
12429	fn stack(it: *CallAbiIterator, isel: *Select, wip_vi: Value.Index) void {
12430	// C.12
12431	it.nsaa = @intCast(wip_vi.alignment(isel).forward(it.nsaa));
12432	const parent_vi = switch (wip_vi.parent(isel)) {
12433	.unallocated, .stack_slot => wip_vi,
12434	.address, .constant => unreachable,
12435	.value => |parent_vi| parent_vi,
12436	};
12437	switch (parent_vi.parent(isel)) {
12438	.unallocated => parent_vi.setParent(isel, .{ .stack_slot = .{
12439	.base = .sp,
12440	.offset = it.nsaa,
12441	} }),
12442	.stack_slot => {},
12443	.address, .value, .constant => unreachable,
12444	}
12445	it.nsaa += @intCast(wip_vi.size(isel));
12446	}
12447
12448	fn integer(it: *CallAbiIterator, isel: *Select, wip_vi: Value.Index) void {
12449	assert(wip_vi.size(isel) <= 8);
12450	const natural_alignment = wip_vi.alignment(isel);
12451	assert(natural_alignment.order(.@"16").compare(.lte));
12452	wip_vi.setAlignment(isel, natural_alignment.maxStrict(.@"8"));
12453	if (it.ngrn == ngrn_end) return it.stack(isel, wip_vi);
12454	wip_vi.setHint(isel, it.ngrn);
12455	it.ngrn = @fromBackingInt(@intCast(@backingInt(it.ngrn) + 1));
12456	}
12457
12458	fn integers(it: *CallAbiIterator, isel: *Select, wip_vi: Value.Index, part_sizes: [2]u64) void {
12459	assert(wip_vi.size(isel) <= 16);
12460	const natural_alignment = wip_vi.alignment(isel);
12461	assert(natural_alignment.order(.@"16").compare(.lte));
12462	wip_vi.setAlignment(isel, natural_alignment.maxStrict(.@"8"));
12463	// C.8
12464	if (natural_alignment == .@"16") it.ngrn = @fromBackingInt(@intCast(std.mem.alignForward(
12465	@typeInfo(Register.Alias).@"enum".tag_type,
12466	@backingInt(it.ngrn),
12467	2,
12468	)));
12469	if (it.ngrn == ngrn_end) return it.stack(isel, wip_vi);
12470	wip_vi.setParts(isel, part_sizes.len);
12471	for (0.., part_sizes) |part_index, part_size|
12472	it.integer(isel, wip_vi.addPart(isel, 8 * part_index, part_size));
12473	}
12474
12475	fn vector(it: *CallAbiIterator, isel: *Select, wip_vi: Value.Index) void {
12476	assert(wip_vi.size(isel) <= 16);
12477	const natural_alignment = wip_vi.alignment(isel);
12478	assert(natural_alignment.order(.@"16").compare(.lte));
12479	wip_vi.setAlignment(isel, natural_alignment.maxStrict(.@"8"));
12480	wip_vi.setIsVector(isel);
12481	if (it.nsrn == nsrn_end) return it.stack(isel, wip_vi);
12482	wip_vi.setHint(isel, it.nsrn);
12483	it.nsrn = @fromBackingInt(@intCast(@backingInt(it.nsrn) + 1));
12484	}
12485
12486	fn vectors(
12487	it: *CallAbiIterator,
12488	isel: *Select,
12489	wip_vi: Value.Index,
12490	fdt: FundamentalDataType,
12491	parts_len: Value.PartsLen,
12492	) void {
12493	const fdt_log2_size = fdt.log2Size();
12494	assert(wip_vi.size(isel) == @shlExact(@as(u9, parts_len), fdt_log2_size));
12495	const natural_alignment = wip_vi.alignment(isel);
12496	assert(natural_alignment.order(.@"16").compare(.lte));
12497	wip_vi.setAlignment(isel, natural_alignment.maxStrict(.@"8"));
12498	if (@backingInt(it.nsrn) > @backingInt(nsrn_end) - parts_len) return it.stack(isel, wip_vi);
12499	if (parts_len == 1) return it.vector(isel, wip_vi);
12500	wip_vi.setParts(isel, parts_len);
12501	const fdt_size = @as(u64, 1) << fdt_log2_size;
12502	for (0..parts_len) |part_index|
12503	it.vector(isel, wip_vi.addPart(isel, part_index << fdt_log2_size, fdt_size));
12504	}
12505
12506	fn indirect(it: *CallAbiIterator, isel: *Select, wip_vi: Value.Index) void {
12507	const wip_address_vi = isel.initValue(.usize);
12508	wip_vi.setParent(isel, .{ .address = wip_address_vi });
12509	it.integer(isel, wip_address_vi);
12510	}
12511	};
12512
12513	const Air = @import("../../Air.zig");
12514	const assert = std.debug.assert;
12515	const codegen = @import("../../codegen.zig");
12516	const Constant = @import("../../Value.zig");
12517	const InternPool = @import("../../InternPool.zig");
12518	const Module = @import("../../Module.zig");
12519	const Register = codegen.aarch64.encoding.Register;
12520	const Select = @This();
12521	const std = @import("std");
12522	const tracking_log = std.log.scoped(.tracking);
12523	const wip_mir_log = std.log.scoped(.@"wip-mir");
12524	const Zcu = @import("../../Zcu.zig");
12525	const ZigType = @import("../../Type.zig");