Commit 89cd42ee806cff64a21cd4eef6c87f0f7337f413 | zig

author	Andrew Kelley <andrew@ziglang.org> 2025-07-27 11:42:09-07:00
committer	GitHub <noreply@github.com> 2025-07-27 11:42:09-07:00
log	89cd42ee806cff64a21cd4eef6c87f0f7337f413
tree	a58ca0c2486476a333a2fedcf1458ebe42d77e31
parent	da408bd6fce9e07daccd35f9257e6819f58fc767
parent	b26e732bd0a33161b079202e9df9dda4b918b2bb
signature	Signed by PGP key B5690EEEBB952194

Merge pull request #24600 from jacobly0/aarch64

aarch64: continued progress

7 files changed, 513 insertions(+), 180 deletions(-)

src
- codegen
  - aarch64.zig
  - aarch64
    Select.zig
    encoding.zig
test
- behavior

src/codegen/aarch64.zig+31-21

...	...	@@ -19,8 +19,12 @@ pub fn generate(
19	19	) !Mir {
20	20	const zcu = pt.zcu;
21	21	const gpa = zcu.gpa;
	22	const ip = &zcu.intern_pool;
22	23	const func = zcu.funcInfo(func_index);
23		const func_type = zcu.intern_pool.indexToKey(func.ty).func_type;
	24	const func_zir = func.zir_body_inst.resolveFull(ip).?;
	25	const file = zcu.fileByIndex(func_zir.file);
	26	const named_params_len = file.zir.?.getParamBody(func_zir.inst).len;
	27	const func_type = ip.indexToKey(func.ty).func_type;
24	28	assert(liveness.* == null);
25	29
26	30	const mod = zcu.navFileScope(func.owner_nav).mod.?;
...	...	@@ -61,23 +65,32 @@ pub fn generate(
61	65	.values = .empty,
62	66	};
63	67	defer isel.deinit();
	68	const is_sysv = !isel.target.os.tag.isDarwin() and isel.target.os.tag != .windows;
	69	const is_sysv_var_args = is_sysv and func_type.is_var_args;
64	70
65	71	const air_main_body = air.getMainBody();
66	72	var param_it: Select.CallAbiIterator = .init;
67	73	const air_args = for (air_main_body, 0..) \|air_inst_index, body_index\| {
68	74	if (air.instructions.items(.tag)[@intFromEnum(air_inst_index)] != .arg) break air_main_body[0..body_index];
69		const param_ty = air.instructions.items(.data)[@intFromEnum(air_inst_index)].arg.ty.toType();
70		const param_vi = try param_it.param(&isel, param_ty);
	75	const arg = air.instructions.items(.data)[@intFromEnum(air_inst_index)].arg;
	76	const param_ty = arg.ty.toType();
	77	const param_vi = param_vi: {
	78	if (arg.zir_param_index >= named_params_len) {
	79	assert(func_type.is_var_args);
	80	if (!is_sysv) break :param_vi try param_it.nonSysvVarArg(&isel, param_ty);
	81	}
	82	break :param_vi try param_it.param(&isel, param_ty);
	83	};
71	84	tracking_log.debug("${d} <- %{d}", .{ @intFromEnum(param_vi.?), @intFromEnum(air_inst_index) });
72	85	try isel.live_values.putNoClobber(gpa, air_inst_index, param_vi.?);
73	86	} else unreachable;
74	87
75	88	const saved_gra_start = if (mod.strip) param_it.ngrn else Select.CallAbiIterator.ngrn_start;
76		const saved_gra_end = if (func_type.is_var_args) Select.CallAbiIterator.ngrn_end else param_it.ngrn;
	89	const saved_gra_end = if (is_sysv_var_args) Select.CallAbiIterator.ngrn_end else param_it.ngrn;
77	90	const saved_gra_len = @intFromEnum(saved_gra_end) - @intFromEnum(saved_gra_start);
78	91
79	92	const saved_vra_start = if (mod.strip) param_it.nsrn else Select.CallAbiIterator.nsrn_start;
80		const saved_vra_end = if (func_type.is_var_args) Select.CallAbiIterator.nsrn_end else param_it.nsrn;
	93	const saved_vra_end = if (is_sysv_var_args) Select.CallAbiIterator.nsrn_end else param_it.nsrn;
81	94	const saved_vra_len = @intFromEnum(saved_vra_end) - @intFromEnum(saved_vra_start);
82	95
83	96	const frame_record = 2;
...	...	@@ -85,11 +98,16 @@ pub fn generate(
85	98	.base = .fp,
86	99	.offset = 8 * std.mem.alignForward(u7, frame_record + saved_gra_len, 2),
87	100	};
88		isel.va_list = .{
89		.__stack = named_stack_args.withOffset(param_it.nsaa),
90		.__gr_top = named_stack_args,
91		.__vr_top = .{ .base = .fp, .offset = 0 },
92		};
	101	const stack_var_args = named_stack_args.withOffset(param_it.nsaa);
	102	const gr_top = named_stack_args;
	103	const vr_top: Select.Value.Indirect = .{ .base = .fp, .offset = 0 };
	104	isel.va_list = if (is_sysv) .{ .sysv = .{
	105	.__stack = stack_var_args,
	106	.__gr_top = gr_top,
	107	.__vr_top = vr_top,
	108	.__gr_offs = @as(i32, @intFromEnum(Select.CallAbiIterator.ngrn_end) - @intFromEnum(param_it.ngrn)) * -8,
	109	.__vr_offs = @as(i32, @intFromEnum(Select.CallAbiIterator.nsrn_end) - @intFromEnum(param_it.nsrn)) * -16,
	110	} } else .{ .other = stack_var_args };
93	111
94	112	// translate arg locations from caller-based to callee-based
95	113	for (air_args) \|air_inst_index\| {
...	...	@@ -106,11 +124,9 @@ pub fn generate(
106	124	const first_passed_part_vi = part_it.next().?;
107	125	const hint_ra = first_passed_part_vi.hint(&isel).?;
108	126	passed_vi.setParent(&isel, .{ .stack_slot = if (hint_ra.isVector())
109		isel.va_list.__vr_top.withOffset(@as(i8, -16) *
110		(@intFromEnum(saved_vra_end) - @intFromEnum(hint_ra)))
	127	vr_top.withOffset(@as(i8, -16) * (@intFromEnum(saved_vra_end) - @intFromEnum(hint_ra)))
111	128	else
112		isel.va_list.__gr_top.withOffset(@as(i8, -8) *
113		(@intFromEnum(saved_gra_end) - @intFromEnum(hint_ra))) });
	129	gr_top.withOffset(@as(i8, -8) * (@intFromEnum(saved_gra_end) - @intFromEnum(hint_ra))) });
114	130	},
115	131	.stack_slot => \|stack_slot\| {
116	132	assert(stack_slot.base == .sp);
...	...	@@ -152,13 +168,7 @@ pub fn generate(
152	168	isel.verify(true);
153	169
154	170	const prologue = isel.instructions.items.len;
155		const epilogue = try isel.layout(
156		param_it,
157		func_type.is_var_args,
158		saved_gra_len,
159		saved_vra_len,
160		mod,
161		);
	171	const epilogue = try isel.layout(param_it, is_sysv_var_args, saved_gra_len, saved_vra_len, mod);
162	172
163	173	const instructions = try isel.instructions.toOwnedSlice(gpa);
164	174	var mir: Mir = .{

src/codegen/aarch64/Select.zig+462-120

...	...	@@ -28,10 +28,15 @@ literal_relocs: std.ArrayListUnmanaged(codegen.aarch64.Mir.Reloc.Literal),
28	28
29	29	// Stack Frame
30	30	returns: bool,
31		va_list: struct {
32		__stack: Value.Indirect,
33		__gr_top: Value.Indirect,
34		__vr_top: Value.Indirect,
	31	va_list: union(enum) {
	32	other: Value.Indirect,
	33	sysv: struct {
	34	__stack: Value.Indirect,
	35	__gr_top: Value.Indirect,
	36	__vr_top: Value.Indirect,
	37	__gr_offs: i32,
	38	__vr_offs: i32,
	39	},
35	40	},
36	41	stack_size: u24,
37	42	stack_align: InternPool.Alignment,
...	...	@@ -408,13 +413,7 @@ pub fn analyze(isel: *Select, air_body: []const Air.Inst.Index) !void {
408	413
409	414	air_body_index += 1;
410	415	},
411		.breakpoint,
412		.dbg_stmt,
413		.dbg_empty_stmt,
414		.dbg_var_ptr,
415		.dbg_var_val,
416		.dbg_arg_inline,
417		=> {
	416	.breakpoint, .dbg_stmt, .dbg_empty_stmt, .dbg_var_ptr, .dbg_var_val, .dbg_arg_inline, .c_va_end => {
418	417	air_body_index += 1;
419	418	air_inst_index = air_body[air_body_index];
420	419	continue :air_tag air_tags[@intFromEnum(air_inst_index)];
...	...	@@ -428,23 +427,43 @@ pub fn analyze(isel: *Select, air_body: []const Air.Inst.Index) !void {
428	427	const extra = isel.air.extraData(Air.Call, pl_op.payload);
429	428	const args: []const Air.Inst.Ref = @ptrCast(isel.air.extra.items[extra.end..][0..extra.data.args_len]);
430	429	isel.saved_registers.insert(.lr);
	430	const callee_ty = isel.air.typeOf(pl_op.operand, ip);
	431	const func_info = switch (ip.indexToKey(callee_ty.toIntern())) {
	432	else => unreachable,
	433	.func_type => \|func_type\| func_type,
	434	.ptr_type => \|ptr_type\| ip.indexToKey(ptr_type.child).func_type,
	435	};
431	436
432	437	try isel.analyzeUse(pl_op.operand);
433	438	var param_it: CallAbiIterator = .init;
434		for (args) \|arg\| {
	439	for (args, 0..) \|arg, arg_index\| {
435	440	const restore_values_len = isel.values.items.len;
436	441	defer isel.values.shrinkRetainingCapacity(restore_values_len);
437		const param_vi = try param_it.param(isel, isel.air.typeOf(arg, ip)) orelse continue;
438		const param_parent = param_vi.parent(isel);
439		switch (switch (param_parent) {
440		.unallocated, .stack_slot => param_parent,
	442	const param_vi = param_vi: {
	443	const param_ty = isel.air.typeOf(arg, ip);
	444	if (arg_index >= func_info.param_types.len) {
	445	assert(func_info.is_var_args);
	446	switch (isel.va_list) {
	447	.other => break :param_vi try param_it.nonSysvVarArg(isel, param_ty),
	448	.sysv => {},
	449	}
	450	}
	451	break :param_vi try param_it.param(isel, param_ty);
	452	} orelse continue;
	453	defer param_vi.deref(isel);
	454	const passed_vi = switch (param_vi.parent(isel)) {
	455	.unallocated, .stack_slot => param_vi,
441	456	.value, .constant => unreachable,
442		.address => \|address_vi\| address_vi.parent(isel),
443		}) {
	457	.address => \|address_vi\| address_vi,
	458	};
	459	switch (passed_vi.parent(isel)) {
444	460	.unallocated => {},
445	461	.stack_slot => \|stack_slot\| {
446	462	assert(stack_slot.base == .sp);
447		isel.stack_size = @max(isel.stack_size, stack_slot.offset);
	463	isel.stack_size = @max(
	464	isel.stack_size,
	465	stack_slot.offset + @as(u24, @intCast(passed_vi.size(isel))),
	466	);
448	467	},
449	468	.value, .constant, .address => unreachable,
450	469	}
...	...	@@ -802,7 +821,7 @@ pub fn analyze(isel: *Select, air_body: []const Air.Inst.Index) !void {
802	821	air_inst_index = air_body[air_body_index];
803	822	continue :air_tag air_tags[@intFromEnum(air_inst_index)];
804	823	},
805		.set_err_return_trace, .c_va_end => {
	824	.set_err_return_trace => {
806	825	const un_op = air_data[@intFromEnum(air_inst_index)].un_op;
807	826
808	827	try isel.analyzeUse(un_op);
...	...	@@ -2474,6 +2493,7 @@ pub fn body(isel: *Select, air_body: []const Air.Inst.Index) error{ OutOfMemory,
2474	2493	}
2475	2494	if (air.next()) \|next_air_tag\| continue :air_tag next_air_tag;
2476	2495	},
	2496	.inferred_alloc, .inferred_alloc_comptime => unreachable,
2477	2497	.assembly => {
2478	2498	const ty_pl = air.data(air.inst_index).ty_pl;
2479	2499	const extra = isel.air.extraData(Air.Asm, ty_pl.payload);
...	...	@@ -3389,6 +3409,12 @@ pub fn body(isel: *Select, air_body: []const Air.Inst.Index) error{ OutOfMemory,
3389	3409	const pl_op = air.data(air.inst_index).pl_op;
3390	3410	const extra = isel.air.extraData(Air.Call, pl_op.payload);
3391	3411	const args: []const Air.Inst.Ref = @ptrCast(isel.air.extra.items[extra.end..][0..extra.data.args_len]);
	3412	const callee_ty = isel.air.typeOf(pl_op.operand, ip);
	3413	const func_info = switch (ip.indexToKey(callee_ty.toIntern())) {
	3414	else => unreachable,
	3415	.func_type => \|func_type\| func_type,
	3416	.ptr_type => \|ptr_type\| ip.indexToKey(ptr_type.child).func_type,
	3417	};
3392	3418
3393	3419	try call.prepareReturn(isel);
3394	3420	const maybe_def_ret_vi = isel.live_values.fetchRemove(air.inst_index);
...	...	@@ -3455,41 +3481,50 @@ pub fn body(isel: *Select, air_body: []const Air.Inst.Index) error{ OutOfMemory,
3455	3481	ret_addr_vi.hint(isel).?,
3456	3482	);
3457	3483	var param_it: CallAbiIterator = .init;
3458		for (args) \|arg\| {
3459		const param_vi = try param_it.param(isel, isel.air.typeOf(arg, ip)) orelse continue;
	3484	for (args, 0..) \|arg, arg_index\| {
	3485	const param_ty = isel.air.typeOf(arg, ip);
	3486	const param_vi = param_vi: {
	3487	if (arg_index >= func_info.param_types.len) {
	3488	assert(func_info.is_var_args);
	3489	switch (isel.va_list) {
	3490	.other => break :param_vi try param_it.nonSysvVarArg(isel, param_ty),
	3491	.sysv => {},
	3492	}
	3493	}
	3494	break :param_vi try param_it.param(isel, param_ty);
	3495	} orelse continue;
3460	3496	defer param_vi.deref(isel);
3461	3497	const arg_vi = try isel.use(arg);
3462		const passed_vi = switch (param_vi.parent(isel)) {
3463		.unallocated, .stack_slot => param_vi,
3464		.value, .constant => unreachable,
3465		.address => \|address_vi\| {
3466		try call.paramAddress(isel, arg_vi, address_vi.hint(isel).?);
3467		continue;
	3498	switch (param_vi.parent(isel)) {
	3499	.unallocated => if (param_vi.hint(isel)) \|param_ra\| {
	3500	try call.paramLiveOut(isel, arg_vi, param_ra);
	3501	} else {
	3502	var param_part_it = param_vi.parts(isel);
	3503	var arg_part_it = arg_vi.parts(isel);
	3504	if (arg_part_it.only()) \|_\| {
	3505	try isel.values.ensureUnusedCapacity(gpa, param_part_it.remaining);
	3506	arg_vi.setParts(isel, param_part_it.remaining);
	3507	while (param_part_it.next()) \|param_part_vi\| _ = arg_vi.addPart(
	3508	isel,
	3509	param_part_vi.get(isel).offset_from_parent,
	3510	param_part_vi.size(isel),
	3511	);
	3512	param_part_it = param_vi.parts(isel);
	3513	arg_part_it = arg_vi.parts(isel);
	3514	}
	3515	while (param_part_it.next()) \|param_part_vi\| {
	3516	const arg_part_vi = arg_part_it.next().?;
	3517	assert(arg_part_vi.get(isel).offset_from_parent ==
	3518	param_part_vi.get(isel).offset_from_parent);
	3519	assert(arg_part_vi.size(isel) == param_part_vi.size(isel));
	3520	try call.paramLiveOut(isel, arg_part_vi, param_part_vi.hint(isel).?);
	3521	}
3468	3522	},
3469		};
3470		if (passed_vi.hint(isel)) \|param_ra\| {
3471		try call.paramLiveOut(isel, arg_vi, param_ra);
3472		} else {
3473		var param_part_it = passed_vi.parts(isel);
3474		var arg_part_it = arg_vi.parts(isel);
3475		if (arg_part_it.only()) \|_\| {
3476		try isel.values.ensureUnusedCapacity(gpa, param_part_it.remaining);
3477		arg_vi.setParts(isel, param_part_it.remaining);
3478		while (param_part_it.next()) \|param_part_vi\| _ = arg_vi.addPart(
3479		isel,
3480		param_part_vi.get(isel).offset_from_parent,
3481		param_part_vi.size(isel),
3482		);
3483		param_part_it = passed_vi.parts(isel);
3484		arg_part_it = arg_vi.parts(isel);
3485		}
3486		while (param_part_it.next()) \|param_part_vi\| {
3487		const arg_part_vi = arg_part_it.next().?;
3488		assert(arg_part_vi.get(isel).offset_from_parent ==
3489		param_part_vi.get(isel).offset_from_parent);
3490		assert(arg_part_vi.size(isel) == param_part_vi.size(isel));
3491		try call.paramLiveOut(isel, arg_part_vi, param_part_vi.hint(isel).?);
3492		}
	3523	.stack_slot => \|stack_slot\| try arg_vi.store(isel, param_ty, stack_slot.base, .{
	3524	.offset = @intCast(stack_slot.offset),
	3525	}),
	3526	.value, .constant => unreachable,
	3527	.address => \|address_vi\| try call.paramAddress(isel, arg_vi, address_vi.hint(isel).?),
3493	3528	}
3494	3529	}
3495	3530	try call.finishParams(isel);
...	...	@@ -4828,9 +4863,7 @@ pub fn body(isel: *Select, air_body: []const Air.Inst.Index) error{ OutOfMemory,
4828	4863
4829	4864	if (air.next()) \|next_air_tag\| continue :air_tag next_air_tag;
4830	4865	},
4831		.dbg_stmt => {
4832		if (air.next()) \|next_air_tag\| continue :air_tag next_air_tag;
4833		},
	4866	.dbg_stmt => if (air.next()) \|next_air_tag\| continue :air_tag next_air_tag,
4834	4867	.dbg_empty_stmt => {
4835	4868	try isel.emit(.nop());
4836	4869	if (air.next()) \|next_air_tag\| continue :air_tag next_air_tag;
...	...	@@ -7079,6 +7112,7 @@ pub fn body(isel: *Select, air_body: []const Air.Inst.Index) error{ OutOfMemory,
7079	7112	}
7080	7113	if (air.next()) \|next_air_tag\| continue :air_tag next_air_tag;
7081	7114	},
	7115	.wasm_memory_size, .wasm_memory_grow => unreachable,
7082	7116	.cmp_lt_errors_len => {
7083	7117	if (isel.live_values.fetchRemove(air.inst_index)) \|is_vi\| unused: {
7084	7118	defer is_vi.value.deref(isel);
...	...	@@ -7135,16 +7169,266 @@ pub fn body(isel: *Select, air_body: []const Air.Inst.Index) error{ OutOfMemory,
7135	7169	}
7136	7170	if (air.next()) \|next_air_tag\| continue :air_tag next_air_tag;
7137	7171	},
7138		.inferred_alloc,
7139		.inferred_alloc_comptime,
7140		.int_from_float_safe,
7141		.int_from_float_optimized_safe,
7142		.wasm_memory_size,
7143		.wasm_memory_grow,
7144		.work_item_id,
7145		.work_group_size,
7146		.work_group_id,
7147		=> unreachable,
	7172	.c_va_arg => {
	7173	const maybe_arg_vi = isel.live_values.fetchRemove(air.inst_index);
	7174	defer if (maybe_arg_vi) \|arg_vi\| arg_vi.value.deref(isel);
	7175	const ty_op = air.data(air.inst_index).ty_op;
	7176	const ty = ty_op.ty.toType();
	7177	var param_it: CallAbiIterator = .init;
	7178	const param_vi = try param_it.param(isel, ty);
	7179	defer param_vi.?.deref(isel);
	7180	const passed_vi = switch (param_vi.?.parent(isel)) {
	7181	.unallocated => param_vi.?,
	7182	.stack_slot, .value, .constant => unreachable,
	7183	.address => \|address_vi\| address_vi,
	7184	};
	7185	const passed_size: u5 = @intCast(passed_vi.alignment(isel).forward(passed_vi.size(isel)));
	7186	const passed_is_vector = passed_vi.isVector(isel);
	7187
	7188	const va_list_ptr_vi = try isel.use(ty_op.operand);
	7189	const va_list_ptr_mat = try va_list_ptr_vi.matReg(isel);
	7190	const offs_ra = try isel.allocIntReg();
	7191	defer isel.freeReg(offs_ra);
	7192	const stack_ra = try isel.allocIntReg();
	7193	defer isel.freeReg(stack_ra);
	7194
	7195	var part_vis: [2]Value.Index = undefined;
	7196	var arg_part_ras: [2]?Register.Alias = @splat(null);
	7197	const parts_len = parts_len: {
	7198	var parts_len: u2 = 0;
	7199	var part_it = passed_vi.parts(isel);
	7200	while (part_it.next()) \|part_vi\| : (parts_len += 1) {
	7201	part_vis[parts_len] = part_vi;
	7202	const arg_vi = maybe_arg_vi orelse continue;
	7203	const part_offset, const part_size = part_vi.position(isel);
	7204	var arg_part_it = arg_vi.value.field(ty, part_offset, part_size);
	7205	const arg_part_vi = try arg_part_it.only(isel);
	7206	arg_part_ras[parts_len] = try arg_part_vi.?.defReg(isel);
	7207	}
	7208	break :parts_len parts_len;
	7209	};
	7210
	7211	const done_label = isel.instructions.items.len;
	7212	try isel.emit(.str(stack_ra.x(), .{ .unsigned_offset = .{
	7213	.base = va_list_ptr_mat.ra.x(),
	7214	.offset = 0,
	7215	} }));
	7216	try isel.emit(switch (parts_len) {
	7217	else => unreachable,
	7218	1 => if (arg_part_ras[0]) \|arg_part_ra\| switch (part_vis[0].size(isel)) {
	7219	else => unreachable,
	7220	1 => if (arg_part_ra.isVector()) .ldr(arg_part_ra.b(), .{ .post_index = .{
	7221	.base = stack_ra.x(),
	7222	.index = passed_size,
	7223	} }) else switch (part_vis[0].signedness(isel)) {
	7224	.signed => .ldrsb(arg_part_ra.w(), .{ .post_index = .{
	7225	.base = stack_ra.x(),
	7226	.index = passed_size,
	7227	} }),
	7228	.unsigned => .ldrb(arg_part_ra.w(), .{ .post_index = .{
	7229	.base = stack_ra.x(),
	7230	.index = passed_size,
	7231	} }),
	7232	},
	7233	2 => if (arg_part_ra.isVector()) .ldr(arg_part_ra.h(), .{ .post_index = .{
	7234	.base = stack_ra.x(),
	7235	.index = passed_size,
	7236	} }) else switch (part_vis[0].signedness(isel)) {
	7237	.signed => .ldrsh(arg_part_ra.w(), .{ .post_index = .{
	7238	.base = stack_ra.x(),
	7239	.index = passed_size,
	7240	} }),
	7241	.unsigned => .ldrh(arg_part_ra.w(), .{ .post_index = .{
	7242	.base = stack_ra.x(),
	7243	.index = passed_size,
	7244	} }),
	7245	},
	7246	4 => .ldr(if (arg_part_ra.isVector()) arg_part_ra.s() else arg_part_ra.w(), .{ .post_index = .{
	7247	.base = stack_ra.x(),
	7248	.index = passed_size,
	7249	} }),
	7250	8 => .ldr(if (arg_part_ra.isVector()) arg_part_ra.d() else arg_part_ra.x(), .{ .post_index = .{
	7251	.base = stack_ra.x(),
	7252	.index = passed_size,
	7253	} }),
	7254	16 => .ldr(arg_part_ra.q(), .{ .post_index = .{
	7255	.base = stack_ra.x(),
	7256	.index = passed_size,
	7257	} }),
	7258	} else .add(stack_ra.x(), stack_ra.x(), .{ .immediate = passed_size }),
	7259	2 => if (arg_part_ras[0] != null or arg_part_ras[1] != null) .ldp(
	7260	@as(Register.Alias, arg_part_ras[0] orelse .zr).x(),
	7261	@as(Register.Alias, arg_part_ras[1] orelse .zr).x(),
	7262	.{ .post_index = .{
	7263	.base = stack_ra.x(),
	7264	.index = passed_size,
	7265	} },
	7266	) else .add(stack_ra.x(), stack_ra.x(), .{ .immediate = passed_size }),
	7267	});
	7268	try isel.emit(.ldr(stack_ra.x(), .{ .unsigned_offset = .{
	7269	.base = va_list_ptr_mat.ra.x(),
	7270	.offset = 0,
	7271	} }));
	7272	switch (isel.va_list) {
	7273	.other => {},
	7274	.sysv => {
	7275	const stack_label = isel.instructions.items.len;
	7276	try isel.emit(.b(
	7277	@intCast((isel.instructions.items.len + 1 - done_label) << 2),
	7278	));
	7279	switch (parts_len) {
	7280	else => unreachable,
	7281	1 => if (arg_part_ras[0]) \|arg_part_ra\| try isel.emit(switch (part_vis[0].size(isel)) {
	7282	else => unreachable,
	7283	1 => if (arg_part_ra.isVector()) .ldr(arg_part_ra.b(), .{ .extended_register = .{
	7284	.base = stack_ra.x(),
	7285	.index = offs_ra.w(),
	7286	.extend = .{ .sxtw = 0 },
	7287	} }) else switch (part_vis[0].signedness(isel)) {
	7288	.signed => .ldrsb(arg_part_ra.w(), .{ .extended_register = .{
	7289	.base = stack_ra.x(),
	7290	.index = offs_ra.w(),
	7291	.extend = .{ .sxtw = 0 },
	7292	} }),
	7293	.unsigned => .ldrb(arg_part_ra.w(), .{ .extended_register = .{
	7294	.base = stack_ra.x(),
	7295	.index = offs_ra.w(),
	7296	.extend = .{ .sxtw = 0 },
	7297	} }),
	7298	},
	7299	2 => if (arg_part_ra.isVector()) .ldr(arg_part_ra.h(), .{ .extended_register = .{
	7300	.base = stack_ra.x(),
	7301	.index = offs_ra.w(),
	7302	.extend = .{ .sxtw = 0 },
	7303	} }) else switch (part_vis[0].signedness(isel)) {
	7304	.signed => .ldrsh(arg_part_ra.w(), .{ .extended_register = .{
	7305	.base = stack_ra.x(),
	7306	.index = offs_ra.w(),
	7307	.extend = .{ .sxtw = 0 },
	7308	} }),
	7309	.unsigned => .ldrh(arg_part_ra.w(), .{ .extended_register = .{
	7310	.base = stack_ra.x(),
	7311	.index = offs_ra.w(),
	7312	.extend = .{ .sxtw = 0 },
	7313	} }),
	7314	},
	7315	4 => .ldr(if (arg_part_ra.isVector()) arg_part_ra.s() else arg_part_ra.w(), .{ .extended_register = .{
	7316	.base = stack_ra.x(),
	7317	.index = offs_ra.w(),
	7318	.extend = .{ .sxtw = 0 },
	7319	} }),
	7320	8 => .ldr(if (arg_part_ra.isVector()) arg_part_ra.d() else arg_part_ra.x(), .{ .extended_register = .{
	7321	.base = stack_ra.x(),
	7322	.index = offs_ra.w(),
	7323	.extend = .{ .sxtw = 0 },
	7324	} }),
	7325	16 => .ldr(arg_part_ra.q(), .{ .extended_register = .{
	7326	.base = stack_ra.x(),
	7327	.index = offs_ra.w(),
	7328	.extend = .{ .sxtw = 0 },
	7329	} }),
	7330	}),
	7331	2 => if (arg_part_ras[0] != null or arg_part_ras[1] != null) {
	7332	try isel.emit(.ldp(
	7333	@as(Register.Alias, arg_part_ras[0] orelse .zr).x(),
	7334	@as(Register.Alias, arg_part_ras[1] orelse .zr).x(),
	7335	.{ .base = stack_ra.x() },
	7336	));
	7337	try isel.emit(.add(stack_ra.x(), stack_ra.x(), .{ .extended_register = .{
	7338	.register = offs_ra.w(),
	7339	.extend = .{ .sxtw = 0 },
	7340	} }));
	7341	},
	7342	}
	7343	try isel.emit(.ldr(stack_ra.x(), .{ .unsigned_offset = .{
	7344	.base = va_list_ptr_mat.ra.x(),
	7345	.offset = if (passed_is_vector) 16 else 8,
	7346	} }));
	7347	try isel.emit(.@"b."(
	7348	.gt,
	7349	@intCast((isel.instructions.items.len + 1 - stack_label) << 2),
	7350	));
	7351	try isel.emit(.str(stack_ra.w(), .{ .unsigned_offset = .{
	7352	.base = va_list_ptr_mat.ra.x(),
	7353	.offset = if (passed_is_vector) 28 else 24,
	7354	} }));
	7355	try isel.emit(.adds(stack_ra.w(), offs_ra.w(), .{ .immediate = passed_size }));
	7356	try isel.emit(.tbz(
	7357	offs_ra.w(),
	7358	31,
	7359	@intCast((isel.instructions.items.len + 1 - stack_label) << 2),
	7360	));
	7361	try isel.emit(.ldr(offs_ra.w(), .{ .unsigned_offset = .{
	7362	.base = va_list_ptr_mat.ra.x(),
	7363	.offset = if (passed_is_vector) 28 else 24,
	7364	} }));
	7365	},
	7366	}
	7367	try va_list_ptr_mat.finish(isel);
	7368	if (air.next()) \|next_air_tag\| continue :air_tag next_air_tag;
	7369	},
	7370	.c_va_copy => {
	7371	if (isel.live_values.fetchRemove(air.inst_index)) \|va_list_vi\| {
	7372	defer va_list_vi.value.deref(isel);
	7373	const ty_op = air.data(air.inst_index).ty_op;
	7374	const va_list_ptr_vi = try isel.use(ty_op.operand);
	7375	const va_list_ptr_mat = try va_list_ptr_vi.matReg(isel);
	7376	_ = try va_list_vi.value.load(isel, ty_op.ty.toType(), va_list_ptr_mat.ra, .{});
	7377	try va_list_ptr_mat.finish(isel);
	7378	}
	7379	if (air.next()) \|next_air_tag\| continue :air_tag next_air_tag;
	7380	},
	7381	.c_va_end => if (air.next()) \|next_air_tag\| continue :air_tag next_air_tag,
	7382	.c_va_start => {
	7383	if (isel.live_values.fetchRemove(air.inst_index)) \|va_list_vi\| {
	7384	defer va_list_vi.value.deref(isel);
	7385	const ty = air.data(air.inst_index).ty;
	7386	switch (isel.va_list) {
	7387	.other => \|va_list\| if (try va_list_vi.value.defReg(isel)) \|va_list_ra\| try isel.emit(.add(
	7388	va_list_ra.x(),
	7389	va_list.base.x(),
	7390	.{ .immediate = @intCast(va_list.offset) },
	7391	)),
	7392	.sysv => \|va_list\| {
	7393	var vr_offs_it = va_list_vi.value.field(ty, 28, 4);
	7394	const vr_offs_vi = try vr_offs_it.only(isel);
	7395	if (try vr_offs_vi.?.defReg(isel)) \|vr_offs_ra\| try isel.movImmediate(
	7396	vr_offs_ra.w(),
	7397	@as(u32, @bitCast(va_list.__vr_offs)),
	7398	);
	7399	var gr_offs_it = va_list_vi.value.field(ty, 24, 4);
	7400	const gr_offs_vi = try gr_offs_it.only(isel);
	7401	if (try gr_offs_vi.?.defReg(isel)) \|gr_offs_ra\| try isel.movImmediate(
	7402	gr_offs_ra.w(),
	7403	@as(u32, @bitCast(va_list.__gr_offs)),
	7404	);
	7405	var vr_top_it = va_list_vi.value.field(ty, 16, 8);
	7406	const vr_top_vi = try vr_top_it.only(isel);
	7407	if (try vr_top_vi.?.defReg(isel)) \|vr_top_ra\| try isel.emit(.add(
	7408	vr_top_ra.x(),
	7409	va_list.__vr_top.base.x(),
	7410	.{ .immediate = @intCast(va_list.__vr_top.offset) },
	7411	));
	7412	var gr_top_it = va_list_vi.value.field(ty, 8, 8);
	7413	const gr_top_vi = try gr_top_it.only(isel);
	7414	if (try gr_top_vi.?.defReg(isel)) \|gr_top_ra\| try isel.emit(.add(
	7415	gr_top_ra.x(),
	7416	va_list.__gr_top.base.x(),
	7417	.{ .immediate = @intCast(va_list.__gr_top.offset) },
	7418	));
	7419	var stack_it = va_list_vi.value.field(ty, 0, 8);
	7420	const stack_vi = try stack_it.only(isel);
	7421	if (try stack_vi.?.defReg(isel)) \|stack_ra\| try isel.emit(.add(
	7422	stack_ra.x(),
	7423	va_list.__stack.base.x(),
	7424	.{ .immediate = @intCast(va_list.__stack.offset) },
	7425	));
	7426	},
	7427	}
	7428	}
	7429	if (air.next()) \|next_air_tag\| continue :air_tag next_air_tag;
	7430	},
	7431	.work_item_id, .work_group_size, .work_group_id => unreachable,
7148	7432	}
7149	7433	assert(air.body_index == 0);
7150	7434	}
...	...	@@ -7225,7 +7509,7 @@ pub fn verify(isel: *Select, check_values: bool) void {
7225	7509	pub fn layout(
7226	7510	isel: *Select,
7227	7511	incoming: CallAbiIterator,
7228		have_va: bool,
	7512	is_sysv_var_args: bool,
7229	7513	saved_gra_len: u7,
7230	7514	saved_vra_len: u7,
7231	7515	mod: *const Package.Module,
...	...	@@ -7236,8 +7520,6 @@ pub fn layout(
7236	7520	wip_mir_log.debug("{f}<body>:\n", .{nav.fqn.fmt(ip)});
7237	7521
7238	7522	const stack_size: u24 = @intCast(InternPool.Alignment.@"16".forward(isel.stack_size));
7239		const stack_size_lo: u12 = @truncate(stack_size >> 0);
7240		const stack_size_hi: u12 = @truncate(stack_size >> 12);
7241	7523
7242	7524	var saves_buf: [10 + 8 + 8 + 2 + 8]struct {
7243	7525	class: enum { integer, vector },
...	...	@@ -7315,7 +7597,7 @@ pub fn layout(
7315	7597
7316	7598	// incoming vr arguments
7317	7599	save_ra = if (mod.strip) incoming.nsrn else CallAbiIterator.nsrn_start;
7318		while (save_ra != if (have_va) CallAbiIterator.nsrn_end else incoming.nsrn) : (save_ra = @enumFromInt(@intFromEnum(save_ra) + 1)) {
	7600	while (save_ra != if (is_sysv_var_args) CallAbiIterator.nsrn_end else incoming.nsrn) : (save_ra = @enumFromInt(@intFromEnum(save_ra) + 1)) {
7319	7601	saves_size = std.mem.alignForward(u10, saves_size, 16);
7320	7602	saves_buf[saves_len] = .{
7321	7603	.class = .vector,
...	...	@@ -7370,7 +7652,7 @@ pub fn layout(
7370	7652	1 => saves_size += 8,
7371	7653	}
7372	7654	save_ra = if (mod.strip) incoming.ngrn else CallAbiIterator.ngrn_start;
7373		while (save_ra != if (have_va) CallAbiIterator.ngrn_end else incoming.ngrn) : (save_ra = @enumFromInt(@intFromEnum(save_ra) + 1)) {
	7655	while (save_ra != if (is_sysv_var_args) CallAbiIterator.ngrn_end else incoming.ngrn) : (save_ra = @enumFromInt(@intFromEnum(save_ra) + 1)) {
7374	7656	saves_size = std.mem.alignForward(u10, saves_size, 8);
7375	7657	saves_buf[saves_len] = .{
7376	7658	.class = .integer,
...	...	@@ -7434,6 +7716,8 @@ pub fn layout(
7434	7716	.fp
7435	7717	else
7436	7718	.ip0;
	7719	const stack_size_lo: u12 = @truncate(stack_size >> 0);
	7720	const stack_size_hi: u12 = @truncate(stack_size >> 12);
7437	7721	if (mod.stack_check) {
7438	7722	if (stack_size_hi > 2) {
7439	7723	try isel.movImmediate(.ip1, stack_size_hi);
...	...	@@ -7481,6 +7765,7 @@ pub fn layout(
7481	7765	if (isel.returns) {
7482	7766	try isel.emit(.ret(.lr));
7483	7767	var save_index: usize = 0;
	7768	var first_offset: ?u10 = null;
7484	7769	while (save_index < saves.len) {
7485	7770	if (save_index + 2 <= saves.len and saves[save_index + 1].needs_restore and
7486	7771	saves[save_index + 0].class == saves[save_index + 1].class and
...	...	@@ -7489,46 +7774,51 @@ pub fn layout(
7489	7774	try isel.emit(.ldp(
7490	7775	saves[save_index + 0].register,
7491	7776	saves[save_index + 1].register,
7492		switch (saves[save_index + 0].offset) {
7493		0 => .{ .post_index = .{
7494		.base = .sp,
7495		.index = @intCast(saves_size),
7496		} },
7497		else => \|offset\| .{ .signed_offset = .{
	7777	if (first_offset) \|offset\| .{ .signed_offset = .{
	7778	.base = .sp,
	7779	.offset = @intCast(saves[save_index + 0].offset - offset),
	7780	} } else form: {
	7781	first_offset = @intCast(saves[save_index + 0].offset);
	7782	break :form .{ .post_index = .{
7498	7783	.base = .sp,
7499		.offset = @intCast(offset),
7500		} },
	7784	.index = @intCast(saves_size - first_offset.?),
	7785	} };
7501	7786	},
7502	7787	));
7503	7788	save_index += 2;
7504	7789	} else if (saves[save_index].needs_restore) {
7505	7790	try isel.emit(.ldr(
7506	7791	saves[save_index].register,
7507		switch (saves[save_index].offset) {
7508		0 => .{ .post_index = .{
7509		.base = .sp,
7510		.index = @intCast(saves_size),
7511		} },
7512		else => \|offset\| .{ .unsigned_offset = .{
	7792	if (first_offset) \|offset\| .{ .unsigned_offset = .{
	7793	.base = .sp,
	7794	.offset = saves[save_index + 0].offset - offset,
	7795	} } else form: {
	7796	const offset = saves[save_index + 0].offset;
	7797	first_offset = offset;
	7798	break :form .{ .post_index = .{
7513	7799	.base = .sp,
7514		.offset = @intCast(offset),
7515		} },
	7800	.index = @intCast(saves_size - offset),
	7801	} };
7516	7802	},
7517	7803	));
7518	7804	save_index += 1;
7519	7805	} else save_index += 1;
7520	7806	}
7521		if (isel.stack_align != .@"16" or (stack_size_lo > 0 and stack_size_hi > 0)) {
7522		try isel.emit(switch (frame_record_offset) {
7523		0 => .add(.sp, .fp, .{ .immediate = 0 }),
7524		else => \|offset\| .sub(.sp, .fp, .{ .immediate = offset }),
7525		});
	7807	const offset = stack_size + first_offset.?;
	7808	const offset_lo: u12 = @truncate(offset >> 0);
	7809	const offset_hi: u12 = @truncate(offset >> 12);
	7810	if (isel.stack_align != .@"16" or (offset_lo > 0 and offset_hi > 0)) {
	7811	const fp_offset = @as(i11, first_offset.?) - frame_record_offset;
	7812	try isel.emit(if (fp_offset >= 0)
	7813	.add(.sp, .fp, .{ .immediate = @intCast(fp_offset) })
	7814	else
	7815	.sub(.sp, .fp, .{ .immediate = @intCast(-fp_offset) }));
7526	7816	} else {
7527		if (stack_size_hi > 0) try isel.emit(.add(.sp, .sp, .{
7528		.shifted_immediate = .{ .immediate = stack_size_hi, .lsl = .@"12" },
	7817	if (offset_hi > 0) try isel.emit(.add(.sp, .sp, .{
	7818	.shifted_immediate = .{ .immediate = offset_hi, .lsl = .@"12" },
7529	7819	}));
7530		if (stack_size_lo > 0) try isel.emit(.add(.sp, .sp, .{
7531		.immediate = stack_size_lo,
	7820	if (offset_lo > 0) try isel.emit(.add(.sp, .sp, .{
	7821	.immediate = offset_lo,
7532	7822	}));
7533	7823	}
7534	7824	wip_mir_log.debug("{f}<epilogue>:\n", .{nav.fqn.fmt(ip)});
...	...	@@ -9493,6 +9783,11 @@ pub const Value = struct {
9493	9783	return it.vi;
9494	9784	}
9495	9785
	9786	pub fn peek(it: PartIterator) ?Value.Index {
	9787	var it_mut = it;
	9788	return it_mut.next();
	9789	}
	9790
9496	9791	pub fn only(it: PartIterator) ?Value.Index {
9497	9792	return if (it.remaining == 1) it.vi else null;
9498	9793	}
...	...	@@ -10119,11 +10414,12 @@ pub const Value = struct {
10119	10414	} },
10120	10415	.error_union => \|error_union\| {
10121	10416	const error_union_type = ip.indexToKey(error_union.ty).error_union_type;
	10417	const error_set_ty: ZigType = .fromInterned(error_union_type.error_set_type);
10122	10418	const payload_ty: ZigType = .fromInterned(error_union_type.payload_type);
10123		if (!ip.isNoReturn(error_union_type.error_set_type) and
10124		offset == codegen.errUnionErrorOffset(payload_ty, zcu))
10125		{
10126		offset = 0;
	10419	const error_set_offset = codegen.errUnionErrorOffset(payload_ty, zcu);
	10420	const error_set_size = error_set_ty.abiSize(zcu);
	10421	if (offset >= error_set_offset and offset + size <= error_set_offset + error_set_size) {
	10422	offset -= error_set_offset;
10127	10423	continue :constant_key switch (error_union.val) {
10128	10424	.err_name => \|err_name\| .{ .err = .{
10129	10425	.ty = error_union_type.error_set_type,
...	...	@@ -10135,15 +10431,18 @@ pub const Value = struct {
10135	10431	} },
10136	10432	};
10137	10433	}
10138		assert(payload_ty.hasRuntimeBitsIgnoreComptime(zcu));
10139		offset -= @intCast(codegen.errUnionPayloadOffset(payload_ty, zcu));
10140		switch (error_union.val) {
10141		.err_name => continue :constant_key .{ .undef = error_union_type.payload_type },
10142		.payload => \|payload\| {
10143		constant = payload;
10144		constant_key = ip.indexToKey(payload);
10145		continue :constant_key constant_key;
10146		},
	10434	const payload_offset = codegen.errUnionPayloadOffset(payload_ty, zcu);
	10435	const payload_size = payload_ty.abiSize(zcu);
	10436	if (offset >= payload_offset and offset + size <= payload_offset + payload_size) {
	10437	offset -= payload_offset;
	10438	switch (error_union.val) {
	10439	.err_name => continue :constant_key .{ .undef = error_union_type.payload_type },
	10440	.payload => \|payload\| {
	10441	constant = payload;
	10442	constant_key = ip.indexToKey(payload);
	10443	continue :constant_key constant_key;
	10444	},
	10445	}
10147	10446	}
10148	10447	},
10149	10448	.enum_tag => \|enum_tag\| continue :constant_key .{ .int = ip.indexToKey(enum_tag.int).int },
...	...	@@ -10680,7 +10979,17 @@ fn hasRepeatedByteRepr(isel: *Select, constant: Constant) error{OutOfMemory}!?u8
10680	10979	fn writeToMemory(isel: *Select, constant: Constant, buffer: []u8) error{OutOfMemory}!bool {
10681	10980	const zcu = isel.pt.zcu;
10682	10981	const ip = &zcu.intern_pool;
10683		switch (ip.indexToKey(constant.toIntern())) {
	10982	if (try isel.writeKeyToMemory(ip.indexToKey(constant.toIntern()), buffer)) return true;
	10983	constant.writeToMemory(isel.pt, buffer) catch \|err\| switch (err) {
	10984	error.OutOfMemory => return error.OutOfMemory,
	10985	error.ReinterpretDeclRef, error.Unimplemented, error.IllDefinedMemoryLayout => return false,
	10986	};
	10987	return true;
	10988	}
	10989	fn writeKeyToMemory(isel: *Select, constant_key: InternPool.Key, buffer: []u8) error{OutOfMemory}!bool {
	10990	const zcu = isel.pt.zcu;
	10991	const ip = &zcu.intern_pool;
	10992	switch (constant_key) {
10684	10993	.int_type,
10685	10994	.ptr_type,
10686	10995	.array_type,
...	...	@@ -10702,6 +11011,37 @@ fn writeToMemory(isel: *Select, constant: Constant, buffer: []u8) error{OutOfMem
10702	11011	.empty_enum_value,
10703	11012	.memoized_call,
10704	11013	=> unreachable, // not a runtime value
	11014	.err => \|err\| {
	11015	const error_int = ip.getErrorValueIfExists(err.name).?;
	11016	switch (buffer.len) {
	11017	else => unreachable,
	11018	inline 1...4 => \|size\| std.mem.writeInt(
	11019	@Type(.{ .int = .{ .signedness = .unsigned, .bits = 8 * size } }),
	11020	buffer[0..size],
	11021	@intCast(error_int),
	11022	isel.target.cpu.arch.endian(),
	11023	),
	11024	}
	11025	},
	11026	.error_union => \|error_union\| {
	11027	const error_union_type = ip.indexToKey(error_union.ty).error_union_type;
	11028	const error_set_ty: ZigType = .fromInterned(error_union_type.error_set_type);
	11029	const payload_ty: ZigType = .fromInterned(error_union_type.payload_type);
	11030	const error_set = buffer[@intCast(codegen.errUnionErrorOffset(payload_ty, zcu))..][0..@intCast(error_set_ty.abiSize(zcu))];
	11031	switch (error_union.val) {
	11032	.err_name => \|err_name\| if (!try isel.writeKeyToMemory(.{ .err = .{
	11033	.ty = error_set_ty.toIntern(),
	11034	.name = err_name,
	11035	} }, error_set)) return false,
	11036	.payload => \|payload\| {
	11037	if (!try isel.writeToMemory(
	11038	.fromInterned(payload),
	11039	buffer[@intCast(codegen.errUnionPayloadOffset(payload_ty, zcu))..][0..@intCast(payload_ty.abiSize(zcu))],
	11040	)) return false;
	11041	@memset(error_set, 0);
	11042	},
	11043	}
	11044	},
10705	11045	.opt => \|opt\| {
10706	11046	const child_size: usize = @intCast(ZigType.fromInterned(ip.indexToKey(opt.ty).opt_type).abiSize(zcu));
10707	11047	switch (opt.val) {
...	...	@@ -10713,7 +11053,6 @@ fn writeToMemory(isel: *Select, constant: Constant, buffer: []u8) error{OutOfMem
10713	11053	if (!ZigType.fromInterned(opt.ty).optionalReprIsPayload(zcu)) buffer[child_size] = @intFromBool(true);
10714	11054	},
10715	11055	}
10716		return true;
10717	11056	},
10718	11057	.aggregate => \|aggregate\| switch (ip.indexToKey(aggregate.ty)) {
10719	11058	else => unreachable,
...	...	@@ -10732,9 +11071,8 @@ fn writeToMemory(isel: *Select, constant: Constant, buffer: []u8) error{OutOfMem
10732	11071	elem_offset += elem_size;
10733	11072	},
10734	11073	}
10735		return true;
10736	11074	},
10737		.vector_type => {},
	11075	.vector_type => return false,
10738	11076	.struct_type => {
10739	11077	const loaded_struct = ip.loadStructType(aggregate.ty);
10740	11078	switch (loaded_struct.layout) {
...	...	@@ -10757,9 +11095,8 @@ fn writeToMemory(isel: *Select, constant: Constant, buffer: []u8) error{OutOfMem
10757	11095	}), buffer[@intCast(field_offset)..][0..@intCast(field_size)])) return false;
10758	11096	field_offset += field_size;
10759	11097	}
10760		return true;
10761	11098	},
10762		.@"extern", .@"packed" => {},
	11099	.@"extern", .@"packed" => return false,
10763	11100	}
10764	11101	},
10765	11102	.tuple_type => \|tuple_type\| {
...	...	@@ -10776,15 +11113,10 @@ fn writeToMemory(isel: *Select, constant: Constant, buffer: []u8) error{OutOfMem
10776	11113	}), buffer[@intCast(field_offset)..][0..@intCast(field_size)])) return false;
10777	11114	field_offset += field_size;
10778	11115	}
10779		return true;
10780	11116	},
10781	11117	},
10782		else => {},
	11118	else => return false,
10783	11119	}
10784		constant.writeToMemory(isel.pt, buffer) catch \|err\| switch (err) {
10785		error.OutOfMemory => return error.OutOfMemory,
10786		error.ReinterpretDeclRef, error.Unimplemented, error.IllDefinedMemoryLayout => return false,
10787		};
10788	11120	return true;
10789	11121	}
10790	11122
...	...	@@ -11607,6 +11939,16 @@ pub const CallAbiIterator = struct {
11607	11939	return wip_vi.ref(isel);
11608	11940	}
11609	11941
	11942	pub fn nonSysvVarArg(it: CallAbiIterator, isel: Select, ty: ZigType) !?Value.Index {
	11943	const ngrn = it.ngrn;
	11944	defer it.ngrn = ngrn;
	11945	it.ngrn = ngrn_end;
	11946	const nsrn = it.nsrn;
	11947	defer it.nsrn = nsrn;
	11948	it.nsrn = nsrn_end;
	11949	return it.param(isel, ty);
	11950	}
	11951
11610	11952	pub fn ret(it: CallAbiIterator, isel: Select, ty: ZigType) !?Value.Index {
11611	11953	const wip_vi = try it.param(isel, ty) orelse return null;
11612	11954	switch (wip_vi.parent(isel)) {

src/codegen/aarch64/encoding.zig+20-20

...	...	@@ -10089,26 +10089,26 @@ pub const Instruction = packed union {
10089	10089	},
10090	10090	} } } };
10091	10091	},
10092		.signed_offset => \|signed_offset\| {
10093		assert(signed_offset.base.format.integer == .doubleword);
10094		return .{ .load_store = .{ .register_pair_offset = .{ .integer = .{
	10092	.pre_index => \|pre_index\| {
	10093	assert(pre_index.base.format.integer == .doubleword);
	10094	return .{ .load_store = .{ .register_pair_pre_indexed = .{ .integer = .{
10095	10095	.ldp = .{
10096	10096	.Rt = t1.alias.encode(.{}),
10097		.Rn = signed_offset.base.alias.encode(.{ .sp = true }),
	10097	.Rn = pre_index.base.alias.encode(.{ .sp = true }),
10098	10098	.Rt2 = t2.alias.encode(.{}),
10099		.imm7 = @intCast(@shrExact(signed_offset.offset, @as(u2, 2) + @intFromEnum(sf))),
	10099	.imm7 = @intCast(@shrExact(pre_index.index, @as(u2, 2) + @intFromEnum(sf))),
10100	10100	.sf = sf,
10101	10101	},
10102	10102	} } } };
10103	10103	},
10104		.pre_index => \|pre_index\| {
10105		assert(pre_index.base.format.integer == .doubleword);
10106		return .{ .load_store = .{ .register_pair_pre_indexed = .{ .integer = .{
	10104	.signed_offset => \|signed_offset\| {
	10105	assert(signed_offset.base.format.integer == .doubleword);
	10106	return .{ .load_store = .{ .register_pair_offset = .{ .integer = .{
10107	10107	.ldp = .{
10108	10108	.Rt = t1.alias.encode(.{}),
10109		.Rn = pre_index.base.alias.encode(.{ .sp = true }),
	10109	.Rn = signed_offset.base.alias.encode(.{ .sp = true }),
10110	10110	.Rt2 = t2.alias.encode(.{}),
10111		.imm7 = @intCast(@shrExact(pre_index.index, @as(u2, 2) + @intFromEnum(sf))),
	10111	.imm7 = @intCast(@shrExact(signed_offset.offset, @as(u2, 2) + @intFromEnum(sf))),
10112	10112	.sf = sf,
10113	10113	},
10114	10114	} } } };
...	...	@@ -11473,26 +11473,26 @@ pub const Instruction = packed union {
11473	11473	},
11474	11474	} } } };
11475	11475	},
11476		.signed_offset => \|signed_offset\| {
11477		assert(signed_offset.base.format.integer == .doubleword);
11478		return .{ .load_store = .{ .register_pair_offset = .{ .integer = .{
	11476	.pre_index => \|pre_index\| {
	11477	assert(pre_index.base.format.integer == .doubleword);
	11478	return .{ .load_store = .{ .register_pair_pre_indexed = .{ .integer = .{
11479	11479	.stp = .{
11480	11480	.Rt = t1.alias.encode(.{}),
11481		.Rn = signed_offset.base.alias.encode(.{ .sp = true }),
	11481	.Rn = pre_index.base.alias.encode(.{ .sp = true }),
11482	11482	.Rt2 = t2.alias.encode(.{}),
11483		.imm7 = @intCast(@shrExact(signed_offset.offset, @as(u2, 2) + @intFromEnum(sf))),
	11483	.imm7 = @intCast(@shrExact(pre_index.index, @as(u2, 2) + @intFromEnum(sf))),
11484	11484	.sf = sf,
11485	11485	},
11486	11486	} } } };
11487	11487	},
11488		.pre_index => \|pre_index\| {
11489		assert(pre_index.base.format.integer == .doubleword);
11490		return .{ .load_store = .{ .register_pair_pre_indexed = .{ .integer = .{
	11488	.signed_offset => \|signed_offset\| {
	11489	assert(signed_offset.base.format.integer == .doubleword);
	11490	return .{ .load_store = .{ .register_pair_offset = .{ .integer = .{
11491	11491	.stp = .{
11492	11492	.Rt = t1.alias.encode(.{}),
11493		.Rn = pre_index.base.alias.encode(.{ .sp = true }),
	11493	.Rn = signed_offset.base.alias.encode(.{ .sp = true }),
11494	11494	.Rt2 = t2.alias.encode(.{}),
11495		.imm7 = @intCast(@shrExact(pre_index.index, @as(u2, 2) + @intFromEnum(sf))),
	11495	.imm7 = @intCast(@shrExact(signed_offset.offset, @as(u2, 2) + @intFromEnum(sf))),
11496	11496	.sf = sf,
11497	11497	},
11498	11498	} } } };

test/behavior/enum.zig-1

...	...	@@ -926,7 +926,6 @@ test "enum literal casting to tagged union" {
926	926	const Bar = enum { A, B, C, D };
927	927
928	928	test "enum literal casting to error union with payload enum" {
929		if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
930	929	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
931	930
932	931	var bar: error{B}!Bar = undefined;

test/behavior/error.zig-12

...	...	@@ -145,14 +145,11 @@ test "implicit cast to optional to error union to return result loc" {
145	145	}
146	146
147	147	test "fn returning empty error set can be passed as fn returning any error" {
148		if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
149
150	148	entry();
151	149	comptime entry();
152	150	}
153	151
154	152	test "fn returning empty error set can be passed as fn returning any error - pointer" {
155		if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
156	153	if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest;
157	154
158	155	entryPtr();
...	...	@@ -404,7 +401,6 @@ fn intLiteral(str: []const u8) !?i64 {
404	401	}
405	402
406	403	test "nested error union function call in optional unwrap" {
407		if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
408	404	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
409	405	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
410	406
...	...	@@ -482,7 +478,6 @@ test "optional error set is the same size as error set" {
482	478	}
483	479
484	480	test "nested catch" {
485		if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
486	481	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
487	482
488	483	const S = struct {
...	...	@@ -698,7 +693,6 @@ test "coerce error set to the current inferred error set" {
698	693	}
699	694
700	695	test "error union payload is properly aligned" {
701		if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
702	696	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
703	697	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
704	698	if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest;
...	...	@@ -757,7 +751,6 @@ test "simple else prong allowed even when all errors handled" {
757	751	}
758	752
759	753	test "pointer to error union payload" {
760		if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
761	754	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
762	755	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
763	756	if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest;
...	...	@@ -845,7 +838,6 @@ test "alignment of wrapping an error union payload" {
845	838	}
846	839
847	840	test "compare error union and error set" {
848		if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
849	841	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
850	842
851	843	var a: anyerror = error.Foo;
...	...	@@ -1034,8 +1026,6 @@ test "errorCast to adhoc inferred error set" {
1034	1026	}
1035	1027
1036	1028	test "@errorCast from error set to error union" {
1037		if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
1038
1039	1029	const S = struct {
1040	1030	fn doTheTest(set: error{ A, B }) error{A}!i32 {
1041	1031	return @errorCast(set);
...	...	@@ -1046,8 +1036,6 @@ test "@errorCast from error set to error union" {
1046	1036	}
1047	1037
1048	1038	test "@errorCast from error union to error union" {
1049		if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
1050
1051	1039	const S = struct {
1052	1040	fn doTheTest(set: error{ A, B }!i32) error{A}!i32 {
1053	1041	return @errorCast(set);

test/behavior/var_args.zig-4

...	...	@@ -92,7 +92,6 @@ fn doNothingWithFirstArg(args: anytype) void {
92	92	}
93	93
94	94	test "simple variadic function" {
95		if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
96	95	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
97	96	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
98	97	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
...	...	@@ -154,7 +153,6 @@ test "simple variadic function" {
154	153	}
155	154
156	155	test "coerce reference to var arg" {
157		if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
158	156	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
159	157	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
160	158	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
...	...	@@ -234,7 +232,6 @@ test "variadic functions" {
234	232	}
235	233
236	234	test "copy VaList" {
237		if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
238	235	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
239	236	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
240	237	if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest;
...	...	@@ -269,7 +266,6 @@ test "copy VaList" {
269	266	}
270	267
271	268	test "unused VaList arg" {
272		if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
273	269	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
274	270	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
275	271	if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest;

test/behavior/while.zig-2

...	...	@@ -174,7 +174,6 @@ test "while with optional as condition with else" {
174	174	}
175	175
176	176	test "while with error union condition" {
177		if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
178	177	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
179	178	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
180	179	if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest;
...	...	@@ -306,7 +305,6 @@ test "while optional 2 break statements and an else" {
306	305	}
307	306
308	307	test "while error 2 break statements and an else" {
309		if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
310	308	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
311	309	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
312	310