Commit 36b65ab59e5e514ad06a11cde96b87c565d86ac8 | zig

author	Mathieu Suen <mathieusuen@yahoo.fr> 2026-01-20 15:15:57+01:00
committer	Matthew Lugg <mlugg@mlugg.co.uk> 2026-02-06 13:06:49+00:00
log	36b65ab59e5e514ad06a11cde96b87c565d86ac8
tree	33e1edfa980caefe0eac488562585ae544946432
parent	d84a638e8b6ffeb95dfafef59e6305bd0e139d4e
signature	Commit is signed but in an unrecognized format.

Air: add "unwrap" functions for loading extra data

14 files changed, 861 insertions(+), 890 deletions(-)

src
- Air.zig
- Air
  - Liveness.zig
  - Liveness
    Verify.zig
  - print.zig
  - types_resolved.zig
- Sema.zig
- codegen
  - aarch64
    Select.zig
  - c.zig
  - llvm.zig
  - riscv64
    CodeGen.zig
  - sparc64
    CodeGen.zig
  - spirv
    CodeGen.zig
  - wasm
    CodeGen.zig
  - x86_64
    CodeGen.zig

src/Air.zig+254-13

...	...	@@ -281,16 +281,21 @@ pub const Inst = struct {
281	281	/// also supports enums and pointers.
282	282	/// Uses the `ty_op` field.
283	283	bitcast,
284		/// Uses the `ty_pl` field with payload `Block`. A block runs its body which always ends
285		/// with a `noreturn` instruction, so the only way to proceed to the code after the `block`
286		/// is to encounter a `br` that targets this `block`. If the `block` type is `noreturn`,
	284	/// A block runs its body which always ends with a `noreturn` instruction,
	285	/// so the only way to proceed to the code after the `block` is to encounter a `br`
	286	/// that targets this `block`. If the `block` type is `noreturn`,
287	287	/// then there do not exist any `br` instructions targeting this `block`.
	288	/// Uses the `ty_pl` field with payload `Block`.
	289	///
	290	/// See `unwrapBlock` for a way to load this tag's data.
288	291	block,
289	292	/// A labeled block of code that loops forever. The body must be `noreturn`: loops
290	293	/// occur through an explicit `repeat` instruction pointing back to this one.
291	294	/// Result type is always `noreturn`; no instructions in a block follow this one.
292	295	/// There is always at least one `repeat` instruction referencing the loop.
293	296	/// Uses the `ty_pl` field. Payload is `Block`.
	297	///
	298	/// See `unwrapBlock` for a way to load this tag's data.
294	299	loop,
295	300	/// Sends control flow back to the beginning of a parent `loop` body.
296	301	/// Uses the `repeat` field.
...	...	@@ -319,6 +324,8 @@ pub const Inst = struct {
319	324	/// Result type is the return type of the function being called.
320	325	/// Uses the `pl_op` field with the `Call` payload. operand is the callee.
321	326	/// Triggers `resolveTypeLayout` on the return type of the callee.
	327	///
	328	/// See `unwrapCall` for a way to load this tag's data.
322	329	call,
323	330	/// Same as `call` except with the `always_tail` attribute.
324	331	call_always_tail,
...	...	@@ -436,14 +443,20 @@ pub const Inst = struct {
436	443	/// Conditional branch.
437	444	/// Result type is always noreturn; no instructions in a block follow this one.
438	445	/// Uses the `pl_op` field. Operand is the condition. Payload is `CondBr`.
	446	///
	447	/// See `unwrapCondBr` for a way to load this tags's data.
439	448	cond_br,
440	449	/// Switch branch.
441	450	/// Result type is always noreturn; no instructions in a block follow this one.
442	451	/// Uses the `pl_op` field. Operand is the condition. Payload is `SwitchBr`.
	452	///
	453	/// See `unwrapSwitch` for a way to load this tags's data.
443	454	switch_br,
444	455	/// Switch branch which can dispatch back to itself with a different operand.
445	456	/// Result type is always noreturn; no instructions in a block follow this one.
446	457	/// Uses the `pl_op` field. Operand is the condition. Payload is `SwitchBr`.
	458	///
	459	/// See `unwrapSwitch` for a way to load this tags's data.
447	460	loop_switch_br,
448	461	/// Dispatches back to a branch of a parent `loop_switch_br`.
449	462	/// Result type is always noreturn; no instructions in a block follow this one.
...	...	@@ -458,6 +471,8 @@ pub const Inst = struct {
458	471	/// payload value, as if `unwrap_errunion_payload` was executed on the operand.
459	472	/// The error branch is considered to have a branch hint of `.unlikely`.
460	473	/// Uses the `pl_op` field. Payload is `Try`.
	474	///
	475	/// See `unwrapTry` for a way to load this tag's data.
461	476	@"try",
462	477	/// Same as `try` except the error branch hint is `.cold`.
463	478	try_cold,
...	...	@@ -465,6 +480,8 @@ pub const Inst = struct {
465	480	/// result is a pointer to the payload. Result is as if `unwrap_errunion_payload_ptr`
466	481	/// was executed on the operand.
467	482	/// Uses the `ty_pl` field. Payload is `TryPtr`.
	483	///
	484	/// See `unwrapTryPtr` for a way to load this tag's data.
468	485	try_ptr,
469	486	/// Same as `try_ptr` except the error branch hint is `.cold`.
470	487	try_ptr_cold,
...	...	@@ -476,6 +493,8 @@ pub const Inst = struct {
476	493	dbg_empty_stmt,
477	494	/// A block that represents an inlined function call.
478	495	/// Uses the `ty_pl` field. Payload is `DbgInlineBlock`.
	496	///
	497	/// See `unwrapBlock` for a way to load this tag's data.
479	498	dbg_inline_block,
480	499	/// Marks the beginning of a local variable. The operand is a pointer pointing
481	500	/// to the storage for the variable. The local may be a const or a var.
...	...	@@ -715,7 +734,7 @@ pub const Inst = struct {
715	734	/// Uses the `ty_pl` field, where the payload index points to:
716	735	/// 1. mask_elem: ShuffleOneMask // for each `mask_len`, which comes from `ty_pl.ty`
717	736	/// 2. operand: Ref // guaranteed not to be an interned value
718		/// See `unwrapShuffleOne`.
	737	/// See `unwrapShuffleOne` for a way to load this tag's data.
719	738	shuffle_one,
720	739	/// Constructs a vector by selecting elements from two vectors based on a mask. Each mask
721	740	/// element is either an index into one of the vectors, or "undef".
...	...	@@ -723,7 +742,7 @@ pub const Inst = struct {
723	742	/// 1. mask_elem: ShuffleOneMask // for each `mask_len`, which comes from `ty_pl.ty`
724	743	/// 2. operand_a: Ref // guaranteed not to be an interned value
725	744	/// 3. operand_b: Ref // guaranteed not to be an interned value
726		/// See `unwrapShuffleTwo`.
	745	/// See `unwrapShuffleTwo` for a way to load this tag's data..
727	746	shuffle_two,
728	747	/// Constructs a vector element-wise from `a` or `b` based on `pred`.
729	748	/// Uses the `pl_op` field with `pred` as operand, and payload `Bin`.
...	...	@@ -944,6 +963,8 @@ pub const Inst = struct {
944	963	/// The calling convention is given by `func.@"callconv"(target)`.
945	964	/// The return type (and hence the result type of this instruction) is `func.returnType()`.
946	965	/// The parameter types are the types of the arguments given in `Air.Call`.
	966	///
	967	/// See `unwrapCompilerRtCall` for a way to load this tag's data.
947	968	legalize_compiler_rt_call,
948	969
949	970	pub fn fromCmpOp(op: std.math.CompareOperator, optimized: bool) Tag {
...	...	@@ -1445,18 +1466,18 @@ pub const ShuffleTwoMask = enum(u32) {
1445	1466	/// Trailing:
1446	1467	/// 0. `Inst.Ref` for every outputs_len
1447	1468	/// 1. `Inst.Ref` for every inputs_len
1448		/// 2. for every outputs_len
	1469	/// 2. A number of u32 elements follow according to the equation `(source_len + 3) / 4`.
	1470	/// Memory starting at this position is reinterpreted as the source bytes.
	1471	/// 3. for every outputs_len
1449	1472	/// - constraint: memory at this position is reinterpreted as a null
1450	1473	/// terminated string.
1451	1474	/// - name: memory at this position is reinterpreted as a null
1452	1475	/// terminated string. pad to the next u32 after the null byte.
1453		/// 3. for every inputs_len
	1476	/// 4. for every inputs_len
1454	1477	/// - constraint: memory at this position is reinterpreted as a null
1455	1478	/// terminated string.
1456	1479	/// - name: memory at this position is reinterpreted as a null
1457	1480	/// terminated string. pad to the next u32 after the null byte.
1458		/// 4. A number of u32 elements follow according to the equation `(source_len + 3) / 4`.
1459		/// Memory starting at this position is reinterpreted as the source bytes.
1460	1481	pub const Asm = struct {
1461	1482	/// Length of the assembly source in bytes.
1462	1483	source_len: u32,
...	...	@@ -2157,11 +2178,229 @@ pub fn unwrapSwitch(air: *const Air, switch_inst: Inst.Index) UnwrappedSwitch {
2157	2178	};
2158	2179	}
2159	2180
2160		pub fn unwrapShuffleOne(air: const Air, zcu: const Zcu, inst_index: Inst.Index) struct {
	2181	pub const UnwrappedDbgInlineBlock = struct {
	2182	func: InternPool.Index,
	2183	body: []const Inst.Index,
	2184	ty: Type,
	2185	};
	2186
	2187	pub fn unwrapDbgBlock(air: *const Air, inst_index: Inst.Index) UnwrappedDbgInlineBlock {
	2188	const data = air.instructions.items(.data)[@intFromEnum(inst_index)];
	2189	const tag = air.instructions.items(.tag)[@intFromEnum(inst_index)];
	2190	assert(tag == .dbg_inline_block);
	2191	const payload = data.ty_pl.payload;
	2192	const extra = air.extraData(Air.DbgInlineBlock, payload);
	2193	return .{
	2194	.func = extra.data.func,
	2195	.ty = data.ty_pl.ty.toType(),
	2196	.body = @ptrCast(air.extra.items[extra.end..][0..extra.data.body_len]),
	2197	};
	2198	}
	2199
	2200	pub const UnwrappedBlock = struct {
	2201	body: []const Inst.Index,
	2202	ty: Type,
	2203	};
	2204
	2205	pub fn unwrapBlock(air: *const Air, inst_index: Inst.Index) UnwrappedBlock {
	2206	const data = air.instructions.items(.data)[@intFromEnum(inst_index)];
	2207	const tag = air.instructions.items(.tag)[@intFromEnum(inst_index)];
	2208	const payload = switch (tag) {
	2209	.block, .loop => data.ty_pl.payload,
	2210	else => unreachable,
	2211	};
	2212	const extra = air.extraData(Air.Block, payload);
	2213	return .{
	2214	.ty = data.ty_pl.ty.toType(),
	2215	.body = @ptrCast(air.extra.items[extra.end..][0..extra.data.body_len]),
	2216	};
	2217	}
	2218
	2219	pub const UnwrappedCall = struct {
	2220	callee: Inst.Ref,
	2221	args: []const Air.Inst.Ref,
	2222	};
	2223
	2224	pub fn unwrapCall(air: *const Air, inst_index: Inst.Index) UnwrappedCall {
	2225	const data = air.instructions.items(.data)[@intFromEnum(inst_index)];
	2226	const tag = air.instructions.items(.tag)[@intFromEnum(inst_index)];
	2227	const payload = switch (tag) {
	2228	.call, .call_always_tail, .call_never_tail, .call_never_inline => data.pl_op.payload,
	2229	else => unreachable,
	2230	};
	2231	const extra = air.extraData(Air.Call, payload);
	2232	return .{
	2233	.callee = data.pl_op.operand,
	2234	.args = @ptrCast(air.extra.items[extra.end..][0..extra.data.args_len]),
	2235	};
	2236	}
	2237
	2238	pub const UnwrappedCompilerRtCall = struct {
	2239	func: CompilerRtFunc,
	2240	args: []const Air.Inst.Ref,
	2241	};
	2242
	2243	pub fn unwrapCompilerRtCall(air: *const Air, inst_index: Inst.Index) UnwrappedCompilerRtCall {
	2244	const data = air.instructions.items(.data)[@intFromEnum(inst_index)];
	2245	const tag = air.instructions.items(.tag)[@intFromEnum(inst_index)];
	2246	assert(tag == .legalize_compiler_rt_call);
	2247	const payload = data.legalize_compiler_rt_call.payload;
	2248	const extra = air.extraData(Air.Call, payload);
	2249	return .{
	2250	.func = data.legalize_compiler_rt_call.func,
	2251	.args = @ptrCast(air.extra.items[extra.end..][0..extra.data.args_len]),
	2252	};
	2253	}
	2254
	2255	pub const UnwrappedCondBr = struct {
	2256	condition: Inst.Ref,
	2257	then_body: []const Inst.Index,
	2258	else_body: []const Inst.Index,
	2259	branch_hints: CondBr.BranchHints,
	2260	};
	2261
	2262	pub fn unwrapCondBr(air: *const Air, inst_index: Inst.Index) UnwrappedCondBr {
	2263	const data = air.instructions.items(.data)[@intFromEnum(inst_index)];
	2264	const tag = air.instructions.items(.tag)[@intFromEnum(inst_index)];
	2265	assert(tag == .cond_br);
	2266	const payload = data.pl_op.payload;
	2267	const extra = air.extraData(Air.CondBr, payload);
	2268	return .{
	2269	.condition = data.pl_op.operand,
	2270	.then_body = @ptrCast(air.extra.items[extra.end..][0..extra.data.then_body_len]),
	2271	.else_body = @ptrCast(air.extra.items[extra.end + extra.data.then_body_len ..][0..extra.data.else_body_len]),
	2272	.branch_hints = extra.data.branch_hints,
	2273	};
	2274	}
	2275
	2276	pub const UnwrappedTry = struct {
	2277	error_union: Inst.Ref,
	2278	else_body: []const Inst.Index,
	2279	};
	2280
	2281	pub fn unwrapTry(air: *const Air, inst_index: Inst.Index) UnwrappedTry {
	2282	const data = air.instructions.items(.data)[@intFromEnum(inst_index)];
	2283	const tag = air.instructions.items(.tag)[@intFromEnum(inst_index)];
	2284	assert(tag == .@"try" or tag == .try_cold);
	2285	const payload = data.pl_op.payload;
	2286	const extra = air.extraData(Air.Try, payload);
	2287	return .{
	2288	.error_union = data.pl_op.operand,
	2289	.else_body = @ptrCast(air.extra.items[extra.end..][0..extra.data.body_len]),
	2290	};
	2291	}
	2292
	2293	pub const UnwrappedTryPtr = struct {
	2294	error_union_payload_ptr_ty: Inst.Ref,
	2295	error_union_ptr: Inst.Ref,
	2296	else_body: []const Inst.Index,
	2297	};
	2298
	2299	pub fn unwrapTryPtr(air: *const Air, inst_index: Inst.Index) UnwrappedTryPtr {
	2300	const data = air.instructions.items(.data)[@intFromEnum(inst_index)];
	2301	const tag = air.instructions.items(.tag)[@intFromEnum(inst_index)];
	2302	assert(tag == .try_ptr or tag == .try_ptr_cold);
	2303	const payload = data.ty_pl.payload;
	2304	const extra = air.extraData(Air.TryPtr, payload);
	2305	return .{
	2306	.error_union_ptr = extra.data.ptr,
	2307	.error_union_payload_ptr_ty = data.ty_pl.ty,
	2308	.else_body = @ptrCast(air.extra.items[extra.end..][0..extra.data.body_len]),
	2309	};
	2310	}
	2311
	2312	pub const UnwrappedAsm = struct {
	2313	outputs: []const Air.Inst.Ref,
	2314	inputs: []const Air.Inst.Ref,
	2315	source: [:0]u8,
	2316	input_constraint_names: []const u32,
	2317	output_constraint_names: []const u32,
	2318	clobbers: InternPool.Index,
	2319	is_volatile: bool,
	2320
	2321	const AsmIterator = struct {
	2322	current: u32,
	2323	operands: []const Air.Inst.Ref,
	2324	constraint_names: []const u32,
	2325
	2326	pub fn next(self: *AsmIterator) ?struct { constraint: []const u8, operand: Inst.Ref, name: []const u8, index: u32 } {
	2327	if (self.current >= self.operands.len) {
	2328	return null;
	2329	}
	2330	defer {
	2331	self.current += 1;
	2332	}
	2333
	2334	const constraint_name = std.mem.sliceAsBytes(self.constraint_names);
	2335	const constraint = std.mem.sliceTo(constraint_name, 0);
	2336	const name = std.mem.sliceTo(constraint_name[constraint.len + 1 ..], 0);
	2337	// This equation accounts for the fact that even if we have exactly 4 bytes
	2338	// for the string, we still use the next u32 for the null terminator.
	2339	const next_offset = std.math.divCeil(usize, constraint.len + 1 + name.len + 1, @sizeOf(u32)) catch unreachable;
	2340	self.constraint_names = self.constraint_names[next_offset..];
	2341
	2342	return .{
	2343	.constraint = constraint,
	2344	.operand = self.operands[self.current],
	2345	.name = name,
	2346	.index = self.current,
	2347	};
	2348	}
	2349	};
	2350
	2351	pub fn iterateInputs(self: *const UnwrappedAsm) AsmIterator {
	2352	return .{
	2353	.current = 0,
	2354	.operands = self.inputs,
	2355	.constraint_names = self.input_constraint_names,
	2356	};
	2357	}
	2358
	2359	pub fn iterateOutputs(self: *const UnwrappedAsm) AsmIterator {
	2360	return .{
	2361	.current = 0,
	2362	.operands = self.outputs,
	2363	.constraint_names = self.output_constraint_names,
	2364	};
	2365	}
	2366	};
	2367
	2368	pub fn unwrapAsm(air: *const Air, inst_index: Inst.Index) UnwrappedAsm {
	2369	const data = air.instructions.items(.data)[@intFromEnum(inst_index)];
	2370	const tag = air.instructions.items(.tag)[@intFromEnum(inst_index)];
	2371	assert(tag == .assembly);
	2372	const payload = data.ty_pl.payload;
	2373	const extra = air.extraData(Air.Asm, payload);
	2374	const source_start = extra.end + extra.data.flags.outputs_len + extra.data.inputs_len;
	2375	const output_constraint_name_start = source_start + (extra.data.source_len / 4) + 1;
	2376	const output_constraint_name = air.extra.items[output_constraint_name_start..];
	2377	const outputs: []Inst.Ref = @ptrCast(air.extra.items[extra.end..][0..extra.data.flags.outputs_len]);
	2378	// Get the input names and constraints offset place after the output.
	2379	var it = UnwrappedAsm.AsmIterator{
	2380	.current = 0,
	2381	.constraint_names = output_constraint_name,
	2382	.operands = outputs,
	2383	};
	2384	while (it.next()) \|_\| {}
	2385
	2386	return .{
	2387	.clobbers = extra.data.clobbers,
	2388	.is_volatile = extra.data.flags.is_volatile,
	2389	.inputs = @ptrCast(air.extra.items[extra.end + extra.data.flags.outputs_len ..][0..extra.data.inputs_len]),
	2390	.outputs = outputs,
	2391	.source = std.mem.sliceAsBytes(air.extra.items[source_start..])[0..extra.data.source_len :0],
	2392	.output_constraint_names = output_constraint_name,
	2393	.input_constraint_names = it.constraint_names,
	2394	};
	2395	}
	2396
	2397	pub const UnwrappedShuffleOne = struct {
2161	2398	result_ty: Type,
2162	2399	operand: Inst.Ref,
2163	2400	mask: []const ShuffleOneMask,
2164		} {
	2401	};
	2402
	2403	pub fn unwrapShuffleOne(air: const Air, zcu: const Zcu, inst_index: Inst.Index) UnwrappedShuffleOne {
2165	2404	const inst = air.instructions.get(@intFromEnum(inst_index));
2166	2405	switch (inst.tag) {
2167	2406	.shuffle_one => {},
...	...	@@ -2177,12 +2416,14 @@ pub fn unwrapShuffleOne(air: const Air, zcu: const Zcu, inst_index: Inst.Index
2177	2416	};
2178	2417	}
2179	2418
2180		pub fn unwrapShuffleTwo(air: const Air, zcu: const Zcu, inst_index: Inst.Index) struct {
	2419	pub const UnwrappedShuffleTwo = struct {
2181	2420	result_ty: Type,
2182	2421	operand_a: Inst.Ref,
2183	2422	operand_b: Inst.Ref,
2184	2423	mask: []const ShuffleTwoMask,
2185		} {
	2424	};
	2425
	2426	pub fn unwrapShuffleTwo(air: const Air, zcu: const Zcu, inst_index: Inst.Index) UnwrappedShuffleTwo {
2186	2427	const inst = air.instructions.get(@intFromEnum(inst_index));
2187	2428	switch (inst.tag) {
2188	2429	.shuffle_two => {},

src/Air/Liveness.zig+39-52

...	...	@@ -17,6 +17,7 @@ const trace = @import("../tracy.zig").trace;
17	17	const Air = @import("../Air.zig");
18	18	const InternPool = @import("../InternPool.zig");
19	19	const Zcu = @import("../Zcu.zig");
	20	const Type = @import("../Type.zig");
20	21
21	22	pub const Verify = @import("Liveness/Verify.zig");
22	23
...	...	@@ -609,13 +610,11 @@ fn analyzeInst(
609	610	},
610	611
611	612	.call, .call_always_tail, .call_never_tail, .call_never_inline => {
612		const inst_data = inst_datas[@intFromEnum(inst)].pl_op;
613		const callee = inst_data.operand;
614		const extra = a.air.extraData(Air.Call, inst_data.payload);
615		const args = @as([]const Air.Inst.Ref, @ptrCast(a.air.extra.items[extra.end..][0..extra.data.args_len]));
	613	const call = a.air.unwrapCall(inst);
	614	const args = call.args;
616	615	if (args.len + 1 <= bpi - 1) {
617	616	var buf = [1]Air.Inst.Ref{.none} ** (bpi - 1);
618		buf[0] = callee;
	617	buf[0] = call.callee;
619	618	@memcpy(buf[1..][0..args.len], args);
620	619	return analyzeOperands(a, pass, data, inst, buf);
621	620	}
...	...	@@ -627,7 +626,7 @@ fn analyzeInst(
627	626	i -= 1;
628	627	try big.feed(args[i]);
629	628	}
630		try big.feed(callee);
	629	try big.feed(call.callee);
631	630	return big.finish();
632	631	},
633	632	.select => {
...	...	@@ -708,18 +707,15 @@ fn analyzeInst(
708	707	.switch_dispatch => return analyzeInstSwitchDispatch(a, pass, data, inst),
709	708
710	709	.assembly => {
711		const extra = a.air.extraData(Air.Asm, inst_datas[@intFromEnum(inst)].ty_pl.payload);
712		const outputs_len = extra.data.flags.outputs_len;
713		var extra_i: usize = extra.end;
714		const outputs = @as([]const Air.Inst.Ref, @ptrCast(a.air.extra.items[extra_i..][0..outputs_len]));
715		extra_i += outputs.len;
716		const inputs = @as([]const Air.Inst.Ref, @ptrCast(a.air.extra.items[extra_i..][0..extra.data.inputs_len]));
717		extra_i += inputs.len;
	710	const unwrapped_asm = a.air.unwrapAsm(inst);
	711
	712	const outputs = unwrapped_asm.outputs;
	713	const inputs = unwrapped_asm.inputs;
718	714
719	715	const num_operands = simple: {
720	716	var buf = [1]Air.Inst.Ref{.none} ** (bpi - 1);
721	717	var buf_index: usize = 0;
722		for (outputs) \|output\| {
	718	for (unwrapped_asm.outputs) \|output\| {
723	719	if (output != .none) {
724	720	if (buf_index < buf.len) buf[buf_index] = output;
725	721	buf_index += 1;
...	...	@@ -748,15 +744,13 @@ fn analyzeInst(
748	744	}
749	745	return big.finish();
750	746	},
751
752		inline .block, .dbg_inline_block => \|comptime_tag\| {
753		const ty_pl = inst_datas[@intFromEnum(inst)].ty_pl;
754		const extra = a.air.extraData(switch (comptime_tag) {
755		.block => Air.Block,
756		.dbg_inline_block => Air.DbgInlineBlock,
757		else => unreachable,
758		}, ty_pl.payload);
759		return analyzeInstBlock(a, pass, data, inst, ty_pl.ty, @ptrCast(a.air.extra.items[extra.end..][0..extra.data.body_len]));
	747	.dbg_inline_block => {
	748	const block = a.air.unwrapDbgBlock(inst);
	749	return analyzeInstBlock(a, pass, data, inst, block.ty, block.body);
	750	},
	751	.block => {
	752	const block = a.air.unwrapBlock(inst);
	753	return analyzeInstBlock(a, pass, data, inst, block.ty, block.body);
760	754	},
761	755	.loop => return analyzeInstLoop(a, pass, data, inst),
762	756
...	...	@@ -778,8 +772,8 @@ fn analyzeInst(
778	772	},
779	773
780	774	.legalize_compiler_rt_call => {
781		const extra = a.air.extraData(Air.Call, inst_datas[@intFromEnum(inst)].legalize_compiler_rt_call.payload);
782		const args: []const Air.Inst.Ref = @ptrCast(a.air.extra.items[extra.end..][0..extra.data.args_len]);
	775	const rt_call = a.air.unwrapCompilerRtCall(inst);
	776	const args = rt_call.args;
783	777	if (args.len <= bpi - 1) {
784	778	var buf: [bpi - 1]Air.Inst.Ref = @splat(.none);
785	779	@memcpy(buf[0..args.len], args);
...	...	@@ -972,7 +966,7 @@ fn analyzeInstBlock(
972	966	comptime pass: LivenessPass,
973	967	data: *LivenessPassData(pass),
974	968	inst: Air.Inst.Index,
975		ty: Air.Inst.Ref,
	969	ty: Type,
976	970	body: []const Air.Inst.Index,
977	971	) !void {
978	972	const gpa = a.gpa;
...	...	@@ -1005,7 +999,7 @@ fn analyzeInstBlock(
1005	999
1006	1000	// If the block is noreturn, block deaths not only aren't useful, they're impossible to
1007	1001	// find: there could be more stuff alive after the block than before it!
1008		if (!a.intern_pool.isNoReturn(ty.toType().toIntern())) {
	1002	if (!a.intern_pool.isNoReturn(ty.toIntern())) {
1009	1003	// The block kills the difference in the live sets
1010	1004	const block_scope = data.block_scopes.get(inst).?;
1011	1005	const num_deaths = data.live_set.count() - block_scope.live_set.count();
...	...	@@ -1139,9 +1133,8 @@ fn analyzeInstLoop(
1139	1133	data: *LivenessPassData(pass),
1140	1134	inst: Air.Inst.Index,
1141	1135	) !void {
1142		const inst_datas = a.air.instructions.items(.data);
1143		const extra = a.air.extraData(Air.Block, inst_datas[@intFromEnum(inst)].ty_pl.payload);
1144		const body: []const Air.Inst.Index = @ptrCast(a.air.extra.items[extra.end..][0..extra.data.body_len]);
	1136	const block = a.air.unwrapBlock(inst);
	1137	const body = block.body;
1145	1138	const gpa = a.gpa;
1146	1139
1147	1140	try analyzeOperands(a, pass, data, inst, .{ .none, .none, .none });
...	...	@@ -1187,44 +1180,38 @@ fn analyzeInstCondBr(
1187	1180	inst: Air.Inst.Index,
1188	1181	comptime inst_type: enum { cond_br, @"try", try_ptr },
1189	1182	) !void {
1190		const inst_datas = a.air.instructions.items(.data);
1191	1183	const gpa = a.gpa;
1192	1184
1193		const extra = switch (inst_type) {
1194		.cond_br => a.air.extraData(Air.CondBr, inst_datas[@intFromEnum(inst)].pl_op.payload),
1195		.@"try" => a.air.extraData(Air.Try, inst_datas[@intFromEnum(inst)].pl_op.payload),
1196		.try_ptr => a.air.extraData(Air.TryPtr, inst_datas[@intFromEnum(inst)].ty_pl.payload),
	1185	const unwrapped_cond = switch (inst_type) {
	1186	.cond_br => a.air.unwrapCondBr(inst),
	1187	.@"try" => a.air.unwrapTry(inst),
	1188	.try_ptr => a.air.unwrapTryPtr(inst),
1197	1189	};
1198	1190
1199	1191	const condition = switch (inst_type) {
1200		.cond_br, .@"try" => inst_datas[@intFromEnum(inst)].pl_op.operand,
1201		.try_ptr => extra.data.ptr,
	1192	.cond_br => unwrapped_cond.condition,
	1193	.@"try" => unwrapped_cond.error_union,
	1194	.try_ptr => unwrapped_cond.error_union_ptr,
1202	1195	};
1203	1196
1204		const then_body: []const Air.Inst.Index = switch (inst_type) {
1205		.cond_br => @ptrCast(a.air.extra.items[extra.end..][0..extra.data.then_body_len]),
1206		else => &.{}, // we won't use this
	1197	const then_body = switch (inst_type) {
	1198	.cond_br => unwrapped_cond.then_body,
	1199	// The "then body" is just the remainder of this block
	1200	else => &.{},
1207	1201	};
1208	1202
1209		const else_body: []const Air.Inst.Index = @ptrCast(switch (inst_type) {
1210		.cond_br => a.air.extra.items[extra.end + then_body.len ..][0..extra.data.else_body_len],
1211		.@"try", .try_ptr => a.air.extra.items[extra.end..][0..extra.data.body_len],
1212		});
	1203	const else_body = switch (inst_type) {
	1204	.cond_br, .@"try", .try_ptr => unwrapped_cond.else_body,
	1205	};
1213	1206
1214	1207	switch (pass) {
1215	1208	.loop_analysis => {
1216		switch (inst_type) {
1217		.cond_br => try analyzeBody(a, pass, data, then_body),
1218		.@"try", .try_ptr => {},
1219		}
	1209	try analyzeBody(a, pass, data, then_body);
1220	1210	try analyzeBody(a, pass, data, else_body);
1221	1211	},
1222	1212
1223	1213	.main_analysis => {
1224		switch (inst_type) {
1225		.cond_br => try analyzeBody(a, pass, data, then_body),
1226		.@"try", .try_ptr => {}, // The "then body" is just the remainder of this block
1227		}
	1214	try analyzeBody(a, pass, data, then_body);
1228	1215	var then_live = data.live_set.move();
1229	1216	defer then_live.deinit(gpa);
1230	1217

src/Air/Liveness/Verify.zig+24-46

...	...	@@ -345,37 +345,26 @@ fn verifyBody(self: *Verify, body: []const Air.Inst.Index) Error!void {
345	345	try self.verifyInst(inst);
346	346	},
347	347	.call, .call_always_tail, .call_never_tail, .call_never_inline => {
348		const pl_op = data[@intFromEnum(inst)].pl_op;
349		const extra = self.air.extraData(Air.Call, pl_op.payload);
350		const args = @as(
351		[]const Air.Inst.Ref,
352		@ptrCast(self.air.extra.items[extra.end..][0..extra.data.args_len]),
353		);
	348	const call = self.air.unwrapCall(inst);
	349	const args = call.args;
354	350
355	351	var bt = self.liveness.iterateBigTomb(inst);
356		try self.verifyOperand(inst, pl_op.operand, bt.feed());
	352	try self.verifyOperand(inst, call.callee, bt.feed());
357	353	for (args) \|arg\| {
358	354	try self.verifyOperand(inst, arg, bt.feed());
359	355	}
360	356	try self.verifyInst(inst);
361	357	},
362	358	.assembly => {
363		const ty_pl = data[@intFromEnum(inst)].ty_pl;
364		const extra = self.air.extraData(Air.Asm, ty_pl.payload);
365		const outputs_len = extra.data.flags.outputs_len;
366		var extra_i = extra.end;
367		const outputs: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[extra_i..][0..outputs_len]);
368		extra_i += outputs.len;
369		const inputs: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[extra_i..][0..extra.data.inputs_len]);
370		extra_i += inputs.len;
	359	const unwrapped_asm = self.air.unwrapAsm(inst);
371	360
372	361	var bt = self.liveness.iterateBigTomb(inst);
373		for (outputs) \|output\| {
	362	for (unwrapped_asm.outputs) \|output\| {
374	363	if (output != .none) {
375	364	try self.verifyOperand(inst, output, bt.feed());
376	365	}
377	366	}
378		for (inputs) \|input\| {
	367	for (unwrapped_asm.inputs) \|input\| {
379	368	try self.verifyOperand(inst, input, bt.feed());
380	369	}
381	370	try self.verifyInst(inst);
...	...	@@ -383,13 +372,12 @@ fn verifyBody(self: *Verify, body: []const Air.Inst.Index) Error!void {
383	372
384	373	// control flow
385	374	.@"try", .try_cold => {
386		const pl_op = data[@intFromEnum(inst)].pl_op;
387		const extra = self.air.extraData(Air.Try, pl_op.payload);
388		const try_body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[extra.end..][0..extra.data.body_len]);
	375	const unwrapped_try = self.air.unwrapTry(inst);
	376	const try_body = unwrapped_try.else_body;
389	377
390	378	const cond_br_liveness = self.liveness.getCondBr(inst);
391	379
392		try self.verifyOperand(inst, pl_op.operand, self.liveness.operandDies(inst, 0));
	380	try self.verifyOperand(inst, unwrapped_try.error_union, self.liveness.operandDies(inst, 0));
393	381
394	382	var live = try self.live.clone(self.gpa);
395	383	defer live.deinit(self.gpa);
...	...	@@ -405,13 +393,12 @@ fn verifyBody(self: *Verify, body: []const Air.Inst.Index) Error!void {
405	393	try self.verifyInst(inst);
406	394	},
407	395	.try_ptr, .try_ptr_cold => {
408		const ty_pl = data[@intFromEnum(inst)].ty_pl;
409		const extra = self.air.extraData(Air.TryPtr, ty_pl.payload);
410		const try_body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[extra.end..][0..extra.data.body_len]);
	396	const unwrapped_try = self.air.unwrapTryPtr(inst);
	397	const try_body = unwrapped_try.else_body;
411	398
412	399	const cond_br_liveness = self.liveness.getCondBr(inst);
413	400
414		try self.verifyOperand(inst, extra.data.ptr, self.liveness.operandDies(inst, 0));
	401	try self.verifyOperand(inst, unwrapped_try.error_union_ptr, self.liveness.operandDies(inst, 0));
415	402
416	403	var live = try self.live.clone(self.gpa);
417	404	defer live.deinit(self.gpa);
...	...	@@ -458,17 +445,11 @@ fn verifyBody(self: *Verify, body: []const Air.Inst.Index) Error!void {
458	445	.block, .dbg_inline_block => \|tag\| {
459	446	const ty_pl = data[@intFromEnum(inst)].ty_pl;
460	447	const block_ty = ty_pl.ty.toType();
461		const block_body: []const Air.Inst.Index = @ptrCast(switch (tag) {
462		inline .block, .dbg_inline_block => \|comptime_tag\| body: {
463		const extra = self.air.extraData(switch (comptime_tag) {
464		.block => Air.Block,
465		.dbg_inline_block => Air.DbgInlineBlock,
466		else => unreachable,
467		}, ty_pl.payload);
468		break :body self.air.extra.items[extra.end..][0..extra.data.body_len];
469		},
	448	const block_body = switch (tag) {
	449	.block => self.air.unwrapBlock(inst).body,
	450	.dbg_inline_block => self.air.unwrapDbgBlock(inst).body,
470	451	else => unreachable,
471		});
	452	};
472	453	const block_liveness = self.liveness.getBlock(inst);
473	454
474	455	var orig_live = try self.live.clone(self.gpa);
...	...	@@ -501,9 +482,7 @@ fn verifyBody(self: *Verify, body: []const Air.Inst.Index) Error!void {
501	482	try self.verifyInstOperands(inst, .{ .none, .none, .none });
502	483	},
503	484	.loop => {
504		const ty_pl = data[@intFromEnum(inst)].ty_pl;
505		const extra = self.air.extraData(Air.Block, ty_pl.payload);
506		const loop_body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[extra.end..][0..extra.data.body_len]);
	485	const block = self.air.unwrapBlock(inst);
507	486
508	487	// The same stuff should be alive after the loop as before it.
509	488	const gop = try self.loops.getOrPut(self.gpa, inst);
...	...	@@ -514,18 +493,17 @@ fn verifyBody(self: *Verify, body: []const Air.Inst.Index) Error!void {
514	493	}
515	494	gop.value_ptr.* = try self.live.clone(self.gpa);
516	495
517		try self.verifyBody(loop_body);
	496	try self.verifyBody(block.body);
518	497
519	498	try self.verifyInstOperands(inst, .{ .none, .none, .none });
520	499	},
521	500	.cond_br => {
522		const pl_op = data[@intFromEnum(inst)].pl_op;
523		const extra = self.air.extraData(Air.CondBr, pl_op.payload);
524		const then_body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[extra.end..][0..extra.data.then_body_len]);
525		const else_body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[extra.end + then_body.len ..][0..extra.data.else_body_len]);
	501	const cond_br = self.air.unwrapCondBr(inst);
	502	const then_body = cond_br.then_body;
	503	const else_body = cond_br.else_body;
526	504	const cond_br_liveness = self.liveness.getCondBr(inst);
527	505
528		try self.verifyOperand(inst, pl_op.operand, self.liveness.operandDies(inst, 0));
	506	try self.verifyOperand(inst, cond_br.condition, self.liveness.operandDies(inst, 0));
529	507
530	508	var live = try self.live.clone(self.gpa);
531	509	defer live.deinit(self.gpa);
...	...	@@ -589,8 +567,8 @@ fn verifyBody(self: *Verify, body: []const Air.Inst.Index) Error!void {
589	567	try self.verifyInstOperands(inst, .{ pl_op.operand, bin.lhs, bin.rhs });
590	568	},
591	569	.legalize_compiler_rt_call => {
592		const extra = self.air.extraData(Air.Call, data[@intFromEnum(inst)].legalize_compiler_rt_call.payload);
593		const args: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[extra.end..][0..extra.data.args_len]);
	570	const rt_call = self.air.unwrapCompilerRtCall(inst);
	571	const args = rt_call.args;
594	572	var bt = self.liveness.iterateBigTomb(inst);
595	573	for (args) \|arg\| {
596	574	try self.verifyOperand(inst, arg, bt.feed());

src/Air/print.zig+52-86

...	...	@@ -395,25 +395,17 @@ const Writer = struct {
395	395	fn writeBlock(w: Writer, s: std.Io.Writer, tag: Air.Inst.Tag, inst: Air.Inst.Index) Error!void {
396	396	const ty_pl = w.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
397	397	try w.writeType(s, ty_pl.ty.toType());
398		const body: []const Air.Inst.Index = @ptrCast(switch (tag) {
399		inline .block, .dbg_inline_block => \|comptime_tag\| body: {
400		const extra = w.air.extraData(switch (comptime_tag) {
401		.block => Air.Block,
402		.dbg_inline_block => Air.DbgInlineBlock,
403		else => unreachable,
404		}, ty_pl.payload);
405		switch (comptime_tag) {
406		.block => {},
407		.dbg_inline_block => {
408		try s.writeAll(", ");
409		try w.writeInstRef(s, Air.internedToRef(extra.data.func), false);
410		},
411		else => unreachable,
412		}
413		break :body w.air.extra.items[extra.end..][0..extra.data.body_len];
	398
	399	const body = switch (tag) {
	400	.block => w.air.unwrapBlock(inst).body,
	401	.dbg_inline_block => body: {
	402	const dbg_block = w.air.unwrapDbgBlock(inst);
	403	try s.writeAll(", ");
	404	try w.writeInstRef(s, Air.internedToRef(dbg_block.func), false);
	405	break :body dbg_block.body;
414	406	},
415	407	else => unreachable,
416		});
	408	};
417	409	if (w.skip_body) return s.writeAll(", ...");
418	410	const liveness_block: Air.Liveness.BlockSlices = if (w.liveness) \|liveness\|
419	411	liveness.getBlock(inst)
...	...	@@ -434,16 +426,14 @@ const Writer = struct {
434	426	}
435	427
436	428	fn writeLoop(w: Writer, s: std.Io.Writer, inst: Air.Inst.Index) Error!void {
437		const ty_pl = w.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
438		const extra = w.air.extraData(Air.Block, ty_pl.payload);
439		const body: []const Air.Inst.Index = @ptrCast(w.air.extra.items[extra.end..][0..extra.data.body_len]);
	429	const block = w.air.unwrapBlock(inst);
440	430
441		try w.writeType(s, ty_pl.ty.toType());
	431	try w.writeType(s, block.ty);
442	432	if (w.skip_body) return s.writeAll(", ...");
443	433	try s.writeAll(", {\n");
444	434	const old_indent = w.indent;
445	435	w.indent += 2;
446		try w.writeBody(s, body);
	436	try w.writeBody(s, block.body);
447	437	w.indent = old_indent;
448	438	try s.splatByteAll(' ', w.indent);
449	439	try s.writeAll("}");
...	...	@@ -532,11 +522,10 @@ const Writer = struct {
532	522	}
533	523
534	524	fn writeLegalizeCompilerRtCall(w: Writer, s: std.Io.Writer, inst: Air.Inst.Index) Error!void {
535		const inst_data = w.air.instructions.items(.data)[@intFromEnum(inst)].legalize_compiler_rt_call;
536		const extra = w.air.extraData(Air.Call, inst_data.payload);
537		const args: []const Air.Inst.Ref = @ptrCast(w.air.extra.items[extra.end..][0..extra.data.args_len]);
	525	const rt_call = w.air.unwrapCompilerRtCall(inst);
	526	const args = rt_call.args;
538	527
539		try s.print("{t}, [", .{inst_data.func});
	528	try s.print("{t}, [", .{rt_call.func});
540	529	for (args, 0..) \|arg, i\| {
541	530	if (i != 0) try s.writeAll(", ");
542	531	try w.writeOperand(s, inst, i, arg);
...	...	@@ -666,11 +655,8 @@ const Writer = struct {
666	655	}
667	656
668	657	fn writeAssembly(w: Writer, s: std.Io.Writer, inst: Air.Inst.Index) Error!void {
669		const ty_pl = w.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
670		const extra = w.air.extraData(Air.Asm, ty_pl.payload);
671		const is_volatile = extra.data.flags.is_volatile;
672		const outputs_len = extra.data.flags.outputs_len;
673		var extra_i: usize = extra.end;
	658	const unwrapped_asm = w.air.unwrapAsm(inst);
	659	const is_volatile = unwrapped_asm.is_volatile;
674	660	var op_index: usize = 0;
675	661
676	662	const ret_ty = w.typeOfIndex(inst);
...	...	@@ -680,49 +666,33 @@ const Writer = struct {
680	666	try s.writeAll(", volatile");
681	667	}
682	668
683		const outputs = @as([]const Air.Inst.Ref, @ptrCast(w.air.extra.items[extra_i..][0..outputs_len]));
684		extra_i += outputs.len;
685		const inputs = @as([]const Air.Inst.Ref, @ptrCast(w.air.extra.items[extra_i..][0..extra.data.inputs_len]));
686		extra_i += inputs.len;
687
688		for (outputs) \|output\| {
689		const extra_bytes = std.mem.sliceAsBytes(w.air.extra.items[extra_i..]);
690		const constraint = std.mem.sliceTo(extra_bytes, 0);
691		const name = std.mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0);
692
693		// This equation accounts for the fact that even if we have exactly 4 bytes
694		// for the strings and their null terminators, we still use the next u32
695		// for the null terminator.
696		extra_i += (constraint.len + name.len + (2 + 3)) / 4;
697
698		if (output == .none) {
	669	var it = unwrapped_asm.iterateOutputs();
	670	while (it.next()) \|out\| {
	671	const name = out.name;
	672	const constraint = out.constraint;
	673	if (out.operand == .none) {
699	674	try s.print(", [{s}] -> {s}", .{ name, constraint });
700	675	} else {
701	676	try s.print(", [{s}] out {s} = (", .{ name, constraint });
702		try w.writeOperand(s, inst, op_index, output);
	677	try w.writeOperand(s, inst, op_index, out.operand);
703	678	op_index += 1;
704	679	try s.writeByte(')');
705	680	}
706	681	}
707	682
708		for (inputs) \|input\| {
709		const extra_bytes = std.mem.sliceAsBytes(w.air.extra.items[extra_i..]);
710		const constraint = std.mem.sliceTo(extra_bytes, 0);
711		const name = std.mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0);
712		// This equation accounts for the fact that even if we have exactly 4 bytes
713		// for the strings and their null terminators, we still use the next u32
714		// for the null terminator.
715		extra_i += (constraint.len + name.len + 1) / 4 + 1;
716
	683	it = unwrapped_asm.iterateInputs();
	684	while (it.next()) \|in\| {
	685	const name = in.name;
	686	const constraint = in.constraint;
717	687	try s.print(", [{s}] in {s} = (", .{ name, constraint });
718		try w.writeOperand(s, inst, op_index, input);
	688	try w.writeOperand(s, inst, op_index, in.operand);
719	689	op_index += 1;
720	690	try s.writeByte(')');
721	691	}
722	692
723	693	const zcu = w.pt.zcu;
724	694	const ip = &zcu.intern_pool;
725		const aggregate = ip.indexToKey(extra.data.clobbers).aggregate;
	695	const aggregate = ip.indexToKey(unwrapped_asm.clobbers).aggregate;
726	696	const struct_type: Type = .fromInterned(aggregate.ty);
727	697	switch (aggregate.storage) {
728	698	.elems => \|elems\| for (elems, 0..) \|elem, i\| {
...	...	@@ -750,7 +720,7 @@ const Writer = struct {
750	720	try s.print(", {x}", .{bytes});
751	721	},
752	722	}
753		const asm_source = std.mem.sliceAsBytes(w.air.extra.items[extra_i..])[0..extra.data.source_len];
	723	const asm_source = unwrapped_asm.source;
754	724	try s.print(", \"{f}\"", .{std.zig.fmtString(asm_source)});
755	725	}
756	726
...	...	@@ -767,10 +737,9 @@ const Writer = struct {
767	737	}
768	738
769	739	fn writeCall(w: Writer, s: std.Io.Writer, inst: Air.Inst.Index) Error!void {
770		const pl_op = w.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
771		const extra = w.air.extraData(Air.Call, pl_op.payload);
772		const args = @as([]const Air.Inst.Ref, @ptrCast(w.air.extra.items[extra.end..][0..extra.data.args_len]));
773		try w.writeOperand(s, inst, 0, pl_op.operand);
	740	const call = w.air.unwrapCall(inst);
	741	const args = call.args;
	742	try w.writeOperand(s, inst, 0, call.callee);
774	743	try s.writeAll(", [");
775	744	for (args, 0..) \|arg, i\| {
776	745	if (i != 0) try s.writeAll(", ");
...	...	@@ -792,15 +761,14 @@ const Writer = struct {
792	761	}
793	762
794	763	fn writeTry(w: Writer, s: std.Io.Writer, inst: Air.Inst.Index) Error!void {
795		const pl_op = w.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
796		const extra = w.air.extraData(Air.Try, pl_op.payload);
797		const body: []const Air.Inst.Index = @ptrCast(w.air.extra.items[extra.end..][0..extra.data.body_len]);
	764	const unwrapped_try = w.air.unwrapTry(inst);
	765	const body = unwrapped_try.else_body;
798	766	const liveness_condbr: Air.Liveness.CondBrSlices = if (w.liveness) \|liveness\|
799	767	liveness.getCondBr(inst)
800	768	else
801	769	.{ .then_deaths = &.{}, .else_deaths = &.{} };
802	770
803		try w.writeOperand(s, inst, 0, pl_op.operand);
	771	try w.writeOperand(s, inst, 0, unwrapped_try.error_union);
804	772	if (w.skip_body) return s.writeAll(", ...");
805	773	try s.writeAll(", {\n");
806	774	const old_indent = w.indent;
...	...	@@ -826,18 +794,17 @@ const Writer = struct {
826	794	}
827	795
828	796	fn writeTryPtr(w: Writer, s: std.Io.Writer, inst: Air.Inst.Index) Error!void {
829		const ty_pl = w.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
830		const extra = w.air.extraData(Air.TryPtr, ty_pl.payload);
831		const body: []const Air.Inst.Index = @ptrCast(w.air.extra.items[extra.end..][0..extra.data.body_len]);
	797	const unwrapped_try = w.air.unwrapTryPtr(inst);
	798	const body = unwrapped_try.else_body;
832	799	const liveness_condbr: Air.Liveness.CondBrSlices = if (w.liveness) \|liveness\|
833	800	liveness.getCondBr(inst)
834	801	else
835	802	.{ .then_deaths = &.{}, .else_deaths = &.{} };
836	803
837		try w.writeOperand(s, inst, 0, extra.data.ptr);
	804	try w.writeOperand(s, inst, 0, unwrapped_try.error_union_ptr);
838	805
839	806	try s.writeAll(", ");
840		try w.writeType(s, ty_pl.ty.toType());
	807	try w.writeType(s, unwrapped_try.error_union_payload_ptr_ty.toType());
841	808	if (w.skip_body) return s.writeAll(", ...");
842	809	try s.writeAll(", {\n");
843	810	const old_indent = w.indent;
...	...	@@ -863,23 +830,22 @@ const Writer = struct {
863	830	}
864	831
865	832	fn writeCondBr(w: Writer, s: std.Io.Writer, inst: Air.Inst.Index) Error!void {
866		const pl_op = w.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
867		const extra = w.air.extraData(Air.CondBr, pl_op.payload);
868		const then_body: []const Air.Inst.Index = @ptrCast(w.air.extra.items[extra.end..][0..extra.data.then_body_len]);
869		const else_body: []const Air.Inst.Index = @ptrCast(w.air.extra.items[extra.end + then_body.len ..][0..extra.data.else_body_len]);
	833	const cond_br = w.air.unwrapCondBr(inst);
	834	const then_body = cond_br.then_body;
	835	const else_body = cond_br.else_body;
870	836	const liveness_condbr: Air.Liveness.CondBrSlices = if (w.liveness) \|liveness\|
871	837	liveness.getCondBr(inst)
872	838	else
873	839	.{ .then_deaths = &.{}, .else_deaths = &.{} };
874	840
875		try w.writeOperand(s, inst, 0, pl_op.operand);
	841	try w.writeOperand(s, inst, 0, cond_br.condition);
876	842	if (w.skip_body) return s.writeAll(", ...");
877	843	try s.writeAll(",");
878		if (extra.data.branch_hints.true != .none) {
879		try s.print(" {s}", .{@tagName(extra.data.branch_hints.true)});
	844	if (cond_br.branch_hints.true != .none) {
	845	try s.print(" {s}", .{@tagName(cond_br.branch_hints.true)});
880	846	}
881		if (extra.data.branch_hints.then_cov != .none) {
882		try s.print(" {s}", .{@tagName(extra.data.branch_hints.then_cov)});
	847	if (cond_br.branch_hints.then_cov != .none) {
	848	try s.print(" {s}", .{@tagName(cond_br.branch_hints.then_cov)});
883	849	}
884	850	try s.writeAll(" {\n");
885	851	const old_indent = w.indent;
...	...	@@ -897,11 +863,11 @@ const Writer = struct {
897	863	try w.writeBody(s, then_body);
898	864	try s.splatByteAll(' ', old_indent);
899	865	try s.writeAll("},");
900		if (extra.data.branch_hints.false != .none) {
901		try s.print(" {s}", .{@tagName(extra.data.branch_hints.false)});
	866	if (cond_br.branch_hints.false != .none) {
	867	try s.print(" {s}", .{@tagName(cond_br.branch_hints.false)});
902	868	}
903		if (extra.data.branch_hints.else_cov != .none) {
904		try s.print(" {s}", .{@tagName(extra.data.branch_hints.else_cov)});
	869	if (cond_br.branch_hints.else_cov != .none) {
	870	try s.print(" {s}", .{@tagName(cond_br.branch_hints.else_cov)});
905	871	}
906	872	try s.writeAll(" {\n");
907	873

src/Air/types_resolved.zig+26-26

...	...	@@ -170,21 +170,21 @@ fn checkBody(air: Air, body: []const Air.Inst.Index, zcu: *Zcu) bool {
170	170	.block,
171	171	.loop,
172	172	=> {
173		const extra = air.extraData(Air.Block, data.ty_pl.payload);
174		if (!checkType(data.ty_pl.ty.toType(), zcu)) return false;
	173	const block = air.unwrapBlock(inst);
	174	if (!checkType(block.ty, zcu)) return false;
175	175	if (!checkBody(
176	176	air,
177		@ptrCast(air.extra.items[extra.end..][0..extra.data.body_len]),
	177	block.body,
178	178	zcu,
179	179	)) return false;
180	180	},
181	181
182	182	.dbg_inline_block => {
183		const extra = air.extraData(Air.DbgInlineBlock, data.ty_pl.payload);
184		if (!checkType(data.ty_pl.ty.toType(), zcu)) return false;
	183	const block = air.unwrapDbgBlock(inst);
	184	if (!checkType(block.ty, zcu)) return false;
185	185	if (!checkBody(
186	186	air,
187		@ptrCast(air.extra.items[extra.end..][0..extra.data.body_len]),
	187	block.body,
188	188	zcu,
189	189	)) return false;
190	190	},
...	...	@@ -342,9 +342,9 @@ fn checkBody(air: Air, body: []const Air.Inst.Index, zcu: *Zcu) bool {
342	342	.call_never_tail,
343	343	.call_never_inline,
344	344	=> {
345		const extra = air.extraData(Air.Call, data.pl_op.payload);
346		const args: []const Air.Inst.Ref = @ptrCast(air.extra.items[extra.end..][0..extra.data.args_len]);
347		if (!checkRef(data.pl_op.operand, zcu)) return false;
	345	const call = air.unwrapCall(inst);
	346	const args = call.args;
	347	if (!checkRef(call.callee, zcu)) return false;
348	348	for (args) \|arg\| if (!checkRef(arg, zcu)) return false;
349	349	},
350	350
...	...	@@ -356,37 +356,37 @@ fn checkBody(air: Air, body: []const Air.Inst.Index, zcu: *Zcu) bool {
356	356	},
357	357
358	358	.@"try", .try_cold => {
359		const extra = air.extraData(Air.Try, data.pl_op.payload);
360		if (!checkRef(data.pl_op.operand, zcu)) return false;
	359	const unwrapped_try = air.unwrapTry(inst);
	360	if (!checkRef(unwrapped_try.error_union, zcu)) return false;
361	361	if (!checkBody(
362	362	air,
363		@ptrCast(air.extra.items[extra.end..][0..extra.data.body_len]),
	363	unwrapped_try.else_body,
364	364	zcu,
365	365	)) return false;
366	366	},
367	367
368	368	.try_ptr, .try_ptr_cold => {
369		const extra = air.extraData(Air.TryPtr, data.ty_pl.payload);
370		if (!checkType(data.ty_pl.ty.toType(), zcu)) return false;
371		if (!checkRef(extra.data.ptr, zcu)) return false;
	369	const unwrapped_try = air.unwrapTryPtr(inst);
	370	if (!checkType(unwrapped_try.error_union_payload_ptr_ty.toType(), zcu)) return false;
	371	if (!checkRef(unwrapped_try.error_union_ptr, zcu)) return false;
372	372	if (!checkBody(
373	373	air,
374		@ptrCast(air.extra.items[extra.end..][0..extra.data.body_len]),
	374	unwrapped_try.else_body,
375	375	zcu,
376	376	)) return false;
377	377	},
378	378
379	379	.cond_br => {
380		const extra = air.extraData(Air.CondBr, data.pl_op.payload);
381		if (!checkRef(data.pl_op.operand, zcu)) return false;
	380	const cond_br = air.unwrapCondBr(inst);
	381	if (!checkRef(cond_br.condition, zcu)) return false;
382	382	if (!checkBody(
383	383	air,
384		@ptrCast(air.extra.items[extra.end..][0..extra.data.then_body_len]),
	384	cond_br.then_body,
385	385	zcu,
386	386	)) return false;
387	387	if (!checkBody(
388	388	air,
389		@ptrCast(air.extra.items[extra.end + extra.data.then_body_len ..][0..extra.data.else_body_len]),
	389	cond_br.else_body,
390	390	zcu,
391	391	)) return false;
392	392	},
...	...	@@ -407,20 +407,20 @@ fn checkBody(air: Air, body: []const Air.Inst.Index, zcu: *Zcu) bool {
407	407	},
408	408
409	409	.assembly => {
410		const extra = air.extraData(Air.Asm, data.ty_pl.payload);
	410	const unwrapped_asm = air.unwrapAsm(inst);
411	411	if (!checkType(data.ty_pl.ty.toType(), zcu)) return false;
412	412	// Luckily, we only care about the inputs and outputs, so we don't have to do
413	413	// the whole null-terminated string dance.
414		const outputs_len = extra.data.flags.outputs_len;
415		const outputs: []const Air.Inst.Ref = @ptrCast(air.extra.items[extra.end..][0..outputs_len]);
416		const inputs: []const Air.Inst.Ref = @ptrCast(air.extra.items[extra.end + outputs_len ..][0..extra.data.inputs_len]);
	414	const outputs = unwrapped_asm.outputs;
	415	const inputs = unwrapped_asm.inputs;
	416
417	417	for (outputs) \|output\| if (output != .none and !checkRef(output, zcu)) return false;
418	418	for (inputs) \|input\| if (input != .none and !checkRef(input, zcu)) return false;
419	419	},
420	420
421	421	.legalize_compiler_rt_call => {
422		const extra = air.extraData(Air.Call, data.legalize_compiler_rt_call.payload);
423		const args: []const Air.Inst.Ref = @ptrCast(air.extra.items[extra.end..][0..extra.data.args_len]);
	422	const rt_call = air.unwrapCompilerRtCall(inst);
	423	const args = rt_call.args;
424	424	for (args) \|arg\| if (!checkRef(arg, zcu)) return false;
425	425	},
426	426

src/Sema.zig+6-6

...	...	@@ -16226,6 +16226,12 @@ fn zirAsm(
16226	16226	});
16227	16227	sema.appendRefsAssumeCapacity(out_args);
16228	16228	sema.appendRefsAssumeCapacity(args);
	16229	{
	16230	const buffer = mem.sliceAsBytes(sema.air_extra.unusedCapacitySlice());
	16231	@memcpy(buffer[0..asm_source.len], asm_source);
	16232	buffer[asm_source.len] = 0;
	16233	sema.air_extra.items.len += asm_source.len / 4 + 1;
	16234	}
16229	16235	for (outputs) \|o\| {
16230	16236	const buffer = mem.sliceAsBytes(sema.air_extra.unusedCapacitySlice());
16231	16237	@memcpy(buffer[0..o.c.len], o.c);
...	...	@@ -16242,12 +16248,6 @@ fn zirAsm(
16242	16248	buffer[input.c.len + 1 + input.n.len] = 0;
16243	16249	sema.air_extra.items.len += (input.c.len + input.n.len + (2 + 3)) / 4;
16244	16250	}
16245		{
16246		const buffer = mem.sliceAsBytes(sema.air_extra.unusedCapacitySlice());
16247		@memcpy(buffer[0..asm_source.len], asm_source);
16248		buffer[asm_source.len] = 0;
16249		sema.air_extra.items.len += asm_source.len / 4 + 1;
16250		}
16251	16251	return asm_air;
16252	16252	}
16253	16253

src/codegen/aarch64/Select.zig+86-112

...	...	@@ -274,10 +274,9 @@ pub fn analyze(isel: *Select, air_body: []const Air.Inst.Index) !void {
274	274	},
275	275	.assembly => {
276	276	const ty_pl = air_data[@intFromEnum(air_inst_index)].ty_pl;
277		const extra = isel.air.extraData(Air.Asm, ty_pl.payload);
278		const operands: []const Air.Inst.Ref = @ptrCast(isel.air.extra.items[extra.end..][0 .. extra.data.flags.outputs_len + extra.data.inputs_len]);
	277	const unwrapped_asm = isel.air.unwrapAsm(air_inst_index);
279	278
280		for (operands) \|operand\| if (operand != .none) try isel.analyzeUse(operand);
	279	for (unwrapped_asm.outputs) \|operand\| if (operand != .none) try isel.analyzeUse(operand);
281	280	if (ty_pl.ty != .void_type) try isel.def_order.putNoClobber(gpa, air_inst_index, {});
282	281
283	282	air_body_index += 1;
...	...	@@ -355,23 +354,23 @@ pub fn analyze(isel: *Select, air_body: []const Air.Inst.Index) !void {
355	354	continue :air_tag air_tags[@intFromEnum(air_inst_index)];
356	355	},
357	356	inline .block, .dbg_inline_block => \|air_tag\| {
358		const ty_pl = air_data[@intFromEnum(air_inst_index)].ty_pl;
359		const extra = isel.air.extraData(switch (air_tag) {
	357	const air_body_block = switch (air_tag) {
360	358	else => comptime unreachable,
361		.block => Air.Block,
362		.dbg_inline_block => Air.DbgInlineBlock,
363		}, ty_pl.payload);
364		const result_ty = ty_pl.ty.toInterned().?;
	359	.block => isel.air.unwrapBlock(air_inst_index),
	360	.dbg_inline_block => isel.air.unwrapDbgBlock(air_inst_index),
	361	};
	362
	363	const result_ty = air_body_block.ty.toIntern();
365	364
366	365	if (result_ty == .noreturn_type) {
367		try isel.analyze(@ptrCast(isel.air.extra.items[extra.end..][0..extra.data.body_len]));
	366	try isel.analyze(air_body_block.body);
368	367
369	368	air_body_index += 1;
370	369	break :air_tag;
371	370	}
372	371
373	372	assert(!(try isel.blocks.getOrPut(gpa, air_inst_index)).found_existing);
374		try isel.analyze(@ptrCast(isel.air.extra.items[extra.end..][0..extra.data.body_len]));
	373	try isel.analyze(air_body_block.body);
375	374	const block_entry = isel.blocks.pop().?;
376	375	assert(block_entry.key == air_inst_index);
377	376
...	...	@@ -382,8 +381,7 @@ pub fn analyze(isel: *Select, air_body: []const Air.Inst.Index) !void {
382	381	continue :air_tag air_tags[@intFromEnum(air_inst_index)];
383	382	},
384	383	.loop => {
385		const ty_pl = air_data[@intFromEnum(air_inst_index)].ty_pl;
386		const extra = isel.air.extraData(Air.Block, ty_pl.payload);
	384	const air_body_block = isel.air.unwrapBlock(air_inst_index);
387	385
388	386	const initial_dom_start = isel.dom_start;
389	387	const initial_dom_len = isel.dom_len;
...	...	@@ -399,7 +397,7 @@ pub fn analyze(isel: *Select, air_body: []const Air.Inst.Index) !void {
399	397	.repeat_list = undefined,
400	398	});
401	399	try isel.dom.appendNTimes(gpa, 0, std.math.divCeil(usize, isel.dom_len, @bitSizeOf(DomInt)) catch unreachable);
402		try isel.analyze(@ptrCast(isel.air.extra.items[extra.end..][0..extra.data.body_len]));
	400	try isel.analyze(air_body_block.body);
403	401	for (
404	402	isel.dom.items[initial_dom_start..].ptr,
405	403	isel.dom.items[isel.dom_start..][0 .. std.math.divCeil(usize, initial_dom_len, @bitSizeOf(DomInt)) catch unreachable],
...	...	@@ -429,18 +427,17 @@ pub fn analyze(isel: *Select, air_body: []const Air.Inst.Index) !void {
429	427	.call_never_tail,
430	428	.call_never_inline,
431	429	=> {
432		const pl_op = air_data[@intFromEnum(air_inst_index)].pl_op;
433		const extra = isel.air.extraData(Air.Call, pl_op.payload);
434		const args: []const Air.Inst.Ref = @ptrCast(isel.air.extra.items[extra.end..][0..extra.data.args_len]);
	430	const air_call = isel.air.unwrapCall(air_inst_index);
	431	const args = air_call.args;
435	432	isel.saved_registers.insert(.lr);
436		const callee_ty = isel.air.typeOf(pl_op.operand, ip);
	433	const callee_ty = isel.air.typeOf(air_call.callee, ip);
437	434	const func_info = switch (ip.indexToKey(callee_ty.toIntern())) {
438	435	else => unreachable,
439	436	.func_type => \|func_type\| func_type,
440	437	.ptr_type => \|ptr_type\| ip.indexToKey(ptr_type.child).func_type,
441	438	};
442	439
443		try isel.analyzeUse(pl_op.operand);
	440	try isel.analyzeUse(air_call.callee);
444	441	var param_it: CallAbiIterator = .init;
445	442	for (args, 0..) \|arg, arg_index\| {
446	443	const restore_values_len = isel.values.items.len;
...	...	@@ -549,13 +546,12 @@ pub fn analyze(isel: *Select, air_body: []const Air.Inst.Index) !void {
549	546	continue :air_tag air_tags[@intFromEnum(air_inst_index)];
550	547	},
551	548	.cond_br => {
552		const pl_op = air_data[@intFromEnum(air_inst_index)].pl_op;
553		const extra = isel.air.extraData(Air.CondBr, pl_op.payload);
	549	const cond_br = isel.air.unwrapCondBr(air_inst_index);
554	550
555		try isel.analyzeUse(pl_op.operand);
	551	try isel.analyzeUse(cond_br.condition);
556	552
557		try isel.analyze(@ptrCast(isel.air.extra.items[extra.end..][0..extra.data.then_body_len]));
558		try isel.analyze(@ptrCast(isel.air.extra.items[extra.end + extra.data.then_body_len ..][0..extra.data.else_body_len]));
	553	try isel.analyze(cond_br.then_body);
	554	try isel.analyze(cond_br.else_body);
559	555
560	556	air_body_index += 1;
561	557	},
...	...	@@ -610,11 +606,10 @@ pub fn analyze(isel: *Select, air_body: []const Air.Inst.Index) !void {
610	606	air_body_index += 1;
611	607	},
612	608	.@"try", .try_cold => {
613		const pl_op = air_data[@intFromEnum(air_inst_index)].pl_op;
614		const extra = isel.air.extraData(Air.Try, pl_op.payload);
	609	const unwrapped_try = isel.air.unwrapTry(air_inst_index);
615	610
616		try isel.analyzeUse(pl_op.operand);
617		try isel.analyze(@ptrCast(isel.air.extra.items[extra.end..][0..extra.data.body_len]));
	611	try isel.analyzeUse(unwrapped_try.error_union);
	612	try isel.analyze(unwrapped_try.else_body);
618	613	try isel.def_order.putNoClobber(gpa, air_inst_index, {});
619	614
620	615	air_body_index += 1;
...	...	@@ -622,11 +617,10 @@ pub fn analyze(isel: *Select, air_body: []const Air.Inst.Index) !void {
622	617	continue :air_tag air_tags[@intFromEnum(air_inst_index)];
623	618	},
624	619	.try_ptr, .try_ptr_cold => {
625		const ty_pl = air_data[@intFromEnum(air_inst_index)].ty_pl;
626		const extra = isel.air.extraData(Air.TryPtr, ty_pl.payload);
	620	const unwrapped_try = isel.air.unwrapTryPtr(air_inst_index);
627	621
628		try isel.analyzeUse(extra.data.ptr);
629		try isel.analyze(@ptrCast(isel.air.extra.items[extra.end..][0..extra.data.body_len]));
	622	try isel.analyzeUse(unwrapped_try.error_union_ptr);
	623	try isel.analyze(unwrapped_try.else_body);
630	624	try isel.def_order.putNoClobber(gpa, air_inst_index, {});
631	625
632	626	air_body_index += 1;
...	...	@@ -2698,12 +2692,8 @@ pub fn body(isel: *Select, air_body: []const Air.Inst.Index) error{ OutOfMemory,
2698	2692	.inferred_alloc, .inferred_alloc_comptime => unreachable,
2699	2693	.assembly => {
2700	2694	const ty_pl = air.data(air.inst_index).ty_pl;
2701		const extra = isel.air.extraData(Air.Asm, ty_pl.payload);
2702		var extra_index = extra.end;
2703		const outputs: []const Air.Inst.Ref = @ptrCast(isel.air.extra.items[extra_index..][0..extra.data.flags.outputs_len]);
2704		extra_index += outputs.len;
2705		const inputs: []const Air.Inst.Ref = @ptrCast(isel.air.extra.items[extra_index..][0..extra.data.inputs_len]);
2706		extra_index += inputs.len;
	2695	const unwrapped_asm = isel.air.unwrapAsm(air.inst_index);
	2696	const inputs = unwrapped_asm.inputs;
2707	2697
2708	2698	var as: codegen.aarch64.Assemble = .{
2709	2699	.source = undefined,
...	...	@@ -2711,15 +2701,12 @@ pub fn body(isel: *Select, air_body: []const Air.Inst.Index) error{ OutOfMemory,
2711	2701	};
2712	2702	defer as.operands.deinit(gpa);
2713	2703
2714		for (outputs) \|output\| {
2715		const extra_bytes = std.mem.sliceAsBytes(isel.air.extra.items[extra_index..]);
2716		const constraint = std.mem.sliceTo(std.mem.sliceAsBytes(isel.air.extra.items[extra_index..]), 0);
2717		const name = std.mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0);
2718		// This equation accounts for the fact that even if we have exactly 4 bytes
2719		// for the string, we still use the next u32 for the null terminator.
2720		extra_index += (constraint.len + name.len + (2 + 3)) / 4;
	2704	var it = unwrapped_asm.iterateOutputs();
	2705	while (it.next()) \|output\| {
	2706	const constraint = output.constraint;
	2707	const name = output.name;
2721	2708
2722		switch (output) {
	2709	switch (output.operand) {
2723	2710	else => return isel.fail("invalid constraint: '{s}'", .{constraint}),
2724	2711	.none => if (std.mem.startsWith(u8, constraint, "={") and std.mem.endsWith(u8, constraint, "}")) {
2725	2712	const output_reg = Register.parse(constraint["={".len .. constraint.len - "}".len]) orelse
...	...	@@ -2760,54 +2747,51 @@ pub fn body(isel: *Select, air_body: []const Air.Inst.Index) error{ OutOfMemory,
2760	2747	}
2761	2748
2762	2749	const input_mats = try gpa.alloc(Value.Materialize, inputs.len);
	2750	var index: u32 = 0;
2763	2751	defer gpa.free(input_mats);
2764		const inputs_extra_index = extra_index;
2765		for (inputs, input_mats) \|input, *input_mat\| {
2766		const extra_bytes = std.mem.sliceAsBytes(isel.air.extra.items[extra_index..]);
2767		const constraint = std.mem.sliceTo(extra_bytes, 0);
2768		const name = std.mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0);
2769		// This equation accounts for the fact that even if we have exactly 4 bytes
2770		// for the string, we still use the next u32 for the null terminator.
2771		extra_index += (constraint.len + name.len + (2 + 3)) / 4;
	2752	it = unwrapped_asm.iterateInputs();
	2753	while (it.next()) \|input\| : (index += 1) {
	2754	const constraint = input.constraint;
	2755	const name = input.name;
2772	2756
2773	2757	if (std.mem.startsWith(u8, constraint, "{") and std.mem.endsWith(u8, constraint, "}")) {
2774	2758	const input_reg = Register.parse(constraint["{".len .. constraint.len - "}".len]) orelse
2775	2759	return isel.fail("invalid constraint: '{s}'", .{constraint});
2776		input_mat.* = .{ .vi = try isel.use(input), .ra = input_reg.alias };
	2760	input_mats[index] = .{ .vi = try isel.use(input.operand), .ra = input_reg.alias };
2777	2761	if (!std.mem.eql(u8, name, "_")) {
2778	2762	const operand_gop = try as.operands.getOrPut(gpa, name);
2779	2763	if (operand_gop.found_existing) return isel.fail("duplicate input name: '{s}'", .{name});
2780		const input_ty = isel.air.typeOf(input, ip);
	2764	const input_ty = isel.air.typeOf(input.operand, ip);
2781	2765	operand_gop.value_ptr.* = .{ .register = switch (input_ty.abiSize(zcu)) {
2782	2766	0 => unreachable,
2783	2767	1...4 => input_reg.alias.w(),
2784	2768	5...8 => input_reg.alias.x(),
2785	2769	else => return isel.fail("too big input type: '{f}'", .{
2786		isel.fmtType(isel.air.typeOf(input, ip)),
	2770	isel.fmtType(isel.air.typeOf(input.operand, ip)),
2787	2771	}),
2788	2772	} };
2789	2773	}
2790	2774	} else if (std.mem.eql(u8, constraint, "r")) {
2791		const input_vi = try isel.use(input);
2792		input_mat.* = try input_vi.matReg(isel);
	2775	const input_vi = try isel.use(input.operand);
	2776	input_mats[index] = try input_vi.matReg(isel);
2793	2777	if (!std.mem.eql(u8, name, "_")) {
2794	2778	const operand_gop = try as.operands.getOrPut(gpa, name);
2795	2779	if (operand_gop.found_existing) return isel.fail("duplicate input name: '{s}'", .{name});
2796	2780	operand_gop.value_ptr.* = .{ .register = switch (input_vi.size(isel)) {
2797	2781	0 => unreachable,
2798		1...4 => input_mat.ra.w(),
2799		5...8 => input_mat.ra.x(),
	2782	1...4 => input_mats[index].ra.w(),
	2783	5...8 => input_mats[index].ra.x(),
2800	2784	else => return isel.fail("too big input type: '{f}'", .{
2801		isel.fmtType(isel.air.typeOf(input, ip)),
	2785	isel.fmtType(isel.air.typeOf(input.operand, ip)),
2802	2786	}),
2803	2787	} };
2804	2788	}
2805	2789	} else if (std.mem.eql(u8, name, "_")) {
2806		input_mat.vi = try isel.use(input);
	2790	input_mats[index].vi = try isel.use(input.operand);
2807	2791	} else return isel.fail("invalid constraint: '{s}'", .{constraint});
2808	2792	}
2809	2793
2810		const clobbers = ip.indexToKey(extra.data.clobbers).aggregate;
	2794	const clobbers = ip.indexToKey(unwrapped_asm.clobbers).aggregate;
2811	2795	const clobbers_ty: ZigType = .fromInterned(clobbers.ty);
2812	2796	for (0..clobbers_ty.structFieldCount(zcu)) \|field_index\| {
2813	2797	switch (switch (clobbers.storage) {
...	...	@@ -2858,7 +2842,7 @@ pub fn body(isel: *Select, air_body: []const Air.Inst.Index) error{ OutOfMemory,
2858	2842	}
2859	2843	}
2860	2844
2861		as.source = std.mem.sliceAsBytes(isel.air.extra.items[extra_index..])[0..extra.data.source_len :0];
	2845	as.source = unwrapped_asm.source;
2862	2846	const asm_start = isel.instructions.items.len;
2863	2847	while (as.nextInstruction() catch \|err\| switch (err) {
2864	2848	error.InvalidSyntax => {
...	...	@@ -2872,21 +2856,18 @@ pub fn body(isel: *Select, air_body: []const Air.Inst.Index) error{ OutOfMemory,
2872	2856	}) \|instruction\| try isel.emit(instruction);
2873	2857	std.mem.reverse(codegen.aarch64.encoding.Instruction, isel.instructions.items[asm_start..]);
2874	2858
2875		extra_index = inputs_extra_index;
2876		for (input_mats) \|input_mat\| {
2877		const extra_bytes = std.mem.sliceAsBytes(isel.air.extra.items[extra_index..]);
2878		const constraint = std.mem.sliceTo(extra_bytes, 0);
2879		const name = std.mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0);
2880		// This equation accounts for the fact that even if we have exactly 4 bytes
2881		// for the string, we still use the next u32 for the null terminator.
2882		extra_index += (constraint.len + name.len + (2 + 3)) / 4;
	2859	it = unwrapped_asm.iterateInputs();
	2860	index = 0;
	2861	while (it.next()) \|input\| : (index += 1) {
	2862	const constraint = input.constraint;
	2863	const name = input.name;
2883	2864
2884	2865	if (std.mem.startsWith(u8, constraint, "{") and std.mem.endsWith(u8, constraint, "}")) {
2885		try input_mat.vi.liveOut(isel, input_mat.ra);
	2866	try input_mats[index].vi.liveOut(isel, input_mats[index].ra);
2886	2867	} else if (std.mem.eql(u8, constraint, "r")) {
2887		try input_mat.finish(isel);
	2868	try input_mats[index].finish(isel);
2888	2869	} else if (std.mem.eql(u8, name, "_")) {
2889		try input_mat.vi.mat(isel);
	2870	try input_mats[index].vi.mat(isel);
2890	2871	} else unreachable;
2891	2872	}
2892	2873
...	...	@@ -3515,16 +3496,16 @@ pub fn body(isel: *Select, air_body: []const Air.Inst.Index) error{ OutOfMemory,
3515	3496	if (air.next()) \|next_air_tag\| continue :air_tag next_air_tag;
3516	3497	},
3517	3498	.block => {
3518		const ty_pl = air.data(air.inst_index).ty_pl;
3519		const extra = isel.air.extraData(Air.Block, ty_pl.payload);
3520		try isel.block(air.inst_index, ty_pl.ty.toType(), @ptrCast(
3521		isel.air.extra.items[extra.end..][0..extra.data.body_len],
3522		));
	3499	const unwrapped_block = isel.air.unwrapBlock(air.inst_index);
	3500	try isel.block(
	3501	air.inst_index,
	3502	unwrapped_block.ty,
	3503	unwrapped_block.body,
	3504	);
3523	3505	if (air.next()) \|next_air_tag\| continue :air_tag next_air_tag;
3524	3506	},
3525	3507	.loop => {
3526		const ty_pl = air.data(air.inst_index).ty_pl;
3527		const extra = isel.air.extraData(Air.Block, ty_pl.payload);
	3508	const unwrapped_block = isel.air.unwrapBlock(air.inst_index);
3528	3509	const loops = isel.loops.values();
3529	3510	const loop_index = isel.loops.getIndex(air.inst_index).?;
3530	3511	const loop = &loops[loop_index];
...	...	@@ -3558,7 +3539,7 @@ pub fn body(isel: *Select, air_body: []const Air.Inst.Index) error{ OutOfMemory,
3558	3539
3559	3540	loop.live_registers = isel.live_registers;
3560	3541	loop.repeat_list = Loop.empty_list;
3561		try isel.body(@ptrCast(isel.air.extra.items[extra.end..][0..extra.data.body_len]));
	3542	try isel.body(unwrapped_block.body);
3562	3543	try isel.merge(&loop.live_registers, .{ .fill_extra = true });
3563	3544
3564	3545	var repeat_label = loop.repeat_list;
...	...	@@ -3608,10 +3589,9 @@ pub fn body(isel: *Select, air_body: []const Air.Inst.Index) error{ OutOfMemory,
3608	3589	if (air.next()) \|next_air_tag\| continue :air_tag next_air_tag;
3609	3590	},
3610	3591	.call => {
3611		const pl_op = air.data(air.inst_index).pl_op;
3612		const extra = isel.air.extraData(Air.Call, pl_op.payload);
3613		const args: []const Air.Inst.Ref = @ptrCast(isel.air.extra.items[extra.end..][0..extra.data.args_len]);
3614		const callee_ty = isel.air.typeOf(pl_op.operand, ip);
	3592	const air_call = isel.air.unwrapCall(air.inst_index);
	3593	const args = air_call.args;
	3594	const callee_ty = isel.air.typeOf(air_call.callee, ip);
3615	3595	const func_info = switch (ip.indexToKey(callee_ty.toIntern())) {
3616	3596	else => unreachable,
3617	3597	.func_type => \|func_type\| func_type,
...	...	@@ -3649,7 +3629,7 @@ pub fn body(isel: *Select, air_body: []const Air.Inst.Index) error{ OutOfMemory,
3649	3629	try call.finishReturn(isel);
3650	3630
3651	3631	try call.prepareCallee(isel);
3652		if (pl_op.operand.toInterned()) \|ct_callee\| {
	3632	if (air_call.callee.toInterned()) \|ct_callee\| {
3653	3633	try isel.nav_relocs.append(gpa, switch (ip.indexToKey(ct_callee)) {
3654	3634	else => unreachable,
3655	3635	inline .@"extern", .func => \|func\| .{
...	...	@@ -3666,7 +3646,7 @@ pub fn body(isel: *Select, air_body: []const Air.Inst.Index) error{ OutOfMemory,
3666	3646	});
3667	3647	try isel.emit(.bl(0));
3668	3648	} else {
3669		const callee_vi = try isel.use(pl_op.operand);
	3649	const callee_vi = try isel.use(air_call.callee);
3670	3650	const callee_mat = try callee_vi.matReg(isel);
3671	3651	try isel.emit(.blr(callee_mat.ra.x()));
3672	3652	try callee_mat.finish(isel);
...	...	@@ -4523,16 +4503,15 @@ pub fn body(isel: *Select, air_body: []const Air.Inst.Index) error{ OutOfMemory,
4523	4503	if (air.next()) \|next_air_tag\| continue :air_tag next_air_tag;
4524	4504	},
4525	4505	.cond_br => {
4526		const pl_op = air.data(air.inst_index).pl_op;
4527		const extra = isel.air.extraData(Air.CondBr, pl_op.payload);
	4506	const cond_br = isel.air.unwrapCondBr(air.inst_index);
4528	4507
4529		try isel.body(@ptrCast(isel.air.extra.items[extra.end + extra.data.then_body_len ..][0..extra.data.else_body_len]));
	4508	try isel.body(cond_br.then_body);
4530	4509	const else_label = isel.instructions.items.len;
4531	4510	const else_live_registers = isel.live_registers;
4532		try isel.body(@ptrCast(isel.air.extra.items[extra.end..][0..extra.data.then_body_len]));
	4511	try isel.body(cond_br.else_body);
4533	4512	try isel.merge(&else_live_registers, .{});
4534	4513
4535		const cond_vi = try isel.use(pl_op.operand);
	4514	const cond_vi = try isel.use(cond_br.condition);
4536	4515	const cond_mat = try cond_vi.matReg(isel);
4537	4516	try isel.emit(.tbz(
4538	4517	cond_mat.ra.x(),
...	...	@@ -4819,13 +4798,12 @@ pub fn body(isel: *Select, air_body: []const Air.Inst.Index) error{ OutOfMemory,
4819	4798	if (air.next()) \|next_air_tag\| continue :air_tag next_air_tag;
4820	4799	},
4821	4800	.@"try", .try_cold => {
4822		const pl_op = air.data(air.inst_index).pl_op;
4823		const extra = isel.air.extraData(Air.Try, pl_op.payload);
4824		const error_union_ty = isel.air.typeOf(pl_op.operand, ip);
	4801	const unwrapped_try = isel.air.unwrapTry(air.inst_index);
	4802	const error_union_ty = isel.air.typeOf(unwrapped_try.error_union, &zcu.intern_pool);
4825	4803	const error_union_info = ip.indexToKey(error_union_ty.toIntern()).error_union_type;
4826	4804	const payload_ty: ZigType = .fromInterned(error_union_info.payload_type);
4827	4805
4828		const error_union_vi = try isel.use(pl_op.operand);
	4806	const error_union_vi = try isel.use(unwrapped_try.error_union);
4829	4807	if (isel.live_values.fetchRemove(air.inst_index)) \|payload_vi\| {
4830	4808	defer payload_vi.value.deref(isel);
4831	4809
...	...	@@ -4840,7 +4818,7 @@ pub fn body(isel: *Select, air_body: []const Air.Inst.Index) error{ OutOfMemory,
4840	4818
4841	4819	const cont_label = isel.instructions.items.len;
4842	4820	const cont_live_registers = isel.live_registers;
4843		try isel.body(@ptrCast(isel.air.extra.items[extra.end..][0..extra.data.body_len]));
	4821	try isel.body(unwrapped_try.else_body);
4844	4822	try isel.merge(&cont_live_registers, .{});
4845	4823
4846	4824	var error_set_part_it = error_union_vi.field(
...	...	@@ -4859,18 +4837,17 @@ pub fn body(isel: *Select, air_body: []const Air.Inst.Index) error{ OutOfMemory,
4859	4837	if (air.next()) \|next_air_tag\| continue :air_tag next_air_tag;
4860	4838	},
4861	4839	.try_ptr, .try_ptr_cold => {
4862		const ty_pl = air.data(air.inst_index).ty_pl;
4863		const extra = isel.air.extraData(Air.TryPtr, ty_pl.payload);
4864		const error_union_ty = isel.air.typeOf(extra.data.ptr, ip).childType(zcu);
	4840	const unwrapped_try = isel.air.unwrapTryPtr(air.inst_index);
	4841	const error_union_ty = isel.air.typeOf(unwrapped_try.error_union_ptr, ip).childType(zcu);
4865	4842	const error_union_info = ip.indexToKey(error_union_ty.toIntern()).error_union_type;
4866	4843	const payload_ty: ZigType = .fromInterned(error_union_info.payload_type);
4867	4844
4868		const error_union_ptr_vi = try isel.use(extra.data.ptr);
	4845	const error_union_ptr_vi = try isel.use(unwrapped_try.error_union_ptr);
4869	4846	const error_union_ptr_mat = try error_union_ptr_vi.matReg(isel);
4870	4847	if (isel.live_values.fetchRemove(air.inst_index)) \|payload_ptr_vi\| unused: {
4871	4848	defer payload_ptr_vi.value.deref(isel);
4872		switch (codegen.errUnionPayloadOffset(ty_pl.ty.toType().childType(zcu), zcu)) {
4873		0 => try payload_ptr_vi.value.move(isel, extra.data.ptr),
	4849	switch (codegen.errUnionPayloadOffset(unwrapped_try.error_union_payload_ptr_ty.toType().childType(zcu), zcu)) {
	4850	0 => try payload_ptr_vi.value.move(isel, unwrapped_try.error_union_ptr),
4874	4851	else => \|payload_offset\| {
4875	4852	const payload_ptr_ra = try payload_ptr_vi.value.defReg(isel) orelse break :unused;
4876	4853	const lo12: u12 = @truncate(payload_offset >> 0);
...	...	@@ -4887,7 +4864,7 @@ pub fn body(isel: *Select, air_body: []const Air.Inst.Index) error{ OutOfMemory,
4887	4864
4888	4865	const cont_label = isel.instructions.items.len;
4889	4866	const cont_live_registers = isel.live_registers;
4890		try isel.body(@ptrCast(isel.air.extra.items[extra.end..][0..extra.data.body_len]));
	4867	try isel.body(unwrapped_try.else_body);
4891	4868	try isel.merge(&cont_live_registers, .{});
4892	4869
4893	4870	const error_set_ra = try isel.allocIntReg();
...	...	@@ -4913,11 +4890,8 @@ pub fn body(isel: *Select, air_body: []const Air.Inst.Index) error{ OutOfMemory,
4913	4890	if (air.next()) \|next_air_tag\| continue :air_tag next_air_tag;
4914	4891	},
4915	4892	.dbg_inline_block => {
4916		const ty_pl = air.data(air.inst_index).ty_pl;
4917		const extra = isel.air.extraData(Air.DbgInlineBlock, ty_pl.payload);
4918		try isel.block(air.inst_index, ty_pl.ty.toType(), @ptrCast(
4919		isel.air.extra.items[extra.end..][0..extra.data.body_len],
4920		));
	4893	const dbg_block = isel.air.unwrapDbgBlock(air.inst_index);
	4894	try isel.block(air.inst_index, dbg_block.ty, dbg_block.body);
4921	4895	if (air.next()) \|next_air_tag\| continue :air_tag next_air_tag;
4922	4896	},
4923	4897	.dbg_var_ptr, .dbg_var_val, .dbg_arg_inline => {

src/codegen/c.zig+65-96

...	...	@@ -4622,9 +4622,8 @@ fn airCall(
4622	4622	const gpa = f.object.dg.gpa;
4623	4623	const w = &f.object.code.writer;
4624	4624
4625		const pl_op = f.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
4626		const extra = f.air.extraData(Air.Call, pl_op.payload);
4627		const args: []const Air.Inst.Ref = @ptrCast(f.air.extra.items[extra.end..][0..extra.data.args_len]);
	4625	const call = f.air.unwrapCall(inst);
	4626	const args = call.args;
4628	4627
4629	4628	const resolved_args = try gpa.alloc(CValue, args.len);
4630	4629	defer gpa.free(resolved_args);
...	...	@@ -4653,15 +4652,15 @@ fn airCall(
4653	4652	}
4654	4653	}
4655	4654
4656		const callee = try f.resolveInst(pl_op.operand);
	4655	const callee = try f.resolveInst(call.callee);
4657	4656
4658	4657	{
4659	4658	var bt = iterateBigTomb(f, inst);
4660		try bt.feed(pl_op.operand);
	4659	try bt.feed(call.callee);
4661	4660	for (args) \|arg\| try bt.feed(arg);
4662	4661	}
4663	4662
4664		const callee_ty = f.typeOf(pl_op.operand);
	4663	const callee_ty = f.typeOf(call.callee);
4665	4664	const callee_is_ptr = switch (callee_ty.zigTypeTag(zcu)) {
4666	4665	.@"fn" => false,
4667	4666	.pointer => true,
...	...	@@ -4698,7 +4697,7 @@ fn airCall(
4698	4697
4699	4698	callee: {
4700	4699	known: {
4701		const callee_val = (try f.air.value(pl_op.operand, pt)) orelse break :known;
	4700	const callee_val = (try f.air.value(call.callee, pt)) orelse break :known;
4702	4701	const fn_nav, const need_cast = switch (ip.indexToKey(callee_val.toIntern())) {
4703	4702	.@"extern" => \|@"extern"\| .{ @"extern".owner_nav, false },
4704	4703	.func => \|func\| .{ func.owner_nav, Type.fromInterned(func.ty).fnCallingConvention(zcu) != .naked and
...	...	@@ -4796,13 +4795,12 @@ fn airDbgInlineBlock(f: *Function, inst: Air.Inst.Index) !CValue {
4796	4795	const pt = f.object.dg.pt;
4797	4796	const zcu = pt.zcu;
4798	4797	const ip = &zcu.intern_pool;
4799		const ty_pl = f.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
4800		const extra = f.air.extraData(Air.DbgInlineBlock, ty_pl.payload);
4801		const owner_nav = ip.getNav(zcu.funcInfo(extra.data.func).owner_nav);
	4798	const block = f.air.unwrapDbgBlock(inst);
	4799	const owner_nav = ip.getNav(zcu.funcInfo(block.func).owner_nav);
4802	4800	const w = &f.object.code.writer;
4803	4801	try w.print("/* inline:{f} */", .{owner_nav.fqn.fmt(&zcu.intern_pool)});
4804	4802	try f.object.newline();
4805		return lowerBlock(f, inst, @ptrCast(f.air.extra.items[extra.end..][0..extra.data.body_len]));
	4803	return lowerBlock(f, inst, block.body);
4806	4804	}
4807	4805
4808	4806	fn airDbgVar(f: *Function, inst: Air.Inst.Index) !CValue {
...	...	@@ -4822,9 +4820,8 @@ fn airDbgVar(f: *Function, inst: Air.Inst.Index) !CValue {
4822	4820	}
4823	4821
4824	4822	fn airBlock(f: *Function, inst: Air.Inst.Index) !CValue {
4825		const ty_pl = f.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
4826		const extra = f.air.extraData(Air.Block, ty_pl.payload);
4827		return lowerBlock(f, inst, @ptrCast(f.air.extra.items[extra.end..][0..extra.data.body_len]));
	4823	const block = f.air.unwrapBlock(inst);
	4824	return lowerBlock(f, inst, block.body);
4828	4825	}
4829	4826
4830	4827	fn lowerBlock(f: *Function, inst: Air.Inst.Index, body: []const Air.Inst.Index) !CValue {
...	...	@@ -4873,21 +4870,19 @@ fn lowerBlock(f: *Function, inst: Air.Inst.Index, body: []const Air.Inst.Index)
4873	4870	}
4874	4871
4875	4872	fn airTry(f: *Function, inst: Air.Inst.Index) !CValue {
4876		const pl_op = f.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
4877		const extra = f.air.extraData(Air.Try, pl_op.payload);
4878		const body: []const Air.Inst.Index = @ptrCast(f.air.extra.items[extra.end..][0..extra.data.body_len]);
4879		const err_union_ty = f.typeOf(pl_op.operand);
4880		return lowerTry(f, inst, pl_op.operand, body, err_union_ty, false);
	4873	const pt = f.object.dg.pt;
	4874	const unwrapped_try = f.air.unwrapTry(inst);
	4875	const body = unwrapped_try.else_body;
	4876	const err_union_ty = f.air.typeOf(unwrapped_try.error_union, &pt.zcu.intern_pool);
	4877	return lowerTry(f, inst, unwrapped_try.error_union, body, err_union_ty, false);
4881	4878	}
4882	4879
4883	4880	fn airTryPtr(f: *Function, inst: Air.Inst.Index) !CValue {
4884	4881	const pt = f.object.dg.pt;
4885		const zcu = pt.zcu;
4886		const ty_pl = f.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
4887		const extra = f.air.extraData(Air.TryPtr, ty_pl.payload);
4888		const body: []const Air.Inst.Index = @ptrCast(f.air.extra.items[extra.end..][0..extra.data.body_len]);
4889		const err_union_ty = f.typeOf(extra.data.ptr).childType(zcu);
4890		return lowerTry(f, inst, extra.data.ptr, body, err_union_ty, true);
	4882	const unwrapped_try = f.air.unwrapTryPtr(inst);
	4883	const body = unwrapped_try.else_body;
	4884	const err_union_ty = f.air.typeOf(unwrapped_try.error_union_ptr, &pt.zcu.intern_pool).childType(pt.zcu);
	4885	return lowerTry(f, inst, unwrapped_try.error_union_ptr, body, err_union_ty, true);
4891	4886	}
4892	4887
4893	4888	fn lowerTry(
...	...	@@ -5216,9 +5211,7 @@ fn airUnreach(o: *Object) !void {
5216	5211	}
5217	5212
5218	5213	fn airLoop(f: *Function, inst: Air.Inst.Index) !void {
5219		const ty_pl = f.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
5220		const loop = f.air.extraData(Air.Block, ty_pl.payload);
5221		const body: []const Air.Inst.Index = @ptrCast(f.air.extra.items[loop.end..][0..loop.data.body_len]);
	5214	const block = f.air.unwrapBlock(inst);
5222	5215	const w = &f.object.code.writer;
5223	5216
5224	5217	// `repeat` instructions matching this loop will branch to
...	...	@@ -5227,16 +5220,15 @@ fn airLoop(f: *Function, inst: Air.Inst.Index) !void {
5227	5220	// construct at all!
5228	5221	try w.print("zig_loop_{d}:", .{@intFromEnum(inst)});
5229	5222	try f.object.newline();
5230		try genBodyInner(f, body); // no need to restore state, we're noreturn
	5223	try genBodyInner(f, block.body); // no need to restore state, we're noreturn
5231	5224	}
5232	5225
5233	5226	fn airCondBr(f: *Function, inst: Air.Inst.Index) !void {
5234		const pl_op = f.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
5235		const cond = try f.resolveInst(pl_op.operand);
5236		try reap(f, inst, &.{pl_op.operand});
5237		const extra = f.air.extraData(Air.CondBr, pl_op.payload);
5238		const then_body: []const Air.Inst.Index = @ptrCast(f.air.extra.items[extra.end..][0..extra.data.then_body_len]);
5239		const else_body: []const Air.Inst.Index = @ptrCast(f.air.extra.items[extra.end + then_body.len ..][0..extra.data.else_body_len]);
	5227	const cond_br = f.air.unwrapCondBr(inst);
	5228	const cond = try f.resolveInst(cond_br.condition);
	5229	try reap(f, inst, &.{cond_br.condition});
	5230	const then_body = cond_br.then_body;
	5231	const else_body = cond_br.else_body;
5240	5232	const liveness_condbr = f.liveness.getCondBr(inst);
5241	5233	const w = &f.object.code.writer;
5242	5234
...	...	@@ -5439,16 +5431,11 @@ fn asmInputNeedsLocal(f: *Function, constraint: []const u8, value: CValue) bool
5439	5431	fn airAsm(f: *Function, inst: Air.Inst.Index) !CValue {
5440	5432	const pt = f.object.dg.pt;
5441	5433	const zcu = pt.zcu;
5442		const ty_pl = f.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
5443		const extra = f.air.extraData(Air.Asm, ty_pl.payload);
5444		const is_volatile = extra.data.flags.is_volatile;
5445		const outputs_len = extra.data.flags.outputs_len;
	5434	const unwrapped_asm = f.air.unwrapAsm(inst);
	5435	const is_volatile = unwrapped_asm.is_volatile;
5446	5436	const gpa = f.object.dg.gpa;
5447		var extra_i: usize = extra.end;
5448		const outputs: []const Air.Inst.Ref = @ptrCast(f.air.extra.items[extra_i..][0..outputs_len]);
5449		extra_i += outputs.len;
5450		const inputs: []const Air.Inst.Ref = @ptrCast(f.air.extra.items[extra_i..][0..extra.data.inputs_len]);
5451		extra_i += inputs.len;
	5437	const outputs = unwrapped_asm.outputs;
	5438	const inputs = unwrapped_asm.inputs;
5452	5439
5453	5440	const result = result: {
5454	5441	const w = &f.object.code.writer;
...	...	@@ -5469,14 +5456,9 @@ fn airAsm(f: *Function, inst: Air.Inst.Index) !CValue {
5469	5456	} else .none;
5470	5457
5471	5458	const locals_begin: LocalIndex = @intCast(f.locals.items.len);
5472		const constraints_extra_begin = extra_i;
5473		for (outputs) \|output\| {
5474		const extra_bytes = mem.sliceAsBytes(f.air.extra.items[extra_i..]);
5475		const constraint = mem.sliceTo(extra_bytes, 0);
5476		const name = mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0);
5477		// This equation accounts for the fact that even if we have exactly 4 bytes
5478		// for the string, we still use the next u32 for the null terminator.
5479		extra_i += (constraint.len + name.len + (2 + 3)) / 4;
	5459	var it = unwrapped_asm.iterateOutputs();
	5460	while (it.next()) \|output\| {
	5461	const constraint = output.constraint;
5480	5462
5481	5463	if (constraint.len < 2 or constraint[0] != '=' or
5482	5464	(constraint[1] == '{' and constraint[constraint.len - 1] != '}'))
...	...	@@ -5486,7 +5468,7 @@ fn airAsm(f: *Function, inst: Air.Inst.Index) !CValue {
5486	5468
5487	5469	const is_reg = constraint[1] == '{';
5488	5470	if (is_reg) {
5489		const output_ty = if (output == .none) inst_ty else f.typeOf(output).childType(zcu);
	5471	const output_ty = if (output.operand == .none) inst_ty else f.typeOf(output.operand).childType(zcu);
5490	5472	try w.writeAll("register ");
5491	5473	const output_local = try f.allocLocalValue(.{
5492	5474	.ctype = try f.ctypeFromType(output_ty, .complete),
...	...	@@ -5505,13 +5487,10 @@ fn airAsm(f: *Function, inst: Air.Inst.Index) !CValue {
5505	5487	try f.object.newline();
5506	5488	}
5507	5489	}
5508		for (inputs) \|input\| {
5509		const extra_bytes = mem.sliceAsBytes(f.air.extra.items[extra_i..]);
5510		const constraint = mem.sliceTo(extra_bytes, 0);
5511		const name = mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0);
5512		// This equation accounts for the fact that even if we have exactly 4 bytes
5513		// for the string, we still use the next u32 for the null terminator.
5514		extra_i += (constraint.len + name.len + (2 + 3)) / 4;
	5490
	5491	it = unwrapped_asm.iterateInputs();
	5492	while (it.next()) \|input\| {
	5493	const constraint = input.constraint;
5515	5494
5516	5495	if (constraint.len < 1 or mem.indexOfScalar(u8, "=+&%", constraint[0]) != null or
5517	5496	(constraint[0] == '{' and constraint[constraint.len - 1] != '}'))
...	...	@@ -5520,9 +5499,9 @@ fn airAsm(f: *Function, inst: Air.Inst.Index) !CValue {
5520	5499	}
5521	5500
5522	5501	const is_reg = constraint[0] == '{';
5523		const input_val = try f.resolveInst(input);
	5502	const input_val = try f.resolveInst(input.operand);
5524	5503	if (asmInputNeedsLocal(f, constraint, input_val)) {
5525		const input_ty = f.typeOf(input);
	5504	const input_ty = f.typeOf(input.operand);
5526	5505	if (is_reg) try w.writeAll("register ");
5527	5506	const input_local = try f.allocLocalValue(.{
5528	5507	.ctype = try f.ctypeFromType(input_ty, .complete),
...	...	@@ -5545,7 +5524,7 @@ fn airAsm(f: *Function, inst: Air.Inst.Index) !CValue {
5545	5524	}
5546	5525
5547	5526	{
5548		const asm_source = mem.sliceAsBytes(f.air.extra.items[extra_i..])[0..extra.data.source_len];
	5527	const asm_source = unwrapped_asm.source;
5549	5528
5550	5529	var stack = std.heap.stackFallback(256, f.object.dg.gpa);
5551	5530	const allocator = stack.get();
...	...	@@ -5599,18 +5578,15 @@ fn airAsm(f: *Function, inst: Air.Inst.Index) !CValue {
5599	5578	try w.print("({f}", .{fmtStringLiteral(fixed_asm_source[0..dst_i], null)});
5600	5579	}
5601	5580
5602		extra_i = constraints_extra_begin;
5603	5581	var locals_index = locals_begin;
5604	5582	try w.writeByte(':');
5605		for (outputs, 0..) \|output, index\| {
5606		const extra_bytes = mem.sliceAsBytes(f.air.extra.items[extra_i..]);
5607		const constraint = mem.sliceTo(extra_bytes, 0);
5608		const name = mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0);
5609		// This equation accounts for the fact that even if we have exactly 4 bytes
5610		// for the string, we still use the next u32 for the null terminator.
5611		extra_i += (constraint.len + name.len + (2 + 3)) / 4;
5612
5613		if (index > 0) try w.writeByte(',');
	5583
	5584	it = unwrapped_asm.iterateOutputs();
	5585	while (it.next()) \|output\| {
	5586	const constraint = output.constraint;
	5587	const name = output.name;
	5588
	5589	if (output.index > 0) try w.writeByte(',');
5614	5590	try w.writeByte(' ');
5615	5591	if (!mem.eql(u8, name, "_")) try w.print("[{s}]", .{name});
5616	5592	const is_reg = constraint[1] == '{';
...	...	@@ -5618,28 +5594,26 @@ fn airAsm(f: *Function, inst: Air.Inst.Index) !CValue {
5618	5594	if (is_reg) {
5619	5595	try f.writeCValue(w, .{ .local = locals_index }, .Other);
5620	5596	locals_index += 1;
5621		} else if (output == .none) {
	5597	} else if (output.operand == .none) {
5622	5598	try f.writeCValue(w, inst_local, .FunctionArgument);
5623	5599	} else {
5624		try f.writeCValueDeref(w, try f.resolveInst(output));
	5600	try f.writeCValueDeref(w, try f.resolveInst(output.operand));
5625	5601	}
5626	5602	try w.writeByte(')');
5627	5603	}
5628	5604	try w.writeByte(':');
5629		for (inputs, 0..) \|input, index\| {
5630		const extra_bytes = mem.sliceAsBytes(f.air.extra.items[extra_i..]);
5631		const constraint = mem.sliceTo(extra_bytes, 0);
5632		const name = mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0);
5633		// This equation accounts for the fact that even if we have exactly 4 bytes
5634		// for the string, we still use the next u32 for the null terminator.
5635		extra_i += (constraint.len + name.len + (2 + 3)) / 4;
5636
5637		if (index > 0) try w.writeByte(',');
	5605
	5606	it = unwrapped_asm.iterateInputs();
	5607	while (it.next()) \|input\| {
	5608	const constraint = input.constraint;
	5609	const name = input.name;
	5610
	5611	if (input.index > 0) try w.writeByte(',');
5638	5612	try w.writeByte(' ');
5639	5613	if (!mem.eql(u8, name, "_")) try w.print("[{s}]", .{name});
5640	5614
5641	5615	const is_reg = constraint[0] == '{';
5642		const input_val = try f.resolveInst(input);
	5616	const input_val = try f.resolveInst(input.operand);
5643	5617	try w.print("{f}(", .{fmtStringLiteral(if (is_reg) "r" else constraint, null)});
5644	5618	try f.writeCValue(w, if (asmInputNeedsLocal(f, constraint, input_val)) local: {
5645	5619	const input_local_idx = locals_index;
...	...	@@ -5650,7 +5624,7 @@ fn airAsm(f: *Function, inst: Air.Inst.Index) !CValue {
5650	5624	}
5651	5625	try w.writeByte(':');
5652	5626	const ip = &zcu.intern_pool;
5653		const aggregate = ip.indexToKey(extra.data.clobbers).aggregate;
	5627	const aggregate = ip.indexToKey(unwrapped_asm.clobbers).aggregate;
5654	5628	const struct_type: Type = .fromInterned(aggregate.ty);
5655	5629	switch (aggregate.storage) {
5656	5630	.elems => \|elems\| for (elems, 0..) \|elem, i\| switch (elem) {
...	...	@@ -5697,22 +5671,17 @@ fn airAsm(f: *Function, inst: Air.Inst.Index) !CValue {
5697	5671	try w.writeAll(");");
5698	5672	try f.object.newline();
5699	5673
5700		extra_i = constraints_extra_begin;
5701	5674	locals_index = locals_begin;
5702		for (outputs) \|output\| {
5703		const extra_bytes = mem.sliceAsBytes(f.air.extra.items[extra_i..]);
5704		const constraint = mem.sliceTo(extra_bytes, 0);
5705		const name = mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0);
5706		// This equation accounts for the fact that even if we have exactly 4 bytes
5707		// for the string, we still use the next u32 for the null terminator.
5708		extra_i += (constraint.len + name.len + (2 + 3)) / 4;
	5675	it = unwrapped_asm.iterateOutputs();
	5676	while (it.next()) \|output\| {
	5677	const constraint = output.constraint;
5709	5678
5710	5679	const is_reg = constraint[1] == '{';
5711	5680	if (is_reg) {
5712		try f.writeCValueDeref(w, if (output == .none)
	5681	try f.writeCValueDeref(w, if (output.operand == .none)
5713	5682	.{ .local_ref = inst_local.new_local }
5714	5683	else
5715		try f.resolveInst(output));
	5684	try f.resolveInst(output.operand));
5716	5685	try w.writeAll(" = ");
5717	5686	try f.writeCValue(w, .{ .local = locals_index }, .Other);
5718	5687	locals_index += 1;

src/codegen/llvm.zig+62-85

...	...	@@ -5258,14 +5258,13 @@ pub const FuncGen = struct {
5258	5258	};
5259	5259
5260	5260	fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.builtin.CallModifier) !Builder.Value {
5261		const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
5262		const extra = self.air.extraData(Air.Call, pl_op.payload);
5263		const args: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[extra.end..][0..extra.data.args_len]);
	5261	const air_call = self.air.unwrapCall(inst);
	5262	const args = air_call.args;
5264	5263	const o = self.ng.object;
5265	5264	const pt = self.ng.pt;
5266	5265	const zcu = pt.zcu;
5267	5266	const ip = &zcu.intern_pool;
5268		const callee_ty = self.typeOf(pl_op.operand);
	5267	const callee_ty = self.typeOf(air_call.callee);
5269	5268	const zig_fn_ty = switch (callee_ty.zigTypeTag(zcu)) {
5270	5269	.@"fn" => callee_ty,
5271	5270	.pointer => callee_ty.childType(zcu),
...	...	@@ -5273,7 +5272,7 @@ pub const FuncGen = struct {
5273	5272	};
5274	5273	const fn_info = zcu.typeToFunc(zig_fn_ty).?;
5275	5274	const return_type = Type.fromInterned(fn_info.return_type);
5276		const llvm_fn = try self.resolveInst(pl_op.operand);
	5275	const llvm_fn = try self.resolveInst(air_call.callee);
5277	5276	const target = zcu.getTarget();
5278	5277	const sret = firstParamSRet(fn_info, zcu, target);
5279	5278
...	...	@@ -5934,9 +5933,8 @@ pub const FuncGen = struct {
5934	5933	}
5935	5934
5936	5935	fn airBlock(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
5937		const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
5938		const extra = self.air.extraData(Air.Block, ty_pl.payload);
5939		return self.lowerBlock(inst, null, @ptrCast(self.air.extra.items[extra.end..][0..extra.data.body_len]));
	5936	const block = self.air.unwrapBlock(inst);
	5937	return self.lowerBlock(inst, null, block.body);
5940	5938	}
5941	5939
5942	5940	fn lowerBlock(
...	...	@@ -6216,11 +6214,10 @@ pub const FuncGen = struct {
6216	6214	}
6217	6215
6218	6216	fn airCondBr(self: *FuncGen, inst: Air.Inst.Index) !void {
6219		const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
6220		const cond = try self.resolveInst(pl_op.operand);
6221		const extra = self.air.extraData(Air.CondBr, pl_op.payload);
6222		const then_body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[extra.end..][0..extra.data.then_body_len]);
6223		const else_body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[extra.end + then_body.len ..][0..extra.data.else_body_len]);
	6217	const cond_br = self.air.unwrapCondBr(inst);
	6218	const cond = try self.resolveInst(cond_br.condition);
	6219	const then_body = cond_br.then_body;
	6220	const else_body = cond_br.else_body;
6224	6221
6225	6222	const Hint = enum {
6226	6223	none,
...	...	@@ -6230,22 +6227,22 @@ pub const FuncGen = struct {
6230	6227	then_cold,
6231	6228	else_cold,
6232	6229	};
6233		const hint: Hint = switch (extra.data.branch_hints.true) {
6234		.none => switch (extra.data.branch_hints.false) {
	6230	const hint: Hint = switch (cond_br.branch_hints.true) {
	6231	.none => switch (cond_br.branch_hints.false) {
6235	6232	.none => .none,
6236	6233	.likely => .else_likely,
6237	6234	.unlikely => .then_likely,
6238	6235	.cold => .else_cold,
6239	6236	.unpredictable => .unpredictable,
6240	6237	},
6241		.likely => switch (extra.data.branch_hints.false) {
	6238	.likely => switch (cond_br.branch_hints.false) {
6242	6239	.none => .then_likely,
6243	6240	.likely => .unpredictable,
6244	6241	.unlikely => .then_likely,
6245	6242	.cold => .else_cold,
6246	6243	.unpredictable => .unpredictable,
6247	6244	},
6248		.unlikely => switch (extra.data.branch_hints.false) {
	6245	.unlikely => switch (cond_br.branch_hints.false) {
6249	6246	.none => .else_likely,
6250	6247	.likely => .else_likely,
6251	6248	.unlikely => .unpredictable,
...	...	@@ -6267,35 +6264,33 @@ pub const FuncGen = struct {
6267	6264
6268	6265	self.wip.cursor = .{ .block = then_block };
6269	6266	if (hint == .then_cold) _ = try self.wip.callIntrinsicAssumeCold();
6270		try self.genBodyDebugScope(null, then_body, extra.data.branch_hints.then_cov);
	6267	try self.genBodyDebugScope(null, then_body, cond_br.branch_hints.then_cov);
6271	6268
6272	6269	self.wip.cursor = .{ .block = else_block };
6273	6270	if (hint == .else_cold) _ = try self.wip.callIntrinsicAssumeCold();
6274		try self.genBodyDebugScope(null, else_body, extra.data.branch_hints.else_cov);
	6271	try self.genBodyDebugScope(null, else_body, cond_br.branch_hints.else_cov);
6275	6272
6276	6273	// No need to reset the insert cursor since this instruction is noreturn.
6277	6274	}
6278	6275
6279	6276	fn airTry(self: *FuncGen, inst: Air.Inst.Index, err_cold: bool) !Builder.Value {
6280		const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
6281		const err_union = try self.resolveInst(pl_op.operand);
6282		const extra = self.air.extraData(Air.Try, pl_op.payload);
6283		const body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[extra.end..][0..extra.data.body_len]);
6284		const err_union_ty = self.typeOf(pl_op.operand);
	6277	const unwrapped_try = self.air.unwrapTry(inst);
	6278	const err_union = try self.resolveInst(unwrapped_try.error_union);
	6279	const body = unwrapped_try.else_body;
	6280	const err_union_ty = self.typeOf(unwrapped_try.error_union);
6285	6281	const is_unused = self.liveness.isUnused(inst);
6286	6282	return lowerTry(self, err_union, body, err_union_ty, false, false, is_unused, err_cold);
6287	6283	}
6288	6284
6289	6285	fn airTryPtr(self: *FuncGen, inst: Air.Inst.Index, err_cold: bool) !Builder.Value {
6290	6286	const zcu = self.ng.pt.zcu;
6291		const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
6292		const extra = self.air.extraData(Air.TryPtr, ty_pl.payload);
6293		const err_union_ptr = try self.resolveInst(extra.data.ptr);
6294		const body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[extra.end..][0..extra.data.body_len]);
6295		const err_union_ty = self.typeOf(extra.data.ptr).childType(zcu);
	6287	const unwrapped_try = self.air.unwrapTryPtr(inst);
	6288	const err_union_ptr = try self.resolveInst(unwrapped_try.error_union_ptr);
	6289	const body = unwrapped_try.else_body;
	6290	const err_union_ty = self.typeOf(unwrapped_try.error_union_ptr).childType(zcu);
6296	6291	const is_unused = self.liveness.isUnused(inst);
6297	6292
6298		self.maybeMarkAllowZeroAccess(self.typeOf(extra.data.ptr).ptrInfo(zcu));
	6293	self.maybeMarkAllowZeroAccess(self.typeOf(unwrapped_try.error_union_ptr).ptrInfo(zcu));
6299	6294
6300	6295	return lowerTry(self, err_union_ptr, body, err_union_ty, true, true, is_unused, err_cold);
6301	6296	}
...	...	@@ -6627,9 +6622,8 @@ pub const FuncGen = struct {
6627	6622	}
6628	6623
6629	6624	fn airLoop(self: *FuncGen, inst: Air.Inst.Index) !void {
6630		const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
6631		const loop = self.air.extraData(Air.Block, ty_pl.payload);
6632		const body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[loop.end..][0..loop.data.body_len]);
	6625	const block = self.air.unwrapBlock(inst);
	6626	const body = block.body;
6633	6627	const loop_block = try self.wip.block(1, "Loop"); // `airRepeat` will increment incoming each time
6634	6628	_ = try self.wip.br(loop_block);
6635	6629
...	...	@@ -7137,10 +7131,9 @@ pub const FuncGen = struct {
7137	7131	}
7138	7132
7139	7133	fn airDbgInlineBlock(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
7140		const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
7141		const extra = self.air.extraData(Air.DbgInlineBlock, ty_pl.payload);
	7134	const block = self.air.unwrapDbgBlock(inst);
7142	7135	self.arg_inline_index = 0;
7143		return self.lowerBlock(inst, extra.data.func, @ptrCast(self.air.extra.items[extra.end..][0..extra.data.body_len]));
	7136	return self.lowerBlock(inst, block.func, block.body);
7144	7137	}
7145	7138
7146	7139	fn airDbgVarPtr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
...	...	@@ -7262,17 +7255,12 @@ pub const FuncGen = struct {
7262	7255	// this implementation feeds the inline assembly code directly to LLVM.
7263	7256
7264	7257	const o = self.ng.object;
7265		const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
7266		const extra = self.air.extraData(Air.Asm, ty_pl.payload);
7267		const is_volatile = extra.data.flags.is_volatile;
7268		const outputs_len = extra.data.flags.outputs_len;
	7258	const unwrapped_asm = self.air.unwrapAsm(inst);
	7259	const is_volatile = unwrapped_asm.is_volatile;
7269	7260	const gpa = self.gpa;
7270		var extra_i: usize = extra.end;
7271	7261
7272		const outputs: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[extra_i..][0..outputs_len]);
7273		extra_i += outputs.len;
7274		const inputs: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[extra_i..][0..extra.data.inputs_len]);
7275		extra_i += inputs.len;
	7262	const outputs = unwrapped_asm.outputs;
	7263	const inputs = unwrapped_asm.inputs;
7276	7264
7277	7265	var llvm_constraints: std.ArrayList(u8) = .empty;
7278	7266	defer llvm_constraints.deinit(gpa);
...	...	@@ -7305,14 +7293,10 @@ pub const FuncGen = struct {
7305	7293	var name_map: std.StringArrayHashMapUnmanaged(u16) = .empty;
7306	7294	try name_map.ensureUnusedCapacity(arena, max_param_count);
7307	7295
7308		var rw_extra_i = extra_i;
7309		for (outputs, llvm_ret_indirect, llvm_rw_vals) \|output, is_indirect, llvm_rw_val\| {
7310		const extra_bytes = std.mem.sliceAsBytes(self.air.extra.items[extra_i..]);
7311		const constraint = std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra.items[extra_i..]), 0);
7312		const name = std.mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0);
7313		// This equation accounts for the fact that even if we have exactly 4 bytes
7314		// for the string, we still use the next u32 for the null terminator.
7315		extra_i += (constraint.len + name.len + (2 + 3)) / 4;
	7296	var it = unwrapped_asm.iterateOutputs();
	7297	while (it.next()) \|output\| {
	7298	const constraint = output.constraint;
	7299	const name = output.name;
7316	7300
7317	7301	try llvm_constraints.ensureUnusedCapacity(gpa, constraint.len + 3);
7318	7302	if (total_i != 0) {
...	...	@@ -7320,15 +7304,15 @@ pub const FuncGen = struct {
7320	7304	}
7321	7305	llvm_constraints.appendAssumeCapacity('=');
7322	7306
7323		if (output != .none) {
7324		const output_inst = try self.resolveInst(output);
7325		const output_ty = self.typeOf(output);
	7307	if (output.operand != .none) {
	7308	const output_inst = try self.resolveInst(output.operand);
	7309	const output_ty = self.typeOf(output.operand);
7326	7310	assert(output_ty.zigTypeTag(zcu) == .pointer);
7327	7311	const elem_llvm_ty = try o.lowerPtrElemTy(pt, output_ty.childType(zcu));
7328	7312
7329	7313	switch (constraint[0]) {
7330	7314	'=' => {},
7331		'+' => llvm_rw_val.* = output_inst,
	7315	'+' => llvm_rw_vals[output.index] = output_inst,
7332	7316	else => return self.todo("unsupported output constraint on output type '{c}'", .{
7333	7317	constraint[0],
7334	7318	}),
...	...	@@ -7337,8 +7321,8 @@ pub const FuncGen = struct {
7337	7321	self.maybeMarkAllowZeroAccess(output_ty.ptrInfo(zcu));
7338	7322
7339	7323	// Pass any non-return outputs indirectly, if the constraint accepts a memory location
7340		is_indirect.* = constraintAllowsMemory(constraint);
7341		if (is_indirect.*) {
	7324	llvm_ret_indirect[output.index] = constraintAllowsMemory(constraint);
	7325	if (llvm_ret_indirect[output.index]) {
7342	7326	// Pass the result by reference as an indirect output (e.g. "=*m")
7343	7327	llvm_constraints.appendAssumeCapacity('*');
7344	7328
...	...	@@ -7359,7 +7343,7 @@ pub const FuncGen = struct {
7359	7343	}),
7360	7344	}
7361	7345
7362		is_indirect.* = false;
	7346	llvm_ret_indirect[output.index] = false;
7363	7347
7364	7348	const ret_ty = self.typeOfIndex(inst);
7365	7349	llvm_ret_types[llvm_ret_i] = try o.lowerType(pt, ret_ty);
...	...	@@ -7387,16 +7371,13 @@ pub const FuncGen = struct {
7387	7371	total_i += 1;
7388	7372	}
7389	7373
7390		for (inputs) \|input\| {
7391		const extra_bytes = std.mem.sliceAsBytes(self.air.extra.items[extra_i..]);
7392		const constraint = std.mem.sliceTo(extra_bytes, 0);
7393		const name = std.mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0);
7394		// This equation accounts for the fact that even if we have exactly 4 bytes
7395		// for the string, we still use the next u32 for the null terminator.
7396		extra_i += (constraint.len + name.len + (2 + 3)) / 4;
	7374	it = unwrapped_asm.iterateInputs();
	7375	while (it.next()) \|input\| {
	7376	const constraint = input.constraint;
	7377	const name = input.name;
7397	7378
7398		const arg_llvm_value = try self.resolveInst(input);
7399		const arg_ty = self.typeOf(input);
	7379	const arg_llvm_value = try self.resolveInst(input.operand);
	7380	const arg_ty = self.typeOf(input.operand);
7400	7381	const is_by_ref = isByRef(arg_ty, zcu);
7401	7382	if (is_by_ref) {
7402	7383	if (constraintAllowsMemory(constraint)) {
...	...	@@ -7452,27 +7433,23 @@ pub const FuncGen = struct {
7452	7433	total_i += 1;
7453	7434	}
7454	7435
7455		for (outputs, llvm_ret_indirect, llvm_rw_vals, 0..) \|output, is_indirect, llvm_rw_val, output_index\| {
7456		const extra_bytes = std.mem.sliceAsBytes(self.air.extra.items[rw_extra_i..]);
7457		const constraint = std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra.items[rw_extra_i..]), 0);
7458		const name = std.mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0);
7459		// This equation accounts for the fact that even if we have exactly 4 bytes
7460		// for the string, we still use the next u32 for the null terminator.
7461		rw_extra_i += (constraint.len + name.len + (2 + 3)) / 4;
	7436	it = unwrapped_asm.iterateOutputs();
	7437	while (it.next()) \|output\| {
	7438	const constraint = output.constraint;
7462	7439
7463	7440	if (constraint[0] != '+') continue;
7464	7441
7465		const rw_ty = self.typeOf(output);
	7442	const rw_ty = self.typeOf(output.operand);
7466	7443	const llvm_elem_ty = try o.lowerPtrElemTy(pt, rw_ty.childType(zcu));
7467		if (is_indirect) {
7468		llvm_param_values[llvm_param_i] = llvm_rw_val;
7469		llvm_param_types[llvm_param_i] = llvm_rw_val.typeOfWip(&self.wip);
	7444	if (llvm_ret_indirect[output.index]) {
	7445	llvm_param_values[llvm_param_i] = llvm_rw_vals[output.index];
	7446	llvm_param_types[llvm_param_i] = llvm_rw_vals[output.index].typeOfWip(&self.wip);
7470	7447	} else {
7471	7448	const alignment = rw_ty.abiAlignment(zcu).toLlvm();
7472	7449	const loaded = try self.wip.load(
7473	7450	if (rw_ty.isVolatilePtr(zcu)) .@"volatile" else .normal,
7474	7451	llvm_elem_ty,
7475		llvm_rw_val,
	7452	llvm_rw_vals[output.index],
7476	7453	alignment,
7477	7454	"",
7478	7455	);
...	...	@@ -7480,18 +7457,18 @@ pub const FuncGen = struct {
7480	7457	llvm_param_types[llvm_param_i] = llvm_elem_ty;
7481	7458	}
7482	7459
7483		try llvm_constraints.print(gpa, ",{d}", .{output_index});
	7460	try llvm_constraints.print(gpa, ",{d}", .{output.index});
7484	7461
7485	7462	// In the case of indirect inputs, LLVM requires the callsite to have
7486	7463	// an elementtype(<ty>) attribute.
7487		llvm_param_attrs[llvm_param_i] = if (is_indirect) llvm_elem_ty else .none;
	7464	llvm_param_attrs[llvm_param_i] = if (llvm_ret_indirect[output.index]) llvm_elem_ty else .none;
7488	7465
7489	7466	llvm_param_i += 1;
7490	7467	total_i += 1;
7491	7468	}
7492	7469
7493	7470	const ip = &zcu.intern_pool;
7494		const aggregate = ip.indexToKey(extra.data.clobbers).aggregate;
	7471	const aggregate = ip.indexToKey(unwrapped_asm.clobbers).aggregate;
7495	7472	const struct_type: Type = .fromInterned(aggregate.ty);
7496	7473	if (total_i != 0) try llvm_constraints.append(gpa, ',');
7497	7474	switch (aggregate.storage) {
...	...	@@ -7539,7 +7516,7 @@ pub const FuncGen = struct {
7539	7516
7540	7517	if (std.mem.endsWith(u8, llvm_constraints.items, ",")) llvm_constraints.items.len -= 1;
7541	7518
7542		const asm_source = std.mem.sliceAsBytes(self.air.extra.items[extra_i..])[0..extra.data.source_len];
	7519	const asm_source = unwrapped_asm.source;
7543	7520
7544	7521	// hackety hacks until stage2 has proper inline asm in the frontend.
7545	7522	var rendered_template = std.array_list.Managed(u8).init(gpa);

src/codegen/riscv64/CodeGen.zig+48-74

...	...	@@ -3627,11 +3627,11 @@ fn airRuntimeNavPtr(func: *Func, inst: Air.Inst.Index) !void {
3627	3627	}
3628	3628
3629	3629	fn airTry(func: *Func, inst: Air.Inst.Index) !void {
3630		const pl_op = func.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
3631		const extra = func.air.extraData(Air.Try, pl_op.payload);
3632		const body: []const Air.Inst.Index = @ptrCast(func.air.extra.items[extra.end..][0..extra.data.body_len]);
3633		const operand_ty = func.typeOf(pl_op.operand);
3634		const result = try func.genTry(inst, pl_op.operand, body, operand_ty, false);
	3630	const zcu = func.pt.zcu;
	3631	const unwrapped_try = func.air.unwrapTry(inst);
	3632	const body = unwrapped_try.else_body;
	3633	const operand_ty = func.air.typeOf(unwrapped_try.error_union, &zcu.intern_pool);
	3634	const result = try func.genTry(inst, unwrapped_try.error_union, body, operand_ty, false);
3635	3635	return func.finishAir(inst, result, .{ .none, .none, .none });
3636	3636	}
3637	3637
...	...	@@ -4801,10 +4801,8 @@ fn airFrameAddress(func: *Func, inst: Air.Inst.Index) !void {
4801	4801
4802	4802	fn airCall(func: *Func, inst: Air.Inst.Index, modifier: std.builtin.CallModifier) !void {
4803	4803	if (modifier == .always_tail) return func.fail("TODO implement tail calls for riscv64", .{});
4804		const pl_op = func.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
4805		const callee = pl_op.operand;
4806		const extra = func.air.extraData(Air.Call, pl_op.payload);
4807		const arg_refs: []const Air.Inst.Ref = @ptrCast(func.air.extra.items[extra.end..][0..extra.data.args_len]);
	4804	const call = func.air.unwrapCall(inst);
	4805	const arg_refs = call.args;
4808	4806
4809	4807	const expected_num_args = 8;
4810	4808	const ExpectedContents = extern struct {
...	...	@@ -4822,10 +4820,10 @@ fn airCall(func: *Func, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
4822	4820	defer allocator.free(arg_vals);
4823	4821	for (arg_vals, arg_refs) \|arg_val, arg_ref\| arg_val. = .{ .air_ref = arg_ref };
4824	4822
4825		const call_ret = try func.genCall(.{ .air = callee }, arg_tys, arg_vals);
	4823	const call_ret = try func.genCall(.{ .air = call.callee }, arg_tys, arg_vals);
4826	4824
4827	4825	var bt = func.liveness.iterateBigTomb(inst);
4828		try func.feed(&bt, pl_op.operand);
	4826	try func.feed(&bt, call.callee);
4829	4827	for (arg_refs) \|arg_ref\| try func.feed(&bt, arg_ref);
4830	4828
4831	4829	const result = if (func.liveness.isUnused(inst)) .unreach else call_ret;
...	...	@@ -5218,9 +5216,8 @@ fn airDbgStmt(func: *Func, inst: Air.Inst.Index) !void {
5218	5216	}
5219	5217
5220	5218	fn airDbgInlineBlock(func: *Func, inst: Air.Inst.Index) !void {
5221		const ty_pl = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
5222		const extra = func.air.extraData(Air.DbgInlineBlock, ty_pl.payload);
5223		try func.lowerBlock(inst, @ptrCast(func.air.extra.items[extra.end..][0..extra.data.body_len]));
	5219	const block = func.air.unwrapDbgBlock(inst);
	5220	try func.lowerBlock(inst, block.body);
5224	5221	}
5225	5222
5226	5223	fn airDbgVar(func: *Func, inst: Air.Inst.Index) InnerError!void {
...	...	@@ -5271,19 +5268,18 @@ fn genVarDbgInfo(
5271	5268	}
5272	5269
5273	5270	fn airCondBr(func: *Func, inst: Air.Inst.Index) !void {
5274		const pl_op = func.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
5275		const cond = try func.resolveInst(pl_op.operand);
5276		const cond_ty = func.typeOf(pl_op.operand);
5277		const extra = func.air.extraData(Air.CondBr, pl_op.payload);
5278		const then_body: []const Air.Inst.Index = @ptrCast(func.air.extra.items[extra.end..][0..extra.data.then_body_len]);
5279		const else_body: []const Air.Inst.Index = @ptrCast(func.air.extra.items[extra.end + then_body.len ..][0..extra.data.else_body_len]);
	5271	const cond_br = func.air.unwrapCondBr(inst);
	5272	const cond = try func.resolveInst(cond_br.condition);
	5273	const cond_ty = func.typeOf(cond_br.condition);
	5274	const then_body = cond_br.then_body;
	5275	const else_body = cond_br.else_body;
5280	5276	const liveness_cond_br = func.liveness.getCondBr(inst);
5281	5277
5282	5278	// If the condition dies here in this condbr instruction, process
5283	5279	// that death now instead of later as this has an effect on
5284	5280	// whether it needs to be spilled in the branches
5285	5281	if (func.liveness.operandDies(inst, 0)) {
5286		if (pl_op.operand.toIndex()) \|op_inst\| try func.processDeath(op_inst);
	5282	if (cond_br.condition.toIndex()) \|op_inst\| try func.processDeath(op_inst);
5287	5283	}
5288	5284
5289	5285	func.scope_generation += 1;
...	...	@@ -5633,10 +5629,7 @@ fn airIsNonErrPtr(func: *Func, inst: Air.Inst.Index) !void {
5633	5629
5634	5630	fn airLoop(func: *Func, inst: Air.Inst.Index) !void {
5635	5631	// A loop is a setup to be able to jump back to the beginning.
5636		const ty_pl = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
5637		const loop = func.air.extraData(Air.Block, ty_pl.payload);
5638		const body: []const Air.Inst.Index = @ptrCast(func.air.extra.items[loop.end..][0..loop.data.body_len]);
5639
	5632	const body = func.air.unwrapBlock(inst);
5640	5633	func.scope_generation += 1;
5641	5634	const state = try func.saveState();
5642	5635
...	...	@@ -5646,7 +5639,7 @@ fn airLoop(func: *Func, inst: Air.Inst.Index) !void {
5646	5639	});
5647	5640	defer assert(func.loops.remove(inst));
5648	5641
5649		try func.genBody(body);
	5642	try func.genBody(body.body);
5650	5643
5651	5644	func.finishAirBookkeeping();
5652	5645	}
...	...	@@ -5663,9 +5656,8 @@ fn jump(func: *Func, index: Mir.Inst.Index) !Mir.Inst.Index {
5663	5656	}
5664	5657
5665	5658	fn airBlock(func: *Func, inst: Air.Inst.Index) !void {
5666		const ty_pl = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
5667		const extra = func.air.extraData(Air.Block, ty_pl.payload);
5668		try func.lowerBlock(inst, @ptrCast(func.air.extra.items[extra.end..][0..extra.data.body_len]));
	5659	const block = func.air.unwrapBlock(inst);
	5660	try func.lowerBlock(inst, block.body);
5669	5661	}
5670	5662
5671	5663	fn lowerBlock(func: *Func, inst: Air.Inst.Index, body: []const Air.Inst.Index) !void {
...	...	@@ -6053,15 +6045,9 @@ fn airBoolOp(func: *Func, inst: Air.Inst.Index) !void {
6053	6045	}
6054	6046
6055	6047	fn airAsm(func: *Func, inst: Air.Inst.Index) !void {
6056		const ty_pl = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
6057		const extra = func.air.extraData(Air.Asm, ty_pl.payload);
6058		const outputs_len = extra.data.flags.outputs_len;
6059		var extra_i: usize = extra.end;
6060		const outputs: []const Air.Inst.Ref =
6061		@ptrCast(func.air.extra.items[extra_i..][0..outputs_len]);
6062		extra_i += outputs.len;
6063		const inputs: []const Air.Inst.Ref = @ptrCast(func.air.extra.items[extra_i..][0..extra.data.inputs_len]);
6064		extra_i += inputs.len;
	6048	const unwrapped_asm = func.air.unwrapAsm(inst);
	6049	const outputs = unwrapped_asm.outputs;
	6050	const inputs = unwrapped_asm.inputs;
6065	6051
6066	6052	var result: MCValue = .none;
6067	6053	var args = std.array_list.Managed(MCValue).init(func.gpa);
...	...	@@ -6076,19 +6062,15 @@ fn airAsm(func: *Func, inst: Air.Inst.Index) !void {
6076	6062	try arg_map.ensureTotalCapacity(@intCast(outputs.len + inputs.len));
6077	6063	defer arg_map.deinit();
6078	6064
6079		var outputs_extra_i = extra_i;
6080		for (outputs) \|output\| {
6081		const extra_bytes = mem.sliceAsBytes(func.air.extra.items[extra_i..]);
6082		const constraint = mem.sliceTo(mem.sliceAsBytes(func.air.extra.items[extra_i..]), 0);
6083		const name = mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0);
6084		// This equation accounts for the fact that even if we have exactly 4 bytes
6085		// for the string, we still use the next u32 for the null terminator.
6086		extra_i += (constraint.len + name.len + (2 + 3)) / 4;
	6065	var it = unwrapped_asm.iterateOutputs();
	6066	while (it.next()) \|output\| {
	6067	const constraint = output.constraint;
	6068	const name = output.name;
6087	6069
6088	6070	const is_read = switch (constraint[0]) {
6089	6071	'=' => false,
6090	6072	'+' => read: {
6091		if (output == .none) return func.fail(
	6073	if (output.operand == .none) return func.fail(
6092	6074	"read-write constraint unsupported for asm result: '{s}'",
6093	6075	.{constraint},
6094	6076	);
...	...	@@ -6100,7 +6082,7 @@ fn airAsm(func: *Func, inst: Air.Inst.Index) !void {
6100	6082	const rest = constraint[@as(usize, 1) + @intFromBool(is_early_clobber) ..];
6101	6083	const arg_mcv: MCValue = arg_mcv: {
6102	6084	const arg_maybe_reg: ?Register = if (mem.eql(u8, rest, "m"))
6103		if (output != .none) null else return func.fail(
	6085	if (output.operand != .none) null else return func.fail(
6104	6086	"memory constraint unsupported for asm result: '{s}'",
6105	6087	.{constraint},
6106	6088	)
...	...	@@ -6115,7 +6097,7 @@ fn airAsm(func: *Func, inst: Air.Inst.Index) !void {
6115	6097	break :arg_mcv args.items[index];
6116	6098	} else return func.fail("invalid constraint: '{s}'", .{constraint});
6117	6099	break :arg_mcv if (arg_maybe_reg) \|reg\| .{ .register = reg } else arg: {
6118		const ptr_mcv = try func.resolveInst(output);
	6100	const ptr_mcv = try func.resolveInst(output.operand);
6119	6101	switch (ptr_mcv) {
6120	6102	.immediate => \|addr\| if (math.cast(i32, @as(i64, @bitCast(addr)))) \|_\|
6121	6103	break :arg ptr_mcv.deref(),
...	...	@@ -6131,20 +6113,17 @@ fn airAsm(func: *Func, inst: Air.Inst.Index) !void {
6131	6113	if (!mem.eql(u8, name, "_"))
6132	6114	arg_map.putAssumeCapacityNoClobber(name, @intCast(args.items.len));
6133	6115	args.appendAssumeCapacity(arg_mcv);
6134		if (output == .none) result = arg_mcv;
6135		if (is_read) try func.load(arg_mcv, .{ .air_ref = output }, func.typeOf(output));
	6116	if (output.operand == .none) result = arg_mcv;
	6117	if (is_read) try func.load(arg_mcv, .{ .air_ref = output.operand }, func.typeOf(output.operand));
6136	6118	}
6137	6119
6138		for (inputs) \|input\| {
6139		const input_bytes = mem.sliceAsBytes(func.air.extra.items[extra_i..]);
6140		const constraint = mem.sliceTo(input_bytes, 0);
6141		const name = mem.sliceTo(input_bytes[constraint.len + 1 ..], 0);
6142		// This equation accounts for the fact that even if we have exactly 4 bytes
6143		// for the string, we still use the next u32 for the null terminator.
6144		extra_i += (constraint.len + name.len + (2 + 3)) / 4;
	6120	it = unwrapped_asm.iterateInputs();
	6121	while (it.next()) \|input\| {
	6122	const constraint = input.constraint;
	6123	const name = input.name;
6145	6124
6146		const ty = func.typeOf(input);
6147		const input_mcv = try func.resolveInst(input);
	6125	const ty = func.typeOf(input.operand);
	6126	const input_mcv = try func.resolveInst(input.operand);
6148	6127	const arg_mcv: MCValue = if (mem.eql(u8, constraint, "X"))
6149	6128	input_mcv
6150	6129	else if (mem.startsWith(u8, constraint, "{") and mem.endsWith(u8, constraint, "}")) arg: {
...	...	@@ -6171,7 +6150,7 @@ fn airAsm(func: *Func, inst: Air.Inst.Index) !void {
6171	6150
6172	6151	const zcu = func.pt.zcu;
6173	6152	const ip = &zcu.intern_pool;
6174		const aggregate = ip.indexToKey(extra.data.clobbers).aggregate;
	6153	const aggregate = ip.indexToKey(unwrapped_asm.clobbers).aggregate;
6175	6154	const struct_type: Type = .fromInterned(aggregate.ty);
6176	6155	switch (aggregate.storage) {
6177	6156	.elems => \|elems\| for (elems, 0..) \|elem, i\| {
...	...	@@ -6231,7 +6210,7 @@ fn airAsm(func: *Func, inst: Air.Inst.Index) !void {
6231	6210	labels.deinit(func.gpa);
6232	6211	}
6233	6212
6234		const asm_source = std.mem.sliceAsBytes(func.air.extra.items[extra_i..])[0..extra.data.source_len];
	6213	const asm_source = unwrapped_asm.source;
6235	6214	var line_it = mem.tokenizeAny(u8, asm_source, "\n\r;");
6236	6215	next_line: while (line_it.next()) \|line\| {
6237	6216	var mnem_it = mem.tokenizeAny(u8, line, " \t");
...	...	@@ -6499,19 +6478,14 @@ fn airAsm(func: *Func, inst: Air.Inst.Index) !void {
6499	6478	while (label_it.next()) \|label\| if (label.value_ptr.pending_relocs.items.len > 0)
6500	6479	return func.fail("undefined label: '{s}'", .{label.key_ptr.*});
6501	6480
6502		for (outputs, args.items[0..outputs.len]) \|output, arg_mcv\| {
6503		const extra_bytes = mem.sliceAsBytes(func.air.extra.items[outputs_extra_i..]);
6504		const constraint =
6505		mem.sliceTo(mem.sliceAsBytes(func.air.extra.items[outputs_extra_i..]), 0);
6506		const name = mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0);
6507		// This equation accounts for the fact that even if we have exactly 4 bytes
6508		// for the string, we still use the next u32 for the null terminator.
6509		outputs_extra_i += (constraint.len + name.len + (2 + 3)) / 4;
6510
6511		if (output == .none) continue;
6512		if (arg_mcv != .register) continue;
	6481	it = unwrapped_asm.iterateOutputs();
	6482	while (it.next()) \|output\| {
	6483	const constraint = output.constraint;
	6484
	6485	if (output.operand == .none) continue;
	6486	if (args.items[output.index] != .register) continue;
6513	6487	if (constraint.len == 2 and std.ascii.isDigit(constraint[1])) continue;
6514		try func.store(.{ .air_ref = output }, arg_mcv, func.typeOf(output));
	6488	try func.store(.{ .air_ref = output.operand }, args.items[output.index], func.typeOf(output.operand));
6515	6489	}
6516	6490
6517	6491	simple: {

src/codegen/sparc64/CodeGen.zig+40-64

...	...	@@ -877,15 +877,10 @@ fn airArrayToSlice(self: *Self, inst: Air.Inst.Index) !void {
877	877	}
878	878
879	879	fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
880		const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
881		const extra = self.air.extraData(Air.Asm, ty_pl.payload);
882		const is_volatile = extra.data.flags.is_volatile;
883		const outputs_len = extra.data.flags.outputs_len;
884		var extra_i: usize = extra.end;
885		const outputs: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[extra_i .. extra_i + outputs_len]);
886		extra_i += outputs.len;
887		const inputs: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[extra_i .. extra_i + extra.data.inputs_len]);
888		extra_i += inputs.len;
	880	const unwrapped_asm = self.air.unwrapAsm(inst);
	881	const is_volatile = unwrapped_asm.is_volatile;
	882	const outputs = unwrapped_asm.outputs;
	883	const inputs = unwrapped_asm.inputs;
889	884
890	885	const dead = !is_volatile and self.liveness.isUnused(inst);
891	886	const result: MCValue = if (dead) .dead else result: {
...	...	@@ -893,27 +888,18 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
893	888	return self.fail("TODO implement codegen for asm with more than 1 output", .{});
894	889	}
895	890
896		const output_constraint: ?[]const u8 = for (outputs) \|output\| {
897		if (output != .none) {
	891	var it = unwrapped_asm.iterateOutputs();
	892	const output_constraint: ?[]const u8 = while (it.next()) \|output\| {
	893	if (output.operand != .none) {
898	894	return self.fail("TODO implement codegen for non-expr asm", .{});
899	895	}
900		const extra_bytes = std.mem.sliceAsBytes(self.air.extra.items[extra_i..]);
901		const constraint = std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra.items[extra_i..]), 0);
902		const name = std.mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0);
903		// This equation accounts for the fact that even if we have exactly 4 bytes
904		// for the string, we still use the next u32 for the null terminator.
905		extra_i += (constraint.len + name.len + (2 + 3)) / 4;
906
907		break constraint;
	896
	897	break output.constraint;
908	898	} else null;
909	899
910		for (inputs) \|input\| {
911		const input_bytes = std.mem.sliceAsBytes(self.air.extra.items[extra_i..]);
912		const constraint = std.mem.sliceTo(input_bytes, 0);
913		const name = std.mem.sliceTo(input_bytes[constraint.len + 1 ..], 0);
914		// This equation accounts for the fact that even if we have exactly 4 bytes
915		// for the string, we still use the next u32 for the null terminator.
916		extra_i += (constraint.len + name.len + (2 + 3)) / 4;
	900	it = unwrapped_asm.iterateInputs();
	901	while (it.next()) \|input\| {
	902	const constraint = input.constraint;
917	903
918	904	if (constraint.len < 3 or constraint[0] != '{' or constraint[constraint.len - 1] != '}') {
919	905	return self.fail("unrecognized asm input constraint: '{s}'", .{constraint});
...	...	@@ -922,15 +908,15 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
922	908	const reg = parseRegName(reg_name) orelse
923	909	return self.fail("unrecognized register: '{s}'", .{reg_name});
924	910
925		const arg_mcv = try self.resolveInst(input);
	911	const arg_mcv = try self.resolveInst(input.operand);
926	912	try self.register_manager.getReg(reg, null);
927		try self.genSetReg(self.typeOf(input), reg, arg_mcv);
	913	try self.genSetReg(self.typeOf(input.operand), reg, arg_mcv);
928	914	}
929	915
930	916	// TODO honor the clobbers
931		_ = extra.data.clobbers;
	917	_ = unwrapped_asm.clobbers;
932	918
933		const asm_source = std.mem.sliceAsBytes(self.air.extra.items[extra_i..])[0..extra.data.source_len];
	919	const asm_source = unwrapped_asm.source;
934	920
935	921	if (mem.eql(u8, asm_source, "ta 0x6d")) {
936	922	_ = try self.addInst(.{
...	...	@@ -1109,9 +1095,8 @@ fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void {
1109	1095	}
1110	1096
1111	1097	fn airBlock(self: *Self, inst: Air.Inst.Index) !void {
1112		const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
1113		const extra = self.air.extraData(Air.Block, ty_pl.payload);
1114		try self.lowerBlock(inst, @ptrCast(self.air.extra.items[extra.end..][0..extra.data.body_len]));
	1098	const block = self.air.unwrapBlock(inst);
	1099	try self.lowerBlock(inst, block.body);
1115	1100	}
1116	1101
1117	1102	fn lowerBlock(self: *Self, inst: Air.Inst.Index, body: []const Air.Inst.Index) !void {
...	...	@@ -1276,11 +1261,9 @@ fn airByteSwap(self: *Self, inst: Air.Inst.Index) !void {
1276	1261	fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier) !void {
1277	1262	if (modifier == .always_tail) return self.fail("TODO implement tail calls for {}", .{self.target.cpu.arch});
1278	1263
1279		const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
1280		const callee = pl_op.operand;
1281		const extra = self.air.extraData(Air.Call, pl_op.payload);
1282		const args: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[extra.end .. extra.end + extra.data.args_len]);
1283		const ty = self.typeOf(callee);
	1264	const call = self.air.unwrapCall(inst);
	1265	const args = call.args;
	1266	const ty = self.typeOf(call.callee);
1284	1267	const pt = self.pt;
1285	1268	const zcu = pt.zcu;
1286	1269	const ip = &zcu.intern_pool;
...	...	@@ -1327,7 +1310,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
1327	1310
1328	1311	// Due to incremental compilation, how function calls are generated depends
1329	1312	// on linking.
1330		if (try self.air.value(callee, pt)) \|func_value\| switch (ip.indexToKey(func_value.toIntern())) {
	1313	if (try self.air.value(call.callee, pt)) \|func_value\| switch (ip.indexToKey(func_value.toIntern())) {
1331	1314	.func => {
1332	1315	return self.fail("TODO implement calling functions", .{});
1333	1316	},
...	...	@@ -1339,7 +1322,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
1339	1322	},
1340	1323	} else {
1341	1324	assert(ty.zigTypeTag(zcu) == .pointer);
1342		const mcv = try self.resolveInst(callee);
	1325	const mcv = try self.resolveInst(call.callee);
1343	1326	try self.genSetReg(ty, .o7, mcv);
1344	1327
1345	1328	_ = try self.addInst(.{
...	...	@@ -1365,13 +1348,13 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
1365	1348
1366	1349	if (args.len + 1 <= Air.Liveness.bpi - 1) {
1367	1350	var buf = [1]Air.Inst.Ref{.none} ** (Air.Liveness.bpi - 1);
1368		buf[0] = callee;
	1351	buf[0] = call.callee;
1369	1352	@memcpy(buf[1..][0..args.len], args);
1370	1353	return self.finishAir(inst, result, buf);
1371	1354	}
1372	1355
1373	1356	var bt = try self.iterateBigTomb(inst, 1 + args.len);
1374		bt.feed(callee);
	1357	bt.feed(call.callee);
1375	1358	for (args) \|arg\| {
1376	1359	bt.feed(arg);
1377	1360	}
...	...	@@ -1451,9 +1434,7 @@ fn airCmpLtErrorsLen(self: *Self, inst: Air.Inst.Index) !void {
1451	1434	}
1452	1435
1453	1436	fn airCmpxchg(self: *Self, inst: Air.Inst.Index) !void {
1454		const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
1455		const extra = self.air.extraData(Air.Block, ty_pl.payload);
1456		_ = extra;
	1437	_ = inst;
1457	1438
1458	1439	return self.fail("TODO implement airCmpxchg for {}", .{
1459	1440	self.target.cpu.arch,
...	...	@@ -1461,11 +1442,10 @@ fn airCmpxchg(self: *Self, inst: Air.Inst.Index) !void {
1461	1442	}
1462	1443
1463	1444	fn airCondBr(self: *Self, inst: Air.Inst.Index) !void {
1464		const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
1465		const condition = try self.resolveInst(pl_op.operand);
1466		const extra = self.air.extraData(Air.CondBr, pl_op.payload);
1467		const then_body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[extra.end..][0..extra.data.then_body_len]);
1468		const else_body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[extra.end + then_body.len ..][0..extra.data.else_body_len]);
	1445	const cond_br = self.air.unwrapCondBr(inst);
	1446	const condition = try self.resolveInst(cond_br.condition);
	1447	const then_body = cond_br.then_body;
	1448	const else_body = cond_br.else_body;
1469	1449	const liveness_condbr = self.liveness.getCondBr(inst);
1470	1450
1471	1451	// Here we emit a branch to the false section.
...	...	@@ -1475,7 +1455,7 @@ fn airCondBr(self: *Self, inst: Air.Inst.Index) !void {
1475	1455	// that death now instead of later as this has an effect on
1476	1456	// whether it needs to be spilled in the branches
1477	1457	if (self.liveness.operandDies(inst, 0)) {
1478		if (pl_op.operand.toIndex()) \|op_index\| {
	1458	if (cond_br.condition.toIndex()) \|op_index\| {
1479	1459	self.processDeath(op_index);
1480	1460	}
1481	1461	}
...	...	@@ -1613,10 +1593,9 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void {
1613	1593	}
1614	1594
1615	1595	fn airDbgInlineBlock(self: *Self, inst: Air.Inst.Index) !void {
1616		const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
1617		const extra = self.air.extraData(Air.DbgInlineBlock, ty_pl.payload);
	1596	const block = self.air.unwrapDbgBlock(inst);
1618	1597	// TODO emit debug info for function change
1619		try self.lowerBlock(inst, @ptrCast(self.air.extra.items[extra.end..][0..extra.data.body_len]));
	1598	try self.lowerBlock(inst, block.body);
1620	1599	}
1621	1600
1622	1601	fn airDbgStmt(self: *Self, inst: Air.Inst.Index) !void {
...	...	@@ -1780,12 +1759,10 @@ fn airLoad(self: *Self, inst: Air.Inst.Index) !void {
1780	1759
1781	1760	fn airLoop(self: *Self, inst: Air.Inst.Index) !void {
1782	1761	// A loop is a setup to be able to jump back to the beginning.
1783		const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
1784		const loop = self.air.extraData(Air.Block, ty_pl.payload);
1785		const body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[loop.end .. loop.end + loop.data.body_len]);
	1762	const block = self.air.unwrapBlock(inst);
1786	1763	const start: u32 = @intCast(self.mir_instructions.len);
1787	1764
1788		try self.genBody(body);
	1765	try self.genBody(block.body);
1789	1766	try self.jump(start);
1790	1767
1791	1768	return self.finishAirBookkeeping();
...	...	@@ -2606,12 +2583,11 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {
2606	2583	}
2607	2584
2608	2585	fn airTry(self: *Self, inst: Air.Inst.Index) !void {
2609		const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
2610		const extra = self.air.extraData(Air.Try, pl_op.payload);
2611		const body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[extra.end..][0..extra.data.body_len]);
	2586	const unwrapped_try = self.air.unwrapTry(inst);
	2587	const body = unwrapped_try.else_body;
2612	2588	const result: MCValue = result: {
2613		const error_union_ty = self.typeOf(pl_op.operand);
2614		const error_union = try self.resolveInst(pl_op.operand);
	2589	const error_union_ty = self.air.typeOf(unwrapped_try.error_union, &self.pt.zcu.intern_pool);
	2590	const error_union = try self.resolveInst(unwrapped_try.error_union);
2615	2591	const is_err_result = try self.isErr(error_union_ty, error_union);
2616	2592	const reloc = try self.condBr(is_err_result);
2617	2593
...	...	@@ -2620,7 +2596,7 @@ fn airTry(self: *Self, inst: Air.Inst.Index) !void {
2620	2596	try self.performReloc(reloc);
2621	2597	break :result try self.errUnionPayload(error_union, error_union_ty);
2622	2598	};
2623		return self.finishAir(inst, result, .{ pl_op.operand, .none, .none });
	2599	return self.finishAir(inst, result, .{ unwrapped_try.error_union, .none, .none });
2624	2600	}
2625	2601
2626	2602	fn airUnaryMath(self: *Self, inst: Air.Inst.Index) !void {

src/codegen/spirv/CodeGen.zig+47-79

...	...	@@ -5002,9 +5002,8 @@ fn genStructuredBody(
5002	5002	}
5003	5003
5004	5004	fn airBlock(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
5005		const inst_datas = cg.air.instructions.items(.data);
5006		const extra = cg.air.extraData(Air.Block, inst_datas[@intFromEnum(inst)].ty_pl.payload);
5007		return cg.lowerBlock(inst, @ptrCast(cg.air.extra.items[extra.end..][0..extra.data.body_len]));
	5005	const block = cg.air.unwrapBlock(inst);
	5006	return cg.lowerBlock(inst, block.body);
5008	5007	}
5009	5008
5010	5009	fn lowerBlock(cg: *CodeGen, inst: Air.Inst.Index, body: []const Air.Inst.Index) !?Id {
...	...	@@ -5188,11 +5187,10 @@ fn airBr(cg: *CodeGen, inst: Air.Inst.Index) !void {
5188	5187
5189	5188	fn airCondBr(cg: *CodeGen, inst: Air.Inst.Index) !void {
5190	5189	const gpa = cg.module.gpa;
5191		const pl_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
5192		const cond_br = cg.air.extraData(Air.CondBr, pl_op.payload);
5193		const then_body: []const Air.Inst.Index = @ptrCast(cg.air.extra.items[cond_br.end..][0..cond_br.data.then_body_len]);
5194		const else_body: []const Air.Inst.Index = @ptrCast(cg.air.extra.items[cond_br.end + then_body.len ..][0..cond_br.data.else_body_len]);
5195		const condition_id = try cg.resolve(pl_op.operand);
	5190	const cond_br = cg.air.unwrapCondBr(inst);
	5191	const then_body = cond_br.then_body;
	5192	const else_body = cond_br.else_body;
	5193	const condition_id = try cg.resolve(cond_br.condition);
5196	5194
5197	5195	const then_label = cg.module.allocId();
5198	5196	const else_label = cg.module.allocId();
...	...	@@ -5251,9 +5249,7 @@ fn airCondBr(cg: *CodeGen, inst: Air.Inst.Index) !void {
5251	5249
5252	5250	fn airLoop(cg: *CodeGen, inst: Air.Inst.Index) !void {
5253	5251	const gpa = cg.module.gpa;
5254		const ty_pl = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
5255		const loop = cg.air.extraData(Air.Block, ty_pl.payload);
5256		const body: []const Air.Inst.Index = @ptrCast(cg.air.extra.items[loop.end..][0..loop.data.body_len]);
	5252	const block = cg.air.unwrapBlock(inst);
5257	5253
5258	5254	const body_label = cg.module.allocId();
5259	5255
...	...	@@ -5284,7 +5280,7 @@ fn airLoop(cg: *CodeGen, inst: Air.Inst.Index) !void {
5284	5280	const next_block = try cg.genStructuredBody(.{ .loop = .{
5285	5281	.merge_label = merge_label,
5286	5282	.continue_label = continue_label,
5287		} }, body);
	5283	} }, block.body);
5288	5284	try cg.structuredBreak(next_block);
5289	5285
5290	5286	try cg.beginSpvBlock(continue_label);
...	...	@@ -5294,7 +5290,7 @@ fn airLoop(cg: *CodeGen, inst: Air.Inst.Index) !void {
5294	5290	.unstructured => {
5295	5291	try cg.body.emit(gpa, .OpBranch, .{ .target_label = body_label });
5296	5292	try cg.beginSpvBlock(body_label);
5297		try cg.genBody(body);
	5293	try cg.genBody(block.body);
5298	5294
5299	5295	try cg.body.emit(gpa, .OpBranch, .{ .target_label = body_label });
5300	5296	},
...	...	@@ -5375,12 +5371,11 @@ fn airRetLoad(cg: *CodeGen, inst: Air.Inst.Index) !void {
5375	5371	fn airTry(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
5376	5372	const gpa = cg.module.gpa;
5377	5373	const zcu = cg.module.zcu;
5378		const pl_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
5379		const err_union_id = try cg.resolve(pl_op.operand);
5380		const extra = cg.air.extraData(Air.Try, pl_op.payload);
5381		const body: []const Air.Inst.Index = @ptrCast(cg.air.extra.items[extra.end..][0..extra.data.body_len]);
	5374	const unwrapped_try = cg.air.unwrapTry(inst);
	5375	const body = unwrapped_try.else_body;
5382	5376
5383		const err_union_ty = cg.typeOf(pl_op.operand);
	5377	const err_union_id = try cg.resolve(unwrapped_try.error_union);
	5378	const err_union_ty = cg.air.typeOf(unwrapped_try.error_union, &zcu.intern_pool);
5384	5379	const payload_ty = cg.typeOfIndex(inst);
5385	5380
5386	5381	const bool_ty_id = try cg.resolveType(.bool, .direct);
...	...	@@ -5882,12 +5877,11 @@ fn airDbgStmt(cg: *CodeGen, inst: Air.Inst.Index) !void {
5882	5877
5883	5878	fn airDbgInlineBlock(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
5884	5879	const zcu = cg.module.zcu;
5885		const inst_datas = cg.air.instructions.items(.data);
5886		const extra = cg.air.extraData(Air.DbgInlineBlock, inst_datas[@intFromEnum(inst)].ty_pl.payload);
	5880	const block = cg.air.unwrapDbgBlock(inst);
5887	5881	const old_base_line = cg.base_line;
5888	5882	defer cg.base_line = old_base_line;
5889		cg.base_line = zcu.navSrcLine(zcu.funcInfo(extra.data.func).owner_nav);
5890		return cg.lowerBlock(inst, @ptrCast(cg.air.extra.items[extra.end..][0..extra.data.body_len]));
	5883	cg.base_line = zcu.navSrcLine(zcu.funcInfo(block.func).owner_nav);
	5884	return cg.lowerBlock(inst, block.body);
5891	5885	}
5892	5886
5893	5887	fn airDbgVar(cg: *CodeGen, inst: Air.Inst.Index) !void {
...	...	@@ -5900,52 +5894,34 @@ fn airDbgVar(cg: *CodeGen, inst: Air.Inst.Index) !void {
5900	5894	fn airAssembly(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
5901	5895	const gpa = cg.module.gpa;
5902	5896	const zcu = cg.module.zcu;
5903		const ty_pl = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
5904		const extra = cg.air.extraData(Air.Asm, ty_pl.payload);
	5897	const unwrapped_asm = cg.air.unwrapAsm(inst);
5905	5898
5906		const is_volatile = extra.data.flags.is_volatile;
5907		const outputs_len = extra.data.flags.outputs_len;
	5899	const is_volatile = unwrapped_asm.is_volatile;
	5900	const outputs_len = unwrapped_asm.outputs.len;
5908	5901
5909	5902	if (!is_volatile and cg.liveness.isUnused(inst)) return null;
5910	5903
5911		var extra_i: usize = extra.end;
5912		const outputs: []const Air.Inst.Ref = @ptrCast(cg.air.extra.items[extra_i..][0..outputs_len]);
5913		extra_i += outputs.len;
5914		const inputs: []const Air.Inst.Ref = @ptrCast(cg.air.extra.items[extra_i..][0..extra.data.inputs_len]);
5915		extra_i += inputs.len;
5916
5917		if (outputs.len > 1) {
	5904	if (outputs_len > 1) {
5918	5905	return cg.todo("implement inline asm with more than 1 output", .{});
5919	5906	}
5920	5907
5921	5908	var ass: Assembler = .{ .cg = cg };
5922	5909	defer ass.deinit();
5923	5910
5924		var output_extra_i = extra_i;
5925		for (outputs) \|output\| {
5926		if (output != .none) {
	5911	var it = unwrapped_asm.iterateOutputs();
	5912	while (it.next()) \|out\| {
	5913	if (out.operand != .none) {
5927	5914	return cg.todo("implement inline asm with non-returned output", .{});
5928	5915	}
5929		const extra_bytes = std.mem.sliceAsBytes(cg.air.extra.items[extra_i..]);
5930		const constraint = std.mem.sliceTo(std.mem.sliceAsBytes(cg.air.extra.items[extra_i..]), 0);
5931		const name = std.mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0);
5932		extra_i += (constraint.len + name.len + (2 + 3)) / 4;
5933		// TODO: Record output and use it somewhere.
5934	5916	}
5935	5917
5936		for (inputs) \|input\| {
5937		const extra_bytes = std.mem.sliceAsBytes(cg.air.extra.items[extra_i..]);
5938		const constraint = std.mem.sliceTo(extra_bytes, 0);
5939		const name = std.mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0);
5940		// This equation accounts for the fact that even if we have exactly 4 bytes
5941		// for the string, we still use the next u32 for the null terminator.
5942		extra_i += (constraint.len + name.len + (2 + 3)) / 4;
5943
5944		const input_ty = cg.typeOf(input);
	5918	it = unwrapped_asm.iterateInputs();
	5919	while (it.next()) \|in\| {
	5920	const input_ty = cg.typeOf(in.operand);
5945	5921
5946		if (std.mem.eql(u8, constraint, "c")) {
	5922	if (std.mem.eql(u8, in.constraint, "c")) {
5947	5923	// constant
5948		const val = (try cg.air.value(input, cg.pt)) orelse {
	5924	const val = (try cg.air.value(in.operand, cg.pt)) orelse {
5949	5925	return cg.fail("assembly inputs with 'c' constraint have to be compile-time known", .{});
5950	5926	};
5951	5927
...	...	@@ -5971,37 +5947,36 @@ fn airAssembly(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
5971	5947
5972	5948	.undef => return cg.fail("assembly input with 'c' constraint cannot be undefined", .{}),
5973	5949
5974		.int => try ass.value_map.put(gpa, name, .{ .constant = @intCast(val.toUnsignedInt(zcu)) }),
5975		.enum_literal => \|str\| try ass.value_map.put(gpa, name, .{ .string = str.toSlice(ip) }),
	5950	.int => try ass.value_map.put(gpa, in.name, .{ .constant = @intCast(val.toUnsignedInt(zcu)) }),
	5951	.enum_literal => \|str\| try ass.value_map.put(gpa, in.name, .{ .string = str.toSlice(ip) }),
5976	5952
5977	5953	else => unreachable, // TODO
5978	5954	}
5979		} else if (std.mem.eql(u8, constraint, "t")) {
	5955	} else if (std.mem.eql(u8, in.constraint, "t")) {
5980	5956	// type
5981	5957	if (input_ty.zigTypeTag(zcu) == .type) {
5982	5958	// This assembly input is a type instead of a value.
5983	5959	// That's fine for now, just make sure to resolve it as such.
5984		const val = (try cg.air.value(input, cg.pt)).?;
	5960	const val = (try cg.air.value(in.operand, cg.pt)).?;
5985	5961	const ty_id = try cg.resolveType(val.toType(), .direct);
5986		try ass.value_map.put(gpa, name, .{ .ty = ty_id });
	5962	try ass.value_map.put(gpa, in.name, .{ .ty = ty_id });
5987	5963	} else {
5988	5964	const ty_id = try cg.resolveType(input_ty, .direct);
5989		try ass.value_map.put(gpa, name, .{ .ty = ty_id });
	5965	try ass.value_map.put(gpa, in.name, .{ .ty = ty_id });
5990	5966	}
5991	5967	} else {
5992	5968	if (input_ty.zigTypeTag(zcu) == .type) {
5993	5969	return cg.fail("use the 't' constraint to supply types to SPIR-V inline assembly", .{});
5994	5970	}
5995	5971
5996		const val_id = try cg.resolve(input);
5997		try ass.value_map.put(gpa, name, .{ .value = val_id });
	5972	const val_id = try cg.resolve(in.operand);
	5973	try ass.value_map.put(gpa, in.name, .{ .value = val_id });
5998	5974	}
5999	5975	}
6000
6001	5976	// TODO: do something with clobbers
6002		_ = extra.data.clobbers;
	5977	_ = unwrapped_asm.clobbers;
6003	5978
6004		const asm_source = std.mem.sliceAsBytes(cg.air.extra.items[extra_i..])[0..extra.data.source_len];
	5979	const asm_source = unwrapped_asm.source;
6005	5980
6006	5981	ass.assemble(asm_source) catch \|err\| switch (err) {
6007	5982	error.AssembleFail => {
...	...	@@ -6033,26 +6008,20 @@ fn airAssembly(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
6033	6008	else => \|others\| return others,
6034	6009	};
6035	6010
6036		for (outputs) \|output\| {
6037		_ = output;
6038		const extra_bytes = std.mem.sliceAsBytes(cg.air.extra.items[output_extra_i..]);
6039		const constraint = std.mem.sliceTo(std.mem.sliceAsBytes(cg.air.extra.items[output_extra_i..]), 0);
6040		const name = std.mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0);
6041		output_extra_i += (constraint.len + name.len + (2 + 3)) / 4;
6042
6043		const result = ass.value_map.get(name) orelse return {
6044		return cg.fail("invalid asm output '{s}'", .{name});
	6011	it = unwrapped_asm.iterateOutputs();
	6012	while (it.next()) \|out\| {
	6013	const result = ass.value_map.get(out.name) orelse return {
	6014	return cg.fail("invalid asm output '{s}'", .{out.name});
6045	6015	};
6046
6047	6016	switch (result) {
6048	6017	.just_declared, .unresolved_forward_reference => unreachable,
6049	6018	.ty => return cg.fail("cannot return spir-v type as value from assembly", .{}),
6050	6019	.value => \|ref\| return ref,
6051	6020	.constant, .string => return cg.fail("cannot return constant from assembly", .{}),
6052	6021	}
6053
6054	6022	// TODO: Multiple results
6055	6023	// TODO: Check that the output type from assembly is the same as the type actually expected by Zig.
	6024
6056	6025	}
6057	6026
6058	6027	return null;
...	...	@@ -6063,10 +6032,9 @@ fn airCall(cg: *CodeGen, inst: Air.Inst.Index, modifier: std.builtin.CallModifie
6063	6032
6064	6033	const gpa = cg.module.gpa;
6065	6034	const zcu = cg.module.zcu;
6066		const pl_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
6067		const extra = cg.air.extraData(Air.Call, pl_op.payload);
6068		const args: []const Air.Inst.Ref = @ptrCast(cg.air.extra.items[extra.end..][0..extra.data.args_len]);
6069		const callee_ty = cg.typeOf(pl_op.operand);
	6035	const air_call = cg.air.unwrapCall(inst);
	6036	const args = air_call.args;
	6037	const callee_ty = cg.typeOf(air_call.callee);
6070	6038	const zig_fn_ty = switch (callee_ty.zigTypeTag(zcu)) {
6071	6039	.@"fn" => callee_ty,
6072	6040	.pointer => return cg.fail("cannot call function pointers", .{}),
...	...	@@ -6077,7 +6045,7 @@ fn airCall(cg: *CodeGen, inst: Air.Inst.Index, modifier: std.builtin.CallModifie
6077	6045
6078	6046	const result_type_id = try cg.resolveFnReturnType(.fromInterned(return_type));
6079	6047	const result_id = cg.module.allocId();
6080		const callee_id = try cg.resolve(pl_op.operand);
	6048	const callee_id = try cg.resolve(air_call.callee);
6081	6049
6082	6050	comptime assert(zig_call_abi_ver == 3);
6083	6051

src/codegen/wasm/CodeGen.zig+26-34

...	...	@@ -2137,10 +2137,9 @@ fn airRetLoad(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
2137	2137
2138	2138	fn airCall(cg: *CodeGen, inst: Air.Inst.Index, modifier: std.builtin.CallModifier) InnerError!void {
2139	2139	if (modifier == .always_tail) return cg.fail("TODO implement tail calls for wasm", .{});
2140		const pl_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
2141		const extra = cg.air.extraData(Air.Call, pl_op.payload);
2142		const args: []const Air.Inst.Ref = @ptrCast(cg.air.extra.items[extra.end..][0..extra.data.args_len]);
2143		const ty = cg.typeOf(pl_op.operand);
	2140	const call = cg.air.unwrapCall(inst);
	2141	const args = call.args;
	2142	const ty = cg.typeOf(call.callee);
2144	2143
2145	2144	const pt = cg.pt;
2146	2145	const zcu = pt.zcu;
...	...	@@ -2155,7 +2154,7 @@ fn airCall(cg: *CodeGen, inst: Air.Inst.Index, modifier: std.builtin.CallModifie
2155	2154	const first_param_sret = firstParamSRet(fn_info.cc, Type.fromInterned(fn_info.return_type), zcu, cg.target);
2156	2155
2157	2156	const callee: ?InternPool.Nav.Index = blk: {
2158		const func_val = (try cg.air.value(pl_op.operand, pt)) orelse break :blk null;
	2157	const func_val = (try cg.air.value(call.callee, pt)) orelse break :blk null;
2159	2158
2160	2159	switch (ip.indexToKey(func_val.toIntern())) {
2161	2160	inline .func, .@"extern" => \|x\| break :blk x.owner_nav,
...	...	@@ -2189,7 +2188,7 @@ fn airCall(cg: *CodeGen, inst: Air.Inst.Index, modifier: std.builtin.CallModifie
2189	2188	// in this case we call a function pointer
2190	2189	// so load its value onto the stack
2191	2190	assert(ty.zigTypeTag(zcu) == .pointer);
2192		const operand = try cg.resolveInst(pl_op.operand);
	2191	const operand = try cg.resolveInst(call.callee);
2193	2192	try cg.emitWValue(operand);
2194	2193
2195	2194	try cg.mir_func_tys.put(cg.gpa, fn_ty.toIntern(), {});
...	...	@@ -2233,7 +2232,7 @@ fn airCall(cg: *CodeGen, inst: Air.Inst.Index, modifier: std.builtin.CallModifie
2233	2232	};
2234	2233
2235	2234	var bt = try cg.iterateBigTomb(inst, 1 + args.len);
2236		bt.feed(pl_op.operand);
	2235	bt.feed(call.callee);
2237	2236	for (args) \|arg\| bt.feed(arg);
2238	2237	return bt.finishAir(result_value);
2239	2238	}
...	...	@@ -3335,9 +3334,8 @@ fn emitUndefined(cg: *CodeGen, ty: Type) InnerError!WValue {
3335	3334	}
3336	3335
3337	3336	fn airBlock(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
3338		const ty_pl = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
3339		const extra = cg.air.extraData(Air.Block, ty_pl.payload);
3340		try cg.lowerBlock(inst, ty_pl.ty.toType(), @ptrCast(cg.air.extra.items[extra.end..][0..extra.data.body_len]));
	3337	const block = cg.air.unwrapBlock(inst);
	3338	try cg.lowerBlock(inst, block.ty, block.body);
3341	3339	}
3342	3340
3343	3341	fn lowerBlock(cg: *CodeGen, inst: Air.Inst.Index, block_ty: Type, body: []const Air.Inst.Index) InnerError!void {
...	...	@@ -3381,9 +3379,7 @@ fn endBlock(cg: *CodeGen) !void {
3381	3379	}
3382	3380
3383	3381	fn airLoop(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
3384		const ty_pl = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
3385		const loop = cg.air.extraData(Air.Block, ty_pl.payload);
3386		const body: []const Air.Inst.Index = @ptrCast(cg.air.extra.items[loop.end..][0..loop.data.body_len]);
	3382	const block = cg.air.unwrapBlock(inst);
3387	3383
3388	3384	// result type of loop is always 'noreturn', meaning we can always
3389	3385	// emit the wasm type 'block_empty'.
...	...	@@ -3392,18 +3388,17 @@ fn airLoop(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
3392	3388	try cg.loops.putNoClobber(cg.gpa, inst, cg.block_depth);
3393	3389	defer assert(cg.loops.remove(inst));
3394	3390
3395		try cg.genBody(body);
	3391	try cg.genBody(block.body);
3396	3392	try cg.endBlock();
3397	3393
3398	3394	return cg.finishAir(inst, .none, &.{});
3399	3395	}
3400	3396
3401	3397	fn airCondBr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
3402		const pl_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
3403		const condition = try cg.resolveInst(pl_op.operand);
3404		const extra = cg.air.extraData(Air.CondBr, pl_op.payload);
3405		const then_body: []const Air.Inst.Index = @ptrCast(cg.air.extra.items[extra.end..][0..extra.data.then_body_len]);
3406		const else_body: []const Air.Inst.Index = @ptrCast(cg.air.extra.items[extra.end + then_body.len ..][0..extra.data.else_body_len]);
	3398	const cond_br = cg.air.unwrapCondBr(inst);
	3399	const condition = try cg.resolveInst(cond_br.condition);
	3400	const then_body = cond_br.then_body;
	3401	const else_body = cond_br.else_body;
3407	3402	const liveness_condbr = cg.liveness.getCondBr(inst);
3408	3403
3409	3404	// result type is always noreturn, so use `block_empty` as type.
...	...	@@ -6423,10 +6418,9 @@ fn airDbgStmt(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6423	6418	}
6424	6419
6425	6420	fn airDbgInlineBlock(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6426		const ty_pl = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
6427		const extra = cg.air.extraData(Air.DbgInlineBlock, ty_pl.payload);
	6421	const block = cg.air.unwrapDbgBlock(inst);
6428	6422	// TODO
6429		try cg.lowerBlock(inst, ty_pl.ty.toType(), @ptrCast(cg.air.extra.items[extra.end..][0..extra.data.body_len]));
	6423	try cg.lowerBlock(inst, block.ty, block.body);
6430	6424	}
6431	6425
6432	6426	fn airDbgVar(
...	...	@@ -6441,24 +6435,22 @@ fn airDbgVar(
6441	6435	}
6442	6436
6443	6437	fn airTry(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6444		const pl_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
6445		const err_union = try cg.resolveInst(pl_op.operand);
6446		const extra = cg.air.extraData(Air.Try, pl_op.payload);
6447		const body: []const Air.Inst.Index = @ptrCast(cg.air.extra.items[extra.end..][0..extra.data.body_len]);
6448		const err_union_ty = cg.typeOf(pl_op.operand);
	6438	const unwrapped_try = cg.air.unwrapTry(inst);
	6439	const body = unwrapped_try.else_body;
	6440	const err_union = try cg.resolveInst(unwrapped_try.error_union);
	6441	const err_union_ty = cg.typeOf(unwrapped_try.error_union);
6449	6442	const result = try lowerTry(cg, inst, err_union, body, err_union_ty, false);
6450		return cg.finishAir(inst, result, &.{pl_op.operand});
	6443	return cg.finishAir(inst, result, &.{unwrapped_try.error_union});
6451	6444	}
6452	6445
6453	6446	fn airTryPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6454	6447	const zcu = cg.pt.zcu;
6455		const ty_pl = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
6456		const extra = cg.air.extraData(Air.TryPtr, ty_pl.payload);
6457		const err_union_ptr = try cg.resolveInst(extra.data.ptr);
6458		const body: []const Air.Inst.Index = @ptrCast(cg.air.extra.items[extra.end..][0..extra.data.body_len]);
6459		const err_union_ty = cg.typeOf(extra.data.ptr).childType(zcu);
	6448	const unwrapped_try = cg.air.unwrapTryPtr(inst);
	6449	const err_union_ptr = try cg.resolveInst(unwrapped_try.error_union_ptr);
	6450	const body = unwrapped_try.else_body;
	6451	const err_union_ty = cg.typeOf(unwrapped_try.error_union_ptr).childType(zcu);
6460	6452	const result = try lowerTry(cg, inst, err_union_ptr, body, err_union_ty, true);
6461		return cg.finishAir(inst, result, &.{extra.data.ptr});
	6453	return cg.finishAir(inst, result, &.{unwrapped_try.error_union_ptr});
6462	6454	}
6463	6455
6464	6456	fn lowerTry(

src/codegen/x86_64/CodeGen.zig+86-117

...	...	@@ -67348,21 +67348,19 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
67348	67348	},
67349	67349	.bitcast => try cg.airBitCast(inst),
67350	67350	.block => {
67351		const ty_pl = air_datas[@intFromEnum(inst)].ty_pl;
67352		const block = cg.air.extraData(Air.Block, ty_pl.payload);
	67351	const block = cg.air.unwrapBlock(inst);
67353	67352	if (!cg.mod.strip) try cg.asmPseudo(.pseudo_dbg_enter_block_none);
67354		try cg.lowerBlock(inst, @ptrCast(cg.air.extra.items[block.end..][0..block.data.body_len]));
	67353	try cg.lowerBlock(inst, block.body);
67355	67354	if (!cg.mod.strip) try cg.asmPseudo(.pseudo_dbg_leave_block_none);
67356	67355	},
67357	67356	.loop => {
67358		const ty_pl = air_datas[@intFromEnum(inst)].ty_pl;
67359		const block = cg.air.extraData(Air.Block, ty_pl.payload);
	67357	const block = cg.air.unwrapBlock(inst);
67360	67358	try cg.loops.putNoClobber(cg.gpa, inst, .{
67361	67359	.state = try cg.saveState(),
67362	67360	.target = @intCast(cg.mir_instructions.len),
67363	67361	});
67364	67362	defer assert(cg.loops.remove(inst));
67365		try cg.genBodyBlock(@ptrCast(cg.air.extra.items[block.end..][0..block.data.body_len]));
	67363	try cg.genBodyBlock(block.body);
67366	67364	},
67367	67365	.repeat => {
67368	67366	const repeat = air_datas[@intFromEnum(inst)].repeat;
...	...	@@ -89048,17 +89046,16 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
89048	89046	try cg.asmOpOnly(.{ ._, .nop });
89049	89047	},
89050	89048	.dbg_inline_block => {
89051		const ty_pl = air_datas[@intFromEnum(inst)].ty_pl;
89052		const dbg_inline_block = cg.air.extraData(Air.DbgInlineBlock, ty_pl.payload);
	89049	const dbg_inline_block = cg.air.unwrapDbgBlock(inst);
89053	89050	const old_inline_func = cg.inline_func;
89054	89051	defer cg.inline_func = old_inline_func;
89055		cg.inline_func = dbg_inline_block.data.func;
	89052	cg.inline_func = dbg_inline_block.func;
89056	89053	if (!cg.mod.strip) _ = try cg.addInst(.{
89057	89054	.tag = .pseudo,
89058	89055	.ops = .pseudo_dbg_enter_inline_func,
89059		.data = .{ .ip_index = dbg_inline_block.data.func },
	89056	.data = .{ .ip_index = dbg_inline_block.func },
89060	89057	});
89061		try cg.lowerBlock(inst, @ptrCast(cg.air.extra.items[dbg_inline_block.end..][0..dbg_inline_block.data.body_len]));
	89058	try cg.lowerBlock(inst, dbg_inline_block.body);
89062	89059	if (!cg.mod.strip) _ = try cg.addInst(.{
89063	89060	.tag = .pseudo,
89064	89061	.ops = .pseudo_dbg_leave_inline_func,
...	...	@@ -175916,10 +175913,8 @@ fn genLocalDebugInfo(cg: *CodeGen, air_tag: Air.Inst.Tag, ty: Type, mcv: MCValue
175916	175913	fn airCall(self: *CodeGen, inst: Air.Inst.Index, modifier: std.builtin.CallModifier, opts: CopyOptions) !void {
175917	175914	if (modifier == .always_tail) return self.fail("TODO implement tail calls for x86_64", .{});
175918	175915
175919		const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
175920		const extra = self.air.extraData(Air.Call, pl_op.payload);
175921		const arg_refs: []const Air.Inst.Ref =
175922		@ptrCast(self.air.extra.items[extra.end..][0..extra.data.args_len]);
	175916	const call = self.air.unwrapCall(inst);
	175917	const arg_refs = call.args;
175923	175918
175924	175919	const ExpectedContents = extern struct {
175925	175920	tys: [32][@sizeOf(Type)]u8 align(@alignOf(Type)),
...	...	@@ -175937,10 +175932,10 @@ fn airCall(self: *CodeGen, inst: Air.Inst.Index, modifier: std.builtin.CallModif
175937	175932	defer allocator.free(arg_vals);
175938	175933	for (arg_vals, arg_refs) \|arg_val, arg_ref\| arg_val. = .{ .air_ref = arg_ref };
175939	175934
175940		const ret = try self.genCall(.{ .air = pl_op.operand }, arg_tys, arg_vals, opts);
	175935	const ret = try self.genCall(.{ .air = call.callee }, arg_tys, arg_vals, opts);
175941	175936
175942	175937	var bt = self.liveness.iterateBigTomb(inst);
175943		try self.feed(&bt, pl_op.operand);
	175938	try self.feed(&bt, call.callee);
175944	175939	for (arg_refs) \|arg_ref\| try self.feed(&bt, arg_ref);
175945	175940
175946	175941	const result = if (self.liveness.isUnused(inst)) .unreach else ret;
...	...	@@ -176300,20 +176295,18 @@ fn airRetLoad(self: *CodeGen, inst: Air.Inst.Index) !void {
176300	176295	}
176301	176296
176302	176297	fn airTry(self: *CodeGen, inst: Air.Inst.Index) !void {
176303		const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
176304		const extra = self.air.extraData(Air.Try, pl_op.payload);
176305		const body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[extra.end..][0..extra.data.body_len]);
176306		const operand_ty = self.typeOf(pl_op.operand);
176307		const result = try self.genTry(inst, pl_op.operand, body, operand_ty, false);
	176298	const unwrapped_try = self.air.unwrapTry(inst);
	176299	const body = unwrapped_try.else_body;
	176300	const operand_ty = self.typeOf(unwrapped_try.error_union);
	176301	const result = try self.genTry(inst, unwrapped_try.error_union, body, operand_ty, false);
176308	176302	return self.finishAir(inst, result, .{ .none, .none, .none });
176309	176303	}
176310	176304
176311	176305	fn airTryPtr(self: *CodeGen, inst: Air.Inst.Index) !void {
176312		const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
176313		const extra = self.air.extraData(Air.TryPtr, ty_pl.payload);
176314		const body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[extra.end..][0..extra.data.body_len]);
176315		const operand_ty = self.typeOf(extra.data.ptr);
176316		const result = try self.genTry(inst, extra.data.ptr, body, operand_ty, true);
	176306	const unwrapped_try = self.air.unwrapTryPtr(inst);
	176307	const body = unwrapped_try.else_body;
	176308	const operand_ty = self.typeOf(unwrapped_try.error_union_ptr);
	176309	const result = try self.genTry(inst, unwrapped_try.error_union_ptr, body, operand_ty, true);
176317	176310	return self.finishAir(inst, result, .{ .none, .none, .none });
176318	176311	}
176319	176312
...	...	@@ -176391,21 +176384,20 @@ fn genCondBrMir(self: *CodeGen, ty: Type, mcv: MCValue) !Mir.Inst.Index {
176391	176384	}
176392	176385
176393	176386	fn airCondBr(self: *CodeGen, inst: Air.Inst.Index) !void {
176394		const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
176395		const cond = try self.resolveInst(pl_op.operand);
176396		const cond_ty = self.typeOf(pl_op.operand);
176397		const extra = self.air.extraData(Air.CondBr, pl_op.payload);
176398		const then_body: []const Air.Inst.Index =
176399		@ptrCast(self.air.extra.items[extra.end..][0..extra.data.then_body_len]);
176400		const else_body: []const Air.Inst.Index =
176401		@ptrCast(self.air.extra.items[extra.end + then_body.len ..][0..extra.data.else_body_len]);
	176387	const cond_br = self.air.unwrapCondBr(inst);
	176388	const then_body = cond_br.then_body;
	176389	const else_body = cond_br.else_body;
	176390
	176391	const cond = try self.resolveInst(cond_br.condition);
	176392	const cond_ty = self.typeOf(cond_br.condition);
	176393
176402	176394	const liveness_cond_br = self.liveness.getCondBr(inst);
176403	176395
176404	176396	// If the condition dies here in this condbr instruction, process
176405	176397	// that death now instead of later as this has an effect on
176406	176398	// whether it needs to be spilled in the branches
176407	176399	if (self.liveness.operandDies(inst, 0)) {
176408		if (pl_op.operand.toIndex()) \|op_inst\| try self.processDeath(op_inst, .{});
	176400	if (cond_br.condition.toIndex()) \|op_inst\| try self.processDeath(op_inst, .{});
176409	176401	}
176410	176402
176411	176403	const state = try self.saveState();
...	...	@@ -177124,14 +177116,10 @@ fn airAsm(self: *CodeGen, inst: Air.Inst.Index) !void {
177124	177116	@setEvalBranchQuota(1_100);
177125	177117	const pt = self.pt;
177126	177118	const zcu = pt.zcu;
177127		const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
177128		const extra = self.air.extraData(Air.Asm, ty_pl.payload);
177129		const outputs_len = extra.data.flags.outputs_len;
177130		var extra_i: usize = extra.end;
177131		const outputs: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[extra_i..][0..outputs_len]);
177132		extra_i += outputs.len;
177133		const inputs: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[extra_i..][0..extra.data.inputs_len]);
177134		extra_i += inputs.len;
	177119	const unwrapped_asm = self.air.unwrapAsm(inst);
	177120
	177121	const outputs = unwrapped_asm.outputs;
	177122	const inputs = unwrapped_asm.inputs;
177135	177123
177136	177124	var result: MCValue = .none;
177137	177125	var args: std.array_list.Managed(MCValue) = .init(self.gpa);
...	...	@@ -177146,36 +177134,29 @@ fn airAsm(self: *CodeGen, inst: Air.Inst.Index) !void {
177146	177134	try arg_map.ensureTotalCapacity(@intCast(outputs.len + inputs.len));
177147	177135	defer arg_map.deinit();
177148	177136
177149		var outputs_extra_i = extra_i;
177150		for (outputs) \|output\| {
177151		const extra_bytes = std.mem.sliceAsBytes(self.air.extra.items[extra_i..]);
177152		const constraint = std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra.items[extra_i..]), 0);
177153		const name = std.mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0);
177154		// This equation accounts for the fact that even if we have exactly 4 bytes
177155		// for the string, we still use the next u32 for the null terminator.
177156		extra_i += (constraint.len + name.len + (2 + 3)) / 4;
177157
177158		const maybe_inst = switch (output) {
	177137	var it = unwrapped_asm.iterateOutputs();
	177138	while (it.next()) \|out\| {
	177139	const maybe_inst = switch (out.operand) {
177159	177140	.none => inst,
177160	177141	else => null,
177161	177142	};
177162		const ty = switch (output) {
	177143	const ty = switch (out.operand) {
177163	177144	.none => self.typeOfIndex(inst),
177164		else => self.typeOf(output).childType(zcu),
	177145	else => self.typeOf(out.operand).childType(zcu),
177165	177146	};
177166		const is_read = switch (constraint[0]) {
	177147	const is_read = switch (out.constraint[0]) {
177167	177148	'=' => false,
177168	177149	'+' => read: {
177169		if (output == .none) return self.fail(
	177150	if (out.operand == .none) return self.fail(
177170	177151	"read-write constraint unsupported for asm result: '{s}'",
177171		.{constraint},
	177152	.{out.constraint},
177172	177153	);
177173	177154	break :read true;
177174	177155	},
177175		else => return self.fail("invalid constraint: '{s}'", .{constraint}),
	177156	else => return self.fail("invalid constraint: '{s}'", .{out.constraint}),
177176	177157	};
177177		const is_early_clobber = constraint[1] == '&';
177178		const rest = constraint[@as(usize, 1) + @intFromBool(is_early_clobber) ..];
	177158	const is_early_clobber = out.constraint[1] == '&';
	177159	const rest = out.constraint[@as(usize, 1) + @intFromBool(is_early_clobber) ..];
177179	177160	const arg_mcv: MCValue = arg_mcv: {
177180	177161	const arg_maybe_reg: ?Register = if (std.mem.eql(u8, rest, "r") or
177181	177162	std.mem.eql(u8, rest, "f") or std.mem.eql(u8, rest, "x"))
...	...	@@ -177189,30 +177170,30 @@ fn airAsm(self: *CodeGen, inst: Air.Inst.Index) !void {
177189	177170	@intCast(ty.abiSize(zcu)),
177190	177171	)
177191	177172	else if (std.mem.eql(u8, rest, "m"))
177192		if (output != .none) null else return self.fail(
	177173	if (out.operand != .none) null else return self.fail(
177193	177174	"memory constraint unsupported for asm result: '{s}'",
177194		.{constraint},
	177175	.{out.constraint},
177195	177176	)
177196	177177	else if (std.mem.eql(u8, rest, "g") or
177197	177178	std.mem.eql(u8, rest, "rm") or std.mem.eql(u8, rest, "mr") or
177198	177179	std.mem.eql(u8, rest, "r,m") or std.mem.eql(u8, rest, "m,r"))
177199	177180	self.register_manager.tryAllocReg(maybe_inst, abi.RegisterClass.gp) orelse
177200		if (output != .none)
	177181	if (out.operand != .none)
177201	177182	null
177202	177183	else
177203	177184	return self.fail("ran out of registers lowering inline asm", .{})
177204	177185	else if (std.mem.startsWith(u8, rest, "{") and std.mem.endsWith(u8, rest, "}"))
177205	177186	parseRegName(rest["{".len .. rest.len - "}".len]) orelse
177206		return self.fail("invalid register constraint: '{s}'", .{constraint})
	177187	return self.fail("invalid register constraint: '{s}'", .{out.constraint})
177207	177188	else if (rest.len == 1 and std.ascii.isDigit(rest[0])) {
177208	177189	const index = std.fmt.charToDigit(rest[0], 10) catch unreachable;
177209	177190	if (index >= args.items.len) return self.fail("constraint out of bounds: '{s}'", .{
177210		constraint,
	177191	out.constraint,
177211	177192	});
177212	177193	break :arg_mcv args.items[index];
177213		} else return self.fail("invalid constraint: '{s}'", .{constraint});
	177194	} else return self.fail("invalid constraint: '{s}'", .{out.constraint});
177214	177195	break :arg_mcv if (arg_maybe_reg) \|reg\| .{ .register = reg } else arg: {
177215		const ptr_mcv = try self.resolveInst(output);
	177196	const ptr_mcv = try self.resolveInst(out.operand);
177216	177197	switch (ptr_mcv) {
177217	177198	.immediate => \|addr\| if (std.math.cast(i32, @as(i64, @bitCast(addr)))) \|_\|
177218	177199	break :arg ptr_mcv.deref(),
...	...	@@ -177223,30 +177204,24 @@ fn airAsm(self: *CodeGen, inst: Air.Inst.Index) !void {
177223	177204	};
177224	177205	};
177225	177206	if (arg_mcv.getReg()) \|reg\| if (RegisterManager.indexOfRegIntoTracked(reg)) \|tracked_index\| {
177226		try self.register_manager.getRegIndex(tracked_index, if (output == .none) inst else null);
	177207	try self.register_manager.getRegIndex(tracked_index, if (out.operand == .none) inst else null);
177227	177208	_ = self.register_manager.lockRegIndexAssumeUnused(tracked_index);
177228	177209	};
177229		if (!std.mem.eql(u8, name, "_"))
177230		arg_map.putAssumeCapacityNoClobber(name, @intCast(args.items.len));
	177210	if (!std.mem.eql(u8, out.name, "_"))
	177211	arg_map.putAssumeCapacityNoClobber(out.name, @intCast(args.items.len));
177231	177212	args.appendAssumeCapacity(arg_mcv);
177232		if (output == .none) result = arg_mcv;
177233		if (is_read) try self.load(arg_mcv, self.typeOf(output), .{ .air_ref = output });
	177213	if (out.operand == .none) result = arg_mcv;
	177214	if (is_read) try self.load(arg_mcv, self.typeOf(out.operand), .{ .air_ref = out.operand });
177234	177215	}
177235	177216
177236		for (inputs) \|input\| {
177237		const input_bytes = std.mem.sliceAsBytes(self.air.extra.items[extra_i..]);
177238		const constraint = std.mem.sliceTo(input_bytes, 0);
177239		const name = std.mem.sliceTo(input_bytes[constraint.len + 1 ..], 0);
177240		// This equation accounts for the fact that even if we have exactly 4 bytes
177241		// for the string, we still use the next u32 for the null terminator.
177242		extra_i += (constraint.len + name.len + (2 + 3)) / 4;
177243
177244		const ty = self.typeOf(input);
177245		const input_mcv = try self.resolveInst(input);
177246		const arg_mcv: MCValue = if (std.mem.eql(u8, constraint, "r") or
177247		std.mem.eql(u8, constraint, "f") or std.mem.eql(u8, constraint, "x"))
	177217	it = unwrapped_asm.iterateInputs();
	177218	while (it.next()) \|in\| {
	177219	const ty = self.typeOf(in.operand);
	177220	const input_mcv = try self.resolveInst(in.operand);
	177221	const arg_mcv: MCValue = if (std.mem.eql(u8, in.constraint, "r") or
	177222	std.mem.eql(u8, in.constraint, "f") or std.mem.eql(u8, in.constraint, "x"))
177248	177223	arg: {
177249		const rc = switch (constraint[0]) {
	177224	const rc = switch (in.constraint[0]) {
177250	177225	'r' => abi.RegisterClass.gp,
177251	177226	'f' => abi.RegisterClass.x87,
177252	177227	'x' => abi.RegisterClass.sse,
...	...	@@ -177258,14 +177233,14 @@ fn airAsm(self: *CodeGen, inst: Air.Inst.Index) !void {
177258	177233	const reg = try self.register_manager.allocReg(null, rc);
177259	177234	try self.genSetReg(reg, ty, input_mcv, .{});
177260	177235	break :arg .{ .register = registerAlias(reg, @intCast(ty.abiSize(zcu))) };
177261		} else if (std.mem.eql(u8, constraint, "i") or std.mem.eql(u8, constraint, "n"))
	177236	} else if (std.mem.eql(u8, in.constraint, "i") or std.mem.eql(u8, in.constraint, "n"))
177262	177237	switch (input_mcv) {
177263	177238	.immediate => \|imm\| .{ .immediate = imm },
177264	177239	else => return self.fail("immediate operand requires comptime value: '{s}'", .{
177265		constraint,
	177240	in.constraint,
177266	177241	}),
177267	177242	}
177268		else if (std.mem.eql(u8, constraint, "m")) arg: {
	177243	else if (std.mem.eql(u8, in.constraint, "m")) arg: {
177269	177244	switch (input_mcv) {
177270	177245	.memory => \|addr\| if (std.math.cast(i32, @as(i64, @bitCast(addr)))) \|_\|
177271	177246	break :arg input_mcv,
...	...	@@ -177284,9 +177259,9 @@ fn airAsm(self: *CodeGen, inst: Air.Inst.Index) !void {
177284	177259	};
177285	177260	try self.genSetReg(addr_reg, .usize, input_mcv.address(), .{});
177286	177261	break :arg .{ .indirect = .{ .reg = addr_reg } };
177287		} else if (std.mem.eql(u8, constraint, "g") or
177288		std.mem.eql(u8, constraint, "rm") or std.mem.eql(u8, constraint, "mr") or
177289		std.mem.eql(u8, constraint, "r,m") or std.mem.eql(u8, constraint, "m,r"))
	177262	} else if (std.mem.eql(u8, in.constraint, "g") or
	177263	std.mem.eql(u8, in.constraint, "rm") or std.mem.eql(u8, in.constraint, "mr") or
	177264	std.mem.eql(u8, in.constraint, "r,m") or std.mem.eql(u8, in.constraint, "m,r"))
177290	177265	arg: {
177291	177266	switch (input_mcv) {
177292	177267	.register, .indirect, .load_frame => break :arg input_mcv,
...	...	@@ -177297,30 +177272,30 @@ fn airAsm(self: *CodeGen, inst: Air.Inst.Index) !void {
177297	177272	const temp_mcv = try self.allocTempRegOrMem(ty, true);
177298	177273	try self.genCopy(ty, temp_mcv, input_mcv, .{});
177299	177274	break :arg temp_mcv;
177300		} else if (std.mem.eql(u8, constraint, "X"))
	177275	} else if (std.mem.eql(u8, in.constraint, "X"))
177301	177276	input_mcv
177302		else if (std.mem.startsWith(u8, constraint, "{") and std.mem.endsWith(u8, constraint, "}")) arg: {
177303		const reg = parseRegName(constraint["{".len .. constraint.len - "}".len]) orelse
177304		return self.fail("invalid register constraint: '{s}'", .{constraint});
	177277	else if (std.mem.startsWith(u8, in.constraint, "{") and std.mem.endsWith(u8, in.constraint, "}")) arg: {
	177278	const reg = parseRegName(in.constraint["{".len .. in.constraint.len - "}".len]) orelse
	177279	return self.fail("invalid register constraint: '{s}'", .{in.constraint});
177305	177280	try self.register_manager.getReg(reg, null);
177306	177281	try self.genSetReg(reg, ty, input_mcv, .{});
177307	177282	break :arg .{ .register = reg };
177308		} else if (constraint.len == 1 and std.ascii.isDigit(constraint[0])) arg: {
177309		const index = std.fmt.charToDigit(constraint[0], 10) catch unreachable;
177310		if (index >= args.items.len) return self.fail("constraint out of bounds: '{s}'", .{constraint});
	177283	} else if (in.constraint.len == 1 and std.ascii.isDigit(in.constraint[0])) arg: {
	177284	const index = std.fmt.charToDigit(in.constraint[0], 10) catch unreachable;
	177285	if (index >= args.items.len) return self.fail("constraint out of bounds: '{s}'", .{in.constraint});
177311	177286	try self.genCopy(ty, args.items[index], input_mcv, .{});
177312	177287	break :arg args.items[index];
177313		} else return self.fail("invalid constraint: '{s}'", .{constraint});
	177288	} else return self.fail("invalid constraint: '{s}'", .{in.constraint});
177314	177289	if (arg_mcv.getReg()) \|reg\| if (RegisterManager.indexOfRegIntoTracked(reg)) \|_\| {
177315	177290	_ = self.register_manager.lockReg(reg);
177316	177291	};
177317		if (!std.mem.eql(u8, name, "_"))
177318		arg_map.putAssumeCapacityNoClobber(name, @intCast(args.items.len));
	177292	if (!std.mem.eql(u8, in.name, "_"))
	177293	arg_map.putAssumeCapacityNoClobber(in.name, @intCast(args.items.len));
177319	177294	args.appendAssumeCapacity(arg_mcv);
177320	177295	}
177321	177296
177322	177297	const ip = &zcu.intern_pool;
177323		const aggregate = ip.indexToKey(extra.data.clobbers).aggregate;
	177298	const aggregate = ip.indexToKey(unwrapped_asm.clobbers).aggregate;
177324	177299	const struct_type: Type = .fromInterned(aggregate.ty);
177325	177300	switch (aggregate.storage) {
177326	177301	.elems => \|elems\| for (elems, 0..) \|elem, i\| switch (elem) {
...	...	@@ -177390,7 +177365,7 @@ fn airAsm(self: *CodeGen, inst: Air.Inst.Index) !void {
177390	177365	labels.deinit(self.gpa);
177391	177366	}
177392	177367
177393		const asm_source = std.mem.sliceAsBytes(self.air.extra.items[extra_i..])[0..extra.data.source_len];
	177368	const asm_source = unwrapped_asm.source;
177394	177369	var line_it = std.mem.tokenizeAny(u8, asm_source, "\n\r;");
177395	177370	next_line: while (line_it.next()) \|line\| {
177396	177371	var mnem_it = std.mem.tokenizeAny(u8, line, " \t");
...	...	@@ -177821,19 +177796,13 @@ fn airAsm(self: *CodeGen, inst: Air.Inst.Index) !void {
177821	177796	while (label_it.next()) \|label\| if (label.value_ptr.pending_relocs.items.len > 0)
177822	177797	return self.fail("undefined label: '{s}'", .{label.key_ptr.*});
177823	177798
177824		for (outputs, args.items[0..outputs.len]) \|output, arg_mcv\| {
177825		const extra_bytes = std.mem.sliceAsBytes(self.air.extra.items[outputs_extra_i..]);
177826		const constraint =
177827		std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra.items[outputs_extra_i..]), 0);
177828		const name = std.mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0);
177829		// This equation accounts for the fact that even if we have exactly 4 bytes
177830		// for the string, we still use the next u32 for the null terminator.
177831		outputs_extra_i += (constraint.len + name.len + (2 + 3)) / 4;
177832
177833		if (output == .none) continue;
	177799	it = unwrapped_asm.iterateOutputs();
	177800	while (it.next()) \|out\| {
	177801	const arg_mcv = args.items[it.current - 1];
	177802	if (out.operand == .none) continue;
177834	177803	if (arg_mcv != .register) continue;
177835		if (constraint.len == 2 and std.ascii.isDigit(constraint[1])) continue;
177836		try self.store(self.typeOf(output), .{ .air_ref = output }, arg_mcv, .{});
	177804	if (out.constraint.len == 2 and std.ascii.isDigit(out.constraint[1])) continue;
	177805	try self.store(self.typeOf(out.operand), .{ .air_ref = out.operand }, arg_mcv, .{});
177837	177806	}
177838	177807
177839	177808	simple: {