Commit c04be630d996a0e01fd3cf05e6cade006e4226ce | zig

author	Jacob Young <jacobly@ziglang.org> 2025-05-28 22:45:19-04:00
committer	Jacob Young <jacobly@ziglang.org> 2025-05-29 03:57:48-04:00
log	c04be630d996a0e01fd3cf05e6cade006e4226ce
tree	edd9d55ad05435b91bd3cb95208a76ead2001094
parent	f25212a479c4f26566b6b851e245e49c6f508b96

Legalize: introduce a new pass before liveness

Each target can opt into different sets of legalize features.
By performing these transformations before liveness, instructions
that become unreferenced will have up-to-date liveness information.

36 files changed, 3225 insertions(+), 3086 deletions(-)

CMakeLists.txt
src
- Air.zig
- Air
  - Legalize.zig
  - Liveness.zig
  - Liveness
    Verify.zig
  - types_resolved.zig
- Liveness.zig
- Liveness
  - Verify.zig
- Sema.zig
- Zcu.zig
- Zcu
  - PerThread.zig
- arch
  - aarch64
    CodeGen.zig
  - arm
    CodeGen.zig
  - powerpc
    CodeGen.zig
  - riscv64
    CodeGen.zig
  - sparc64
    CodeGen.zig
    Emit.zig
  - wasm
    CodeGen.zig
  - x86_64
    CodeGen.zig
- codegen.zig
- codegen
- link.zig
- link
  - C.zig
  - Coff.zig
  - Elf.zig
  - Elf
    ZigObject.zig
  - Goff.zig
  - MachO.zig
  - MachO
    ZigObject.zig
  - Plan9.zig
  - SpirV.zig
  - Wasm.zig
  - Xcoff.zig
- print_air.zig

CMakeLists.txt+4-2

 @@ -512,13 +512,15 @@ set(ZIG_STAGE2_SOURCES
     lib/std/zig/llvm/bitcode_writer.zig
     lib/std/zig/llvm/ir.zig
     src/Air.zig
+    src/Air/Legalize.zig
+    src/Air/Liveness.zig
+    src/Air/Liveness/Verify.zig
+    src/Air/types_resolved.zig
     src/Builtin.zig
     src/Compilation.zig
     src/Compilation/Config.zig
     src/DarwinPosixSpawn.zig
     src/InternPool.zig
-    src/Liveness.zig
-    src/Liveness/Verify.zig
     src/Package.zig
     src/Package/Fetch.zig
     src/Package/Fetch/git.zig

src/Air.zig+24-15

 @@ -9,16 +9,19 @@ const builtin = @import("builtin");
 const assert = std.debug.assert;
 const Air = @This();
-const Value = @import("Value.zig");
-const Type = @import("Type.zig");
 const InternPool = @import("InternPool.zig");
+const Type = @import("Type.zig");
+const Value = @import("Value.zig");
 const Zcu = @import("Zcu.zig");
 const types_resolved = @import("Air/types_resolved.zig");
+pub const Legalize = @import("Air/Legalize.zig");
+pub const Liveness = @import("Air/Liveness.zig");
 instructions: std.MultiArrayList(Inst).Slice,
 /// The meaning of this data is determined by `Inst.Tag` value.
 /// The first few indexes are reserved. See `ExtraIndex` for the values.
-extra: []const u32,
+extra: std.ArrayListUnmanaged(u32),
 pub const ExtraIndex = enum(u32) {
     /// Payload index of the main `Block` in the `extra` array.
 @@ -244,22 +247,27 @@ pub const Inst = struct {
         /// Uses the `bin_op` field.
         bit_or,
         /// Shift right. `>>`
+        /// The rhs type may be a scalar version of the lhs type.
         /// Uses the `bin_op` field.
         shr,
         /// Shift right. The shift produces a poison value if it shifts out any non-zero bits.
+        /// The rhs type may be a scalar version of the lhs type.
         /// Uses the `bin_op` field.
         shr_exact,
         /// Shift left. `<<`
+        /// The rhs type may be a scalar version of the lhs type.
         /// Uses the `bin_op` field.
         shl,
         /// Shift left; For unsigned integers, the shift produces a poison value if it shifts
         /// out any non-zero bits. For signed integers, the shift produces a poison value if
         /// it shifts out any bits that disagree with the resultant sign bit.
+        /// The rhs type may be a scalar version of the lhs type.
         /// Uses the `bin_op` field.
         shl_exact,
         /// Saturating integer shift left. `<<|`. The result is the same type as the `lhs`.
         /// The `rhs` must have the same vector shape as the `lhs`, but with any unsigned
         /// integer as the scalar type.
+        /// The rhs type may be a scalar version of the lhs type.
         /// Uses the `bin_op` field.
         shl_sat,
         /// Bitwise XOR. `^`
 @@ -1378,9 +1386,9 @@ pub const UnionInit = struct {
 };
 pub fn getMainBody(air: Air) []const Air.Inst.Index {
-    const body_index = air.extra[@intFromEnum(ExtraIndex.main_block)];
+    const body_index = air.extra.items[@intFromEnum(ExtraIndex.main_block)];
     const extra = air.extraData(Block, body_index);
-    return @ptrCast(air.extra[extra.end..][0..extra.data.body_len]);
+    return @ptrCast(air.extra.items[extra.end..][0..extra.data.body_len]);
 }
 pub fn typeOf(air: *const Air, inst: Air.Inst.Ref, ip: *const InternPool) Type {
 @@ -1656,9 +1664,9 @@ pub fn extraData(air: Air, comptime T: type, index: usize) struct { data: T, end
     var result: T = undefined;
     inline for (fields) |field| {
         @field(result, field.name) = switch (field.type) {
-            u32 => air.extra[i],
+            u32 => air.extra.items[i],
-            InternPool.Index, Inst.Ref => @enumFromInt(air.extra[i]),
+            InternPool.Index, Inst.Ref => @enumFromInt(air.extra.items[i]),
-            i32, CondBr.BranchHints => @bitCast(air.extra[i]),
+            i32, CondBr.BranchHints => @bitCast(air.extra.items[i]),
             else => @compileError("bad field type: " ++ @typeName(field.type)),
         };
         i += 1;
 @@ -1671,7 +1679,7 @@ pub fn extraData(air: Air, comptime T: type, index: usize) struct { data: T, end
 pub fn deinit(air: *Air, gpa: std.mem.Allocator) void {
     air.instructions.deinit(gpa);
-    gpa.free(air.extra);
+    air.extra.deinit(gpa);
     air.* = undefined;
 }
 @@ -1700,7 +1708,7 @@ pub const NullTerminatedString = enum(u32) {
     pub fn toSlice(nts: NullTerminatedString, air: Air) [:0]const u8 {
         if (nts == .none) return "";
-        const bytes = std.mem.sliceAsBytes(air.extra[@intFromEnum(nts)..]);
+        const bytes = std.mem.sliceAsBytes(air.extra.items[@intFromEnum(nts)..]);
         return bytes[0..std.mem.indexOfScalar(u8, bytes, 0).? :0];
     }
 };
 @@ -1943,7 +1951,7 @@ pub const UnwrappedSwitch = struct {
         return us.getHintInner(us.cases_len);
     }
     fn getHintInner(us: UnwrappedSwitch, idx: u32) std.builtin.BranchHint {
-        const bag = us.air.extra[us.branch_hints_start..][idx / 10];
+        const bag = us.air.extra.items[us.branch_hints_start..][idx / 10];
         const bits: u3 = @truncate(bag >> @intCast(3 * (idx % 10)));
         return @enumFromInt(bits);
     }
 @@ -1971,13 +1979,13 @@ pub const UnwrappedSwitch = struct {
             const extra = it.air.extraData(SwitchBr.Case, it.extra_index);
             var extra_index = extra.end;
-            const items: []const Inst.Ref = @ptrCast(it.air.extra[extra_index..][0..extra.data.items_len]);
+            const items: []const Inst.Ref = @ptrCast(it.air.extra.items[extra_index..][0..extra.data.items_len]);
             extra_index += items.len;
             // TODO: ptrcast from []const Inst.Ref to []const [2]Inst.Ref when supported
-            const ranges_ptr: [*]const [2]Inst.Ref = @ptrCast(it.air.extra[extra_index..]);
+            const ranges_ptr: [*]const [2]Inst.Ref = @ptrCast(it.air.extra.items[extra_index..]);
             const ranges: []const [2]Inst.Ref = ranges_ptr[0..extra.data.ranges_len];
             extra_index += ranges.len * 2;
-            const body: []const Inst.Index = @ptrCast(it.air.extra[extra_index..][0..extra.data.body_len]);
+            const body: []const Inst.Index = @ptrCast(it.air.extra.items[extra_index..][0..extra.data.body_len]);
             extra_index += body.len;
             it.extra_index = @intCast(extra_index);
 @@ -1992,7 +2000,7 @@ pub const UnwrappedSwitch = struct {
         /// Returns the body of the "default" (`else`) case.
         pub fn elseBody(it: *CaseIterator) []const Inst.Index {
             assert(it.next_case == it.cases_len);
-            return @ptrCast(it.air.extra[it.extra_index..][0..it.else_body_len]);
+            return @ptrCast(it.air.extra.items[it.extra_index..][0..it.else_body_len]);
         }
         pub const Case = struct {
             idx: u32,
 @@ -2025,6 +2033,7 @@ pub fn unwrapSwitch(air: *const Air, switch_inst: Inst.Index) UnwrappedSwitch {
 pub const typesFullyResolved = types_resolved.typesFullyResolved;
 pub const typeFullyResolved = types_resolved.checkType;
 pub const valFullyResolved = types_resolved.checkVal;
+pub const legalize = Legalize.legalize;
 pub const CoveragePoint = enum(u1) {
     /// Indicates the block is not a place of interest corresponding to

src/Air/Legalize.zig created+147

...	@@ -0,0 +1,147 @@
1	zcu: *const Zcu,
2	air: Air,
3	features: std.enums.EnumSet(Feature),
4
5	pub const Feature = enum {
6	/// Legalize (shift lhs, (splat rhs)) -> (shift lhs, rhs)
7	remove_shift_vector_rhs_splat,
8	/// Legalize reduce of a one element vector to a bitcast
9	reduce_one_elem_to_bitcast,
10	};
11
12	pub const Features = std.enums.EnumFieldStruct(Feature, bool, false);
13
14	pub fn legalize(air: Air, backend: std.builtin.CompilerBackend, zcu: const Zcu) std.mem.Allocator.Error!void {
15	var l: Legalize = .{
16	.zcu = zcu,
17	.air = air.*,
18	.features = features: switch (backend) {
19	.other, .stage1 => unreachable,
20	inline .stage2_llvm,
21	.stage2_c,
22	.stage2_wasm,
23	.stage2_arm,
24	.stage2_x86_64,
25	.stage2_aarch64,
26	.stage2_x86,
27	.stage2_riscv64,
28	.stage2_sparc64,
29	.stage2_spirv64,
30	.stage2_powerpc,
31	=> \|ct_backend\| {
32	const Backend = codegen.importBackend(ct_backend) orelse break :features .initEmpty();
33	break :features if (@hasDecl(Backend, "legalize_features"))
34	.init(Backend.legalize_features)
35	else
36	.initEmpty();
37	},
38	_ => unreachable,
39	},
40	};
41	defer air.* = l.air;
42	if (!l.features.bits.eql(.initEmpty())) try l.legalizeBody(l.air.getMainBody());
43	}
44
45	fn legalizeBody(l: *Legalize, body: []const Air.Inst.Index) std.mem.Allocator.Error!void {
46	const zcu = l.zcu;
47	const ip = &zcu.intern_pool;
48	const tags = l.air.instructions.items(.tag);
49	const data = l.air.instructions.items(.data);
50	for (body) \|inst\| inst: switch (tags[@intFromEnum(inst)]) {
51	else => {},
52
53	.shl,
54	.shl_exact,
55	.shl_sat,
56	.shr,
57	.shr_exact,
58	=> \|air_tag\| if (l.features.contains(.remove_shift_vector_rhs_splat)) done: {
59	const bin_op = data[@intFromEnum(inst)].bin_op;
60	const ty = l.air.typeOf(bin_op.rhs, ip);
61	if (!ty.isVector(zcu)) break :done;
62	if (bin_op.rhs.toInterned()) \|rhs_ip_index\| switch (ip.indexToKey(rhs_ip_index)) {
63	else => {},
64	.aggregate => \|aggregate\| switch (aggregate.storage) {
65	else => {},
66	.repeated_elem => \|splat\| continue :inst l.replaceInst(inst, air_tag, .{ .bin_op = .{
67	.lhs = bin_op.lhs,
68	.rhs = Air.internedToRef(splat),
69	} }),
70	},
71	} else {
72	const rhs_inst = bin_op.rhs.toIndex().?;
73	switch (tags[@intFromEnum(rhs_inst)]) {
74	else => {},
75	.splat => continue :inst l.replaceInst(inst, air_tag, .{ .bin_op = .{
76	.lhs = bin_op.lhs,
77	.rhs = data[@intFromEnum(rhs_inst)].ty_op.operand,
78	} }),
79	}
80	}
81	},
82
83	.reduce,
84	.reduce_optimized,
85	=> if (l.features.contains(.reduce_one_elem_to_bitcast)) done: {
86	const reduce = data[@intFromEnum(inst)].reduce;
87	const vector_ty = l.air.typeOf(reduce.operand, ip);
88	switch (vector_ty.vectorLen(zcu)) {
89	0 => unreachable,
90	1 => continue :inst l.replaceInst(inst, .bitcast, .{ .ty_op = .{
91	.ty = Air.internedToRef(vector_ty.scalarType(zcu).toIntern()),
92	.operand = reduce.operand,
93	} }),
94	else => break :done,
95	}
96	},
97
98	.@"try", .try_cold => {
99	const pl_op = data[@intFromEnum(inst)].pl_op;
100	const extra = l.air.extraData(Air.Try, pl_op.payload);
101	try l.legalizeBody(@ptrCast(l.air.extra.items[extra.end..][0..extra.data.body_len]));
102	},
103	.try_ptr, .try_ptr_cold => {
104	const ty_pl = data[@intFromEnum(inst)].ty_pl;
105	const extra = l.air.extraData(Air.TryPtr, ty_pl.payload);
106	try l.legalizeBody(@ptrCast(l.air.extra.items[extra.end..][0..extra.data.body_len]));
107	},
108	.block, .loop => {
109	const ty_pl = data[@intFromEnum(inst)].ty_pl;
110	const extra = l.air.extraData(Air.Block, ty_pl.payload);
111	try l.legalizeBody(@ptrCast(l.air.extra.items[extra.end..][0..extra.data.body_len]));
112	},
113	.dbg_inline_block => {
114	const ty_pl = data[@intFromEnum(inst)].ty_pl;
115	const extra = l.air.extraData(Air.DbgInlineBlock, ty_pl.payload);
116	try l.legalizeBody(@ptrCast(l.air.extra.items[extra.end..][0..extra.data.body_len]));
117	},
118	.cond_br => {
119	const pl_op = data[@intFromEnum(inst)].pl_op;
120	const extra = l.air.extraData(Air.CondBr, pl_op.payload);
121	try l.legalizeBody(@ptrCast(l.air.extra.items[extra.end..][0..extra.data.then_body_len]));
122	try l.legalizeBody(@ptrCast(l.air.extra.items[extra.end + extra.data.then_body_len ..][0..extra.data.else_body_len]));
123	},
124	.switch_br, .loop_switch_br => {
125	const switch_br = l.air.unwrapSwitch(inst);
126	var it = switch_br.iterateCases();
127	while (it.next()) \|case\| try l.legalizeBody(case.body);
128	try l.legalizeBody(it.elseBody());
129	},
130	};
131	}
132
133	// inline to propagate comptime `tag`s
134	inline fn replaceInst(l: *Legalize, inst: Air.Inst.Index, tag: Air.Inst.Tag, data: Air.Inst.Data) Air.Inst.Tag {
135	const ip = &l.zcu.intern_pool;
136	const orig_ty = if (std.debug.runtime_safety) l.air.typeOfIndex(inst, ip) else {};
137	l.air.instructions.items(.tag)[@intFromEnum(inst)] = tag;
138	l.air.instructions.items(.data)[@intFromEnum(inst)] = data;
139	if (std.debug.runtime_safety) std.debug.assert(l.air.typeOfIndex(inst, ip).toIntern() == orig_ty.toIntern());
140	return tag;
141	}
142
143	const Air = @import("../Air.zig");
144	const codegen = @import("../codegen.zig");
145	const Legalize = @This();
146	const std = @import("std");
147	const Zcu = @import("../Zcu.zig");

src/Air/Liveness.zig created+2050

...	@@ -0,0 +1,2050 @@
1	//! For each AIR instruction, we want to know:
2	//! * Is the instruction unreferenced (e.g. dies immediately)?
3	//! * For each of its operands, does the operand die with this instruction (e.g. is
4	//! this the last reference to it)?
5	//! Some instructions are special, such as:
6	//! * Conditional Branches
7	//! * Switch Branches
8	const std = @import("std");
9	const log = std.log.scoped(.liveness);
10	const assert = std.debug.assert;
11	const Allocator = std.mem.Allocator;
12	const Log2Int = std.math.Log2Int;
13
14	const Liveness = @This();
15	const trace = @import("../tracy.zig").trace;
16	const Air = @import("../Air.zig");
17	const InternPool = @import("../InternPool.zig");
18
19	pub const Verify = @import("Liveness/Verify.zig");
20
21	/// This array is split into sets of 4 bits per AIR instruction.
22	/// The MSB (0bX000) is whether the instruction is unreferenced.
23	/// The LSB (0b000X) is the first operand, and so on, up to 3 operands. A set bit means the
24	/// operand dies after this instruction.
25	/// Instructions which need more data to track liveness have special handling via the
26	/// `special` table.
27	tomb_bits: []usize,
28	/// Sparse table of specially handled instructions. The value is an index into the `extra`
29	/// array. The meaning of the data depends on the AIR tag.
30	/// * `cond_br` - points to a `CondBr` in `extra` at this index.
31	/// * `try`, `try_ptr` - points to a `CondBr` in `extra` at this index. The error path (the block
32	/// in the instruction) is considered the "else" path, and the rest of the block the "then".
33	/// * `switch_br` - points to a `SwitchBr` in `extra` at this index.
34	/// * `loop_switch_br` - points to a `SwitchBr` in `extra` at this index.
35	/// * `block` - points to a `Block` in `extra` at this index.
36	/// * `asm`, `call`, `aggregate_init` - the value is a set of bits which are the extra tomb
37	/// bits of operands.
38	/// The main tomb bits are still used and the extra ones are starting with the lsb of the
39	/// value here.
40	special: std.AutoHashMapUnmanaged(Air.Inst.Index, u32),
41	/// Auxiliary data. The way this data is interpreted is determined contextually.
42	extra: []const u32,
43
44	/// Trailing is the set of instructions whose lifetimes end at the start of the then branch,
45	/// followed by the set of instructions whose lifetimes end at the start of the else branch.
46	pub const CondBr = struct {
47	then_death_count: u32,
48	else_death_count: u32,
49	};
50
51	/// Trailing is:
52	/// * For each case in the same order as in the AIR:
53	/// - case_death_count: u32
54	/// - Air.Inst.Index for each `case_death_count`: set of instructions whose lifetimes
55	/// end at the start of this case.
56	/// * Air.Inst.Index for each `else_death_count`: set of instructions whose lifetimes
57	/// end at the start of the else case.
58	pub const SwitchBr = struct {
59	else_death_count: u32,
60	};
61
62	/// Trailing is the set of instructions which die in the block. Note that these are not additional
63	/// deaths (they are all recorded as normal within the block), but backends may use this information
64	/// as a more efficient way to track which instructions are still alive after a block.
65	pub const Block = struct {
66	death_count: u32,
67	};
68
69	/// Liveness analysis runs in several passes. Each pass iterates backwards over instructions in
70	/// bodies, and recurses into bodies.
71	const LivenessPass = enum {
72	/// In this pass, we perform some basic analysis of loops to gain information the main pass needs.
73	/// In particular, for every `loop` and `loop_switch_br`, we track the following information:
74	/// * Every outer block which the loop body contains a `br` to.
75	/// * Every outer loop which the loop body contains a `repeat` to.
76	/// * Every operand referenced within the loop body but created outside the loop.
77	/// This gives the main analysis pass enough information to determine the full set of
78	/// instructions which need to be alive when a loop repeats. This data is TEMPORARILY stored in
79	/// `a.extra`. It is not re-added to `extra` by the main pass, since it is not useful to
80	/// backends.
81	loop_analysis,
82
83	/// This pass performs the main liveness analysis, setting up tombs and extra data while
84	/// considering control flow etc.
85	main_analysis,
86	};
87
88	/// Each analysis pass may wish to pass data through calls. A pointer to a `LivenessPassData(pass)`
89	/// stored on the stack is passed through calls to `analyzeInst` etc.
90	fn LivenessPassData(comptime pass: LivenessPass) type {
91	return switch (pass) {
92	.loop_analysis => struct {
93	/// The set of blocks which are exited with a `br` instruction at some point within this
94	/// body and which we are currently within. Also includes `loop`s which are the target
95	/// of a `repeat` instruction, and `loop_switch_br`s which are the target of a
96	/// `switch_dispatch` instruction.
97	breaks: std.AutoHashMapUnmanaged(Air.Inst.Index, void) = .empty,
98
99	/// The set of operands for which we have seen at least one usage but not their birth.
100	live_set: std.AutoHashMapUnmanaged(Air.Inst.Index, void) = .empty,
101
102	fn deinit(self: *@This(), gpa: Allocator) void {
103	self.breaks.deinit(gpa);
104	self.live_set.deinit(gpa);
105	}
106	},
107
108	.main_analysis => struct {
109	/// Every `block` and `loop` currently under analysis.
110	block_scopes: std.AutoHashMapUnmanaged(Air.Inst.Index, BlockScope) = .empty,
111
112	/// The set of instructions currently alive in the current control
113	/// flow branch.
114	live_set: std.AutoHashMapUnmanaged(Air.Inst.Index, void) = .empty,
115
116	/// The extra data initialized by the `loop_analysis` pass for this pass to consume.
117	/// Owned by this struct during this pass.
118	old_extra: std.ArrayListUnmanaged(u32) = .empty,
119
120	const BlockScope = struct {
121	/// If this is a `block`, these instructions are alive upon a `br` to this block.
122	/// If this is a `loop`, these instructions are alive upon a `repeat` to this block.
123	live_set: std.AutoHashMapUnmanaged(Air.Inst.Index, void),
124	};
125
126	fn deinit(self: *@This(), gpa: Allocator) void {
127	var it = self.block_scopes.valueIterator();
128	while (it.next()) \|block\| {
129	block.live_set.deinit(gpa);
130	}
131	self.block_scopes.deinit(gpa);
132	self.live_set.deinit(gpa);
133	self.old_extra.deinit(gpa);
134	}
135	},
136	};
137	}
138
139	pub fn analyze(gpa: Allocator, air: Air, intern_pool: *InternPool) Allocator.Error!Liveness {
140	const tracy = trace(@src());
141	defer tracy.end();
142
143	var a: Analysis = .{
144	.gpa = gpa,
145	.air = air,
146	.tomb_bits = try gpa.alloc(
147	usize,
148	(air.instructions.len * bpi + @bitSizeOf(usize) - 1) / @bitSizeOf(usize),
149	),
150	.extra = .{},
151	.special = .{},
152	.intern_pool = intern_pool,
153	};
154	errdefer gpa.free(a.tomb_bits);
155	errdefer a.special.deinit(gpa);
156	defer a.extra.deinit(gpa);
157
158	@memset(a.tomb_bits, 0);
159
160	const main_body = air.getMainBody();
161
162	{
163	var data: LivenessPassData(.loop_analysis) = .{};
164	defer data.deinit(gpa);
165	try analyzeBody(&a, .loop_analysis, &data, main_body);
166	}
167
168	{
169	var data: LivenessPassData(.main_analysis) = .{};
170	defer data.deinit(gpa);
171	data.old_extra = a.extra;
172	a.extra = .{};
173	try analyzeBody(&a, .main_analysis, &data, main_body);
174	assert(data.live_set.count() == 0);
175	}
176
177	return .{
178	.tomb_bits = a.tomb_bits,
179	.special = a.special,
180	.extra = try a.extra.toOwnedSlice(gpa),
181	};
182	}
183
184	pub fn getTombBits(l: Liveness, inst: Air.Inst.Index) Bpi {
185	const usize_index = (@intFromEnum(inst) * bpi) / @bitSizeOf(usize);
186	return @as(Bpi, @truncate(l.tomb_bits[usize_index] >>
187	@as(Log2Int(usize), @intCast((@intFromEnum(inst) % (@bitSizeOf(usize) / bpi)) * bpi))));
188	}
189
190	pub fn isUnused(l: Liveness, inst: Air.Inst.Index) bool {
191	const usize_index = (@intFromEnum(inst) * bpi) / @bitSizeOf(usize);
192	const mask = @as(usize, 1) <<
193	@as(Log2Int(usize), @intCast((@intFromEnum(inst) % (@bitSizeOf(usize) / bpi)) * bpi + (bpi - 1)));
194	return (l.tomb_bits[usize_index] & mask) != 0;
195	}
196
197	pub fn operandDies(l: Liveness, inst: Air.Inst.Index, operand: OperandInt) bool {
198	assert(operand < bpi - 1);
199	const usize_index = (@intFromEnum(inst) * bpi) / @bitSizeOf(usize);
200	const mask = @as(usize, 1) <<
201	@as(Log2Int(usize), @intCast((@intFromEnum(inst) % (@bitSizeOf(usize) / bpi)) * bpi + operand));
202	return (l.tomb_bits[usize_index] & mask) != 0;
203	}
204
205	const OperandCategory = enum {
206	/// The operand lives on, but this instruction cannot possibly mutate memory.
207	none,
208	/// The operand lives on and this instruction can mutate memory.
209	write,
210	/// The operand dies at this instruction.
211	tomb,
212	/// The operand lives on, and this instruction is noreturn.
213	noret,
214	/// This instruction is too complicated for analysis, no information is available.
215	complex,
216	};
217
218	/// Given an instruction that we are examining, and an operand that we are looking for,
219	/// returns a classification.
220	pub fn categorizeOperand(
221	l: Liveness,
222	air: Air,
223	inst: Air.Inst.Index,
224	operand: Air.Inst.Index,
225	ip: *const InternPool,
226	) OperandCategory {
227	const air_tags = air.instructions.items(.tag);
228	const air_datas = air.instructions.items(.data);
229	const operand_ref = operand.toRef();
230	switch (air_tags[@intFromEnum(inst)]) {
231	.add,
232	.add_safe,
233	.add_wrap,
234	.add_sat,
235	.add_optimized,
236	.sub,
237	.sub_safe,
238	.sub_wrap,
239	.sub_sat,
240	.sub_optimized,
241	.mul,
242	.mul_safe,
243	.mul_wrap,
244	.mul_sat,
245	.mul_optimized,
246	.div_float,
247	.div_trunc,
248	.div_floor,
249	.div_exact,
250	.rem,
251	.mod,
252	.bit_and,
253	.bit_or,
254	.xor,
255	.cmp_lt,
256	.cmp_lte,
257	.cmp_eq,
258	.cmp_gte,
259	.cmp_gt,
260	.cmp_neq,
261	.bool_and,
262	.bool_or,
263	.array_elem_val,
264	.slice_elem_val,
265	.ptr_elem_val,
266	.shl,
267	.shl_exact,
268	.shl_sat,
269	.shr,
270	.shr_exact,
271	.min,
272	.max,
273	.div_float_optimized,
274	.div_trunc_optimized,
275	.div_floor_optimized,
276	.div_exact_optimized,
277	.rem_optimized,
278	.mod_optimized,
279	.neg_optimized,
280	.cmp_lt_optimized,
281	.cmp_lte_optimized,
282	.cmp_eq_optimized,
283	.cmp_gte_optimized,
284	.cmp_gt_optimized,
285	.cmp_neq_optimized,
286	=> {
287	const o = air_datas[@intFromEnum(inst)].bin_op;
288	if (o.lhs == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
289	if (o.rhs == operand_ref) return matchOperandSmallIndex(l, inst, 1, .none);
290	return .none;
291	},
292
293	.store,
294	.store_safe,
295	.atomic_store_unordered,
296	.atomic_store_monotonic,
297	.atomic_store_release,
298	.atomic_store_seq_cst,
299	.set_union_tag,
300	.memset,
301	.memset_safe,
302	.memcpy,
303	.memmove,
304	=> {
305	const o = air_datas[@intFromEnum(inst)].bin_op;
306	if (o.lhs == operand_ref) return matchOperandSmallIndex(l, inst, 0, .write);
307	if (o.rhs == operand_ref) return matchOperandSmallIndex(l, inst, 1, .write);
308	return .write;
309	},
310
311	.vector_store_elem => {
312	const o = air_datas[@intFromEnum(inst)].vector_store_elem;
313	const extra = air.extraData(Air.Bin, o.payload).data;
314	if (o.vector_ptr == operand_ref) return matchOperandSmallIndex(l, inst, 0, .write);
315	if (extra.lhs == operand_ref) return matchOperandSmallIndex(l, inst, 1, .none);
316	if (extra.rhs == operand_ref) return matchOperandSmallIndex(l, inst, 2, .none);
317	return .write;
318	},
319
320	.arg,
321	.alloc,
322	.inferred_alloc,
323	.inferred_alloc_comptime,
324	.ret_ptr,
325	.trap,
326	.breakpoint,
327	.repeat,
328	.switch_dispatch,
329	.dbg_stmt,
330	.dbg_empty_stmt,
331	.unreach,
332	.ret_addr,
333	.frame_addr,
334	.wasm_memory_size,
335	.err_return_trace,
336	.save_err_return_trace_index,
337	.tlv_dllimport_ptr,
338	.c_va_start,
339	.work_item_id,
340	.work_group_size,
341	.work_group_id,
342	=> return .none,
343
344	.not,
345	.bitcast,
346	.load,
347	.fpext,
348	.fptrunc,
349	.intcast,
350	.intcast_safe,
351	.trunc,
352	.optional_payload,
353	.optional_payload_ptr,
354	.wrap_optional,
355	.unwrap_errunion_payload,
356	.unwrap_errunion_err,
357	.unwrap_errunion_payload_ptr,
358	.unwrap_errunion_err_ptr,
359	.wrap_errunion_payload,
360	.wrap_errunion_err,
361	.slice_ptr,
362	.slice_len,
363	.ptr_slice_len_ptr,
364	.ptr_slice_ptr_ptr,
365	.struct_field_ptr_index_0,
366	.struct_field_ptr_index_1,
367	.struct_field_ptr_index_2,
368	.struct_field_ptr_index_3,
369	.array_to_slice,
370	.int_from_float,
371	.int_from_float_optimized,
372	.float_from_int,
373	.get_union_tag,
374	.clz,
375	.ctz,
376	.popcount,
377	.byte_swap,
378	.bit_reverse,
379	.splat,
380	.error_set_has_value,
381	.addrspace_cast,
382	.c_va_arg,
383	.c_va_copy,
384	.abs,
385	=> {
386	const o = air_datas[@intFromEnum(inst)].ty_op;
387	if (o.operand == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
388	return .none;
389	},
390
391	.optional_payload_ptr_set,
392	.errunion_payload_ptr_set,
393	=> {
394	const o = air_datas[@intFromEnum(inst)].ty_op;
395	if (o.operand == operand_ref) return matchOperandSmallIndex(l, inst, 0, .write);
396	return .write;
397	},
398
399	.is_null,
400	.is_non_null,
401	.is_null_ptr,
402	.is_non_null_ptr,
403	.is_err,
404	.is_non_err,
405	.is_err_ptr,
406	.is_non_err_ptr,
407	.is_named_enum_value,
408	.tag_name,
409	.error_name,
410	.sqrt,
411	.sin,
412	.cos,
413	.tan,
414	.exp,
415	.exp2,
416	.log,
417	.log2,
418	.log10,
419	.floor,
420	.ceil,
421	.round,
422	.trunc_float,
423	.neg,
424	.cmp_lt_errors_len,
425	.c_va_end,
426	=> {
427	const o = air_datas[@intFromEnum(inst)].un_op;
428	if (o == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
429	return .none;
430	},
431
432	.ret,
433	.ret_safe,
434	.ret_load,
435	=> {
436	const o = air_datas[@intFromEnum(inst)].un_op;
437	if (o == operand_ref) return matchOperandSmallIndex(l, inst, 0, .noret);
438	return .noret;
439	},
440
441	.set_err_return_trace => {
442	const o = air_datas[@intFromEnum(inst)].un_op;
443	if (o == operand_ref) return matchOperandSmallIndex(l, inst, 0, .write);
444	return .write;
445	},
446
447	.add_with_overflow,
448	.sub_with_overflow,
449	.mul_with_overflow,
450	.shl_with_overflow,
451	.ptr_add,
452	.ptr_sub,
453	.ptr_elem_ptr,
454	.slice_elem_ptr,
455	.slice,
456	=> {
457	const ty_pl = air_datas[@intFromEnum(inst)].ty_pl;
458	const extra = air.extraData(Air.Bin, ty_pl.payload).data;
459	if (extra.lhs == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
460	if (extra.rhs == operand_ref) return matchOperandSmallIndex(l, inst, 1, .none);
461	return .none;
462	},
463
464	.dbg_var_ptr,
465	.dbg_var_val,
466	.dbg_arg_inline,
467	=> {
468	const o = air_datas[@intFromEnum(inst)].pl_op.operand;
469	if (o == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
470	return .none;
471	},
472
473	.prefetch => {
474	const prefetch = air_datas[@intFromEnum(inst)].prefetch;
475	if (prefetch.ptr == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
476	return .none;
477	},
478
479	.call, .call_always_tail, .call_never_tail, .call_never_inline => {
480	const inst_data = air_datas[@intFromEnum(inst)].pl_op;
481	const callee = inst_data.operand;
482	const extra = air.extraData(Air.Call, inst_data.payload);
483	const args = @as([]const Air.Inst.Ref, @ptrCast(air.extra.items[extra.end..][0..extra.data.args_len]));
484	if (args.len + 1 <= bpi - 1) {
485	if (callee == operand_ref) return matchOperandSmallIndex(l, inst, 0, .write);
486	for (args, 0..) \|arg, i\| {
487	if (arg == operand_ref) return matchOperandSmallIndex(l, inst, @as(OperandInt, @intCast(i + 1)), .write);
488	}
489	return .write;
490	}
491	var bt = l.iterateBigTomb(inst);
492	if (bt.feed()) {
493	if (callee == operand_ref) return .tomb;
494	} else {
495	if (callee == operand_ref) return .write;
496	}
497	for (args) \|arg\| {
498	if (bt.feed()) {
499	if (arg == operand_ref) return .tomb;
500	} else {
501	if (arg == operand_ref) return .write;
502	}
503	}
504	return .write;
505	},
506	.select => {
507	const pl_op = air_datas[@intFromEnum(inst)].pl_op;
508	const extra = air.extraData(Air.Bin, pl_op.payload).data;
509	if (pl_op.operand == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
510	if (extra.lhs == operand_ref) return matchOperandSmallIndex(l, inst, 1, .none);
511	if (extra.rhs == operand_ref) return matchOperandSmallIndex(l, inst, 2, .none);
512	return .none;
513	},
514	.shuffle => {
515	const extra = air.extraData(Air.Shuffle, air_datas[@intFromEnum(inst)].ty_pl.payload).data;
516	if (extra.a == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
517	if (extra.b == operand_ref) return matchOperandSmallIndex(l, inst, 1, .none);
518	return .none;
519	},
520	.reduce, .reduce_optimized => {
521	const reduce = air_datas[@intFromEnum(inst)].reduce;
522	if (reduce.operand == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
523	return .none;
524	},
525	.cmp_vector, .cmp_vector_optimized => {
526	const extra = air.extraData(Air.VectorCmp, air_datas[@intFromEnum(inst)].ty_pl.payload).data;
527	if (extra.lhs == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
528	if (extra.rhs == operand_ref) return matchOperandSmallIndex(l, inst, 1, .none);
529	return .none;
530	},
531	.aggregate_init => {
532	const ty_pl = air_datas[@intFromEnum(inst)].ty_pl;
533	const aggregate_ty = ty_pl.ty.toType();
534	const len = @as(usize, @intCast(aggregate_ty.arrayLenIp(ip)));
535	const elements = @as([]const Air.Inst.Ref, @ptrCast(air.extra.items[ty_pl.payload..][0..len]));
536
537	if (elements.len <= bpi - 1) {
538	for (elements, 0..) \|elem, i\| {
539	if (elem == operand_ref) return matchOperandSmallIndex(l, inst, @as(OperandInt, @intCast(i)), .none);
540	}
541	return .none;
542	}
543
544	var bt = l.iterateBigTomb(inst);
545	for (elements) \|elem\| {
546	if (bt.feed()) {
547	if (elem == operand_ref) return .tomb;
548	} else {
549	if (elem == operand_ref) return .write;
550	}
551	}
552	return .write;
553	},
554	.union_init => {
555	const extra = air.extraData(Air.UnionInit, air_datas[@intFromEnum(inst)].ty_pl.payload).data;
556	if (extra.init == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
557	return .none;
558	},
559	.struct_field_ptr, .struct_field_val => {
560	const extra = air.extraData(Air.StructField, air_datas[@intFromEnum(inst)].ty_pl.payload).data;
561	if (extra.struct_operand == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
562	return .none;
563	},
564	.field_parent_ptr => {
565	const extra = air.extraData(Air.FieldParentPtr, air_datas[@intFromEnum(inst)].ty_pl.payload).data;
566	if (extra.field_ptr == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
567	return .none;
568	},
569	.cmpxchg_strong, .cmpxchg_weak => {
570	const extra = air.extraData(Air.Cmpxchg, air_datas[@intFromEnum(inst)].ty_pl.payload).data;
571	if (extra.ptr == operand_ref) return matchOperandSmallIndex(l, inst, 0, .write);
572	if (extra.expected_value == operand_ref) return matchOperandSmallIndex(l, inst, 1, .write);
573	if (extra.new_value == operand_ref) return matchOperandSmallIndex(l, inst, 2, .write);
574	return .write;
575	},
576	.mul_add => {
577	const pl_op = air_datas[@intFromEnum(inst)].pl_op;
578	const extra = air.extraData(Air.Bin, pl_op.payload).data;
579	if (extra.lhs == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
580	if (extra.rhs == operand_ref) return matchOperandSmallIndex(l, inst, 1, .none);
581	if (pl_op.operand == operand_ref) return matchOperandSmallIndex(l, inst, 2, .none);
582	return .none;
583	},
584	.atomic_load => {
585	const ptr = air_datas[@intFromEnum(inst)].atomic_load.ptr;
586	if (ptr == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
587	return .none;
588	},
589	.atomic_rmw => {
590	const pl_op = air_datas[@intFromEnum(inst)].pl_op;
591	const extra = air.extraData(Air.AtomicRmw, pl_op.payload).data;
592	if (pl_op.operand == operand_ref) return matchOperandSmallIndex(l, inst, 0, .write);
593	if (extra.operand == operand_ref) return matchOperandSmallIndex(l, inst, 1, .write);
594	return .write;
595	},
596
597	.br => {
598	const br = air_datas[@intFromEnum(inst)].br;
599	if (br.operand == operand_ref) return matchOperandSmallIndex(l, operand, 0, .noret);
600	return .noret;
601	},
602	.assembly => {
603	return .complex;
604	},
605	.block, .dbg_inline_block => \|tag\| {
606	const ty_pl = air_datas[@intFromEnum(inst)].ty_pl;
607	const body: []const Air.Inst.Index = @ptrCast(switch (tag) {
608	inline .block, .dbg_inline_block => \|comptime_tag\| body: {
609	const extra = air.extraData(switch (comptime_tag) {
610	.block => Air.Block,
611	.dbg_inline_block => Air.DbgInlineBlock,
612	else => unreachable,
613	}, ty_pl.payload);
614	break :body air.extra.items[extra.end..][0..extra.data.body_len];
615	},
616	else => unreachable,
617	});
618
619	if (body.len == 1 and air_tags[@intFromEnum(body[0])] == .cond_br) {
620	// Peephole optimization for "panic-like" conditionals, which have
621	// one empty branch and another which calls a `noreturn` function.
622	// This allows us to infer that safety checks do not modify memory,
623	// as far as control flow successors are concerned.
624
625	const inst_data = air_datas[@intFromEnum(body[0])].pl_op;
626	const cond_extra = air.extraData(Air.CondBr, inst_data.payload);
627	if (inst_data.operand == operand_ref and operandDies(l, body[0], 0))
628	return .tomb;
629
630	if (cond_extra.data.then_body_len > 2 or cond_extra.data.else_body_len > 2)
631	return .complex;
632
633	const then_body: []const Air.Inst.Index = @ptrCast(air.extra.items[cond_extra.end..][0..cond_extra.data.then_body_len]);
634	const else_body: []const Air.Inst.Index = @ptrCast(air.extra.items[cond_extra.end + cond_extra.data.then_body_len ..][0..cond_extra.data.else_body_len]);
635	if (then_body.len > 1 and air_tags[@intFromEnum(then_body[1])] != .unreach)
636	return .complex;
637	if (else_body.len > 1 and air_tags[@intFromEnum(else_body[1])] != .unreach)
638	return .complex;
639
640	var operand_live: bool = true;
641	for (&[_]Air.Inst.Index{ then_body[0], else_body[0] }) \|cond_inst\| {
642	if (l.categorizeOperand(air, cond_inst, operand, ip) == .tomb)
643	operand_live = false;
644
645	switch (air_tags[@intFromEnum(cond_inst)]) {
646	.br => { // Breaks immediately back to block
647	const br = air_datas[@intFromEnum(cond_inst)].br;
648	if (br.block_inst != inst)
649	return .complex;
650	},
651	.call => {}, // Calls a noreturn function
652	else => return .complex,
653	}
654	}
655	return if (operand_live) .none else .tomb;
656	}
657
658	return .complex;
659	},
660
661	.@"try",
662	.try_cold,
663	.try_ptr,
664	.try_ptr_cold,
665	.loop,
666	.cond_br,
667	.switch_br,
668	.loop_switch_br,
669	=> return .complex,
670
671	.wasm_memory_grow => {
672	const pl_op = air_datas[@intFromEnum(inst)].pl_op;
673	if (pl_op.operand == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
674	return .none;
675	},
676	}
677	}
678
679	fn matchOperandSmallIndex(
680	l: Liveness,
681	inst: Air.Inst.Index,
682	operand: OperandInt,
683	default: OperandCategory,
684	) OperandCategory {
685	if (operandDies(l, inst, operand)) {
686	return .tomb;
687	} else {
688	return default;
689	}
690	}
691
692	/// Higher level API.
693	pub const CondBrSlices = struct {
694	then_deaths: []const Air.Inst.Index,
695	else_deaths: []const Air.Inst.Index,
696	};
697
698	pub fn getCondBr(l: Liveness, inst: Air.Inst.Index) CondBrSlices {
699	var index: usize = l.special.get(inst) orelse return .{
700	.then_deaths = &.{},
701	.else_deaths = &.{},
702	};
703	const then_death_count = l.extra[index];
704	index += 1;
705	const else_death_count = l.extra[index];
706	index += 1;
707	const then_deaths: []const Air.Inst.Index = @ptrCast(l.extra[index..][0..then_death_count]);
708	index += then_death_count;
709	return .{
710	.then_deaths = then_deaths,
711	.else_deaths = @ptrCast(l.extra[index..][0..else_death_count]),
712	};
713	}
714
715	/// Indexed by case number as they appear in AIR.
716	/// Else is the last element.
717	pub const SwitchBrTable = struct {
718	deaths: []const []const Air.Inst.Index,
719	};
720
721	/// Caller owns the memory.
722	pub fn getSwitchBr(l: Liveness, gpa: Allocator, inst: Air.Inst.Index, cases_len: u32) Allocator.Error!SwitchBrTable {
723	var index: usize = l.special.get(inst) orelse return .{ .deaths = &.{} };
724	const else_death_count = l.extra[index];
725	index += 1;
726
727	var deaths = try gpa.alloc([]const Air.Inst.Index, cases_len);
728	errdefer gpa.free(deaths);
729
730	var case_i: u32 = 0;
731	while (case_i < cases_len - 1) : (case_i += 1) {
732	const case_death_count: u32 = l.extra[index];
733	index += 1;
734	deaths[case_i] = @ptrCast(l.extra[index..][0..case_death_count]);
735	index += case_death_count;
736	}
737	{
738	// Else
739	deaths[case_i] = @ptrCast(l.extra[index..][0..else_death_count]);
740	}
741	return .{ .deaths = deaths };
742	}
743
744	/// Note that this information is technically redundant, but is useful for
745	/// backends nonetheless: see `Block`.
746	pub const BlockSlices = struct {
747	deaths: []const Air.Inst.Index,
748	};
749
750	pub fn getBlock(l: Liveness, inst: Air.Inst.Index) BlockSlices {
751	const index: usize = l.special.get(inst) orelse return .{
752	.deaths = &.{},
753	};
754	const death_count = l.extra[index];
755	const deaths: []const Air.Inst.Index = @ptrCast(l.extra[index + 1 ..][0..death_count]);
756	return .{
757	.deaths = deaths,
758	};
759	}
760
761	pub const LoopSlice = struct {
762	deaths: []const Air.Inst.Index,
763	};
764
765	pub fn deinit(l: *Liveness, gpa: Allocator) void {
766	gpa.free(l.tomb_bits);
767	gpa.free(l.extra);
768	l.special.deinit(gpa);
769	l.* = undefined;
770	}
771
772	pub fn iterateBigTomb(l: Liveness, inst: Air.Inst.Index) BigTomb {
773	return .{
774	.tomb_bits = l.getTombBits(inst),
775	.extra_start = l.special.get(inst) orelse 0,
776	.extra_offset = 0,
777	.extra = l.extra,
778	.bit_index = 0,
779	.reached_end = false,
780	};
781	}
782
783	/// How many tomb bits per AIR instruction.
784	pub const bpi = 4;
785	pub const Bpi = std.meta.Int(.unsigned, bpi);
786	pub const OperandInt = std.math.Log2Int(Bpi);
787
788	/// Useful for decoders of Liveness information.
789	pub const BigTomb = struct {
790	tomb_bits: Liveness.Bpi,
791	bit_index: u32,
792	extra_start: u32,
793	extra_offset: u32,
794	extra: []const u32,
795	reached_end: bool,
796
797	/// Returns whether the next operand dies.
798	pub fn feed(bt: *BigTomb) bool {
799	if (bt.reached_end) return false;
800
801	const this_bit_index = bt.bit_index;
802	bt.bit_index += 1;
803
804	const small_tombs = bpi - 1;
805	if (this_bit_index < small_tombs) {
806	const dies = @as(u1, @truncate(bt.tomb_bits >> @as(Liveness.OperandInt, @intCast(this_bit_index)))) != 0;
807	return dies;
808	}
809
810	const big_bit_index = this_bit_index - small_tombs;
811	while (big_bit_index - bt.extra_offset * 31 >= 31) {
812	if (@as(u1, @truncate(bt.extra[bt.extra_start + bt.extra_offset] >> 31)) != 0) {
813	bt.reached_end = true;
814	return false;
815	}
816	bt.extra_offset += 1;
817	}
818	const dies = @as(u1, @truncate(bt.extra[bt.extra_start + bt.extra_offset] >>
819	@as(u5, @intCast(big_bit_index - bt.extra_offset * 31)))) != 0;
820	return dies;
821	}
822	};
823
824	/// In-progress data; on successful analysis converted into `Liveness`.
825	const Analysis = struct {
826	gpa: Allocator,
827	air: Air,
828	intern_pool: *InternPool,
829	tomb_bits: []usize,
830	special: std.AutoHashMapUnmanaged(Air.Inst.Index, u32),
831	extra: std.ArrayListUnmanaged(u32),
832
833	fn addExtra(a: *Analysis, extra: anytype) Allocator.Error!u32 {
834	const fields = std.meta.fields(@TypeOf(extra));
835	try a.extra.ensureUnusedCapacity(a.gpa, fields.len);
836	return addExtraAssumeCapacity(a, extra);
837	}
838
839	fn addExtraAssumeCapacity(a: *Analysis, extra: anytype) u32 {
840	const fields = std.meta.fields(@TypeOf(extra));
841	const result = @as(u32, @intCast(a.extra.items.len));
842	inline for (fields) \|field\| {
843	a.extra.appendAssumeCapacity(switch (field.type) {
844	u32 => @field(extra, field.name),
845	else => @compileError("bad field type"),
846	});
847	}
848	return result;
849	}
850	};
851
852	fn analyzeBody(
853	a: *Analysis,
854	comptime pass: LivenessPass,
855	data: *LivenessPassData(pass),
856	body: []const Air.Inst.Index,
857	) Allocator.Error!void {
858	var i: usize = body.len;
859	while (i != 0) {
860	i -= 1;
861	const inst = body[i];
862	try analyzeInst(a, pass, data, inst);
863	}
864	}
865
866	fn analyzeInst(
867	a: *Analysis,
868	comptime pass: LivenessPass,
869	data: *LivenessPassData(pass),
870	inst: Air.Inst.Index,
871	) Allocator.Error!void {
872	const ip = a.intern_pool;
873	const inst_tags = a.air.instructions.items(.tag);
874	const inst_datas = a.air.instructions.items(.data);
875
876	switch (inst_tags[@intFromEnum(inst)]) {
877	.add,
878	.add_safe,
879	.add_optimized,
880	.add_wrap,
881	.add_sat,
882	.sub,
883	.sub_safe,
884	.sub_optimized,
885	.sub_wrap,
886	.sub_sat,
887	.mul,
888	.mul_safe,
889	.mul_optimized,
890	.mul_wrap,
891	.mul_sat,
892	.div_float,
893	.div_float_optimized,
894	.div_trunc,
895	.div_trunc_optimized,
896	.div_floor,
897	.div_floor_optimized,
898	.div_exact,
899	.div_exact_optimized,
900	.rem,
901	.rem_optimized,
902	.mod,
903	.mod_optimized,
904	.bit_and,
905	.bit_or,
906	.xor,
907	.cmp_lt,
908	.cmp_lt_optimized,
909	.cmp_lte,
910	.cmp_lte_optimized,
911	.cmp_eq,
912	.cmp_eq_optimized,
913	.cmp_gte,
914	.cmp_gte_optimized,
915	.cmp_gt,
916	.cmp_gt_optimized,
917	.cmp_neq,
918	.cmp_neq_optimized,
919	.bool_and,
920	.bool_or,
921	.store,
922	.store_safe,
923	.array_elem_val,
924	.slice_elem_val,
925	.ptr_elem_val,
926	.shl,
927	.shl_exact,
928	.shl_sat,
929	.shr,
930	.shr_exact,
931	.atomic_store_unordered,
932	.atomic_store_monotonic,
933	.atomic_store_release,
934	.atomic_store_seq_cst,
935	.set_union_tag,
936	.min,
937	.max,
938	.memset,
939	.memset_safe,
940	.memcpy,
941	.memmove,
942	=> {
943	const o = inst_datas[@intFromEnum(inst)].bin_op;
944	return analyzeOperands(a, pass, data, inst, .{ o.lhs, o.rhs, .none });
945	},
946
947	.vector_store_elem => {
948	const o = inst_datas[@intFromEnum(inst)].vector_store_elem;
949	const extra = a.air.extraData(Air.Bin, o.payload).data;
950	return analyzeOperands(a, pass, data, inst, .{ o.vector_ptr, extra.lhs, extra.rhs });
951	},
952
953	.arg,
954	.alloc,
955	.ret_ptr,
956	.breakpoint,
957	.dbg_stmt,
958	.dbg_empty_stmt,
959	.ret_addr,
960	.frame_addr,
961	.wasm_memory_size,
962	.err_return_trace,
963	.save_err_return_trace_index,
964	.tlv_dllimport_ptr,
965	.c_va_start,
966	.work_item_id,
967	.work_group_size,
968	.work_group_id,
969	=> return analyzeOperands(a, pass, data, inst, .{ .none, .none, .none }),
970
971	.inferred_alloc, .inferred_alloc_comptime => unreachable,
972
973	.trap,
974	.unreach,
975	=> return analyzeFuncEnd(a, pass, data, inst, .{ .none, .none, .none }),
976
977	.not,
978	.bitcast,
979	.load,
980	.fpext,
981	.fptrunc,
982	.intcast,
983	.intcast_safe,
984	.trunc,
985	.optional_payload,
986	.optional_payload_ptr,
987	.optional_payload_ptr_set,
988	.errunion_payload_ptr_set,
989	.wrap_optional,
990	.unwrap_errunion_payload,
991	.unwrap_errunion_err,
992	.unwrap_errunion_payload_ptr,
993	.unwrap_errunion_err_ptr,
994	.wrap_errunion_payload,
995	.wrap_errunion_err,
996	.slice_ptr,
997	.slice_len,
998	.ptr_slice_len_ptr,
999	.ptr_slice_ptr_ptr,
1000	.struct_field_ptr_index_0,
1001	.struct_field_ptr_index_1,
1002	.struct_field_ptr_index_2,
1003	.struct_field_ptr_index_3,
1004	.array_to_slice,
1005	.int_from_float,
1006	.int_from_float_optimized,
1007	.float_from_int,
1008	.get_union_tag,
1009	.clz,
1010	.ctz,
1011	.popcount,
1012	.byte_swap,
1013	.bit_reverse,
1014	.splat,
1015	.error_set_has_value,
1016	.addrspace_cast,
1017	.c_va_arg,
1018	.c_va_copy,
1019	.abs,
1020	=> {
1021	const o = inst_datas[@intFromEnum(inst)].ty_op;
1022	return analyzeOperands(a, pass, data, inst, .{ o.operand, .none, .none });
1023	},
1024
1025	.is_null,
1026	.is_non_null,
1027	.is_null_ptr,
1028	.is_non_null_ptr,
1029	.is_err,
1030	.is_non_err,
1031	.is_err_ptr,
1032	.is_non_err_ptr,
1033	.is_named_enum_value,
1034	.tag_name,
1035	.error_name,
1036	.sqrt,
1037	.sin,
1038	.cos,
1039	.tan,
1040	.exp,
1041	.exp2,
1042	.log,
1043	.log2,
1044	.log10,
1045	.floor,
1046	.ceil,
1047	.round,
1048	.trunc_float,
1049	.neg,
1050	.neg_optimized,
1051	.cmp_lt_errors_len,
1052	.set_err_return_trace,
1053	.c_va_end,
1054	=> {
1055	const operand = inst_datas[@intFromEnum(inst)].un_op;
1056	return analyzeOperands(a, pass, data, inst, .{ operand, .none, .none });
1057	},
1058
1059	.ret,
1060	.ret_safe,
1061	.ret_load,
1062	=> {
1063	const operand = inst_datas[@intFromEnum(inst)].un_op;
1064	return analyzeFuncEnd(a, pass, data, inst, .{ operand, .none, .none });
1065	},
1066
1067	.add_with_overflow,
1068	.sub_with_overflow,
1069	.mul_with_overflow,
1070	.shl_with_overflow,
1071	.ptr_add,
1072	.ptr_sub,
1073	.ptr_elem_ptr,
1074	.slice_elem_ptr,
1075	.slice,
1076	=> {
1077	const ty_pl = inst_datas[@intFromEnum(inst)].ty_pl;
1078	const extra = a.air.extraData(Air.Bin, ty_pl.payload).data;
1079	return analyzeOperands(a, pass, data, inst, .{ extra.lhs, extra.rhs, .none });
1080	},
1081
1082	.dbg_var_ptr,
1083	.dbg_var_val,
1084	.dbg_arg_inline,
1085	=> {
1086	const operand = inst_datas[@intFromEnum(inst)].pl_op.operand;
1087	return analyzeOperands(a, pass, data, inst, .{ operand, .none, .none });
1088	},
1089
1090	.prefetch => {
1091	const prefetch = inst_datas[@intFromEnum(inst)].prefetch;
1092	return analyzeOperands(a, pass, data, inst, .{ prefetch.ptr, .none, .none });
1093	},
1094
1095	.call, .call_always_tail, .call_never_tail, .call_never_inline => {
1096	const inst_data = inst_datas[@intFromEnum(inst)].pl_op;
1097	const callee = inst_data.operand;
1098	const extra = a.air.extraData(Air.Call, inst_data.payload);
1099	const args = @as([]const Air.Inst.Ref, @ptrCast(a.air.extra.items[extra.end..][0..extra.data.args_len]));
1100	if (args.len + 1 <= bpi - 1) {
1101	var buf = [1]Air.Inst.Ref{.none} ** (bpi - 1);
1102	buf[0] = callee;
1103	@memcpy(buf[1..][0..args.len], args);
1104	return analyzeOperands(a, pass, data, inst, buf);
1105	}
1106
1107	var big = try AnalyzeBigOperands(pass).init(a, data, inst, args.len + 1);
1108	defer big.deinit();
1109	var i: usize = args.len;
1110	while (i > 0) {
1111	i -= 1;
1112	try big.feed(args[i]);
1113	}
1114	try big.feed(callee);
1115	return big.finish();
1116	},
1117	.select => {
1118	const pl_op = inst_datas[@intFromEnum(inst)].pl_op;
1119	const extra = a.air.extraData(Air.Bin, pl_op.payload).data;
1120	return analyzeOperands(a, pass, data, inst, .{ pl_op.operand, extra.lhs, extra.rhs });
1121	},
1122	.shuffle => {
1123	const extra = a.air.extraData(Air.Shuffle, inst_datas[@intFromEnum(inst)].ty_pl.payload).data;
1124	return analyzeOperands(a, pass, data, inst, .{ extra.a, extra.b, .none });
1125	},
1126	.reduce, .reduce_optimized => {
1127	const reduce = inst_datas[@intFromEnum(inst)].reduce;
1128	return analyzeOperands(a, pass, data, inst, .{ reduce.operand, .none, .none });
1129	},
1130	.cmp_vector, .cmp_vector_optimized => {
1131	const extra = a.air.extraData(Air.VectorCmp, inst_datas[@intFromEnum(inst)].ty_pl.payload).data;
1132	return analyzeOperands(a, pass, data, inst, .{ extra.lhs, extra.rhs, .none });
1133	},
1134	.aggregate_init => {
1135	const ty_pl = inst_datas[@intFromEnum(inst)].ty_pl;
1136	const aggregate_ty = ty_pl.ty.toType();
1137	const len = @as(usize, @intCast(aggregate_ty.arrayLenIp(ip)));
1138	const elements = @as([]const Air.Inst.Ref, @ptrCast(a.air.extra.items[ty_pl.payload..][0..len]));
1139
1140	if (elements.len <= bpi - 1) {
1141	var buf = [1]Air.Inst.Ref{.none} ** (bpi - 1);
1142	@memcpy(buf[0..elements.len], elements);
1143	return analyzeOperands(a, pass, data, inst, buf);
1144	}
1145
1146	var big = try AnalyzeBigOperands(pass).init(a, data, inst, elements.len);
1147	defer big.deinit();
1148	var i: usize = elements.len;
1149	while (i > 0) {
1150	i -= 1;
1151	try big.feed(elements[i]);
1152	}
1153	return big.finish();
1154	},
1155	.union_init => {
1156	const extra = a.air.extraData(Air.UnionInit, inst_datas[@intFromEnum(inst)].ty_pl.payload).data;
1157	return analyzeOperands(a, pass, data, inst, .{ extra.init, .none, .none });
1158	},
1159	.struct_field_ptr, .struct_field_val => {
1160	const extra = a.air.extraData(Air.StructField, inst_datas[@intFromEnum(inst)].ty_pl.payload).data;
1161	return analyzeOperands(a, pass, data, inst, .{ extra.struct_operand, .none, .none });
1162	},
1163	.field_parent_ptr => {
1164	const extra = a.air.extraData(Air.FieldParentPtr, inst_datas[@intFromEnum(inst)].ty_pl.payload).data;
1165	return analyzeOperands(a, pass, data, inst, .{ extra.field_ptr, .none, .none });
1166	},
1167	.cmpxchg_strong, .cmpxchg_weak => {
1168	const extra = a.air.extraData(Air.Cmpxchg, inst_datas[@intFromEnum(inst)].ty_pl.payload).data;
1169	return analyzeOperands(a, pass, data, inst, .{ extra.ptr, extra.expected_value, extra.new_value });
1170	},
1171	.mul_add => {
1172	const pl_op = inst_datas[@intFromEnum(inst)].pl_op;
1173	const extra = a.air.extraData(Air.Bin, pl_op.payload).data;
1174	return analyzeOperands(a, pass, data, inst, .{ extra.lhs, extra.rhs, pl_op.operand });
1175	},
1176	.atomic_load => {
1177	const ptr = inst_datas[@intFromEnum(inst)].atomic_load.ptr;
1178	return analyzeOperands(a, pass, data, inst, .{ ptr, .none, .none });
1179	},
1180	.atomic_rmw => {
1181	const pl_op = inst_datas[@intFromEnum(inst)].pl_op;
1182	const extra = a.air.extraData(Air.AtomicRmw, pl_op.payload).data;
1183	return analyzeOperands(a, pass, data, inst, .{ pl_op.operand, extra.operand, .none });
1184	},
1185
1186	.br => return analyzeInstBr(a, pass, data, inst),
1187	.repeat => return analyzeInstRepeat(a, pass, data, inst),
1188	.switch_dispatch => return analyzeInstSwitchDispatch(a, pass, data, inst),
1189
1190	.assembly => {
1191	const extra = a.air.extraData(Air.Asm, inst_datas[@intFromEnum(inst)].ty_pl.payload);
1192	var extra_i: usize = extra.end;
1193	const outputs = @as([]const Air.Inst.Ref, @ptrCast(a.air.extra.items[extra_i..][0..extra.data.outputs_len]));
1194	extra_i += outputs.len;
1195	const inputs = @as([]const Air.Inst.Ref, @ptrCast(a.air.extra.items[extra_i..][0..extra.data.inputs_len]));
1196	extra_i += inputs.len;
1197
1198	const num_operands = simple: {
1199	var buf = [1]Air.Inst.Ref{.none} ** (bpi - 1);
1200	var buf_index: usize = 0;
1201	for (outputs) \|output\| {
1202	if (output != .none) {
1203	if (buf_index < buf.len) buf[buf_index] = output;
1204	buf_index += 1;
1205	}
1206	}
1207	if (buf_index + inputs.len > buf.len) {
1208	break :simple buf_index + inputs.len;
1209	}
1210	@memcpy(buf[buf_index..][0..inputs.len], inputs);
1211	return analyzeOperands(a, pass, data, inst, buf);
1212	};
1213
1214	var big = try AnalyzeBigOperands(pass).init(a, data, inst, num_operands);
1215	defer big.deinit();
1216	var i: usize = inputs.len;
1217	while (i > 0) {
1218	i -= 1;
1219	try big.feed(inputs[i]);
1220	}
1221	i = outputs.len;
1222	while (i > 0) {
1223	i -= 1;
1224	if (outputs[i] != .none) {
1225	try big.feed(outputs[i]);
1226	}
1227	}
1228	return big.finish();
1229	},
1230
1231	inline .block, .dbg_inline_block => \|comptime_tag\| {
1232	const ty_pl = inst_datas[@intFromEnum(inst)].ty_pl;
1233	const extra = a.air.extraData(switch (comptime_tag) {
1234	.block => Air.Block,
1235	.dbg_inline_block => Air.DbgInlineBlock,
1236	else => unreachable,
1237	}, ty_pl.payload);
1238	return analyzeInstBlock(a, pass, data, inst, ty_pl.ty, @ptrCast(a.air.extra.items[extra.end..][0..extra.data.body_len]));
1239	},
1240	.loop => return analyzeInstLoop(a, pass, data, inst),
1241
1242	.@"try", .try_cold => return analyzeInstCondBr(a, pass, data, inst, .@"try"),
1243	.try_ptr, .try_ptr_cold => return analyzeInstCondBr(a, pass, data, inst, .try_ptr),
1244	.cond_br => return analyzeInstCondBr(a, pass, data, inst, .cond_br),
1245	.switch_br => return analyzeInstSwitchBr(a, pass, data, inst, false),
1246	.loop_switch_br => return analyzeInstSwitchBr(a, pass, data, inst, true),
1247
1248	.wasm_memory_grow => {
1249	const pl_op = inst_datas[@intFromEnum(inst)].pl_op;
1250	return analyzeOperands(a, pass, data, inst, .{ pl_op.operand, .none, .none });
1251	},
1252	}
1253	}
1254
1255	/// Every instruction should hit this (after handling any nested bodies), in every pass. In the
1256	/// initial pass, it is responsible for marking deaths of the (first three) operands and noticing
1257	/// immediate deaths.
1258	fn analyzeOperands(
1259	a: *Analysis,
1260	comptime pass: LivenessPass,
1261	data: *LivenessPassData(pass),
1262	inst: Air.Inst.Index,
1263	operands: [bpi - 1]Air.Inst.Ref,
1264	) Allocator.Error!void {
1265	const gpa = a.gpa;
1266	const ip = a.intern_pool;
1267
1268	switch (pass) {
1269	.loop_analysis => {
1270	_ = data.live_set.remove(inst);
1271
1272	for (operands) \|op_ref\| {
1273	const operand = op_ref.toIndexAllowNone() orelse continue;
1274	_ = try data.live_set.put(gpa, operand, {});
1275	}
1276	},
1277
1278	.main_analysis => {
1279	const usize_index = (@intFromEnum(inst) * bpi) / @bitSizeOf(usize);
1280
1281	// This logic must synchronize with `will_die_immediately` in `AnalyzeBigOperands.init`.
1282	const immediate_death = if (data.live_set.remove(inst)) blk: {
1283	log.debug("[{}] %{}: removed from live set", .{ pass, @intFromEnum(inst) });
1284	break :blk false;
1285	} else blk: {
1286	log.debug("[{}] %{}: immediate death", .{ pass, @intFromEnum(inst) });
1287	break :blk true;
1288	};
1289
1290	var tomb_bits: Bpi = @as(Bpi, @intFromBool(immediate_death)) << (bpi - 1);
1291
1292	// If our result is unused and the instruction doesn't need to be lowered, backends will
1293	// skip the lowering of this instruction, so we don't want to record uses of operands.
1294	// That way, we can mark as many instructions as possible unused.
1295	if (!immediate_death or a.air.mustLower(inst, ip)) {
1296	// Note that it's important we iterate over the operands backwards, so that if a dying
1297	// operand is used multiple times we mark its last use as its death.
1298	var i = operands.len;
1299	while (i > 0) {
1300	i -= 1;
1301	const op_ref = operands[i];
1302	const operand = op_ref.toIndexAllowNone() orelse continue;
1303
1304	const mask = @as(Bpi, 1) << @as(OperandInt, @intCast(i));
1305
1306	if ((try data.live_set.fetchPut(gpa, operand, {})) == null) {
1307	log.debug("[{}] %{}: added %{} to live set (operand dies here)", .{ pass, @intFromEnum(inst), operand });
1308	tomb_bits \|= mask;
1309	}
1310	}
1311	}
1312
1313	a.tomb_bits[usize_index] \|= @as(usize, tomb_bits) <<
1314	@as(Log2Int(usize), @intCast((@intFromEnum(inst) % (@bitSizeOf(usize) / bpi)) * bpi));
1315	},
1316	}
1317	}
1318
1319	/// Like `analyzeOperands`, but for an instruction which returns from a function, so should
1320	/// effectively kill every remaining live value other than its operands.
1321	fn analyzeFuncEnd(
1322	a: *Analysis,
1323	comptime pass: LivenessPass,
1324	data: *LivenessPassData(pass),
1325	inst: Air.Inst.Index,
1326	operands: [bpi - 1]Air.Inst.Ref,
1327	) Allocator.Error!void {
1328	switch (pass) {
1329	.loop_analysis => {
1330	// No operands need to be alive if we're returning from the function, so we don't need
1331	// to touch `breaks` here even though this is sort of like a break to the top level.
1332	},
1333
1334	.main_analysis => {
1335	data.live_set.clearRetainingCapacity();
1336	},
1337	}
1338
1339	return analyzeOperands(a, pass, data, inst, operands);
1340	}
1341
1342	fn analyzeInstBr(
1343	a: *Analysis,
1344	comptime pass: LivenessPass,
1345	data: *LivenessPassData(pass),
1346	inst: Air.Inst.Index,
1347	) !void {
1348	const inst_datas = a.air.instructions.items(.data);
1349	const br = inst_datas[@intFromEnum(inst)].br;
1350	const gpa = a.gpa;
1351
1352	switch (pass) {
1353	.loop_analysis => {
1354	try data.breaks.put(gpa, br.block_inst, {});
1355	},
1356
1357	.main_analysis => {
1358	const block_scope = data.block_scopes.get(br.block_inst).?; // we should always be breaking from an enclosing block
1359
1360	const new_live_set = try block_scope.live_set.clone(gpa);
1361	data.live_set.deinit(gpa);
1362	data.live_set = new_live_set;
1363	},
1364	}
1365
1366	return analyzeOperands(a, pass, data, inst, .{ br.operand, .none, .none });
1367	}
1368
1369	fn analyzeInstRepeat(
1370	a: *Analysis,
1371	comptime pass: LivenessPass,
1372	data: *LivenessPassData(pass),
1373	inst: Air.Inst.Index,
1374	) !void {
1375	const inst_datas = a.air.instructions.items(.data);
1376	const repeat = inst_datas[@intFromEnum(inst)].repeat;
1377	const gpa = a.gpa;
1378
1379	switch (pass) {
1380	.loop_analysis => {
1381	try data.breaks.put(gpa, repeat.loop_inst, {});
1382	},
1383
1384	.main_analysis => {
1385	const block_scope = data.block_scopes.get(repeat.loop_inst).?; // we should always be repeating an enclosing loop
1386
1387	const new_live_set = try block_scope.live_set.clone(gpa);
1388	data.live_set.deinit(gpa);
1389	data.live_set = new_live_set;
1390	},
1391	}
1392
1393	return analyzeOperands(a, pass, data, inst, .{ .none, .none, .none });
1394	}
1395
1396	fn analyzeInstSwitchDispatch(
1397	a: *Analysis,
1398	comptime pass: LivenessPass,
1399	data: *LivenessPassData(pass),
1400	inst: Air.Inst.Index,
1401	) !void {
1402	// This happens to be identical to `analyzeInstBr`, but is separated anyway for clarity.
1403
1404	const inst_datas = a.air.instructions.items(.data);
1405	const br = inst_datas[@intFromEnum(inst)].br;
1406	const gpa = a.gpa;
1407
1408	switch (pass) {
1409	.loop_analysis => {
1410	try data.breaks.put(gpa, br.block_inst, {});
1411	},
1412
1413	.main_analysis => {
1414	const block_scope = data.block_scopes.get(br.block_inst).?; // we should always be repeating an enclosing loop
1415
1416	const new_live_set = try block_scope.live_set.clone(gpa);
1417	data.live_set.deinit(gpa);
1418	data.live_set = new_live_set;
1419	},
1420	}
1421
1422	return analyzeOperands(a, pass, data, inst, .{ br.operand, .none, .none });
1423	}
1424
1425	fn analyzeInstBlock(
1426	a: *Analysis,
1427	comptime pass: LivenessPass,
1428	data: *LivenessPassData(pass),
1429	inst: Air.Inst.Index,
1430	ty: Air.Inst.Ref,
1431	body: []const Air.Inst.Index,
1432	) !void {
1433	const gpa = a.gpa;
1434
1435	// We actually want to do `analyzeOperands` first, since our result logically doesn't
1436	// exist until the block body ends (and we're iterating backwards)
1437	try analyzeOperands(a, pass, data, inst, .{ .none, .none, .none });
1438
1439	switch (pass) {
1440	.loop_analysis => {
1441	try analyzeBody(a, pass, data, body);
1442	_ = data.breaks.remove(inst);
1443	},
1444
1445	.main_analysis => {
1446	log.debug("[{}] %{}: block live set is {}", .{ pass, inst, fmtInstSet(&data.live_set) });
1447	// We can move the live set because the body should have a noreturn
1448	// instruction which overrides the set.
1449	try data.block_scopes.put(gpa, inst, .{
1450	.live_set = data.live_set.move(),
1451	});
1452	defer {
1453	log.debug("[{}] %{}: popped block scope", .{ pass, inst });
1454	var scope = data.block_scopes.fetchRemove(inst).?.value;
1455	scope.live_set.deinit(gpa);
1456	}
1457
1458	log.debug("[{}] %{}: pushed new block scope", .{ pass, inst });
1459	try analyzeBody(a, pass, data, body);
1460
1461	// If the block is noreturn, block deaths not only aren't useful, they're impossible to
1462	// find: there could be more stuff alive after the block than before it!
1463	if (!a.intern_pool.isNoReturn(ty.toType().toIntern())) {
1464	// The block kills the difference in the live sets
1465	const block_scope = data.block_scopes.get(inst).?;
1466	const num_deaths = data.live_set.count() - block_scope.live_set.count();
1467
1468	try a.extra.ensureUnusedCapacity(gpa, num_deaths + std.meta.fields(Block).len);
1469	const extra_index = a.addExtraAssumeCapacity(Block{
1470	.death_count = num_deaths,
1471	});
1472
1473	var measured_num: u32 = 0;
1474	var it = data.live_set.keyIterator();
1475	while (it.next()) \|key\| {
1476	const alive = key.*;
1477	if (!block_scope.live_set.contains(alive)) {
1478	// Dies in block
1479	a.extra.appendAssumeCapacity(@intFromEnum(alive));
1480	measured_num += 1;
1481	}
1482	}
1483	assert(measured_num == num_deaths); // post-live-set should be a subset of pre-live-set
1484	try a.special.put(gpa, inst, extra_index);
1485	log.debug("[{}] %{}: block deaths are {}", .{
1486	pass,
1487	inst,
1488	fmtInstList(@ptrCast(a.extra.items[extra_index + 1 ..][0..num_deaths])),
1489	});
1490	}
1491	},
1492	}
1493	}
1494
1495	fn writeLoopInfo(
1496	a: *Analysis,
1497	data: *LivenessPassData(.loop_analysis),
1498	inst: Air.Inst.Index,
1499	old_breaks: std.AutoHashMapUnmanaged(Air.Inst.Index, void),
1500	old_live: std.AutoHashMapUnmanaged(Air.Inst.Index, void),
1501	) !void {
1502	const gpa = a.gpa;
1503
1504	// `loop`s are guaranteed to have at least one matching `repeat`.
1505	// Similarly, `loop_switch_br`s have a matching `switch_dispatch`.
1506	// However, we no longer care about repeats of this loop for resolving
1507	// which operands must live within it.
1508	assert(data.breaks.remove(inst));
1509
1510	const extra_index: u32 = @intCast(a.extra.items.len);
1511
1512	const num_breaks = data.breaks.count();
1513	try a.extra.ensureUnusedCapacity(gpa, 1 + num_breaks);
1514
1515	a.extra.appendAssumeCapacity(num_breaks);
1516
1517	var it = data.breaks.keyIterator();
1518	while (it.next()) \|key\| {
1519	const block_inst = key.*;
1520	a.extra.appendAssumeCapacity(@intFromEnum(block_inst));
1521	}
1522	log.debug("[{}] %{}: includes breaks to {}", .{ LivenessPass.loop_analysis, inst, fmtInstSet(&data.breaks) });
1523
1524	// Now we put the live operands from the loop body in too
1525	const num_live = data.live_set.count();
1526	try a.extra.ensureUnusedCapacity(gpa, 1 + num_live);
1527
1528	a.extra.appendAssumeCapacity(num_live);
1529	it = data.live_set.keyIterator();
1530	while (it.next()) \|key\| {
1531	const alive = key.*;
1532	a.extra.appendAssumeCapacity(@intFromEnum(alive));
1533	}
1534	log.debug("[{}] %{}: maintain liveness of {}", .{ LivenessPass.loop_analysis, inst, fmtInstSet(&data.live_set) });
1535
1536	try a.special.put(gpa, inst, extra_index);
1537
1538	// Add back operands which were previously alive
1539	it = old_live.keyIterator();
1540	while (it.next()) \|key\| {
1541	const alive = key.*;
1542	try data.live_set.put(gpa, alive, {});
1543	}
1544
1545	// And the same for breaks
1546	it = old_breaks.keyIterator();
1547	while (it.next()) \|key\| {
1548	const block_inst = key.*;
1549	try data.breaks.put(gpa, block_inst, {});
1550	}
1551	}
1552
1553	/// When analyzing a loop in the main pass, sets up `data.live_set` to be the set
1554	/// of operands known to be alive when the loop repeats.
1555	fn resolveLoopLiveSet(
1556	a: *Analysis,
1557	data: *LivenessPassData(.main_analysis),
1558	inst: Air.Inst.Index,
1559	) !void {
1560	const gpa = a.gpa;
1561
1562	const extra_idx = a.special.fetchRemove(inst).?.value;
1563	const num_breaks = data.old_extra.items[extra_idx];
1564	const breaks: []const Air.Inst.Index = @ptrCast(data.old_extra.items[extra_idx + 1 ..][0..num_breaks]);
1565
1566	const num_loop_live = data.old_extra.items[extra_idx + num_breaks + 1];
1567	const loop_live: []const Air.Inst.Index = @ptrCast(data.old_extra.items[extra_idx + num_breaks + 2 ..][0..num_loop_live]);
1568
1569	// This is necessarily not in the same control flow branch, because loops are noreturn
1570	data.live_set.clearRetainingCapacity();
1571
1572	try data.live_set.ensureUnusedCapacity(gpa, @intCast(loop_live.len));
1573	for (loop_live) \|alive\| data.live_set.putAssumeCapacity(alive, {});
1574
1575	log.debug("[{}] %{}: block live set is {}", .{ LivenessPass.main_analysis, inst, fmtInstSet(&data.live_set) });
1576
1577	for (breaks) \|block_inst\| {
1578	// We might break to this block, so include every operand that the block needs alive
1579	const block_scope = data.block_scopes.get(block_inst).?;
1580
1581	var it = block_scope.live_set.keyIterator();
1582	while (it.next()) \|key\| {
1583	const alive = key.*;
1584	try data.live_set.put(gpa, alive, {});
1585	}
1586	}
1587
1588	log.debug("[{}] %{}: loop live set is {}", .{ LivenessPass.main_analysis, inst, fmtInstSet(&data.live_set) });
1589	}
1590
1591	fn analyzeInstLoop(
1592	a: *Analysis,
1593	comptime pass: LivenessPass,
1594	data: *LivenessPassData(pass),
1595	inst: Air.Inst.Index,
1596	) !void {
1597	const inst_datas = a.air.instructions.items(.data);
1598	const extra = a.air.extraData(Air.Block, inst_datas[@intFromEnum(inst)].ty_pl.payload);
1599	const body: []const Air.Inst.Index = @ptrCast(a.air.extra.items[extra.end..][0..extra.data.body_len]);
1600	const gpa = a.gpa;
1601
1602	try analyzeOperands(a, pass, data, inst, .{ .none, .none, .none });
1603
1604	switch (pass) {
1605	.loop_analysis => {
1606	var old_breaks = data.breaks.move();
1607	defer old_breaks.deinit(gpa);
1608
1609	var old_live = data.live_set.move();
1610	defer old_live.deinit(gpa);
1611
1612	try analyzeBody(a, pass, data, body);
1613
1614	try writeLoopInfo(a, data, inst, old_breaks, old_live);
1615	},
1616
1617	.main_analysis => {
1618	try resolveLoopLiveSet(a, data, inst);
1619
1620	// Now, `data.live_set` is the operands which must be alive when the loop repeats.
1621	// Move them into a block scope for corresponding `repeat` instructions to notice.
1622	try data.block_scopes.putNoClobber(gpa, inst, .{
1623	.live_set = data.live_set.move(),
1624	});
1625	defer {
1626	log.debug("[{}] %{}: popped loop block scop", .{ pass, inst });
1627	var scope = data.block_scopes.fetchRemove(inst).?.value;
1628	scope.live_set.deinit(gpa);
1629	}
1630	try analyzeBody(a, pass, data, body);
1631	},
1632	}
1633	}
1634
1635	/// Despite its name, this function is used for analysis of not only `cond_br` instructions, but
1636	/// also `try` and `try_ptr`, which are highly related. The `inst_type` parameter indicates which
1637	/// type of instruction `inst` points to.
1638	fn analyzeInstCondBr(
1639	a: *Analysis,
1640	comptime pass: LivenessPass,
1641	data: *LivenessPassData(pass),
1642	inst: Air.Inst.Index,
1643	comptime inst_type: enum { cond_br, @"try", try_ptr },
1644	) !void {
1645	const inst_datas = a.air.instructions.items(.data);
1646	const gpa = a.gpa;
1647
1648	const extra = switch (inst_type) {
1649	.cond_br => a.air.extraData(Air.CondBr, inst_datas[@intFromEnum(inst)].pl_op.payload),
1650	.@"try" => a.air.extraData(Air.Try, inst_datas[@intFromEnum(inst)].pl_op.payload),
1651	.try_ptr => a.air.extraData(Air.TryPtr, inst_datas[@intFromEnum(inst)].ty_pl.payload),
1652	};
1653
1654	const condition = switch (inst_type) {
1655	.cond_br, .@"try" => inst_datas[@intFromEnum(inst)].pl_op.operand,
1656	.try_ptr => extra.data.ptr,
1657	};
1658
1659	const then_body: []const Air.Inst.Index = switch (inst_type) {
1660	.cond_br => @ptrCast(a.air.extra.items[extra.end..][0..extra.data.then_body_len]),
1661	else => &.{}, // we won't use this
1662	};
1663
1664	const else_body: []const Air.Inst.Index = @ptrCast(switch (inst_type) {
1665	.cond_br => a.air.extra.items[extra.end + then_body.len ..][0..extra.data.else_body_len],
1666	.@"try", .try_ptr => a.air.extra.items[extra.end..][0..extra.data.body_len],
1667	});
1668
1669	switch (pass) {
1670	.loop_analysis => {
1671	switch (inst_type) {
1672	.cond_br => try analyzeBody(a, pass, data, then_body),
1673	.@"try", .try_ptr => {},
1674	}
1675	try analyzeBody(a, pass, data, else_body);
1676	},
1677
1678	.main_analysis => {
1679	switch (inst_type) {
1680	.cond_br => try analyzeBody(a, pass, data, then_body),
1681	.@"try", .try_ptr => {}, // The "then body" is just the remainder of this block
1682	}
1683	var then_live = data.live_set.move();
1684	defer then_live.deinit(gpa);
1685
1686	try analyzeBody(a, pass, data, else_body);
1687	var else_live = data.live_set.move();
1688	defer else_live.deinit(gpa);
1689
1690	// Operands which are alive in one branch but not the other need to die at the start of
1691	// the peer branch.
1692
1693	var then_mirrored_deaths: std.ArrayListUnmanaged(Air.Inst.Index) = .empty;
1694	defer then_mirrored_deaths.deinit(gpa);
1695
1696	var else_mirrored_deaths: std.ArrayListUnmanaged(Air.Inst.Index) = .empty;
1697	defer else_mirrored_deaths.deinit(gpa);
1698
1699	// Note: this invalidates `else_live`, but expands `then_live` to be their union
1700	{
1701	var it = then_live.keyIterator();
1702	while (it.next()) \|key\| {
1703	const death = key.*;
1704	if (else_live.remove(death)) continue; // removing makes the loop below faster
1705
1706	// If this is a `try`, the "then body" (rest of the branch) might have
1707	// referenced our result. We want to avoid killing this value in the else branch
1708	// if that's the case, since it only exists in the (fake) then branch.
1709	switch (inst_type) {
1710	.cond_br => {},
1711	.@"try", .try_ptr => if (death == inst) continue,
1712	}
1713
1714	try else_mirrored_deaths.append(gpa, death);
1715	}
1716	// Since we removed common stuff above, `else_live` is now only operands
1717	// which are only alive in the else branch
1718	it = else_live.keyIterator();
1719	while (it.next()) \|key\| {
1720	const death = key.*;
1721	try then_mirrored_deaths.append(gpa, death);
1722	// Make `then_live` contain the full live set (i.e. union of both)
1723	try then_live.put(gpa, death, {});
1724	}
1725	}
1726
1727	log.debug("[{}] %{}: 'then' branch mirrored deaths are {}", .{ pass, inst, fmtInstList(then_mirrored_deaths.items) });
1728	log.debug("[{}] %{}: 'else' branch mirrored deaths are {}", .{ pass, inst, fmtInstList(else_mirrored_deaths.items) });
1729
1730	data.live_set.deinit(gpa);
1731	data.live_set = then_live.move(); // Really the union of both live sets
1732
1733	log.debug("[{}] %{}: new live set is {}", .{ pass, inst, fmtInstSet(&data.live_set) });
1734
1735	// Write the mirrored deaths to `extra`
1736	const then_death_count = @as(u32, @intCast(then_mirrored_deaths.items.len));
1737	const else_death_count = @as(u32, @intCast(else_mirrored_deaths.items.len));
1738	try a.extra.ensureUnusedCapacity(gpa, std.meta.fields(CondBr).len + then_death_count + else_death_count);
1739	const extra_index = a.addExtraAssumeCapacity(CondBr{
1740	.then_death_count = then_death_count,
1741	.else_death_count = else_death_count,
1742	});
1743	a.extra.appendSliceAssumeCapacity(@ptrCast(then_mirrored_deaths.items));
1744	a.extra.appendSliceAssumeCapacity(@ptrCast(else_mirrored_deaths.items));
1745	try a.special.put(gpa, inst, extra_index);
1746	},
1747	}
1748
1749	try analyzeOperands(a, pass, data, inst, .{ condition, .none, .none });
1750	}
1751
1752	fn analyzeInstSwitchBr(
1753	a: *Analysis,
1754	comptime pass: LivenessPass,
1755	data: *LivenessPassData(pass),
1756	inst: Air.Inst.Index,
1757	is_dispatch_loop: bool,
1758	) !void {
1759	const inst_datas = a.air.instructions.items(.data);
1760	const pl_op = inst_datas[@intFromEnum(inst)].pl_op;
1761	const condition = pl_op.operand;
1762	const switch_br = a.air.unwrapSwitch(inst);
1763	const gpa = a.gpa;
1764	const ncases = switch_br.cases_len;
1765
1766	switch (pass) {
1767	.loop_analysis => {
1768	var old_breaks: std.AutoHashMapUnmanaged(Air.Inst.Index, void) = .empty;
1769	defer old_breaks.deinit(gpa);
1770
1771	var old_live: std.AutoHashMapUnmanaged(Air.Inst.Index, void) = .empty;
1772	defer old_live.deinit(gpa);
1773
1774	if (is_dispatch_loop) {
1775	old_breaks = data.breaks.move();
1776	old_live = data.live_set.move();
1777	}
1778
1779	var it = switch_br.iterateCases();
1780	while (it.next()) \|case\| {
1781	try analyzeBody(a, pass, data, case.body);
1782	}
1783	{ // else
1784	const else_body = it.elseBody();
1785	try analyzeBody(a, pass, data, else_body);
1786	}
1787
1788	if (is_dispatch_loop) {
1789	try writeLoopInfo(a, data, inst, old_breaks, old_live);
1790	}
1791	},
1792
1793	.main_analysis => {
1794	if (is_dispatch_loop) {
1795	try resolveLoopLiveSet(a, data, inst);
1796	try data.block_scopes.putNoClobber(gpa, inst, .{
1797	.live_set = data.live_set.move(),
1798	});
1799	}
1800	defer if (is_dispatch_loop) {
1801	log.debug("[{}] %{}: popped loop block scop", .{ pass, inst });
1802	var scope = data.block_scopes.fetchRemove(inst).?.value;
1803	scope.live_set.deinit(gpa);
1804	};
1805	// This is, all in all, just a messier version of the `cond_br` logic. If you're trying
1806	// to understand it, I encourage looking at `analyzeInstCondBr` first.
1807
1808	const DeathSet = std.AutoHashMapUnmanaged(Air.Inst.Index, void);
1809	const DeathList = std.ArrayListUnmanaged(Air.Inst.Index);
1810
1811	var case_live_sets = try gpa.alloc(std.AutoHashMapUnmanaged(Air.Inst.Index, void), ncases + 1); // +1 for else
1812	defer gpa.free(case_live_sets);
1813
1814	@memset(case_live_sets, .{});
1815	defer for (case_live_sets) \|*live_set\| live_set.deinit(gpa);
1816
1817	var case_it = switch_br.iterateCases();
1818	while (case_it.next()) \|case\| {
1819	try analyzeBody(a, pass, data, case.body);
1820	case_live_sets[case.idx] = data.live_set.move();
1821	}
1822	{ // else
1823	const else_body = case_it.elseBody();
1824	try analyzeBody(a, pass, data, else_body);
1825	case_live_sets[ncases] = data.live_set.move();
1826	}
1827
1828	const mirrored_deaths = try gpa.alloc(DeathList, ncases + 1);
1829	defer gpa.free(mirrored_deaths);
1830
1831	@memset(mirrored_deaths, .{});
1832	defer for (mirrored_deaths) \|*md\| md.deinit(gpa);
1833
1834	{
1835	var all_alive: DeathSet = .{};
1836	defer all_alive.deinit(gpa);
1837
1838	for (case_live_sets) \|*live_set\| {
1839	try all_alive.ensureUnusedCapacity(gpa, live_set.count());
1840	var it = live_set.keyIterator();
1841	while (it.next()) \|key\| {
1842	const alive = key.*;
1843	all_alive.putAssumeCapacity(alive, {});
1844	}
1845	}
1846
1847	for (mirrored_deaths, case_live_sets) \|mirrored, live_set\| {
1848	var it = all_alive.keyIterator();
1849	while (it.next()) \|key\| {
1850	const alive = key.*;
1851	if (!live_set.contains(alive)) {
1852	// Should die at the start of this branch
1853	try mirrored.append(gpa, alive);
1854	}
1855	}
1856	}
1857
1858	for (mirrored_deaths, 0..) \|mirrored, i\| {
1859	log.debug("[{}] %{}: case {} mirrored deaths are {}", .{ pass, inst, i, fmtInstList(mirrored.items) });
1860	}
1861
1862	data.live_set.deinit(gpa);
1863	data.live_set = all_alive.move();
1864
1865	log.debug("[{}] %{}: new live set is {}", .{ pass, inst, fmtInstSet(&data.live_set) });
1866	}
1867
1868	const else_death_count = @as(u32, @intCast(mirrored_deaths[ncases].items.len));
1869	const extra_index = try a.addExtra(SwitchBr{
1870	.else_death_count = else_death_count,
1871	});
1872	for (mirrored_deaths[0..ncases]) \|mirrored\| {
1873	const num = @as(u32, @intCast(mirrored.items.len));
1874	try a.extra.ensureUnusedCapacity(gpa, num + 1);
1875	a.extra.appendAssumeCapacity(num);
1876	a.extra.appendSliceAssumeCapacity(@ptrCast(mirrored.items));
1877	}
1878	try a.extra.ensureUnusedCapacity(gpa, else_death_count);
1879	a.extra.appendSliceAssumeCapacity(@ptrCast(mirrored_deaths[ncases].items));
1880	try a.special.put(gpa, inst, extra_index);
1881	},
1882	}
1883
1884	try analyzeOperands(a, pass, data, inst, .{ condition, .none, .none });
1885	}
1886
1887	fn AnalyzeBigOperands(comptime pass: LivenessPass) type {
1888	return struct {
1889	a: *Analysis,
1890	data: *LivenessPassData(pass),
1891	inst: Air.Inst.Index,
1892
1893	operands_remaining: u32,
1894	small: [bpi - 1]Air.Inst.Ref = .{.none} ** (bpi - 1),
1895	extra_tombs: []u32,
1896
1897	// Only used in `LivenessPass.main_analysis`
1898	will_die_immediately: bool,
1899
1900	const Self = @This();
1901
1902	fn init(
1903	a: *Analysis,
1904	data: *LivenessPassData(pass),
1905	inst: Air.Inst.Index,
1906	total_operands: usize,
1907	) !Self {
1908	const extra_operands = @as(u32, @intCast(total_operands)) -\| (bpi - 1);
1909	const max_extra_tombs = (extra_operands + 30) / 31;
1910
1911	const extra_tombs: []u32 = switch (pass) {
1912	.loop_analysis => &.{},
1913	.main_analysis => try a.gpa.alloc(u32, max_extra_tombs),
1914	};
1915	errdefer a.gpa.free(extra_tombs);
1916
1917	@memset(extra_tombs, 0);
1918
1919	const will_die_immediately: bool = switch (pass) {
1920	.loop_analysis => false, // track everything, since we don't have full liveness information yet
1921	.main_analysis => !data.live_set.contains(inst),
1922	};
1923
1924	return .{
1925	.a = a,
1926	.data = data,
1927	.inst = inst,
1928	.operands_remaining = @as(u32, @intCast(total_operands)),
1929	.extra_tombs = extra_tombs,
1930	.will_die_immediately = will_die_immediately,
1931	};
1932	}
1933
1934	/// Must be called with operands in reverse order.
1935	fn feed(big: *Self, op_ref: Air.Inst.Ref) !void {
1936	const ip = big.a.intern_pool;
1937	// Note that after this, `operands_remaining` becomes the index of the current operand
1938	big.operands_remaining -= 1;
1939
1940	if (big.operands_remaining < bpi - 1) {
1941	big.small[big.operands_remaining] = op_ref;
1942	return;
1943	}
1944
1945	const operand = op_ref.toIndex() orelse return;
1946
1947	// If our result is unused and the instruction doesn't need to be lowered, backends will
1948	// skip the lowering of this instruction, so we don't want to record uses of operands.
1949	// That way, we can mark as many instructions as possible unused.
1950	if (big.will_die_immediately and !big.a.air.mustLower(big.inst, ip)) return;
1951
1952	const extra_byte = (big.operands_remaining - (bpi - 1)) / 31;
1953	const extra_bit = @as(u5, @intCast(big.operands_remaining - (bpi - 1) - extra_byte * 31));
1954
1955	const gpa = big.a.gpa;
1956
1957	switch (pass) {
1958	.loop_analysis => {
1959	_ = try big.data.live_set.put(gpa, operand, {});
1960	},
1961
1962	.main_analysis => {
1963	if ((try big.data.live_set.fetchPut(gpa, operand, {})) == null) {
1964	log.debug("[{}] %{}: added %{} to live set (operand dies here)", .{ pass, big.inst, operand });
1965	big.extra_tombs[extra_byte] \|= @as(u32, 1) << extra_bit;
1966	}
1967	},
1968	}
1969	}
1970
1971	fn finish(big: *Self) !void {
1972	const gpa = big.a.gpa;
1973
1974	std.debug.assert(big.operands_remaining == 0);
1975
1976	switch (pass) {
1977	.loop_analysis => {},
1978
1979	.main_analysis => {
1980	// Note that the MSB is set on the final tomb to indicate the terminal element. This
1981	// allows for an optimisation where we only add as many extra tombs as are needed to
1982	// represent the dying operands. Each pass modifies operand bits and so needs to write
1983	// back, so let's figure out how many extra tombs we really need. Note that we always
1984	// keep at least one.
1985	var num: usize = big.extra_tombs.len;
1986	while (num > 1) {
1987	if (@as(u31, @truncate(big.extra_tombs[num - 1])) != 0) {
1988	// Some operand dies here
1989	break;
1990	}
1991	num -= 1;
1992	}
1993	// Mark final tomb
1994	big.extra_tombs[num - 1] \|= @as(u32, 1) << 31;
1995
1996	const extra_tombs = big.extra_tombs[0..num];
1997
1998	const extra_index = @as(u32, @intCast(big.a.extra.items.len));
1999	try big.a.extra.appendSlice(gpa, extra_tombs);
2000	try big.a.special.put(gpa, big.inst, extra_index);
2001	},
2002	}
2003
2004	try analyzeOperands(big.a, pass, big.data, big.inst, big.small);
2005	}
2006
2007	fn deinit(big: *Self) void {
2008	big.a.gpa.free(big.extra_tombs);
2009	}
2010	};
2011	}
2012
2013	fn fmtInstSet(set: *const std.AutoHashMapUnmanaged(Air.Inst.Index, void)) FmtInstSet {
2014	return .{ .set = set };
2015	}
2016
2017	const FmtInstSet = struct {
2018	set: *const std.AutoHashMapUnmanaged(Air.Inst.Index, void),
2019
2020	pub fn format(val: FmtInstSet, comptime _: []const u8, _: std.fmt.FormatOptions, w: anytype) !void {
2021	if (val.set.count() == 0) {
2022	try w.writeAll("[no instructions]");
2023	return;
2024	}
2025	var it = val.set.keyIterator();
2026	try w.print("%{}", .{it.next().?.*});
2027	while (it.next()) \|key\| {
2028	try w.print(" %{}", .{key.*});
2029	}
2030	}
2031	};
2032
2033	fn fmtInstList(list: []const Air.Inst.Index) FmtInstList {
2034	return .{ .list = list };
2035	}
2036
2037	const FmtInstList = struct {
2038	list: []const Air.Inst.Index,
2039
2040	pub fn format(val: FmtInstList, comptime _: []const u8, _: std.fmt.FormatOptions, w: anytype) !void {
2041	if (val.list.len == 0) {
2042	try w.writeAll("[no instructions]");
2043	return;
2044	}
2045	try w.print("%{}", .{val.list[0]});
2046	for (val.list[1..]) \|inst\| {
2047	try w.print(" %{}", .{inst});
2048	}
2049	}
2050	};

src/Air/Liveness/Verify.zig created+642

...	@@ -0,0 +1,642 @@
1	//! Verifies that Liveness information is valid.
2
3	gpa: std.mem.Allocator,
4	air: Air,
5	liveness: Liveness,
6	live: LiveMap = .{},
7	blocks: std.AutoHashMapUnmanaged(Air.Inst.Index, LiveMap) = .empty,
8	loops: std.AutoHashMapUnmanaged(Air.Inst.Index, LiveMap) = .empty,
9	intern_pool: *const InternPool,
10
11	pub const Error = error{ LivenessInvalid, OutOfMemory };
12
13	pub fn deinit(self: *Verify) void {
14	self.live.deinit(self.gpa);
15	{
16	var it = self.blocks.valueIterator();
17	while (it.next()) \|block\| block.deinit(self.gpa);
18	self.blocks.deinit(self.gpa);
19	}
20	{
21	var it = self.loops.valueIterator();
22	while (it.next()) \|block\| block.deinit(self.gpa);
23	self.loops.deinit(self.gpa);
24	}
25	self.* = undefined;
26	}
27
28	pub fn verify(self: *Verify) Error!void {
29	self.live.clearRetainingCapacity();
30	self.blocks.clearRetainingCapacity();
31	self.loops.clearRetainingCapacity();
32	try self.verifyBody(self.air.getMainBody());
33	// We don't care about `self.live` now, because the loop body was noreturn - everything being dead was checked on `ret` etc
34	assert(self.blocks.count() == 0);
35	assert(self.loops.count() == 0);
36	}
37
38	const LiveMap = std.AutoHashMapUnmanaged(Air.Inst.Index, void);
39
40	fn verifyBody(self: *Verify, body: []const Air.Inst.Index) Error!void {
41	const ip = self.intern_pool;
42	const tags = self.air.instructions.items(.tag);
43	const data = self.air.instructions.items(.data);
44	for (body) \|inst\| {
45	if (self.liveness.isUnused(inst) and !self.air.mustLower(inst, ip)) {
46	// This instruction will not be lowered and should be ignored.
47	continue;
48	}
49
50	switch (tags[@intFromEnum(inst)]) {
51	// no operands
52	.arg,
53	.alloc,
54	.inferred_alloc,
55	.inferred_alloc_comptime,
56	.ret_ptr,
57	.breakpoint,
58	.dbg_stmt,
59	.dbg_empty_stmt,
60	.ret_addr,
61	.frame_addr,
62	.wasm_memory_size,
63	.err_return_trace,
64	.save_err_return_trace_index,
65	.tlv_dllimport_ptr,
66	.c_va_start,
67	.work_item_id,
68	.work_group_size,
69	.work_group_id,
70	=> try self.verifyInstOperands(inst, .{ .none, .none, .none }),
71
72	.trap, .unreach => {
73	try self.verifyInstOperands(inst, .{ .none, .none, .none });
74	// This instruction terminates the function, so everything should be dead
75	if (self.live.count() > 0) return invalid("%{}: instructions still alive", .{inst});
76	},
77
78	// unary
79	.not,
80	.bitcast,
81	.load,
82	.fpext,
83	.fptrunc,
84	.intcast,
85	.intcast_safe,
86	.trunc,
87	.optional_payload,
88	.optional_payload_ptr,
89	.optional_payload_ptr_set,
90	.errunion_payload_ptr_set,
91	.wrap_optional,
92	.unwrap_errunion_payload,
93	.unwrap_errunion_err,
94	.unwrap_errunion_payload_ptr,
95	.unwrap_errunion_err_ptr,
96	.wrap_errunion_payload,
97	.wrap_errunion_err,
98	.slice_ptr,
99	.slice_len,
100	.ptr_slice_len_ptr,
101	.ptr_slice_ptr_ptr,
102	.struct_field_ptr_index_0,
103	.struct_field_ptr_index_1,
104	.struct_field_ptr_index_2,
105	.struct_field_ptr_index_3,
106	.array_to_slice,
107	.int_from_float,
108	.int_from_float_optimized,
109	.float_from_int,
110	.get_union_tag,
111	.clz,
112	.ctz,
113	.popcount,
114	.byte_swap,
115	.bit_reverse,
116	.splat,
117	.error_set_has_value,
118	.addrspace_cast,
119	.c_va_arg,
120	.c_va_copy,
121	.abs,
122	=> {
123	const ty_op = data[@intFromEnum(inst)].ty_op;
124	try self.verifyInstOperands(inst, .{ ty_op.operand, .none, .none });
125	},
126	.is_null,
127	.is_non_null,
128	.is_null_ptr,
129	.is_non_null_ptr,
130	.is_err,
131	.is_non_err,
132	.is_err_ptr,
133	.is_non_err_ptr,
134	.is_named_enum_value,
135	.tag_name,
136	.error_name,
137	.sqrt,
138	.sin,
139	.cos,
140	.tan,
141	.exp,
142	.exp2,
143	.log,
144	.log2,
145	.log10,
146	.floor,
147	.ceil,
148	.round,
149	.trunc_float,
150	.neg,
151	.neg_optimized,
152	.cmp_lt_errors_len,
153	.set_err_return_trace,
154	.c_va_end,
155	=> {
156	const un_op = data[@intFromEnum(inst)].un_op;
157	try self.verifyInstOperands(inst, .{ un_op, .none, .none });
158	},
159	.ret,
160	.ret_safe,
161	.ret_load,
162	=> {
163	const un_op = data[@intFromEnum(inst)].un_op;
164	try self.verifyInstOperands(inst, .{ un_op, .none, .none });
165	// This instruction terminates the function, so everything should be dead
166	if (self.live.count() > 0) return invalid("%{}: instructions still alive", .{inst});
167	},
168	.dbg_var_ptr,
169	.dbg_var_val,
170	.dbg_arg_inline,
171	.wasm_memory_grow,
172	=> {
173	const pl_op = data[@intFromEnum(inst)].pl_op;
174	try self.verifyInstOperands(inst, .{ pl_op.operand, .none, .none });
175	},
176	.prefetch => {
177	const prefetch = data[@intFromEnum(inst)].prefetch;
178	try self.verifyInstOperands(inst, .{ prefetch.ptr, .none, .none });
179	},
180	.reduce,
181	.reduce_optimized,
182	=> {
183	const reduce = data[@intFromEnum(inst)].reduce;
184	try self.verifyInstOperands(inst, .{ reduce.operand, .none, .none });
185	},
186	.union_init => {
187	const ty_pl = data[@intFromEnum(inst)].ty_pl;
188	const extra = self.air.extraData(Air.UnionInit, ty_pl.payload).data;
189	try self.verifyInstOperands(inst, .{ extra.init, .none, .none });
190	},
191	.struct_field_ptr, .struct_field_val => {
192	const ty_pl = data[@intFromEnum(inst)].ty_pl;
193	const extra = self.air.extraData(Air.StructField, ty_pl.payload).data;
194	try self.verifyInstOperands(inst, .{ extra.struct_operand, .none, .none });
195	},
196	.field_parent_ptr => {
197	const ty_pl = data[@intFromEnum(inst)].ty_pl;
198	const extra = self.air.extraData(Air.FieldParentPtr, ty_pl.payload).data;
199	try self.verifyInstOperands(inst, .{ extra.field_ptr, .none, .none });
200	},
201	.atomic_load => {
202	const atomic_load = data[@intFromEnum(inst)].atomic_load;
203	try self.verifyInstOperands(inst, .{ atomic_load.ptr, .none, .none });
204	},
205
206	// binary
207	.add,
208	.add_safe,
209	.add_optimized,
210	.add_wrap,
211	.add_sat,
212	.sub,
213	.sub_safe,
214	.sub_optimized,
215	.sub_wrap,
216	.sub_sat,
217	.mul,
218	.mul_safe,
219	.mul_optimized,
220	.mul_wrap,
221	.mul_sat,
222	.div_float,
223	.div_float_optimized,
224	.div_trunc,
225	.div_trunc_optimized,
226	.div_floor,
227	.div_floor_optimized,
228	.div_exact,
229	.div_exact_optimized,
230	.rem,
231	.rem_optimized,
232	.mod,
233	.mod_optimized,
234	.bit_and,
235	.bit_or,
236	.xor,
237	.cmp_lt,
238	.cmp_lt_optimized,
239	.cmp_lte,
240	.cmp_lte_optimized,
241	.cmp_eq,
242	.cmp_eq_optimized,
243	.cmp_gte,
244	.cmp_gte_optimized,
245	.cmp_gt,
246	.cmp_gt_optimized,
247	.cmp_neq,
248	.cmp_neq_optimized,
249	.bool_and,
250	.bool_or,
251	.store,
252	.store_safe,
253	.array_elem_val,
254	.slice_elem_val,
255	.ptr_elem_val,
256	.shl,
257	.shl_exact,
258	.shl_sat,
259	.shr,
260	.shr_exact,
261	.atomic_store_unordered,
262	.atomic_store_monotonic,
263	.atomic_store_release,
264	.atomic_store_seq_cst,
265	.set_union_tag,
266	.min,
267	.max,
268	.memset,
269	.memset_safe,
270	.memcpy,
271	.memmove,
272	=> {
273	const bin_op = data[@intFromEnum(inst)].bin_op;
274	try self.verifyInstOperands(inst, .{ bin_op.lhs, bin_op.rhs, .none });
275	},
276	.add_with_overflow,
277	.sub_with_overflow,
278	.mul_with_overflow,
279	.shl_with_overflow,
280	.ptr_add,
281	.ptr_sub,
282	.ptr_elem_ptr,
283	.slice_elem_ptr,
284	.slice,
285	=> {
286	const ty_pl = data[@intFromEnum(inst)].ty_pl;
287	const extra = self.air.extraData(Air.Bin, ty_pl.payload).data;
288	try self.verifyInstOperands(inst, .{ extra.lhs, extra.rhs, .none });
289	},
290	.shuffle => {
291	const ty_pl = data[@intFromEnum(inst)].ty_pl;
292	const extra = self.air.extraData(Air.Shuffle, ty_pl.payload).data;
293	try self.verifyInstOperands(inst, .{ extra.a, extra.b, .none });
294	},
295	.cmp_vector,
296	.cmp_vector_optimized,
297	=> {
298	const ty_pl = data[@intFromEnum(inst)].ty_pl;
299	const extra = self.air.extraData(Air.VectorCmp, ty_pl.payload).data;
300	try self.verifyInstOperands(inst, .{ extra.lhs, extra.rhs, .none });
301	},
302	.atomic_rmw => {
303	const pl_op = data[@intFromEnum(inst)].pl_op;
304	const extra = self.air.extraData(Air.AtomicRmw, pl_op.payload).data;
305	try self.verifyInstOperands(inst, .{ pl_op.operand, extra.operand, .none });
306	},
307
308	// ternary
309	.select => {
310	const pl_op = data[@intFromEnum(inst)].pl_op;
311	const extra = self.air.extraData(Air.Bin, pl_op.payload).data;
312	try self.verifyInstOperands(inst, .{ pl_op.operand, extra.lhs, extra.rhs });
313	},
314	.mul_add => {
315	const pl_op = data[@intFromEnum(inst)].pl_op;
316	const extra = self.air.extraData(Air.Bin, pl_op.payload).data;
317	try self.verifyInstOperands(inst, .{ extra.lhs, extra.rhs, pl_op.operand });
318	},
319	.vector_store_elem => {
320	const vector_store_elem = data[@intFromEnum(inst)].vector_store_elem;
321	const extra = self.air.extraData(Air.Bin, vector_store_elem.payload).data;
322	try self.verifyInstOperands(inst, .{ vector_store_elem.vector_ptr, extra.lhs, extra.rhs });
323	},
324	.cmpxchg_strong,
325	.cmpxchg_weak,
326	=> {
327	const ty_pl = data[@intFromEnum(inst)].ty_pl;
328	const extra = self.air.extraData(Air.Cmpxchg, ty_pl.payload).data;
329	try self.verifyInstOperands(inst, .{ extra.ptr, extra.expected_value, extra.new_value });
330	},
331
332	// big tombs
333	.aggregate_init => {
334	const ty_pl = data[@intFromEnum(inst)].ty_pl;
335	const aggregate_ty = ty_pl.ty.toType();
336	const len = @as(usize, @intCast(aggregate_ty.arrayLenIp(ip)));
337	const elements = @as([]const Air.Inst.Ref, @ptrCast(self.air.extra.items[ty_pl.payload..][0..len]));
338
339	var bt = self.liveness.iterateBigTomb(inst);
340	for (elements) \|element\| {
341	try self.verifyOperand(inst, element, bt.feed());
342	}
343	try self.verifyInst(inst);
344	},
345	.call, .call_always_tail, .call_never_tail, .call_never_inline => {
346	const pl_op = data[@intFromEnum(inst)].pl_op;
347	const extra = self.air.extraData(Air.Call, pl_op.payload);
348	const args = @as(
349	[]const Air.Inst.Ref,
350	@ptrCast(self.air.extra.items[extra.end..][0..extra.data.args_len]),
351	);
352
353	var bt = self.liveness.iterateBigTomb(inst);
354	try self.verifyOperand(inst, pl_op.operand, bt.feed());
355	for (args) \|arg\| {
356	try self.verifyOperand(inst, arg, bt.feed());
357	}
358	try self.verifyInst(inst);
359	},
360	.assembly => {
361	const ty_pl = data[@intFromEnum(inst)].ty_pl;
362	const extra = self.air.extraData(Air.Asm, ty_pl.payload);
363	var extra_i = extra.end;
364	const outputs = @as(
365	[]const Air.Inst.Ref,
366	@ptrCast(self.air.extra.items[extra_i..][0..extra.data.outputs_len]),
367	);
368	extra_i += outputs.len;
369	const inputs = @as(
370	[]const Air.Inst.Ref,
371	@ptrCast(self.air.extra.items[extra_i..][0..extra.data.inputs_len]),
372	);
373	extra_i += inputs.len;
374
375	var bt = self.liveness.iterateBigTomb(inst);
376	for (outputs) \|output\| {
377	if (output != .none) {
378	try self.verifyOperand(inst, output, bt.feed());
379	}
380	}
381	for (inputs) \|input\| {
382	try self.verifyOperand(inst, input, bt.feed());
383	}
384	try self.verifyInst(inst);
385	},
386
387	// control flow
388	.@"try", .try_cold => {
389	const pl_op = data[@intFromEnum(inst)].pl_op;
390	const extra = self.air.extraData(Air.Try, pl_op.payload);
391	const try_body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[extra.end..][0..extra.data.body_len]);
392
393	const cond_br_liveness = self.liveness.getCondBr(inst);
394
395	try self.verifyOperand(inst, pl_op.operand, self.liveness.operandDies(inst, 0));
396
397	var live = try self.live.clone(self.gpa);
398	defer live.deinit(self.gpa);
399
400	for (cond_br_liveness.else_deaths) \|death\| try self.verifyDeath(inst, death);
401	try self.verifyBody(try_body);
402
403	self.live.deinit(self.gpa);
404	self.live = live.move();
405
406	for (cond_br_liveness.then_deaths) \|death\| try self.verifyDeath(inst, death);
407
408	try self.verifyInst(inst);
409	},
410	.try_ptr, .try_ptr_cold => {
411	const ty_pl = data[@intFromEnum(inst)].ty_pl;
412	const extra = self.air.extraData(Air.TryPtr, ty_pl.payload);
413	const try_body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[extra.end..][0..extra.data.body_len]);
414
415	const cond_br_liveness = self.liveness.getCondBr(inst);
416
417	try self.verifyOperand(inst, extra.data.ptr, self.liveness.operandDies(inst, 0));
418
419	var live = try self.live.clone(self.gpa);
420	defer live.deinit(self.gpa);
421
422	for (cond_br_liveness.else_deaths) \|death\| try self.verifyDeath(inst, death);
423	try self.verifyBody(try_body);
424
425	self.live.deinit(self.gpa);
426	self.live = live.move();
427
428	for (cond_br_liveness.then_deaths) \|death\| try self.verifyDeath(inst, death);
429
430	try self.verifyInst(inst);
431	},
432	.br => {
433	const br = data[@intFromEnum(inst)].br;
434	const gop = try self.blocks.getOrPut(self.gpa, br.block_inst);
435
436	try self.verifyOperand(inst, br.operand, self.liveness.operandDies(inst, 0));
437	if (gop.found_existing) {
438	try self.verifyMatchingLiveness(br.block_inst, gop.value_ptr.*);
439	} else {
440	gop.value_ptr.* = try self.live.clone(self.gpa);
441	}
442	try self.verifyInst(inst);
443	},
444	.repeat => {
445	const repeat = data[@intFromEnum(inst)].repeat;
446	const expected_live = self.loops.get(repeat.loop_inst) orelse
447	return invalid("%{}: loop %{} not in scope", .{ @intFromEnum(inst), @intFromEnum(repeat.loop_inst) });
448
449	try self.verifyMatchingLiveness(repeat.loop_inst, expected_live);
450	},
451	.switch_dispatch => {
452	const br = data[@intFromEnum(inst)].br;
453
454	try self.verifyOperand(inst, br.operand, self.liveness.operandDies(inst, 0));
455
456	const expected_live = self.loops.get(br.block_inst) orelse
457	return invalid("%{}: loop %{} not in scope", .{ @intFromEnum(inst), @intFromEnum(br.block_inst) });
458
459	try self.verifyMatchingLiveness(br.block_inst, expected_live);
460	},
461	.block, .dbg_inline_block => \|tag\| {
462	const ty_pl = data[@intFromEnum(inst)].ty_pl;
463	const block_ty = ty_pl.ty.toType();
464	const block_body: []const Air.Inst.Index = @ptrCast(switch (tag) {
465	inline .block, .dbg_inline_block => \|comptime_tag\| body: {
466	const extra = self.air.extraData(switch (comptime_tag) {
467	.block => Air.Block,
468	.dbg_inline_block => Air.DbgInlineBlock,
469	else => unreachable,
470	}, ty_pl.payload);
471	break :body self.air.extra.items[extra.end..][0..extra.data.body_len];
472	},
473	else => unreachable,
474	});
475	const block_liveness = self.liveness.getBlock(inst);
476
477	var orig_live = try self.live.clone(self.gpa);
478	defer orig_live.deinit(self.gpa);
479
480	assert(!self.blocks.contains(inst));
481	try self.verifyBody(block_body);
482
483	// Liveness data after the block body is garbage, but we want to
484	// restore it to verify deaths
485	self.live.deinit(self.gpa);
486	self.live = orig_live.move();
487
488	for (block_liveness.deaths) \|death\| try self.verifyDeath(inst, death);
489
490	if (ip.isNoReturn(block_ty.toIntern())) {
491	assert(!self.blocks.contains(inst));
492	} else {
493	var live = self.blocks.fetchRemove(inst).?.value;
494	defer live.deinit(self.gpa);
495
496	try self.verifyMatchingLiveness(inst, live);
497	}
498
499	try self.verifyInstOperands(inst, .{ .none, .none, .none });
500	},
501	.loop => {
502	const ty_pl = data[@intFromEnum(inst)].ty_pl;
503	const extra = self.air.extraData(Air.Block, ty_pl.payload);
504	const loop_body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[extra.end..][0..extra.data.body_len]);
505
506	// The same stuff should be alive after the loop as before it.
507	const gop = try self.loops.getOrPut(self.gpa, inst);
508	if (gop.found_existing) return invalid("%{}: loop already exists", .{@intFromEnum(inst)});
509	defer {
510	var live = self.loops.fetchRemove(inst).?;
511	live.value.deinit(self.gpa);
512	}
513	gop.value_ptr.* = try self.live.clone(self.gpa);
514
515	try self.verifyBody(loop_body);
516
517	try self.verifyInstOperands(inst, .{ .none, .none, .none });
518	},
519	.cond_br => {
520	const pl_op = data[@intFromEnum(inst)].pl_op;
521	const extra = self.air.extraData(Air.CondBr, pl_op.payload);
522	const then_body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[extra.end..][0..extra.data.then_body_len]);
523	const else_body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[extra.end + then_body.len ..][0..extra.data.else_body_len]);
524	const cond_br_liveness = self.liveness.getCondBr(inst);
525
526	try self.verifyOperand(inst, pl_op.operand, self.liveness.operandDies(inst, 0));
527
528	var live = try self.live.clone(self.gpa);
529	defer live.deinit(self.gpa);
530
531	for (cond_br_liveness.then_deaths) \|death\| try self.verifyDeath(inst, death);
532	try self.verifyBody(then_body);
533
534	self.live.deinit(self.gpa);
535	self.live = live.move();
536
537	for (cond_br_liveness.else_deaths) \|death\| try self.verifyDeath(inst, death);
538	try self.verifyBody(else_body);
539
540	try self.verifyInst(inst);
541	},
542	.switch_br, .loop_switch_br => {
543	const switch_br = self.air.unwrapSwitch(inst);
544	const switch_br_liveness = try self.liveness.getSwitchBr(
545	self.gpa,
546	inst,
547	switch_br.cases_len + 1,
548	);
549	defer self.gpa.free(switch_br_liveness.deaths);
550
551	try self.verifyOperand(inst, switch_br.operand, self.liveness.operandDies(inst, 0));
552
553	// Excluding the operand (which we just handled), the same stuff should be alive
554	// after the loop as before it.
555	{
556	const gop = try self.loops.getOrPut(self.gpa, inst);
557	if (gop.found_existing) return invalid("%{}: loop already exists", .{@intFromEnum(inst)});
558	gop.value_ptr.* = self.live.move();
559	}
560	defer {
561	var live = self.loops.fetchRemove(inst).?;
562	live.value.deinit(self.gpa);
563	}
564
565	var it = switch_br.iterateCases();
566	while (it.next()) \|case\| {
567	self.live.deinit(self.gpa);
568	self.live = try self.loops.get(inst).?.clone(self.gpa);
569
570	for (switch_br_liveness.deaths[case.idx]) \|death\| try self.verifyDeath(inst, death);
571	try self.verifyBody(case.body);
572	}
573
574	const else_body = it.elseBody();
575	if (else_body.len > 0) {
576	self.live.deinit(self.gpa);
577	self.live = try self.loops.get(inst).?.clone(self.gpa);
578	for (switch_br_liveness.deaths[switch_br.cases_len]) \|death\| try self.verifyDeath(inst, death);
579	try self.verifyBody(else_body);
580	}
581
582	try self.verifyInst(inst);
583	},
584	}
585	}
586	}
587
588	fn verifyDeath(self: *Verify, inst: Air.Inst.Index, operand: Air.Inst.Index) Error!void {
589	try self.verifyOperand(inst, operand.toRef(), true);
590	}
591
592	fn verifyOperand(self: *Verify, inst: Air.Inst.Index, op_ref: Air.Inst.Ref, dies: bool) Error!void {
593	const operand = op_ref.toIndexAllowNone() orelse {
594	assert(!dies);
595	return;
596	};
597	if (dies) {
598	if (!self.live.remove(operand)) return invalid("%{}: dead operand %{} reused and killed again", .{ inst, operand });
599	} else {
600	if (!self.live.contains(operand)) return invalid("%{}: dead operand %{} reused", .{ inst, operand });
601	}
602	}
603
604	fn verifyInstOperands(
605	self: *Verify,
606	inst: Air.Inst.Index,
607	operands: [Liveness.bpi - 1]Air.Inst.Ref,
608	) Error!void {
609	for (operands, 0..) \|operand, operand_index\| {
610	const dies = self.liveness.operandDies(inst, @as(Liveness.OperandInt, @intCast(operand_index)));
611	try self.verifyOperand(inst, operand, dies);
612	}
613	try self.verifyInst(inst);
614	}
615
616	fn verifyInst(self: *Verify, inst: Air.Inst.Index) Error!void {
617	if (self.liveness.isUnused(inst)) {
618	assert(!self.live.contains(inst));
619	} else {
620	try self.live.putNoClobber(self.gpa, inst, {});
621	}
622	}
623
624	fn verifyMatchingLiveness(self: *Verify, block: Air.Inst.Index, live: LiveMap) Error!void {
625	if (self.live.count() != live.count()) return invalid("%{}: different deaths across branches", .{block});
626	var live_it = self.live.keyIterator();
627	while (live_it.next()) \|live_inst\| if (!live.contains(live_inst.*)) return invalid("%{}: different deaths across branches", .{block});
628	}
629
630	fn invalid(comptime fmt: []const u8, args: anytype) error{LivenessInvalid} {
631	log.err(fmt, args);
632	return error.LivenessInvalid;
633	}
634
635	const std = @import("std");
636	const assert = std.debug.assert;
637	const log = std.log.scoped(.liveness_verify);
638
639	const Air = @import("../../Air.zig");
640	const Liveness = @import("../Liveness.zig");
641	const InternPool = @import("../../InternPool.zig");
642	const Verify = @This();

src/Air/types_resolved.zig+10-10

src/Liveness.zig deleted-2050

...	@@ -1,2050 +0,0 @@
1	//! For each AIR instruction, we want to know:
2	//! * Is the instruction unreferenced (e.g. dies immediately)?
3	//! * For each of its operands, does the operand die with this instruction (e.g. is
4	//! this the last reference to it)?
5	//! Some instructions are special, such as:
6	//! * Conditional Branches
7	//! * Switch Branches
8	const std = @import("std");
9	const log = std.log.scoped(.liveness);
10	const assert = std.debug.assert;
11	const Allocator = std.mem.Allocator;
12	const Log2Int = std.math.Log2Int;
13
14	const Liveness = @This();
15	const trace = @import("tracy.zig").trace;
16	const Air = @import("Air.zig");
17	const InternPool = @import("InternPool.zig");
18
19	pub const Verify = @import("Liveness/Verify.zig");
20
21	/// This array is split into sets of 4 bits per AIR instruction.
22	/// The MSB (0bX000) is whether the instruction is unreferenced.
23	/// The LSB (0b000X) is the first operand, and so on, up to 3 operands. A set bit means the
24	/// operand dies after this instruction.
25	/// Instructions which need more data to track liveness have special handling via the
26	/// `special` table.
27	tomb_bits: []usize,
28	/// Sparse table of specially handled instructions. The value is an index into the `extra`
29	/// array. The meaning of the data depends on the AIR tag.
30	/// * `cond_br` - points to a `CondBr` in `extra` at this index.
31	/// * `try`, `try_ptr` - points to a `CondBr` in `extra` at this index. The error path (the block
32	/// in the instruction) is considered the "else" path, and the rest of the block the "then".
33	/// * `switch_br` - points to a `SwitchBr` in `extra` at this index.
34	/// * `loop_switch_br` - points to a `SwitchBr` in `extra` at this index.
35	/// * `block` - points to a `Block` in `extra` at this index.
36	/// * `asm`, `call`, `aggregate_init` - the value is a set of bits which are the extra tomb
37	/// bits of operands.
38	/// The main tomb bits are still used and the extra ones are starting with the lsb of the
39	/// value here.
40	special: std.AutoHashMapUnmanaged(Air.Inst.Index, u32),
41	/// Auxiliary data. The way this data is interpreted is determined contextually.
42	extra: []const u32,
43
44	/// Trailing is the set of instructions whose lifetimes end at the start of the then branch,
45	/// followed by the set of instructions whose lifetimes end at the start of the else branch.
46	pub const CondBr = struct {
47	then_death_count: u32,
48	else_death_count: u32,
49	};
50
51	/// Trailing is:
52	/// * For each case in the same order as in the AIR:
53	/// - case_death_count: u32
54	/// - Air.Inst.Index for each `case_death_count`: set of instructions whose lifetimes
55	/// end at the start of this case.
56	/// * Air.Inst.Index for each `else_death_count`: set of instructions whose lifetimes
57	/// end at the start of the else case.
58	pub const SwitchBr = struct {
59	else_death_count: u32,
60	};
61
62	/// Trailing is the set of instructions which die in the block. Note that these are not additional
63	/// deaths (they are all recorded as normal within the block), but backends may use this information
64	/// as a more efficient way to track which instructions are still alive after a block.
65	pub const Block = struct {
66	death_count: u32,
67	};
68
69	/// Liveness analysis runs in several passes. Each pass iterates backwards over instructions in
70	/// bodies, and recurses into bodies.
71	const LivenessPass = enum {
72	/// In this pass, we perform some basic analysis of loops to gain information the main pass needs.
73	/// In particular, for every `loop` and `loop_switch_br`, we track the following information:
74	/// * Every outer block which the loop body contains a `br` to.
75	/// * Every outer loop which the loop body contains a `repeat` to.
76	/// * Every operand referenced within the loop body but created outside the loop.
77	/// This gives the main analysis pass enough information to determine the full set of
78	/// instructions which need to be alive when a loop repeats. This data is TEMPORARILY stored in
79	/// `a.extra`. It is not re-added to `extra` by the main pass, since it is not useful to
80	/// backends.
81	loop_analysis,
82
83	/// This pass performs the main liveness analysis, setting up tombs and extra data while
84	/// considering control flow etc.
85	main_analysis,
86	};
87
88	/// Each analysis pass may wish to pass data through calls. A pointer to a `LivenessPassData(pass)`
89	/// stored on the stack is passed through calls to `analyzeInst` etc.
90	fn LivenessPassData(comptime pass: LivenessPass) type {
91	return switch (pass) {
92	.loop_analysis => struct {
93	/// The set of blocks which are exited with a `br` instruction at some point within this
94	/// body and which we are currently within. Also includes `loop`s which are the target
95	/// of a `repeat` instruction, and `loop_switch_br`s which are the target of a
96	/// `switch_dispatch` instruction.
97	breaks: std.AutoHashMapUnmanaged(Air.Inst.Index, void) = .empty,
98
99	/// The set of operands for which we have seen at least one usage but not their birth.
100	live_set: std.AutoHashMapUnmanaged(Air.Inst.Index, void) = .empty,
101
102	fn deinit(self: *@This(), gpa: Allocator) void {
103	self.breaks.deinit(gpa);
104	self.live_set.deinit(gpa);
105	}
106	},
107
108	.main_analysis => struct {
109	/// Every `block` and `loop` currently under analysis.
110	block_scopes: std.AutoHashMapUnmanaged(Air.Inst.Index, BlockScope) = .empty,
111
112	/// The set of instructions currently alive in the current control
113	/// flow branch.
114	live_set: std.AutoHashMapUnmanaged(Air.Inst.Index, void) = .empty,
115
116	/// The extra data initialized by the `loop_analysis` pass for this pass to consume.
117	/// Owned by this struct during this pass.
118	old_extra: std.ArrayListUnmanaged(u32) = .empty,
119
120	const BlockScope = struct {
121	/// If this is a `block`, these instructions are alive upon a `br` to this block.
122	/// If this is a `loop`, these instructions are alive upon a `repeat` to this block.
123	live_set: std.AutoHashMapUnmanaged(Air.Inst.Index, void),
124	};
125
126	fn deinit(self: *@This(), gpa: Allocator) void {
127	var it = self.block_scopes.valueIterator();
128	while (it.next()) \|block\| {
129	block.live_set.deinit(gpa);
130	}
131	self.block_scopes.deinit(gpa);
132	self.live_set.deinit(gpa);
133	self.old_extra.deinit(gpa);
134	}
135	},
136	};
137	}
138
139	pub fn analyze(gpa: Allocator, air: Air, intern_pool: *InternPool) Allocator.Error!Liveness {
140	const tracy = trace(@src());
141	defer tracy.end();
142
143	var a: Analysis = .{
144	.gpa = gpa,
145	.air = air,
146	.tomb_bits = try gpa.alloc(
147	usize,
148	(air.instructions.len * bpi + @bitSizeOf(usize) - 1) / @bitSizeOf(usize),
149	),
150	.extra = .{},
151	.special = .{},
152	.intern_pool = intern_pool,
153	};
154	errdefer gpa.free(a.tomb_bits);
155	errdefer a.special.deinit(gpa);
156	defer a.extra.deinit(gpa);
157
158	@memset(a.tomb_bits, 0);
159
160	const main_body = air.getMainBody();
161
162	{
163	var data: LivenessPassData(.loop_analysis) = .{};
164	defer data.deinit(gpa);
165	try analyzeBody(&a, .loop_analysis, &data, main_body);
166	}
167
168	{
169	var data: LivenessPassData(.main_analysis) = .{};
170	defer data.deinit(gpa);
171	data.old_extra = a.extra;
172	a.extra = .{};
173	try analyzeBody(&a, .main_analysis, &data, main_body);
174	assert(data.live_set.count() == 0);
175	}
176
177	return .{
178	.tomb_bits = a.tomb_bits,
179	.special = a.special,
180	.extra = try a.extra.toOwnedSlice(gpa),
181	};
182	}
183
184	pub fn getTombBits(l: Liveness, inst: Air.Inst.Index) Bpi {
185	const usize_index = (@intFromEnum(inst) * bpi) / @bitSizeOf(usize);
186	return @as(Bpi, @truncate(l.tomb_bits[usize_index] >>
187	@as(Log2Int(usize), @intCast((@intFromEnum(inst) % (@bitSizeOf(usize) / bpi)) * bpi))));
188	}
189
190	pub fn isUnused(l: Liveness, inst: Air.Inst.Index) bool {
191	const usize_index = (@intFromEnum(inst) * bpi) / @bitSizeOf(usize);
192	const mask = @as(usize, 1) <<
193	@as(Log2Int(usize), @intCast((@intFromEnum(inst) % (@bitSizeOf(usize) / bpi)) * bpi + (bpi - 1)));
194	return (l.tomb_bits[usize_index] & mask) != 0;
195	}
196
197	pub fn operandDies(l: Liveness, inst: Air.Inst.Index, operand: OperandInt) bool {
198	assert(operand < bpi - 1);
199	const usize_index = (@intFromEnum(inst) * bpi) / @bitSizeOf(usize);
200	const mask = @as(usize, 1) <<
201	@as(Log2Int(usize), @intCast((@intFromEnum(inst) % (@bitSizeOf(usize) / bpi)) * bpi + operand));
202	return (l.tomb_bits[usize_index] & mask) != 0;
203	}
204
205	const OperandCategory = enum {
206	/// The operand lives on, but this instruction cannot possibly mutate memory.
207	none,
208	/// The operand lives on and this instruction can mutate memory.
209	write,
210	/// The operand dies at this instruction.
211	tomb,
212	/// The operand lives on, and this instruction is noreturn.
213	noret,
214	/// This instruction is too complicated for analysis, no information is available.
215	complex,
216	};
217
218	/// Given an instruction that we are examining, and an operand that we are looking for,
219	/// returns a classification.
220	pub fn categorizeOperand(
221	l: Liveness,
222	air: Air,
223	inst: Air.Inst.Index,
224	operand: Air.Inst.Index,
225	ip: *const InternPool,
226	) OperandCategory {
227	const air_tags = air.instructions.items(.tag);
228	const air_datas = air.instructions.items(.data);
229	const operand_ref = operand.toRef();
230	switch (air_tags[@intFromEnum(inst)]) {
231	.add,
232	.add_safe,
233	.add_wrap,
234	.add_sat,
235	.add_optimized,
236	.sub,
237	.sub_safe,
238	.sub_wrap,
239	.sub_sat,
240	.sub_optimized,
241	.mul,
242	.mul_safe,
243	.mul_wrap,
244	.mul_sat,
245	.mul_optimized,
246	.div_float,
247	.div_trunc,
248	.div_floor,
249	.div_exact,
250	.rem,
251	.mod,
252	.bit_and,
253	.bit_or,
254	.xor,
255	.cmp_lt,
256	.cmp_lte,
257	.cmp_eq,
258	.cmp_gte,
259	.cmp_gt,
260	.cmp_neq,
261	.bool_and,
262	.bool_or,
263	.array_elem_val,
264	.slice_elem_val,
265	.ptr_elem_val,
266	.shl,
267	.shl_exact,
268	.shl_sat,
269	.shr,
270	.shr_exact,
271	.min,
272	.max,
273	.div_float_optimized,
274	.div_trunc_optimized,
275	.div_floor_optimized,
276	.div_exact_optimized,
277	.rem_optimized,
278	.mod_optimized,
279	.neg_optimized,
280	.cmp_lt_optimized,
281	.cmp_lte_optimized,
282	.cmp_eq_optimized,
283	.cmp_gte_optimized,
284	.cmp_gt_optimized,
285	.cmp_neq_optimized,
286	=> {
287	const o = air_datas[@intFromEnum(inst)].bin_op;
288	if (o.lhs == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
289	if (o.rhs == operand_ref) return matchOperandSmallIndex(l, inst, 1, .none);
290	return .none;
291	},
292
293	.store,
294	.store_safe,
295	.atomic_store_unordered,
296	.atomic_store_monotonic,
297	.atomic_store_release,
298	.atomic_store_seq_cst,
299	.set_union_tag,
300	.memset,
301	.memset_safe,
302	.memcpy,
303	.memmove,
304	=> {
305	const o = air_datas[@intFromEnum(inst)].bin_op;
306	if (o.lhs == operand_ref) return matchOperandSmallIndex(l, inst, 0, .write);
307	if (o.rhs == operand_ref) return matchOperandSmallIndex(l, inst, 1, .write);
308	return .write;
309	},
310
311	.vector_store_elem => {
312	const o = air_datas[@intFromEnum(inst)].vector_store_elem;
313	const extra = air.extraData(Air.Bin, o.payload).data;
314	if (o.vector_ptr == operand_ref) return matchOperandSmallIndex(l, inst, 0, .write);
315	if (extra.lhs == operand_ref) return matchOperandSmallIndex(l, inst, 1, .none);
316	if (extra.rhs == operand_ref) return matchOperandSmallIndex(l, inst, 2, .none);
317	return .write;
318	},
319
320	.arg,
321	.alloc,
322	.inferred_alloc,
323	.inferred_alloc_comptime,
324	.ret_ptr,
325	.trap,
326	.breakpoint,
327	.repeat,
328	.switch_dispatch,
329	.dbg_stmt,
330	.dbg_empty_stmt,
331	.unreach,
332	.ret_addr,
333	.frame_addr,
334	.wasm_memory_size,
335	.err_return_trace,
336	.save_err_return_trace_index,
337	.tlv_dllimport_ptr,
338	.c_va_start,
339	.work_item_id,
340	.work_group_size,
341	.work_group_id,
342	=> return .none,
343
344	.not,
345	.bitcast,
346	.load,
347	.fpext,
348	.fptrunc,
349	.intcast,
350	.intcast_safe,
351	.trunc,
352	.optional_payload,
353	.optional_payload_ptr,
354	.wrap_optional,
355	.unwrap_errunion_payload,
356	.unwrap_errunion_err,
357	.unwrap_errunion_payload_ptr,
358	.unwrap_errunion_err_ptr,
359	.wrap_errunion_payload,
360	.wrap_errunion_err,
361	.slice_ptr,
362	.slice_len,
363	.ptr_slice_len_ptr,
364	.ptr_slice_ptr_ptr,
365	.struct_field_ptr_index_0,
366	.struct_field_ptr_index_1,
367	.struct_field_ptr_index_2,
368	.struct_field_ptr_index_3,
369	.array_to_slice,
370	.int_from_float,
371	.int_from_float_optimized,
372	.float_from_int,
373	.get_union_tag,
374	.clz,
375	.ctz,
376	.popcount,
377	.byte_swap,
378	.bit_reverse,
379	.splat,
380	.error_set_has_value,
381	.addrspace_cast,
382	.c_va_arg,
383	.c_va_copy,
384	.abs,
385	=> {
386	const o = air_datas[@intFromEnum(inst)].ty_op;
387	if (o.operand == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
388	return .none;
389	},
390
391	.optional_payload_ptr_set,
392	.errunion_payload_ptr_set,
393	=> {
394	const o = air_datas[@intFromEnum(inst)].ty_op;
395	if (o.operand == operand_ref) return matchOperandSmallIndex(l, inst, 0, .write);
396	return .write;
397	},
398
399	.is_null,
400	.is_non_null,
401	.is_null_ptr,
402	.is_non_null_ptr,
403	.is_err,
404	.is_non_err,
405	.is_err_ptr,
406	.is_non_err_ptr,
407	.is_named_enum_value,
408	.tag_name,
409	.error_name,
410	.sqrt,
411	.sin,
412	.cos,
413	.tan,
414	.exp,
415	.exp2,
416	.log,
417	.log2,
418	.log10,
419	.floor,
420	.ceil,
421	.round,
422	.trunc_float,
423	.neg,
424	.cmp_lt_errors_len,
425	.c_va_end,
426	=> {
427	const o = air_datas[@intFromEnum(inst)].un_op;
428	if (o == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
429	return .none;
430	},
431
432	.ret,
433	.ret_safe,
434	.ret_load,
435	=> {
436	const o = air_datas[@intFromEnum(inst)].un_op;
437	if (o == operand_ref) return matchOperandSmallIndex(l, inst, 0, .noret);
438	return .noret;
439	},
440
441	.set_err_return_trace => {
442	const o = air_datas[@intFromEnum(inst)].un_op;
443	if (o == operand_ref) return matchOperandSmallIndex(l, inst, 0, .write);
444	return .write;
445	},
446
447	.add_with_overflow,
448	.sub_with_overflow,
449	.mul_with_overflow,
450	.shl_with_overflow,
451	.ptr_add,
452	.ptr_sub,
453	.ptr_elem_ptr,
454	.slice_elem_ptr,
455	.slice,
456	=> {
457	const ty_pl = air_datas[@intFromEnum(inst)].ty_pl;
458	const extra = air.extraData(Air.Bin, ty_pl.payload).data;
459	if (extra.lhs == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
460	if (extra.rhs == operand_ref) return matchOperandSmallIndex(l, inst, 1, .none);
461	return .none;
462	},
463
464	.dbg_var_ptr,
465	.dbg_var_val,
466	.dbg_arg_inline,
467	=> {
468	const o = air_datas[@intFromEnum(inst)].pl_op.operand;
469	if (o == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
470	return .none;
471	},
472
473	.prefetch => {
474	const prefetch = air_datas[@intFromEnum(inst)].prefetch;
475	if (prefetch.ptr == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
476	return .none;
477	},
478
479	.call, .call_always_tail, .call_never_tail, .call_never_inline => {
480	const inst_data = air_datas[@intFromEnum(inst)].pl_op;
481	const callee = inst_data.operand;
482	const extra = air.extraData(Air.Call, inst_data.payload);
483	const args = @as([]const Air.Inst.Ref, @ptrCast(air.extra[extra.end..][0..extra.data.args_len]));
484	if (args.len + 1 <= bpi - 1) {
485	if (callee == operand_ref) return matchOperandSmallIndex(l, inst, 0, .write);
486	for (args, 0..) \|arg, i\| {
487	if (arg == operand_ref) return matchOperandSmallIndex(l, inst, @as(OperandInt, @intCast(i + 1)), .write);
488	}
489	return .write;
490	}
491	var bt = l.iterateBigTomb(inst);
492	if (bt.feed()) {
493	if (callee == operand_ref) return .tomb;
494	} else {
495	if (callee == operand_ref) return .write;
496	}
497	for (args) \|arg\| {
498	if (bt.feed()) {
499	if (arg == operand_ref) return .tomb;
500	} else {
501	if (arg == operand_ref) return .write;
502	}
503	}
504	return .write;
505	},
506	.select => {
507	const pl_op = air_datas[@intFromEnum(inst)].pl_op;
508	const extra = air.extraData(Air.Bin, pl_op.payload).data;
509	if (pl_op.operand == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
510	if (extra.lhs == operand_ref) return matchOperandSmallIndex(l, inst, 1, .none);
511	if (extra.rhs == operand_ref) return matchOperandSmallIndex(l, inst, 2, .none);
512	return .none;
513	},
514	.shuffle => {
515	const extra = air.extraData(Air.Shuffle, air_datas[@intFromEnum(inst)].ty_pl.payload).data;
516	if (extra.a == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
517	if (extra.b == operand_ref) return matchOperandSmallIndex(l, inst, 1, .none);
518	return .none;
519	},
520	.reduce, .reduce_optimized => {
521	const reduce = air_datas[@intFromEnum(inst)].reduce;
522	if (reduce.operand == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
523	return .none;
524	},
525	.cmp_vector, .cmp_vector_optimized => {
526	const extra = air.extraData(Air.VectorCmp, air_datas[@intFromEnum(inst)].ty_pl.payload).data;
527	if (extra.lhs == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
528	if (extra.rhs == operand_ref) return matchOperandSmallIndex(l, inst, 1, .none);
529	return .none;
530	},
531	.aggregate_init => {
532	const ty_pl = air_datas[@intFromEnum(inst)].ty_pl;
533	const aggregate_ty = ty_pl.ty.toType();
534	const len = @as(usize, @intCast(aggregate_ty.arrayLenIp(ip)));
535	const elements = @as([]const Air.Inst.Ref, @ptrCast(air.extra[ty_pl.payload..][0..len]));
536
537	if (elements.len <= bpi - 1) {
538	for (elements, 0..) \|elem, i\| {
539	if (elem == operand_ref) return matchOperandSmallIndex(l, inst, @as(OperandInt, @intCast(i)), .none);
540	}
541	return .none;
542	}
543
544	var bt = l.iterateBigTomb(inst);
545	for (elements) \|elem\| {
546	if (bt.feed()) {
547	if (elem == operand_ref) return .tomb;
548	} else {
549	if (elem == operand_ref) return .write;
550	}
551	}
552	return .write;
553	},
554	.union_init => {
555	const extra = air.extraData(Air.UnionInit, air_datas[@intFromEnum(inst)].ty_pl.payload).data;
556	if (extra.init == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
557	return .none;
558	},
559	.struct_field_ptr, .struct_field_val => {
560	const extra = air.extraData(Air.StructField, air_datas[@intFromEnum(inst)].ty_pl.payload).data;
561	if (extra.struct_operand == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
562	return .none;
563	},
564	.field_parent_ptr => {
565	const extra = air.extraData(Air.FieldParentPtr, air_datas[@intFromEnum(inst)].ty_pl.payload).data;
566	if (extra.field_ptr == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
567	return .none;
568	},
569	.cmpxchg_strong, .cmpxchg_weak => {
570	const extra = air.extraData(Air.Cmpxchg, air_datas[@intFromEnum(inst)].ty_pl.payload).data;
571	if (extra.ptr == operand_ref) return matchOperandSmallIndex(l, inst, 0, .write);
572	if (extra.expected_value == operand_ref) return matchOperandSmallIndex(l, inst, 1, .write);
573	if (extra.new_value == operand_ref) return matchOperandSmallIndex(l, inst, 2, .write);
574	return .write;
575	},
576	.mul_add => {
577	const pl_op = air_datas[@intFromEnum(inst)].pl_op;
578	const extra = air.extraData(Air.Bin, pl_op.payload).data;
579	if (extra.lhs == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
580	if (extra.rhs == operand_ref) return matchOperandSmallIndex(l, inst, 1, .none);
581	if (pl_op.operand == operand_ref) return matchOperandSmallIndex(l, inst, 2, .none);
582	return .none;
583	},
584	.atomic_load => {
585	const ptr = air_datas[@intFromEnum(inst)].atomic_load.ptr;
586	if (ptr == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
587	return .none;
588	},
589	.atomic_rmw => {
590	const pl_op = air_datas[@intFromEnum(inst)].pl_op;
591	const extra = air.extraData(Air.AtomicRmw, pl_op.payload).data;
592	if (pl_op.operand == operand_ref) return matchOperandSmallIndex(l, inst, 0, .write);
593	if (extra.operand == operand_ref) return matchOperandSmallIndex(l, inst, 1, .write);
594	return .write;
595	},
596
597	.br => {
598	const br = air_datas[@intFromEnum(inst)].br;
599	if (br.operand == operand_ref) return matchOperandSmallIndex(l, operand, 0, .noret);
600	return .noret;
601	},
602	.assembly => {
603	return .complex;
604	},
605	.block, .dbg_inline_block => \|tag\| {
606	const ty_pl = air_datas[@intFromEnum(inst)].ty_pl;
607	const body: []const Air.Inst.Index = @ptrCast(switch (tag) {
608	inline .block, .dbg_inline_block => \|comptime_tag\| body: {
609	const extra = air.extraData(switch (comptime_tag) {
610	.block => Air.Block,
611	.dbg_inline_block => Air.DbgInlineBlock,
612	else => unreachable,
613	}, ty_pl.payload);
614	break :body air.extra[extra.end..][0..extra.data.body_len];
615	},
616	else => unreachable,
617	});
618
619	if (body.len == 1 and air_tags[@intFromEnum(body[0])] == .cond_br) {
620	// Peephole optimization for "panic-like" conditionals, which have
621	// one empty branch and another which calls a `noreturn` function.
622	// This allows us to infer that safety checks do not modify memory,
623	// as far as control flow successors are concerned.
624
625	const inst_data = air_datas[@intFromEnum(body[0])].pl_op;
626	const cond_extra = air.extraData(Air.CondBr, inst_data.payload);
627	if (inst_data.operand == operand_ref and operandDies(l, body[0], 0))
628	return .tomb;
629
630	if (cond_extra.data.then_body_len > 2 or cond_extra.data.else_body_len > 2)
631	return .complex;
632
633	const then_body: []const Air.Inst.Index = @ptrCast(air.extra[cond_extra.end..][0..cond_extra.data.then_body_len]);
634	const else_body: []const Air.Inst.Index = @ptrCast(air.extra[cond_extra.end + cond_extra.data.then_body_len ..][0..cond_extra.data.else_body_len]);
635	if (then_body.len > 1 and air_tags[@intFromEnum(then_body[1])] != .unreach)
636	return .complex;
637	if (else_body.len > 1 and air_tags[@intFromEnum(else_body[1])] != .unreach)
638	return .complex;
639
640	var operand_live: bool = true;
641	for (&[_]Air.Inst.Index{ then_body[0], else_body[0] }) \|cond_inst\| {
642	if (l.categorizeOperand(air, cond_inst, operand, ip) == .tomb)
643	operand_live = false;
644
645	switch (air_tags[@intFromEnum(cond_inst)]) {
646	.br => { // Breaks immediately back to block
647	const br = air_datas[@intFromEnum(cond_inst)].br;
648	if (br.block_inst != inst)
649	return .complex;
650	},
651	.call => {}, // Calls a noreturn function
652	else => return .complex,
653	}
654	}
655	return if (operand_live) .none else .tomb;
656	}
657
658	return .complex;
659	},
660
661	.@"try",
662	.try_cold,
663	.try_ptr,
664	.try_ptr_cold,
665	.loop,
666	.cond_br,
667	.switch_br,
668	.loop_switch_br,
669	=> return .complex,
670
671	.wasm_memory_grow => {
672	const pl_op = air_datas[@intFromEnum(inst)].pl_op;
673	if (pl_op.operand == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
674	return .none;
675	},
676	}
677	}
678
679	fn matchOperandSmallIndex(
680	l: Liveness,
681	inst: Air.Inst.Index,
682	operand: OperandInt,
683	default: OperandCategory,
684	) OperandCategory {
685	if (operandDies(l, inst, operand)) {
686	return .tomb;
687	} else {
688	return default;
689	}
690	}
691
692	/// Higher level API.
693	pub const CondBrSlices = struct {
694	then_deaths: []const Air.Inst.Index,
695	else_deaths: []const Air.Inst.Index,
696	};
697
698	pub fn getCondBr(l: Liveness, inst: Air.Inst.Index) CondBrSlices {
699	var index: usize = l.special.get(inst) orelse return .{
700	.then_deaths = &.{},
701	.else_deaths = &.{},
702	};
703	const then_death_count = l.extra[index];
704	index += 1;
705	const else_death_count = l.extra[index];
706	index += 1;
707	const then_deaths: []const Air.Inst.Index = @ptrCast(l.extra[index..][0..then_death_count]);
708	index += then_death_count;
709	return .{
710	.then_deaths = then_deaths,
711	.else_deaths = @ptrCast(l.extra[index..][0..else_death_count]),
712	};
713	}
714
715	/// Indexed by case number as they appear in AIR.
716	/// Else is the last element.
717	pub const SwitchBrTable = struct {
718	deaths: []const []const Air.Inst.Index,
719	};
720
721	/// Caller owns the memory.
722	pub fn getSwitchBr(l: Liveness, gpa: Allocator, inst: Air.Inst.Index, cases_len: u32) Allocator.Error!SwitchBrTable {
723	var index: usize = l.special.get(inst) orelse return .{ .deaths = &.{} };
724	const else_death_count = l.extra[index];
725	index += 1;
726
727	var deaths = try gpa.alloc([]const Air.Inst.Index, cases_len);
728	errdefer gpa.free(deaths);
729
730	var case_i: u32 = 0;
731	while (case_i < cases_len - 1) : (case_i += 1) {
732	const case_death_count: u32 = l.extra[index];
733	index += 1;
734	deaths[case_i] = @ptrCast(l.extra[index..][0..case_death_count]);
735	index += case_death_count;
736	}
737	{
738	// Else
739	deaths[case_i] = @ptrCast(l.extra[index..][0..else_death_count]);
740	}
741	return .{ .deaths = deaths };
742	}
743
744	/// Note that this information is technically redundant, but is useful for
745	/// backends nonetheless: see `Block`.
746	pub const BlockSlices = struct {
747	deaths: []const Air.Inst.Index,
748	};
749
750	pub fn getBlock(l: Liveness, inst: Air.Inst.Index) BlockSlices {
751	const index: usize = l.special.get(inst) orelse return .{
752	.deaths = &.{},
753	};
754	const death_count = l.extra[index];
755	const deaths: []const Air.Inst.Index = @ptrCast(l.extra[index + 1 ..][0..death_count]);
756	return .{
757	.deaths = deaths,
758	};
759	}
760
761	pub const LoopSlice = struct {
762	deaths: []const Air.Inst.Index,
763	};
764
765	pub fn deinit(l: *Liveness, gpa: Allocator) void {
766	gpa.free(l.tomb_bits);
767	gpa.free(l.extra);
768	l.special.deinit(gpa);
769	l.* = undefined;
770	}
771
772	pub fn iterateBigTomb(l: Liveness, inst: Air.Inst.Index) BigTomb {
773	return .{
774	.tomb_bits = l.getTombBits(inst),
775	.extra_start = l.special.get(inst) orelse 0,
776	.extra_offset = 0,
777	.extra = l.extra,
778	.bit_index = 0,
779	.reached_end = false,
780	};
781	}
782
783	/// How many tomb bits per AIR instruction.
784	pub const bpi = 4;
785	pub const Bpi = std.meta.Int(.unsigned, bpi);
786	pub const OperandInt = std.math.Log2Int(Bpi);
787
788	/// Useful for decoders of Liveness information.
789	pub const BigTomb = struct {
790	tomb_bits: Liveness.Bpi,
791	bit_index: u32,
792	extra_start: u32,
793	extra_offset: u32,
794	extra: []const u32,
795	reached_end: bool,
796
797	/// Returns whether the next operand dies.
798	pub fn feed(bt: *BigTomb) bool {
799	if (bt.reached_end) return false;
800
801	const this_bit_index = bt.bit_index;
802	bt.bit_index += 1;
803
804	const small_tombs = bpi - 1;
805	if (this_bit_index < small_tombs) {
806	const dies = @as(u1, @truncate(bt.tomb_bits >> @as(Liveness.OperandInt, @intCast(this_bit_index)))) != 0;
807	return dies;
808	}
809
810	const big_bit_index = this_bit_index - small_tombs;
811	while (big_bit_index - bt.extra_offset * 31 >= 31) {
812	if (@as(u1, @truncate(bt.extra[bt.extra_start + bt.extra_offset] >> 31)) != 0) {
813	bt.reached_end = true;
814	return false;
815	}
816	bt.extra_offset += 1;
817	}
818	const dies = @as(u1, @truncate(bt.extra[bt.extra_start + bt.extra_offset] >>
819	@as(u5, @intCast(big_bit_index - bt.extra_offset * 31)))) != 0;
820	return dies;
821	}
822	};
823
824	/// In-progress data; on successful analysis converted into `Liveness`.
825	const Analysis = struct {
826	gpa: Allocator,
827	air: Air,
828	intern_pool: *InternPool,
829	tomb_bits: []usize,
830	special: std.AutoHashMapUnmanaged(Air.Inst.Index, u32),
831	extra: std.ArrayListUnmanaged(u32),
832
833	fn addExtra(a: *Analysis, extra: anytype) Allocator.Error!u32 {
834	const fields = std.meta.fields(@TypeOf(extra));
835	try a.extra.ensureUnusedCapacity(a.gpa, fields.len);
836	return addExtraAssumeCapacity(a, extra);
837	}
838
839	fn addExtraAssumeCapacity(a: *Analysis, extra: anytype) u32 {
840	const fields = std.meta.fields(@TypeOf(extra));
841	const result = @as(u32, @intCast(a.extra.items.len));
842	inline for (fields) \|field\| {
843	a.extra.appendAssumeCapacity(switch (field.type) {
844	u32 => @field(extra, field.name),
845	else => @compileError("bad field type"),
846	});
847	}
848	return result;
849	}
850	};
851
852	fn analyzeBody(
853	a: *Analysis,
854	comptime pass: LivenessPass,
855	data: *LivenessPassData(pass),
856	body: []const Air.Inst.Index,
857	) Allocator.Error!void {
858	var i: usize = body.len;
859	while (i != 0) {
860	i -= 1;
861	const inst = body[i];
862	try analyzeInst(a, pass, data, inst);
863	}
864	}
865
866	fn analyzeInst(
867	a: *Analysis,
868	comptime pass: LivenessPass,
869	data: *LivenessPassData(pass),
870	inst: Air.Inst.Index,
871	) Allocator.Error!void {
872	const ip = a.intern_pool;
873	const inst_tags = a.air.instructions.items(.tag);
874	const inst_datas = a.air.instructions.items(.data);
875
876	switch (inst_tags[@intFromEnum(inst)]) {
877	.add,
878	.add_safe,
879	.add_optimized,
880	.add_wrap,
881	.add_sat,
882	.sub,
883	.sub_safe,
884	.sub_optimized,
885	.sub_wrap,
886	.sub_sat,
887	.mul,
888	.mul_safe,
889	.mul_optimized,
890	.mul_wrap,
891	.mul_sat,
892	.div_float,
893	.div_float_optimized,
894	.div_trunc,
895	.div_trunc_optimized,
896	.div_floor,
897	.div_floor_optimized,
898	.div_exact,
899	.div_exact_optimized,
900	.rem,
901	.rem_optimized,
902	.mod,
903	.mod_optimized,
904	.bit_and,
905	.bit_or,
906	.xor,
907	.cmp_lt,
908	.cmp_lt_optimized,
909	.cmp_lte,
910	.cmp_lte_optimized,
911	.cmp_eq,
912	.cmp_eq_optimized,
913	.cmp_gte,
914	.cmp_gte_optimized,
915	.cmp_gt,
916	.cmp_gt_optimized,
917	.cmp_neq,
918	.cmp_neq_optimized,
919	.bool_and,
920	.bool_or,
921	.store,
922	.store_safe,
923	.array_elem_val,
924	.slice_elem_val,
925	.ptr_elem_val,
926	.shl,
927	.shl_exact,
928	.shl_sat,
929	.shr,
930	.shr_exact,
931	.atomic_store_unordered,
932	.atomic_store_monotonic,
933	.atomic_store_release,
934	.atomic_store_seq_cst,
935	.set_union_tag,
936	.min,
937	.max,
938	.memset,
939	.memset_safe,
940	.memcpy,
941	.memmove,
942	=> {
943	const o = inst_datas[@intFromEnum(inst)].bin_op;
944	return analyzeOperands(a, pass, data, inst, .{ o.lhs, o.rhs, .none });
945	},
946
947	.vector_store_elem => {
948	const o = inst_datas[@intFromEnum(inst)].vector_store_elem;
949	const extra = a.air.extraData(Air.Bin, o.payload).data;
950	return analyzeOperands(a, pass, data, inst, .{ o.vector_ptr, extra.lhs, extra.rhs });
951	},
952
953	.arg,
954	.alloc,
955	.ret_ptr,
956	.breakpoint,
957	.dbg_stmt,
958	.dbg_empty_stmt,
959	.ret_addr,
960	.frame_addr,
961	.wasm_memory_size,
962	.err_return_trace,
963	.save_err_return_trace_index,
964	.tlv_dllimport_ptr,
965	.c_va_start,
966	.work_item_id,
967	.work_group_size,
968	.work_group_id,
969	=> return analyzeOperands(a, pass, data, inst, .{ .none, .none, .none }),
970
971	.inferred_alloc, .inferred_alloc_comptime => unreachable,
972
973	.trap,
974	.unreach,
975	=> return analyzeFuncEnd(a, pass, data, inst, .{ .none, .none, .none }),
976
977	.not,
978	.bitcast,
979	.load,
980	.fpext,
981	.fptrunc,
982	.intcast,
983	.intcast_safe,
984	.trunc,
985	.optional_payload,
986	.optional_payload_ptr,
987	.optional_payload_ptr_set,
988	.errunion_payload_ptr_set,
989	.wrap_optional,
990	.unwrap_errunion_payload,
991	.unwrap_errunion_err,
992	.unwrap_errunion_payload_ptr,
993	.unwrap_errunion_err_ptr,
994	.wrap_errunion_payload,
995	.wrap_errunion_err,
996	.slice_ptr,
997	.slice_len,
998	.ptr_slice_len_ptr,
999	.ptr_slice_ptr_ptr,
1000	.struct_field_ptr_index_0,
1001	.struct_field_ptr_index_1,
1002	.struct_field_ptr_index_2,
1003	.struct_field_ptr_index_3,
1004	.array_to_slice,
1005	.int_from_float,
1006	.int_from_float_optimized,
1007	.float_from_int,
1008	.get_union_tag,
1009	.clz,
1010	.ctz,
1011	.popcount,
1012	.byte_swap,
1013	.bit_reverse,
1014	.splat,
1015	.error_set_has_value,
1016	.addrspace_cast,
1017	.c_va_arg,
1018	.c_va_copy,
1019	.abs,
1020	=> {
1021	const o = inst_datas[@intFromEnum(inst)].ty_op;
1022	return analyzeOperands(a, pass, data, inst, .{ o.operand, .none, .none });
1023	},
1024
1025	.is_null,
1026	.is_non_null,
1027	.is_null_ptr,
1028	.is_non_null_ptr,
1029	.is_err,
1030	.is_non_err,
1031	.is_err_ptr,
1032	.is_non_err_ptr,
1033	.is_named_enum_value,
1034	.tag_name,
1035	.error_name,
1036	.sqrt,
1037	.sin,
1038	.cos,
1039	.tan,
1040	.exp,
1041	.exp2,
1042	.log,
1043	.log2,
1044	.log10,
1045	.floor,
1046	.ceil,
1047	.round,
1048	.trunc_float,
1049	.neg,
1050	.neg_optimized,
1051	.cmp_lt_errors_len,
1052	.set_err_return_trace,
1053	.c_va_end,
1054	=> {
1055	const operand = inst_datas[@intFromEnum(inst)].un_op;
1056	return analyzeOperands(a, pass, data, inst, .{ operand, .none, .none });
1057	},
1058
1059	.ret,
1060	.ret_safe,
1061	.ret_load,
1062	=> {
1063	const operand = inst_datas[@intFromEnum(inst)].un_op;
1064	return analyzeFuncEnd(a, pass, data, inst, .{ operand, .none, .none });
1065	},
1066
1067	.add_with_overflow,
1068	.sub_with_overflow,
1069	.mul_with_overflow,
1070	.shl_with_overflow,
1071	.ptr_add,
1072	.ptr_sub,
1073	.ptr_elem_ptr,
1074	.slice_elem_ptr,
1075	.slice,
1076	=> {
1077	const ty_pl = inst_datas[@intFromEnum(inst)].ty_pl;
1078	const extra = a.air.extraData(Air.Bin, ty_pl.payload).data;
1079	return analyzeOperands(a, pass, data, inst, .{ extra.lhs, extra.rhs, .none });
1080	},
1081
1082	.dbg_var_ptr,
1083	.dbg_var_val,
1084	.dbg_arg_inline,
1085	=> {
1086	const operand = inst_datas[@intFromEnum(inst)].pl_op.operand;
1087	return analyzeOperands(a, pass, data, inst, .{ operand, .none, .none });
1088	},
1089
1090	.prefetch => {
1091	const prefetch = inst_datas[@intFromEnum(inst)].prefetch;
1092	return analyzeOperands(a, pass, data, inst, .{ prefetch.ptr, .none, .none });
1093	},
1094
1095	.call, .call_always_tail, .call_never_tail, .call_never_inline => {
1096	const inst_data = inst_datas[@intFromEnum(inst)].pl_op;
1097	const callee = inst_data.operand;
1098	const extra = a.air.extraData(Air.Call, inst_data.payload);
1099	const args = @as([]const Air.Inst.Ref, @ptrCast(a.air.extra[extra.end..][0..extra.data.args_len]));
1100	if (args.len + 1 <= bpi - 1) {
1101	var buf = [1]Air.Inst.Ref{.none} ** (bpi - 1);
1102	buf[0] = callee;
1103	@memcpy(buf[1..][0..args.len], args);
1104	return analyzeOperands(a, pass, data, inst, buf);
1105	}
1106
1107	var big = try AnalyzeBigOperands(pass).init(a, data, inst, args.len + 1);
1108	defer big.deinit();
1109	var i: usize = args.len;
1110	while (i > 0) {
1111	i -= 1;
1112	try big.feed(args[i]);
1113	}
1114	try big.feed(callee);
1115	return big.finish();
1116	},
1117	.select => {
1118	const pl_op = inst_datas[@intFromEnum(inst)].pl_op;
1119	const extra = a.air.extraData(Air.Bin, pl_op.payload).data;
1120	return analyzeOperands(a, pass, data, inst, .{ pl_op.operand, extra.lhs, extra.rhs });
1121	},
1122	.shuffle => {
1123	const extra = a.air.extraData(Air.Shuffle, inst_datas[@intFromEnum(inst)].ty_pl.payload).data;
1124	return analyzeOperands(a, pass, data, inst, .{ extra.a, extra.b, .none });
1125	},
1126	.reduce, .reduce_optimized => {
1127	const reduce = inst_datas[@intFromEnum(inst)].reduce;
1128	return analyzeOperands(a, pass, data, inst, .{ reduce.operand, .none, .none });
1129	},
1130	.cmp_vector, .cmp_vector_optimized => {
1131	const extra = a.air.extraData(Air.VectorCmp, inst_datas[@intFromEnum(inst)].ty_pl.payload).data;
1132	return analyzeOperands(a, pass, data, inst, .{ extra.lhs, extra.rhs, .none });
1133	},
1134	.aggregate_init => {
1135	const ty_pl = inst_datas[@intFromEnum(inst)].ty_pl;
1136	const aggregate_ty = ty_pl.ty.toType();
1137	const len = @as(usize, @intCast(aggregate_ty.arrayLenIp(ip)));
1138	const elements = @as([]const Air.Inst.Ref, @ptrCast(a.air.extra[ty_pl.payload..][0..len]));
1139
1140	if (elements.len <= bpi - 1) {
1141	var buf = [1]Air.Inst.Ref{.none} ** (bpi - 1);
1142	@memcpy(buf[0..elements.len], elements);
1143	return analyzeOperands(a, pass, data, inst, buf);
1144	}
1145
1146	var big = try AnalyzeBigOperands(pass).init(a, data, inst, elements.len);
1147	defer big.deinit();
1148	var i: usize = elements.len;
1149	while (i > 0) {
1150	i -= 1;
1151	try big.feed(elements[i]);
1152	}
1153	return big.finish();
1154	},
1155	.union_init => {
1156	const extra = a.air.extraData(Air.UnionInit, inst_datas[@intFromEnum(inst)].ty_pl.payload).data;
1157	return analyzeOperands(a, pass, data, inst, .{ extra.init, .none, .none });
1158	},
1159	.struct_field_ptr, .struct_field_val => {
1160	const extra = a.air.extraData(Air.StructField, inst_datas[@intFromEnum(inst)].ty_pl.payload).data;
1161	return analyzeOperands(a, pass, data, inst, .{ extra.struct_operand, .none, .none });
1162	},
1163	.field_parent_ptr => {
1164	const extra = a.air.extraData(Air.FieldParentPtr, inst_datas[@intFromEnum(inst)].ty_pl.payload).data;
1165	return analyzeOperands(a, pass, data, inst, .{ extra.field_ptr, .none, .none });
1166	},
1167	.cmpxchg_strong, .cmpxchg_weak => {
1168	const extra = a.air.extraData(Air.Cmpxchg, inst_datas[@intFromEnum(inst)].ty_pl.payload).data;
1169	return analyzeOperands(a, pass, data, inst, .{ extra.ptr, extra.expected_value, extra.new_value });
1170	},
1171	.mul_add => {
1172	const pl_op = inst_datas[@intFromEnum(inst)].pl_op;
1173	const extra = a.air.extraData(Air.Bin, pl_op.payload).data;
1174	return analyzeOperands(a, pass, data, inst, .{ extra.lhs, extra.rhs, pl_op.operand });
1175	},
1176	.atomic_load => {
1177	const ptr = inst_datas[@intFromEnum(inst)].atomic_load.ptr;
1178	return analyzeOperands(a, pass, data, inst, .{ ptr, .none, .none });
1179	},
1180	.atomic_rmw => {
1181	const pl_op = inst_datas[@intFromEnum(inst)].pl_op;
1182	const extra = a.air.extraData(Air.AtomicRmw, pl_op.payload).data;
1183	return analyzeOperands(a, pass, data, inst, .{ pl_op.operand, extra.operand, .none });
1184	},
1185
1186	.br => return analyzeInstBr(a, pass, data, inst),
1187	.repeat => return analyzeInstRepeat(a, pass, data, inst),
1188	.switch_dispatch => return analyzeInstSwitchDispatch(a, pass, data, inst),
1189
1190	.assembly => {
1191	const extra = a.air.extraData(Air.Asm, inst_datas[@intFromEnum(inst)].ty_pl.payload);
1192	var extra_i: usize = extra.end;
1193	const outputs = @as([]const Air.Inst.Ref, @ptrCast(a.air.extra[extra_i..][0..extra.data.outputs_len]));
1194	extra_i += outputs.len;
1195	const inputs = @as([]const Air.Inst.Ref, @ptrCast(a.air.extra[extra_i..][0..extra.data.inputs_len]));
1196	extra_i += inputs.len;
1197
1198	const num_operands = simple: {
1199	var buf = [1]Air.Inst.Ref{.none} ** (bpi - 1);
1200	var buf_index: usize = 0;
1201	for (outputs) \|output\| {
1202	if (output != .none) {
1203	if (buf_index < buf.len) buf[buf_index] = output;
1204	buf_index += 1;
1205	}
1206	}
1207	if (buf_index + inputs.len > buf.len) {
1208	break :simple buf_index + inputs.len;
1209	}
1210	@memcpy(buf[buf_index..][0..inputs.len], inputs);
1211	return analyzeOperands(a, pass, data, inst, buf);
1212	};
1213
1214	var big = try AnalyzeBigOperands(pass).init(a, data, inst, num_operands);
1215	defer big.deinit();
1216	var i: usize = inputs.len;
1217	while (i > 0) {
1218	i -= 1;
1219	try big.feed(inputs[i]);
1220	}
1221	i = outputs.len;
1222	while (i > 0) {
1223	i -= 1;
1224	if (outputs[i] != .none) {
1225	try big.feed(outputs[i]);
1226	}
1227	}
1228	return big.finish();
1229	},
1230
1231	inline .block, .dbg_inline_block => \|comptime_tag\| {
1232	const ty_pl = inst_datas[@intFromEnum(inst)].ty_pl;
1233	const extra = a.air.extraData(switch (comptime_tag) {
1234	.block => Air.Block,
1235	.dbg_inline_block => Air.DbgInlineBlock,
1236	else => unreachable,
1237	}, ty_pl.payload);
1238	return analyzeInstBlock(a, pass, data, inst, ty_pl.ty, @ptrCast(a.air.extra[extra.end..][0..extra.data.body_len]));
1239	},
1240	.loop => return analyzeInstLoop(a, pass, data, inst),
1241
1242	.@"try", .try_cold => return analyzeInstCondBr(a, pass, data, inst, .@"try"),
1243	.try_ptr, .try_ptr_cold => return analyzeInstCondBr(a, pass, data, inst, .try_ptr),
1244	.cond_br => return analyzeInstCondBr(a, pass, data, inst, .cond_br),
1245	.switch_br => return analyzeInstSwitchBr(a, pass, data, inst, false),
1246	.loop_switch_br => return analyzeInstSwitchBr(a, pass, data, inst, true),
1247
1248	.wasm_memory_grow => {
1249	const pl_op = inst_datas[@intFromEnum(inst)].pl_op;
1250	return analyzeOperands(a, pass, data, inst, .{ pl_op.operand, .none, .none });
1251	},
1252	}
1253	}
1254
1255	/// Every instruction should hit this (after handling any nested bodies), in every pass. In the
1256	/// initial pass, it is responsible for marking deaths of the (first three) operands and noticing
1257	/// immediate deaths.
1258	fn analyzeOperands(
1259	a: *Analysis,
1260	comptime pass: LivenessPass,
1261	data: *LivenessPassData(pass),
1262	inst: Air.Inst.Index,
1263	operands: [bpi - 1]Air.Inst.Ref,
1264	) Allocator.Error!void {
1265	const gpa = a.gpa;
1266	const ip = a.intern_pool;
1267
1268	switch (pass) {
1269	.loop_analysis => {
1270	_ = data.live_set.remove(inst);
1271
1272	for (operands) \|op_ref\| {
1273	const operand = op_ref.toIndexAllowNone() orelse continue;
1274	_ = try data.live_set.put(gpa, operand, {});
1275	}
1276	},
1277
1278	.main_analysis => {
1279	const usize_index = (@intFromEnum(inst) * bpi) / @bitSizeOf(usize);
1280
1281	// This logic must synchronize with `will_die_immediately` in `AnalyzeBigOperands.init`.
1282	const immediate_death = if (data.live_set.remove(inst)) blk: {
1283	log.debug("[{}] %{}: removed from live set", .{ pass, @intFromEnum(inst) });
1284	break :blk false;
1285	} else blk: {
1286	log.debug("[{}] %{}: immediate death", .{ pass, @intFromEnum(inst) });
1287	break :blk true;
1288	};
1289
1290	var tomb_bits: Bpi = @as(Bpi, @intFromBool(immediate_death)) << (bpi - 1);
1291
1292	// If our result is unused and the instruction doesn't need to be lowered, backends will
1293	// skip the lowering of this instruction, so we don't want to record uses of operands.
1294	// That way, we can mark as many instructions as possible unused.
1295	if (!immediate_death or a.air.mustLower(inst, ip)) {
1296	// Note that it's important we iterate over the operands backwards, so that if a dying
1297	// operand is used multiple times we mark its last use as its death.
1298	var i = operands.len;
1299	while (i > 0) {
1300	i -= 1;
1301	const op_ref = operands[i];
1302	const operand = op_ref.toIndexAllowNone() orelse continue;
1303
1304	const mask = @as(Bpi, 1) << @as(OperandInt, @intCast(i));
1305
1306	if ((try data.live_set.fetchPut(gpa, operand, {})) == null) {
1307	log.debug("[{}] %{}: added %{} to live set (operand dies here)", .{ pass, @intFromEnum(inst), operand });
1308	tomb_bits \|= mask;
1309	}
1310	}
1311	}
1312
1313	a.tomb_bits[usize_index] \|= @as(usize, tomb_bits) <<
1314	@as(Log2Int(usize), @intCast((@intFromEnum(inst) % (@bitSizeOf(usize) / bpi)) * bpi));
1315	},
1316	}
1317	}
1318
1319	/// Like `analyzeOperands`, but for an instruction which returns from a function, so should
1320	/// effectively kill every remaining live value other than its operands.
1321	fn analyzeFuncEnd(
1322	a: *Analysis,
1323	comptime pass: LivenessPass,
1324	data: *LivenessPassData(pass),
1325	inst: Air.Inst.Index,
1326	operands: [bpi - 1]Air.Inst.Ref,
1327	) Allocator.Error!void {
1328	switch (pass) {
1329	.loop_analysis => {
1330	// No operands need to be alive if we're returning from the function, so we don't need
1331	// to touch `breaks` here even though this is sort of like a break to the top level.
1332	},
1333
1334	.main_analysis => {
1335	data.live_set.clearRetainingCapacity();
1336	},
1337	}
1338
1339	return analyzeOperands(a, pass, data, inst, operands);
1340	}
1341
1342	fn analyzeInstBr(
1343	a: *Analysis,
1344	comptime pass: LivenessPass,
1345	data: *LivenessPassData(pass),
1346	inst: Air.Inst.Index,
1347	) !void {
1348	const inst_datas = a.air.instructions.items(.data);
1349	const br = inst_datas[@intFromEnum(inst)].br;
1350	const gpa = a.gpa;
1351
1352	switch (pass) {
1353	.loop_analysis => {
1354	try data.breaks.put(gpa, br.block_inst, {});
1355	},
1356
1357	.main_analysis => {
1358	const block_scope = data.block_scopes.get(br.block_inst).?; // we should always be breaking from an enclosing block
1359
1360	const new_live_set = try block_scope.live_set.clone(gpa);
1361	data.live_set.deinit(gpa);
1362	data.live_set = new_live_set;
1363	},
1364	}
1365
1366	return analyzeOperands(a, pass, data, inst, .{ br.operand, .none, .none });
1367	}
1368
1369	fn analyzeInstRepeat(
1370	a: *Analysis,
1371	comptime pass: LivenessPass,
1372	data: *LivenessPassData(pass),
1373	inst: Air.Inst.Index,
1374	) !void {
1375	const inst_datas = a.air.instructions.items(.data);
1376	const repeat = inst_datas[@intFromEnum(inst)].repeat;
1377	const gpa = a.gpa;
1378
1379	switch (pass) {
1380	.loop_analysis => {
1381	try data.breaks.put(gpa, repeat.loop_inst, {});
1382	},
1383
1384	.main_analysis => {
1385	const block_scope = data.block_scopes.get(repeat.loop_inst).?; // we should always be repeating an enclosing loop
1386
1387	const new_live_set = try block_scope.live_set.clone(gpa);
1388	data.live_set.deinit(gpa);
1389	data.live_set = new_live_set;
1390	},
1391	}
1392
1393	return analyzeOperands(a, pass, data, inst, .{ .none, .none, .none });
1394	}
1395
1396	fn analyzeInstSwitchDispatch(
1397	a: *Analysis,
1398	comptime pass: LivenessPass,
1399	data: *LivenessPassData(pass),
1400	inst: Air.Inst.Index,
1401	) !void {
1402	// This happens to be identical to `analyzeInstBr`, but is separated anyway for clarity.
1403
1404	const inst_datas = a.air.instructions.items(.data);
1405	const br = inst_datas[@intFromEnum(inst)].br;
1406	const gpa = a.gpa;
1407
1408	switch (pass) {
1409	.loop_analysis => {
1410	try data.breaks.put(gpa, br.block_inst, {});
1411	},
1412
1413	.main_analysis => {
1414	const block_scope = data.block_scopes.get(br.block_inst).?; // we should always be repeating an enclosing loop
1415
1416	const new_live_set = try block_scope.live_set.clone(gpa);
1417	data.live_set.deinit(gpa);
1418	data.live_set = new_live_set;
1419	},
1420	}
1421
1422	return analyzeOperands(a, pass, data, inst, .{ br.operand, .none, .none });
1423	}
1424
1425	fn analyzeInstBlock(
1426	a: *Analysis,
1427	comptime pass: LivenessPass,
1428	data: *LivenessPassData(pass),
1429	inst: Air.Inst.Index,
1430	ty: Air.Inst.Ref,
1431	body: []const Air.Inst.Index,
1432	) !void {
1433	const gpa = a.gpa;
1434
1435	// We actually want to do `analyzeOperands` first, since our result logically doesn't
1436	// exist until the block body ends (and we're iterating backwards)
1437	try analyzeOperands(a, pass, data, inst, .{ .none, .none, .none });
1438
1439	switch (pass) {
1440	.loop_analysis => {
1441	try analyzeBody(a, pass, data, body);
1442	_ = data.breaks.remove(inst);
1443	},
1444
1445	.main_analysis => {
1446	log.debug("[{}] %{}: block live set is {}", .{ pass, inst, fmtInstSet(&data.live_set) });
1447	// We can move the live set because the body should have a noreturn
1448	// instruction which overrides the set.
1449	try data.block_scopes.put(gpa, inst, .{
1450	.live_set = data.live_set.move(),
1451	});
1452	defer {
1453	log.debug("[{}] %{}: popped block scope", .{ pass, inst });
1454	var scope = data.block_scopes.fetchRemove(inst).?.value;
1455	scope.live_set.deinit(gpa);
1456	}
1457
1458	log.debug("[{}] %{}: pushed new block scope", .{ pass, inst });
1459	try analyzeBody(a, pass, data, body);
1460
1461	// If the block is noreturn, block deaths not only aren't useful, they're impossible to
1462	// find: there could be more stuff alive after the block than before it!
1463	if (!a.intern_pool.isNoReturn(ty.toType().toIntern())) {
1464	// The block kills the difference in the live sets
1465	const block_scope = data.block_scopes.get(inst).?;
1466	const num_deaths = data.live_set.count() - block_scope.live_set.count();
1467
1468	try a.extra.ensureUnusedCapacity(gpa, num_deaths + std.meta.fields(Block).len);
1469	const extra_index = a.addExtraAssumeCapacity(Block{
1470	.death_count = num_deaths,
1471	});
1472
1473	var measured_num: u32 = 0;
1474	var it = data.live_set.keyIterator();
1475	while (it.next()) \|key\| {
1476	const alive = key.*;
1477	if (!block_scope.live_set.contains(alive)) {
1478	// Dies in block
1479	a.extra.appendAssumeCapacity(@intFromEnum(alive));
1480	measured_num += 1;
1481	}
1482	}
1483	assert(measured_num == num_deaths); // post-live-set should be a subset of pre-live-set
1484	try a.special.put(gpa, inst, extra_index);
1485	log.debug("[{}] %{}: block deaths are {}", .{
1486	pass,
1487	inst,
1488	fmtInstList(@ptrCast(a.extra.items[extra_index + 1 ..][0..num_deaths])),
1489	});
1490	}
1491	},
1492	}
1493	}
1494
1495	fn writeLoopInfo(
1496	a: *Analysis,
1497	data: *LivenessPassData(.loop_analysis),
1498	inst: Air.Inst.Index,
1499	old_breaks: std.AutoHashMapUnmanaged(Air.Inst.Index, void),
1500	old_live: std.AutoHashMapUnmanaged(Air.Inst.Index, void),
1501	) !void {
1502	const gpa = a.gpa;
1503
1504	// `loop`s are guaranteed to have at least one matching `repeat`.
1505	// Similarly, `loop_switch_br`s have a matching `switch_dispatch`.
1506	// However, we no longer care about repeats of this loop for resolving
1507	// which operands must live within it.
1508	assert(data.breaks.remove(inst));
1509
1510	const extra_index: u32 = @intCast(a.extra.items.len);
1511
1512	const num_breaks = data.breaks.count();
1513	try a.extra.ensureUnusedCapacity(gpa, 1 + num_breaks);
1514
1515	a.extra.appendAssumeCapacity(num_breaks);
1516
1517	var it = data.breaks.keyIterator();
1518	while (it.next()) \|key\| {
1519	const block_inst = key.*;
1520	a.extra.appendAssumeCapacity(@intFromEnum(block_inst));
1521	}
1522	log.debug("[{}] %{}: includes breaks to {}", .{ LivenessPass.loop_analysis, inst, fmtInstSet(&data.breaks) });
1523
1524	// Now we put the live operands from the loop body in too
1525	const num_live = data.live_set.count();
1526	try a.extra.ensureUnusedCapacity(gpa, 1 + num_live);
1527
1528	a.extra.appendAssumeCapacity(num_live);
1529	it = data.live_set.keyIterator();
1530	while (it.next()) \|key\| {
1531	const alive = key.*;
1532	a.extra.appendAssumeCapacity(@intFromEnum(alive));
1533	}
1534	log.debug("[{}] %{}: maintain liveness of {}", .{ LivenessPass.loop_analysis, inst, fmtInstSet(&data.live_set) });
1535
1536	try a.special.put(gpa, inst, extra_index);
1537
1538	// Add back operands which were previously alive
1539	it = old_live.keyIterator();
1540	while (it.next()) \|key\| {
1541	const alive = key.*;
1542	try data.live_set.put(gpa, alive, {});
1543	}
1544
1545	// And the same for breaks
1546	it = old_breaks.keyIterator();
1547	while (it.next()) \|key\| {
1548	const block_inst = key.*;
1549	try data.breaks.put(gpa, block_inst, {});
1550	}
1551	}
1552
1553	/// When analyzing a loop in the main pass, sets up `data.live_set` to be the set
1554	/// of operands known to be alive when the loop repeats.
1555	fn resolveLoopLiveSet(
1556	a: *Analysis,
1557	data: *LivenessPassData(.main_analysis),
1558	inst: Air.Inst.Index,
1559	) !void {
1560	const gpa = a.gpa;
1561
1562	const extra_idx = a.special.fetchRemove(inst).?.value;
1563	const num_breaks = data.old_extra.items[extra_idx];
1564	const breaks: []const Air.Inst.Index = @ptrCast(data.old_extra.items[extra_idx + 1 ..][0..num_breaks]);
1565
1566	const num_loop_live = data.old_extra.items[extra_idx + num_breaks + 1];
1567	const loop_live: []const Air.Inst.Index = @ptrCast(data.old_extra.items[extra_idx + num_breaks + 2 ..][0..num_loop_live]);
1568
1569	// This is necessarily not in the same control flow branch, because loops are noreturn
1570	data.live_set.clearRetainingCapacity();
1571
1572	try data.live_set.ensureUnusedCapacity(gpa, @intCast(loop_live.len));
1573	for (loop_live) \|alive\| data.live_set.putAssumeCapacity(alive, {});
1574
1575	log.debug("[{}] %{}: block live set is {}", .{ LivenessPass.main_analysis, inst, fmtInstSet(&data.live_set) });
1576
1577	for (breaks) \|block_inst\| {
1578	// We might break to this block, so include every operand that the block needs alive
1579	const block_scope = data.block_scopes.get(block_inst).?;
1580
1581	var it = block_scope.live_set.keyIterator();
1582	while (it.next()) \|key\| {
1583	const alive = key.*;
1584	try data.live_set.put(gpa, alive, {});
1585	}
1586	}
1587
1588	log.debug("[{}] %{}: loop live set is {}", .{ LivenessPass.main_analysis, inst, fmtInstSet(&data.live_set) });
1589	}
1590
1591	fn analyzeInstLoop(
1592	a: *Analysis,
1593	comptime pass: LivenessPass,
1594	data: *LivenessPassData(pass),
1595	inst: Air.Inst.Index,
1596	) !void {
1597	const inst_datas = a.air.instructions.items(.data);
1598	const extra = a.air.extraData(Air.Block, inst_datas[@intFromEnum(inst)].ty_pl.payload);
1599	const body: []const Air.Inst.Index = @ptrCast(a.air.extra[extra.end..][0..extra.data.body_len]);
1600	const gpa = a.gpa;
1601
1602	try analyzeOperands(a, pass, data, inst, .{ .none, .none, .none });
1603
1604	switch (pass) {
1605	.loop_analysis => {
1606	var old_breaks = data.breaks.move();
1607	defer old_breaks.deinit(gpa);
1608
1609	var old_live = data.live_set.move();
1610	defer old_live.deinit(gpa);
1611
1612	try analyzeBody(a, pass, data, body);
1613
1614	try writeLoopInfo(a, data, inst, old_breaks, old_live);
1615	},
1616
1617	.main_analysis => {
1618	try resolveLoopLiveSet(a, data, inst);
1619
1620	// Now, `data.live_set` is the operands which must be alive when the loop repeats.
1621	// Move them into a block scope for corresponding `repeat` instructions to notice.
1622	try data.block_scopes.putNoClobber(gpa, inst, .{
1623	.live_set = data.live_set.move(),
1624	});
1625	defer {
1626	log.debug("[{}] %{}: popped loop block scop", .{ pass, inst });
1627	var scope = data.block_scopes.fetchRemove(inst).?.value;
1628	scope.live_set.deinit(gpa);
1629	}
1630	try analyzeBody(a, pass, data, body);
1631	},
1632	}
1633	}
1634
1635	/// Despite its name, this function is used for analysis of not only `cond_br` instructions, but
1636	/// also `try` and `try_ptr`, which are highly related. The `inst_type` parameter indicates which
1637	/// type of instruction `inst` points to.
1638	fn analyzeInstCondBr(
1639	a: *Analysis,
1640	comptime pass: LivenessPass,
1641	data: *LivenessPassData(pass),
1642	inst: Air.Inst.Index,
1643	comptime inst_type: enum { cond_br, @"try", try_ptr },
1644	) !void {
1645	const inst_datas = a.air.instructions.items(.data);
1646	const gpa = a.gpa;
1647
1648	const extra = switch (inst_type) {
1649	.cond_br => a.air.extraData(Air.CondBr, inst_datas[@intFromEnum(inst)].pl_op.payload),
1650	.@"try" => a.air.extraData(Air.Try, inst_datas[@intFromEnum(inst)].pl_op.payload),
1651	.try_ptr => a.air.extraData(Air.TryPtr, inst_datas[@intFromEnum(inst)].ty_pl.payload),
1652	};
1653
1654	const condition = switch (inst_type) {
1655	.cond_br, .@"try" => inst_datas[@intFromEnum(inst)].pl_op.operand,
1656	.try_ptr => extra.data.ptr,
1657	};
1658
1659	const then_body: []const Air.Inst.Index = switch (inst_type) {
1660	.cond_br => @ptrCast(a.air.extra[extra.end..][0..extra.data.then_body_len]),
1661	else => &.{}, // we won't use this
1662	};
1663
1664	const else_body: []const Air.Inst.Index = @ptrCast(switch (inst_type) {
1665	.cond_br => a.air.extra[extra.end + then_body.len ..][0..extra.data.else_body_len],
1666	.@"try", .try_ptr => a.air.extra[extra.end..][0..extra.data.body_len],
1667	});
1668
1669	switch (pass) {
1670	.loop_analysis => {
1671	switch (inst_type) {
1672	.cond_br => try analyzeBody(a, pass, data, then_body),
1673	.@"try", .try_ptr => {},
1674	}
1675	try analyzeBody(a, pass, data, else_body);
1676	},
1677
1678	.main_analysis => {
1679	switch (inst_type) {
1680	.cond_br => try analyzeBody(a, pass, data, then_body),
1681	.@"try", .try_ptr => {}, // The "then body" is just the remainder of this block
1682	}
1683	var then_live = data.live_set.move();
1684	defer then_live.deinit(gpa);
1685
1686	try analyzeBody(a, pass, data, else_body);
1687	var else_live = data.live_set.move();
1688	defer else_live.deinit(gpa);
1689
1690	// Operands which are alive in one branch but not the other need to die at the start of
1691	// the peer branch.
1692
1693	var then_mirrored_deaths: std.ArrayListUnmanaged(Air.Inst.Index) = .empty;
1694	defer then_mirrored_deaths.deinit(gpa);
1695
1696	var else_mirrored_deaths: std.ArrayListUnmanaged(Air.Inst.Index) = .empty;
1697	defer else_mirrored_deaths.deinit(gpa);
1698
1699	// Note: this invalidates `else_live`, but expands `then_live` to be their union
1700	{
1701	var it = then_live.keyIterator();
1702	while (it.next()) \|key\| {
1703	const death = key.*;
1704	if (else_live.remove(death)) continue; // removing makes the loop below faster
1705
1706	// If this is a `try`, the "then body" (rest of the branch) might have
1707	// referenced our result. We want to avoid killing this value in the else branch
1708	// if that's the case, since it only exists in the (fake) then branch.
1709	switch (inst_type) {
1710	.cond_br => {},
1711	.@"try", .try_ptr => if (death == inst) continue,
1712	}
1713
1714	try else_mirrored_deaths.append(gpa, death);
1715	}
1716	// Since we removed common stuff above, `else_live` is now only operands
1717	// which are only alive in the else branch
1718	it = else_live.keyIterator();
1719	while (it.next()) \|key\| {
1720	const death = key.*;
1721	try then_mirrored_deaths.append(gpa, death);
1722	// Make `then_live` contain the full live set (i.e. union of both)
1723	try then_live.put(gpa, death, {});
1724	}
1725	}
1726
1727	log.debug("[{}] %{}: 'then' branch mirrored deaths are {}", .{ pass, inst, fmtInstList(then_mirrored_deaths.items) });
1728	log.debug("[{}] %{}: 'else' branch mirrored deaths are {}", .{ pass, inst, fmtInstList(else_mirrored_deaths.items) });
1729
1730	data.live_set.deinit(gpa);
1731	data.live_set = then_live.move(); // Really the union of both live sets
1732
1733	log.debug("[{}] %{}: new live set is {}", .{ pass, inst, fmtInstSet(&data.live_set) });
1734
1735	// Write the mirrored deaths to `extra`
1736	const then_death_count = @as(u32, @intCast(then_mirrored_deaths.items.len));
1737	const else_death_count = @as(u32, @intCast(else_mirrored_deaths.items.len));
1738	try a.extra.ensureUnusedCapacity(gpa, std.meta.fields(CondBr).len + then_death_count + else_death_count);
1739	const extra_index = a.addExtraAssumeCapacity(CondBr{
1740	.then_death_count = then_death_count,
1741	.else_death_count = else_death_count,
1742	});
1743	a.extra.appendSliceAssumeCapacity(@ptrCast(then_mirrored_deaths.items));
1744	a.extra.appendSliceAssumeCapacity(@ptrCast(else_mirrored_deaths.items));
1745	try a.special.put(gpa, inst, extra_index);
1746	},
1747	}
1748
1749	try analyzeOperands(a, pass, data, inst, .{ condition, .none, .none });
1750	}
1751
1752	fn analyzeInstSwitchBr(
1753	a: *Analysis,
1754	comptime pass: LivenessPass,
1755	data: *LivenessPassData(pass),
1756	inst: Air.Inst.Index,
1757	is_dispatch_loop: bool,
1758	) !void {
1759	const inst_datas = a.air.instructions.items(.data);
1760	const pl_op = inst_datas[@intFromEnum(inst)].pl_op;
1761	const condition = pl_op.operand;
1762	const switch_br = a.air.unwrapSwitch(inst);
1763	const gpa = a.gpa;
1764	const ncases = switch_br.cases_len;
1765
1766	switch (pass) {
1767	.loop_analysis => {
1768	var old_breaks: std.AutoHashMapUnmanaged(Air.Inst.Index, void) = .empty;
1769	defer old_breaks.deinit(gpa);
1770
1771	var old_live: std.AutoHashMapUnmanaged(Air.Inst.Index, void) = .empty;
1772	defer old_live.deinit(gpa);
1773
1774	if (is_dispatch_loop) {
1775	old_breaks = data.breaks.move();
1776	old_live = data.live_set.move();
1777	}
1778
1779	var it = switch_br.iterateCases();
1780	while (it.next()) \|case\| {
1781	try analyzeBody(a, pass, data, case.body);
1782	}
1783	{ // else
1784	const else_body = it.elseBody();
1785	try analyzeBody(a, pass, data, else_body);
1786	}
1787
1788	if (is_dispatch_loop) {
1789	try writeLoopInfo(a, data, inst, old_breaks, old_live);
1790	}
1791	},
1792
1793	.main_analysis => {
1794	if (is_dispatch_loop) {
1795	try resolveLoopLiveSet(a, data, inst);
1796	try data.block_scopes.putNoClobber(gpa, inst, .{
1797	.live_set = data.live_set.move(),
1798	});
1799	}
1800	defer if (is_dispatch_loop) {
1801	log.debug("[{}] %{}: popped loop block scop", .{ pass, inst });
1802	var scope = data.block_scopes.fetchRemove(inst).?.value;
1803	scope.live_set.deinit(gpa);
1804	};
1805	// This is, all in all, just a messier version of the `cond_br` logic. If you're trying
1806	// to understand it, I encourage looking at `analyzeInstCondBr` first.
1807
1808	const DeathSet = std.AutoHashMapUnmanaged(Air.Inst.Index, void);
1809	const DeathList = std.ArrayListUnmanaged(Air.Inst.Index);
1810
1811	var case_live_sets = try gpa.alloc(std.AutoHashMapUnmanaged(Air.Inst.Index, void), ncases + 1); // +1 for else
1812	defer gpa.free(case_live_sets);
1813
1814	@memset(case_live_sets, .{});
1815	defer for (case_live_sets) \|*live_set\| live_set.deinit(gpa);
1816
1817	var case_it = switch_br.iterateCases();
1818	while (case_it.next()) \|case\| {
1819	try analyzeBody(a, pass, data, case.body);
1820	case_live_sets[case.idx] = data.live_set.move();
1821	}
1822	{ // else
1823	const else_body = case_it.elseBody();
1824	try analyzeBody(a, pass, data, else_body);
1825	case_live_sets[ncases] = data.live_set.move();
1826	}
1827
1828	const mirrored_deaths = try gpa.alloc(DeathList, ncases + 1);
1829	defer gpa.free(mirrored_deaths);
1830
1831	@memset(mirrored_deaths, .{});
1832	defer for (mirrored_deaths) \|*md\| md.deinit(gpa);
1833
1834	{
1835	var all_alive: DeathSet = .{};
1836	defer all_alive.deinit(gpa);
1837
1838	for (case_live_sets) \|*live_set\| {
1839	try all_alive.ensureUnusedCapacity(gpa, live_set.count());
1840	var it = live_set.keyIterator();
1841	while (it.next()) \|key\| {
1842	const alive = key.*;
1843	all_alive.putAssumeCapacity(alive, {});
1844	}
1845	}
1846
1847	for (mirrored_deaths, case_live_sets) \|mirrored, live_set\| {
1848	var it = all_alive.keyIterator();
1849	while (it.next()) \|key\| {
1850	const alive = key.*;
1851	if (!live_set.contains(alive)) {
1852	// Should die at the start of this branch
1853	try mirrored.append(gpa, alive);
1854	}
1855	}
1856	}
1857
1858	for (mirrored_deaths, 0..) \|mirrored, i\| {
1859	log.debug("[{}] %{}: case {} mirrored deaths are {}", .{ pass, inst, i, fmtInstList(mirrored.items) });
1860	}
1861
1862	data.live_set.deinit(gpa);
1863	data.live_set = all_alive.move();
1864
1865	log.debug("[{}] %{}: new live set is {}", .{ pass, inst, fmtInstSet(&data.live_set) });
1866	}
1867
1868	const else_death_count = @as(u32, @intCast(mirrored_deaths[ncases].items.len));
1869	const extra_index = try a.addExtra(SwitchBr{
1870	.else_death_count = else_death_count,
1871	});
1872	for (mirrored_deaths[0..ncases]) \|mirrored\| {
1873	const num = @as(u32, @intCast(mirrored.items.len));
1874	try a.extra.ensureUnusedCapacity(gpa, num + 1);
1875	a.extra.appendAssumeCapacity(num);
1876	a.extra.appendSliceAssumeCapacity(@ptrCast(mirrored.items));
1877	}
1878	try a.extra.ensureUnusedCapacity(gpa, else_death_count);
1879	a.extra.appendSliceAssumeCapacity(@ptrCast(mirrored_deaths[ncases].items));
1880	try a.special.put(gpa, inst, extra_index);
1881	},
1882	}
1883
1884	try analyzeOperands(a, pass, data, inst, .{ condition, .none, .none });
1885	}
1886
1887	fn AnalyzeBigOperands(comptime pass: LivenessPass) type {
1888	return struct {
1889	a: *Analysis,
1890	data: *LivenessPassData(pass),
1891	inst: Air.Inst.Index,
1892
1893	operands_remaining: u32,
1894	small: [bpi - 1]Air.Inst.Ref = .{.none} ** (bpi - 1),
1895	extra_tombs: []u32,
1896
1897	// Only used in `LivenessPass.main_analysis`
1898	will_die_immediately: bool,
1899
1900	const Self = @This();
1901
1902	fn init(
1903	a: *Analysis,
1904	data: *LivenessPassData(pass),
1905	inst: Air.Inst.Index,
1906	total_operands: usize,
1907	) !Self {
1908	const extra_operands = @as(u32, @intCast(total_operands)) -\| (bpi - 1);
1909	const max_extra_tombs = (extra_operands + 30) / 31;
1910
1911	const extra_tombs: []u32 = switch (pass) {
1912	.loop_analysis => &.{},
1913	.main_analysis => try a.gpa.alloc(u32, max_extra_tombs),
1914	};
1915	errdefer a.gpa.free(extra_tombs);
1916
1917	@memset(extra_tombs, 0);
1918
1919	const will_die_immediately: bool = switch (pass) {
1920	.loop_analysis => false, // track everything, since we don't have full liveness information yet
1921	.main_analysis => !data.live_set.contains(inst),
1922	};
1923
1924	return .{
1925	.a = a,
1926	.data = data,
1927	.inst = inst,
1928	.operands_remaining = @as(u32, @intCast(total_operands)),
1929	.extra_tombs = extra_tombs,
1930	.will_die_immediately = will_die_immediately,
1931	};
1932	}
1933
1934	/// Must be called with operands in reverse order.
1935	fn feed(big: *Self, op_ref: Air.Inst.Ref) !void {
1936	const ip = big.a.intern_pool;
1937	// Note that after this, `operands_remaining` becomes the index of the current operand
1938	big.operands_remaining -= 1;
1939
1940	if (big.operands_remaining < bpi - 1) {
1941	big.small[big.operands_remaining] = op_ref;
1942	return;
1943	}
1944
1945	const operand = op_ref.toIndex() orelse return;
1946
1947	// If our result is unused and the instruction doesn't need to be lowered, backends will
1948	// skip the lowering of this instruction, so we don't want to record uses of operands.
1949	// That way, we can mark as many instructions as possible unused.
1950	if (big.will_die_immediately and !big.a.air.mustLower(big.inst, ip)) return;
1951
1952	const extra_byte = (big.operands_remaining - (bpi - 1)) / 31;
1953	const extra_bit = @as(u5, @intCast(big.operands_remaining - (bpi - 1) - extra_byte * 31));
1954
1955	const gpa = big.a.gpa;
1956
1957	switch (pass) {
1958	.loop_analysis => {
1959	_ = try big.data.live_set.put(gpa, operand, {});
1960	},
1961
1962	.main_analysis => {
1963	if ((try big.data.live_set.fetchPut(gpa, operand, {})) == null) {
1964	log.debug("[{}] %{}: added %{} to live set (operand dies here)", .{ pass, big.inst, operand });
1965	big.extra_tombs[extra_byte] \|= @as(u32, 1) << extra_bit;
1966	}
1967	},
1968	}
1969	}
1970
1971	fn finish(big: *Self) !void {
1972	const gpa = big.a.gpa;
1973
1974	std.debug.assert(big.operands_remaining == 0);
1975
1976	switch (pass) {
1977	.loop_analysis => {},
1978
1979	.main_analysis => {
1980	// Note that the MSB is set on the final tomb to indicate the terminal element. This
1981	// allows for an optimisation where we only add as many extra tombs as are needed to
1982	// represent the dying operands. Each pass modifies operand bits and so needs to write
1983	// back, so let's figure out how many extra tombs we really need. Note that we always
1984	// keep at least one.
1985	var num: usize = big.extra_tombs.len;
1986	while (num > 1) {
1987	if (@as(u31, @truncate(big.extra_tombs[num - 1])) != 0) {
1988	// Some operand dies here
1989	break;
1990	}
1991	num -= 1;
1992	}
1993	// Mark final tomb
1994	big.extra_tombs[num - 1] \|= @as(u32, 1) << 31;
1995
1996	const extra_tombs = big.extra_tombs[0..num];
1997
1998	const extra_index = @as(u32, @intCast(big.a.extra.items.len));
1999	try big.a.extra.appendSlice(gpa, extra_tombs);
2000	try big.a.special.put(gpa, big.inst, extra_index);
2001	},
2002	}
2003
2004	try analyzeOperands(big.a, pass, big.data, big.inst, big.small);
2005	}
2006
2007	fn deinit(big: *Self) void {
2008	big.a.gpa.free(big.extra_tombs);
2009	}
2010	};
2011	}
2012
2013	fn fmtInstSet(set: *const std.AutoHashMapUnmanaged(Air.Inst.Index, void)) FmtInstSet {
2014	return .{ .set = set };
2015	}
2016
2017	const FmtInstSet = struct {
2018	set: *const std.AutoHashMapUnmanaged(Air.Inst.Index, void),
2019
2020	pub fn format(val: FmtInstSet, comptime _: []const u8, _: std.fmt.FormatOptions, w: anytype) !void {
2021	if (val.set.count() == 0) {
2022	try w.writeAll("[no instructions]");
2023	return;
2024	}
2025	var it = val.set.keyIterator();
2026	try w.print("%{}", .{it.next().?.*});
2027	while (it.next()) \|key\| {
2028	try w.print(" %{}", .{key.*});
2029	}
2030	}
2031	};
2032
2033	fn fmtInstList(list: []const Air.Inst.Index) FmtInstList {
2034	return .{ .list = list };
2035	}
2036
2037	const FmtInstList = struct {
2038	list: []const Air.Inst.Index,
2039
2040	pub fn format(val: FmtInstList, comptime _: []const u8, _: std.fmt.FormatOptions, w: anytype) !void {
2041	if (val.list.len == 0) {
2042	try w.writeAll("[no instructions]");
2043	return;
2044	}
2045	try w.print("%{}", .{val.list[0]});
2046	for (val.list[1..]) \|inst\| {
2047	try w.print(" %{}", .{inst});
2048	}
2049	}
2050	};

src/Liveness/Verify.zig deleted-642

...	@@ -1,642 +0,0 @@
1	//! Verifies that Liveness information is valid.
2
3	gpa: std.mem.Allocator,
4	air: Air,
5	liveness: Liveness,
6	live: LiveMap = .{},
7	blocks: std.AutoHashMapUnmanaged(Air.Inst.Index, LiveMap) = .empty,
8	loops: std.AutoHashMapUnmanaged(Air.Inst.Index, LiveMap) = .empty,
9	intern_pool: *const InternPool,
10
11	pub const Error = error{ LivenessInvalid, OutOfMemory };
12
13	pub fn deinit(self: *Verify) void {
14	self.live.deinit(self.gpa);
15	{
16	var it = self.blocks.valueIterator();
17	while (it.next()) \|block\| block.deinit(self.gpa);
18	self.blocks.deinit(self.gpa);
19	}
20	{
21	var it = self.loops.valueIterator();
22	while (it.next()) \|block\| block.deinit(self.gpa);
23	self.loops.deinit(self.gpa);
24	}
25	self.* = undefined;
26	}
27
28	pub fn verify(self: *Verify) Error!void {
29	self.live.clearRetainingCapacity();
30	self.blocks.clearRetainingCapacity();
31	self.loops.clearRetainingCapacity();
32	try self.verifyBody(self.air.getMainBody());
33	// We don't care about `self.live` now, because the loop body was noreturn - everything being dead was checked on `ret` etc
34	assert(self.blocks.count() == 0);
35	assert(self.loops.count() == 0);
36	}
37
38	const LiveMap = std.AutoHashMapUnmanaged(Air.Inst.Index, void);
39
40	fn verifyBody(self: *Verify, body: []const Air.Inst.Index) Error!void {
41	const ip = self.intern_pool;
42	const tags = self.air.instructions.items(.tag);
43	const data = self.air.instructions.items(.data);
44	for (body) \|inst\| {
45	if (self.liveness.isUnused(inst) and !self.air.mustLower(inst, ip)) {
46	// This instruction will not be lowered and should be ignored.
47	continue;
48	}
49
50	switch (tags[@intFromEnum(inst)]) {
51	// no operands
52	.arg,
53	.alloc,
54	.inferred_alloc,
55	.inferred_alloc_comptime,
56	.ret_ptr,
57	.breakpoint,
58	.dbg_stmt,
59	.dbg_empty_stmt,
60	.ret_addr,
61	.frame_addr,
62	.wasm_memory_size,
63	.err_return_trace,
64	.save_err_return_trace_index,
65	.tlv_dllimport_ptr,
66	.c_va_start,
67	.work_item_id,
68	.work_group_size,
69	.work_group_id,
70	=> try self.verifyInstOperands(inst, .{ .none, .none, .none }),
71
72	.trap, .unreach => {
73	try self.verifyInstOperands(inst, .{ .none, .none, .none });
74	// This instruction terminates the function, so everything should be dead
75	if (self.live.count() > 0) return invalid("%{}: instructions still alive", .{inst});
76	},
77
78	// unary
79	.not,
80	.bitcast,
81	.load,
82	.fpext,
83	.fptrunc,
84	.intcast,
85	.intcast_safe,
86	.trunc,
87	.optional_payload,
88	.optional_payload_ptr,
89	.optional_payload_ptr_set,
90	.errunion_payload_ptr_set,
91	.wrap_optional,
92	.unwrap_errunion_payload,
93	.unwrap_errunion_err,
94	.unwrap_errunion_payload_ptr,
95	.unwrap_errunion_err_ptr,
96	.wrap_errunion_payload,
97	.wrap_errunion_err,
98	.slice_ptr,
99	.slice_len,
100	.ptr_slice_len_ptr,
101	.ptr_slice_ptr_ptr,
102	.struct_field_ptr_index_0,
103	.struct_field_ptr_index_1,
104	.struct_field_ptr_index_2,
105	.struct_field_ptr_index_3,
106	.array_to_slice,
107	.int_from_float,
108	.int_from_float_optimized,
109	.float_from_int,
110	.get_union_tag,
111	.clz,
112	.ctz,
113	.popcount,
114	.byte_swap,
115	.bit_reverse,
116	.splat,
117	.error_set_has_value,
118	.addrspace_cast,
119	.c_va_arg,
120	.c_va_copy,
121	.abs,
122	=> {
123	const ty_op = data[@intFromEnum(inst)].ty_op;
124	try self.verifyInstOperands(inst, .{ ty_op.operand, .none, .none });
125	},
126	.is_null,
127	.is_non_null,
128	.is_null_ptr,
129	.is_non_null_ptr,
130	.is_err,
131	.is_non_err,
132	.is_err_ptr,
133	.is_non_err_ptr,
134	.is_named_enum_value,
135	.tag_name,
136	.error_name,
137	.sqrt,
138	.sin,
139	.cos,
140	.tan,
141	.exp,
142	.exp2,
143	.log,
144	.log2,
145	.log10,
146	.floor,
147	.ceil,
148	.round,
149	.trunc_float,
150	.neg,
151	.neg_optimized,
152	.cmp_lt_errors_len,
153	.set_err_return_trace,
154	.c_va_end,
155	=> {
156	const un_op = data[@intFromEnum(inst)].un_op;
157	try self.verifyInstOperands(inst, .{ un_op, .none, .none });
158	},
159	.ret,
160	.ret_safe,
161	.ret_load,
162	=> {
163	const un_op = data[@intFromEnum(inst)].un_op;
164	try self.verifyInstOperands(inst, .{ un_op, .none, .none });
165	// This instruction terminates the function, so everything should be dead
166	if (self.live.count() > 0) return invalid("%{}: instructions still alive", .{inst});
167	},
168	.dbg_var_ptr,
169	.dbg_var_val,
170	.dbg_arg_inline,
171	.wasm_memory_grow,
172	=> {
173	const pl_op = data[@intFromEnum(inst)].pl_op;
174	try self.verifyInstOperands(inst, .{ pl_op.operand, .none, .none });
175	},
176	.prefetch => {
177	const prefetch = data[@intFromEnum(inst)].prefetch;
178	try self.verifyInstOperands(inst, .{ prefetch.ptr, .none, .none });
179	},
180	.reduce,
181	.reduce_optimized,
182	=> {
183	const reduce = data[@intFromEnum(inst)].reduce;
184	try self.verifyInstOperands(inst, .{ reduce.operand, .none, .none });
185	},
186	.union_init => {
187	const ty_pl = data[@intFromEnum(inst)].ty_pl;
188	const extra = self.air.extraData(Air.UnionInit, ty_pl.payload).data;
189	try self.verifyInstOperands(inst, .{ extra.init, .none, .none });
190	},
191	.struct_field_ptr, .struct_field_val => {
192	const ty_pl = data[@intFromEnum(inst)].ty_pl;
193	const extra = self.air.extraData(Air.StructField, ty_pl.payload).data;
194	try self.verifyInstOperands(inst, .{ extra.struct_operand, .none, .none });
195	},
196	.field_parent_ptr => {
197	const ty_pl = data[@intFromEnum(inst)].ty_pl;
198	const extra = self.air.extraData(Air.FieldParentPtr, ty_pl.payload).data;
199	try self.verifyInstOperands(inst, .{ extra.field_ptr, .none, .none });
200	},
201	.atomic_load => {
202	const atomic_load = data[@intFromEnum(inst)].atomic_load;
203	try self.verifyInstOperands(inst, .{ atomic_load.ptr, .none, .none });
204	},
205
206	// binary
207	.add,
208	.add_safe,
209	.add_optimized,
210	.add_wrap,
211	.add_sat,
212	.sub,
213	.sub_safe,
214	.sub_optimized,
215	.sub_wrap,
216	.sub_sat,
217	.mul,
218	.mul_safe,
219	.mul_optimized,
220	.mul_wrap,
221	.mul_sat,
222	.div_float,
223	.div_float_optimized,
224	.div_trunc,
225	.div_trunc_optimized,
226	.div_floor,
227	.div_floor_optimized,
228	.div_exact,
229	.div_exact_optimized,
230	.rem,
231	.rem_optimized,
232	.mod,
233	.mod_optimized,
234	.bit_and,
235	.bit_or,
236	.xor,
237	.cmp_lt,
238	.cmp_lt_optimized,
239	.cmp_lte,
240	.cmp_lte_optimized,
241	.cmp_eq,
242	.cmp_eq_optimized,
243	.cmp_gte,
244	.cmp_gte_optimized,
245	.cmp_gt,
246	.cmp_gt_optimized,
247	.cmp_neq,
248	.cmp_neq_optimized,
249	.bool_and,
250	.bool_or,
251	.store,
252	.store_safe,
253	.array_elem_val,
254	.slice_elem_val,
255	.ptr_elem_val,
256	.shl,
257	.shl_exact,
258	.shl_sat,
259	.shr,
260	.shr_exact,
261	.atomic_store_unordered,
262	.atomic_store_monotonic,
263	.atomic_store_release,
264	.atomic_store_seq_cst,
265	.set_union_tag,
266	.min,
267	.max,
268	.memset,
269	.memset_safe,
270	.memcpy,
271	.memmove,
272	=> {
273	const bin_op = data[@intFromEnum(inst)].bin_op;
274	try self.verifyInstOperands(inst, .{ bin_op.lhs, bin_op.rhs, .none });
275	},
276	.add_with_overflow,
277	.sub_with_overflow,
278	.mul_with_overflow,
279	.shl_with_overflow,
280	.ptr_add,
281	.ptr_sub,
282	.ptr_elem_ptr,
283	.slice_elem_ptr,
284	.slice,
285	=> {
286	const ty_pl = data[@intFromEnum(inst)].ty_pl;
287	const extra = self.air.extraData(Air.Bin, ty_pl.payload).data;
288	try self.verifyInstOperands(inst, .{ extra.lhs, extra.rhs, .none });
289	},
290	.shuffle => {
291	const ty_pl = data[@intFromEnum(inst)].ty_pl;
292	const extra = self.air.extraData(Air.Shuffle, ty_pl.payload).data;
293	try self.verifyInstOperands(inst, .{ extra.a, extra.b, .none });
294	},
295	.cmp_vector,
296	.cmp_vector_optimized,
297	=> {
298	const ty_pl = data[@intFromEnum(inst)].ty_pl;
299	const extra = self.air.extraData(Air.VectorCmp, ty_pl.payload).data;
300	try self.verifyInstOperands(inst, .{ extra.lhs, extra.rhs, .none });
301	},
302	.atomic_rmw => {
303	const pl_op = data[@intFromEnum(inst)].pl_op;
304	const extra = self.air.extraData(Air.AtomicRmw, pl_op.payload).data;
305	try self.verifyInstOperands(inst, .{ pl_op.operand, extra.operand, .none });
306	},
307
308	// ternary
309	.select => {
310	const pl_op = data[@intFromEnum(inst)].pl_op;
311	const extra = self.air.extraData(Air.Bin, pl_op.payload).data;
312	try self.verifyInstOperands(inst, .{ pl_op.operand, extra.lhs, extra.rhs });
313	},
314	.mul_add => {
315	const pl_op = data[@intFromEnum(inst)].pl_op;
316	const extra = self.air.extraData(Air.Bin, pl_op.payload).data;
317	try self.verifyInstOperands(inst, .{ extra.lhs, extra.rhs, pl_op.operand });
318	},
319	.vector_store_elem => {
320	const vector_store_elem = data[@intFromEnum(inst)].vector_store_elem;
321	const extra = self.air.extraData(Air.Bin, vector_store_elem.payload).data;
322	try self.verifyInstOperands(inst, .{ vector_store_elem.vector_ptr, extra.lhs, extra.rhs });
323	},
324	.cmpxchg_strong,
325	.cmpxchg_weak,
326	=> {
327	const ty_pl = data[@intFromEnum(inst)].ty_pl;
328	const extra = self.air.extraData(Air.Cmpxchg, ty_pl.payload).data;
329	try self.verifyInstOperands(inst, .{ extra.ptr, extra.expected_value, extra.new_value });
330	},
331
332	// big tombs
333	.aggregate_init => {
334	const ty_pl = data[@intFromEnum(inst)].ty_pl;
335	const aggregate_ty = ty_pl.ty.toType();
336	const len = @as(usize, @intCast(aggregate_ty.arrayLenIp(ip)));
337	const elements = @as([]const Air.Inst.Ref, @ptrCast(self.air.extra[ty_pl.payload..][0..len]));
338
339	var bt = self.liveness.iterateBigTomb(inst);
340	for (elements) \|element\| {
341	try self.verifyOperand(inst, element, bt.feed());
342	}
343	try self.verifyInst(inst);
344	},
345	.call, .call_always_tail, .call_never_tail, .call_never_inline => {
346	const pl_op = data[@intFromEnum(inst)].pl_op;
347	const extra = self.air.extraData(Air.Call, pl_op.payload);
348	const args = @as(
349	[]const Air.Inst.Ref,
350	@ptrCast(self.air.extra[extra.end..][0..extra.data.args_len]),
351	);
352
353	var bt = self.liveness.iterateBigTomb(inst);
354	try self.verifyOperand(inst, pl_op.operand, bt.feed());
355	for (args) \|arg\| {
356	try self.verifyOperand(inst, arg, bt.feed());
357	}
358	try self.verifyInst(inst);
359	},
360	.assembly => {
361	const ty_pl = data[@intFromEnum(inst)].ty_pl;
362	const extra = self.air.extraData(Air.Asm, ty_pl.payload);
363	var extra_i = extra.end;
364	const outputs = @as(
365	[]const Air.Inst.Ref,
366	@ptrCast(self.air.extra[extra_i..][0..extra.data.outputs_len]),
367	);
368	extra_i += outputs.len;
369	const inputs = @as(
370	[]const Air.Inst.Ref,
371	@ptrCast(self.air.extra[extra_i..][0..extra.data.inputs_len]),
372	);
373	extra_i += inputs.len;
374
375	var bt = self.liveness.iterateBigTomb(inst);
376	for (outputs) \|output\| {
377	if (output != .none) {
378	try self.verifyOperand(inst, output, bt.feed());
379	}
380	}
381	for (inputs) \|input\| {
382	try self.verifyOperand(inst, input, bt.feed());
383	}
384	try self.verifyInst(inst);
385	},
386
387	// control flow
388	.@"try", .try_cold => {
389	const pl_op = data[@intFromEnum(inst)].pl_op;
390	const extra = self.air.extraData(Air.Try, pl_op.payload);
391	const try_body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra.end..][0..extra.data.body_len]);
392
393	const cond_br_liveness = self.liveness.getCondBr(inst);
394
395	try self.verifyOperand(inst, pl_op.operand, self.liveness.operandDies(inst, 0));
396
397	var live = try self.live.clone(self.gpa);
398	defer live.deinit(self.gpa);
399
400	for (cond_br_liveness.else_deaths) \|death\| try self.verifyDeath(inst, death);
401	try self.verifyBody(try_body);
402
403	self.live.deinit(self.gpa);
404	self.live = live.move();
405
406	for (cond_br_liveness.then_deaths) \|death\| try self.verifyDeath(inst, death);
407
408	try self.verifyInst(inst);
409	},
410	.try_ptr, .try_ptr_cold => {
411	const ty_pl = data[@intFromEnum(inst)].ty_pl;
412	const extra = self.air.extraData(Air.TryPtr, ty_pl.payload);
413	const try_body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra.end..][0..extra.data.body_len]);
414
415	const cond_br_liveness = self.liveness.getCondBr(inst);
416
417	try self.verifyOperand(inst, extra.data.ptr, self.liveness.operandDies(inst, 0));
418
419	var live = try self.live.clone(self.gpa);
420	defer live.deinit(self.gpa);
421
422	for (cond_br_liveness.else_deaths) \|death\| try self.verifyDeath(inst, death);
423	try self.verifyBody(try_body);
424
425	self.live.deinit(self.gpa);
426	self.live = live.move();
427
428	for (cond_br_liveness.then_deaths) \|death\| try self.verifyDeath(inst, death);
429
430	try self.verifyInst(inst);
431	},
432	.br => {
433	const br = data[@intFromEnum(inst)].br;
434	const gop = try self.blocks.getOrPut(self.gpa, br.block_inst);
435
436	try self.verifyOperand(inst, br.operand, self.liveness.operandDies(inst, 0));
437	if (gop.found_existing) {
438	try self.verifyMatchingLiveness(br.block_inst, gop.value_ptr.*);
439	} else {
440	gop.value_ptr.* = try self.live.clone(self.gpa);
441	}
442	try self.verifyInst(inst);
443	},
444	.repeat => {
445	const repeat = data[@intFromEnum(inst)].repeat;
446	const expected_live = self.loops.get(repeat.loop_inst) orelse
447	return invalid("%{}: loop %{} not in scope", .{ @intFromEnum(inst), @intFromEnum(repeat.loop_inst) });
448
449	try self.verifyMatchingLiveness(repeat.loop_inst, expected_live);
450	},
451	.switch_dispatch => {
452	const br = data[@intFromEnum(inst)].br;
453
454	try self.verifyOperand(inst, br.operand, self.liveness.operandDies(inst, 0));
455
456	const expected_live = self.loops.get(br.block_inst) orelse
457	return invalid("%{}: loop %{} not in scope", .{ @intFromEnum(inst), @intFromEnum(br.block_inst) });
458
459	try self.verifyMatchingLiveness(br.block_inst, expected_live);
460	},
461	.block, .dbg_inline_block => \|tag\| {
462	const ty_pl = data[@intFromEnum(inst)].ty_pl;
463	const block_ty = ty_pl.ty.toType();
464	const block_body: []const Air.Inst.Index = @ptrCast(switch (tag) {
465	inline .block, .dbg_inline_block => \|comptime_tag\| body: {
466	const extra = self.air.extraData(switch (comptime_tag) {
467	.block => Air.Block,
468	.dbg_inline_block => Air.DbgInlineBlock,
469	else => unreachable,
470	}, ty_pl.payload);
471	break :body self.air.extra[extra.end..][0..extra.data.body_len];
472	},
473	else => unreachable,
474	});
475	const block_liveness = self.liveness.getBlock(inst);
476
477	var orig_live = try self.live.clone(self.gpa);
478	defer orig_live.deinit(self.gpa);
479
480	assert(!self.blocks.contains(inst));
481	try self.verifyBody(block_body);
482
483	// Liveness data after the block body is garbage, but we want to
484	// restore it to verify deaths
485	self.live.deinit(self.gpa);
486	self.live = orig_live.move();
487
488	for (block_liveness.deaths) \|death\| try self.verifyDeath(inst, death);
489
490	if (ip.isNoReturn(block_ty.toIntern())) {
491	assert(!self.blocks.contains(inst));
492	} else {
493	var live = self.blocks.fetchRemove(inst).?.value;
494	defer live.deinit(self.gpa);
495
496	try self.verifyMatchingLiveness(inst, live);
497	}
498
499	try self.verifyInstOperands(inst, .{ .none, .none, .none });
500	},
501	.loop => {
502	const ty_pl = data[@intFromEnum(inst)].ty_pl;
503	const extra = self.air.extraData(Air.Block, ty_pl.payload);
504	const loop_body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra.end..][0..extra.data.body_len]);
505
506	// The same stuff should be alive after the loop as before it.
507	const gop = try self.loops.getOrPut(self.gpa, inst);
508	if (gop.found_existing) return invalid("%{}: loop already exists", .{@intFromEnum(inst)});
509	defer {
510	var live = self.loops.fetchRemove(inst).?;
511	live.value.deinit(self.gpa);
512	}
513	gop.value_ptr.* = try self.live.clone(self.gpa);
514
515	try self.verifyBody(loop_body);
516
517	try self.verifyInstOperands(inst, .{ .none, .none, .none });
518	},
519	.cond_br => {
520	const pl_op = data[@intFromEnum(inst)].pl_op;
521	const extra = self.air.extraData(Air.CondBr, pl_op.payload);
522	const then_body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra.end..][0..extra.data.then_body_len]);
523	const else_body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra.end + then_body.len ..][0..extra.data.else_body_len]);
524	const cond_br_liveness = self.liveness.getCondBr(inst);
525
526	try self.verifyOperand(inst, pl_op.operand, self.liveness.operandDies(inst, 0));
527
528	var live = try self.live.clone(self.gpa);
529	defer live.deinit(self.gpa);
530
531	for (cond_br_liveness.then_deaths) \|death\| try self.verifyDeath(inst, death);
532	try self.verifyBody(then_body);
533
534	self.live.deinit(self.gpa);
535	self.live = live.move();
536
537	for (cond_br_liveness.else_deaths) \|death\| try self.verifyDeath(inst, death);
538	try self.verifyBody(else_body);
539
540	try self.verifyInst(inst);
541	},
542	.switch_br, .loop_switch_br => {
543	const switch_br = self.air.unwrapSwitch(inst);
544	const switch_br_liveness = try self.liveness.getSwitchBr(
545	self.gpa,
546	inst,
547	switch_br.cases_len + 1,
548	);
549	defer self.gpa.free(switch_br_liveness.deaths);
550
551	try self.verifyOperand(inst, switch_br.operand, self.liveness.operandDies(inst, 0));
552
553	// Excluding the operand (which we just handled), the same stuff should be alive
554	// after the loop as before it.
555	{
556	const gop = try self.loops.getOrPut(self.gpa, inst);
557	if (gop.found_existing) return invalid("%{}: loop already exists", .{@intFromEnum(inst)});
558	gop.value_ptr.* = self.live.move();
559	}
560	defer {
561	var live = self.loops.fetchRemove(inst).?;
562	live.value.deinit(self.gpa);
563	}
564
565	var it = switch_br.iterateCases();
566	while (it.next()) \|case\| {
567	self.live.deinit(self.gpa);
568	self.live = try self.loops.get(inst).?.clone(self.gpa);
569
570	for (switch_br_liveness.deaths[case.idx]) \|death\| try self.verifyDeath(inst, death);
571	try self.verifyBody(case.body);
572	}
573
574	const else_body = it.elseBody();
575	if (else_body.len > 0) {
576	self.live.deinit(self.gpa);
577	self.live = try self.loops.get(inst).?.clone(self.gpa);
578	for (switch_br_liveness.deaths[switch_br.cases_len]) \|death\| try self.verifyDeath(inst, death);
579	try self.verifyBody(else_body);
580	}
581
582	try self.verifyInst(inst);
583	},
584	}
585	}
586	}
587
588	fn verifyDeath(self: *Verify, inst: Air.Inst.Index, operand: Air.Inst.Index) Error!void {
589	try self.verifyOperand(inst, operand.toRef(), true);
590	}
591
592	fn verifyOperand(self: *Verify, inst: Air.Inst.Index, op_ref: Air.Inst.Ref, dies: bool) Error!void {
593	const operand = op_ref.toIndexAllowNone() orelse {
594	assert(!dies);
595	return;
596	};
597	if (dies) {
598	if (!self.live.remove(operand)) return invalid("%{}: dead operand %{} reused and killed again", .{ inst, operand });
599	} else {
600	if (!self.live.contains(operand)) return invalid("%{}: dead operand %{} reused", .{ inst, operand });
601	}
602	}
603
604	fn verifyInstOperands(
605	self: *Verify,
606	inst: Air.Inst.Index,
607	operands: [Liveness.bpi - 1]Air.Inst.Ref,
608	) Error!void {
609	for (operands, 0..) \|operand, operand_index\| {
610	const dies = self.liveness.operandDies(inst, @as(Liveness.OperandInt, @intCast(operand_index)));
611	try self.verifyOperand(inst, operand, dies);
612	}
613	try self.verifyInst(inst);
614	}
615
616	fn verifyInst(self: *Verify, inst: Air.Inst.Index) Error!void {
617	if (self.liveness.isUnused(inst)) {
618	assert(!self.live.contains(inst));
619	} else {
620	try self.live.putNoClobber(self.gpa, inst, {});
621	}
622	}
623
624	fn verifyMatchingLiveness(self: *Verify, block: Air.Inst.Index, live: LiveMap) Error!void {
625	if (self.live.count() != live.count()) return invalid("%{}: different deaths across branches", .{block});
626	var live_it = self.live.keyIterator();
627	while (live_it.next()) \|live_inst\| if (!live.contains(live_inst.*)) return invalid("%{}: different deaths across branches", .{block});
628	}
629
630	fn invalid(comptime fmt: []const u8, args: anytype) error{LivenessInvalid} {
631	log.err(fmt, args);
632	return error.LivenessInvalid;
633	}
634
635	const std = @import("std");
636	const assert = std.debug.assert;
637	const log = std.log.scoped(.liveness_verify);
638
639	const Air = @import("../Air.zig");
640	const Liveness = @import("../Liveness.zig");
641	const InternPool = @import("../InternPool.zig");
642	const Verify = @This();

src/Sema.zig+2-8

 @@ -756,13 +756,7 @@ pub const Block = struct {
     fn addReduce(block: *Block, operand: Air.Inst.Ref, operation: std.builtin.ReduceOp) !Air.Inst.Ref {
         const sema = block.sema;
         const zcu = sema.pt.zcu;
-        const vector_ty = sema.typeOf(operand);
+        const allow_optimized = switch (sema.typeOf(operand).childType(zcu).zigTypeTag(zcu)) {
-        switch (vector_ty.vectorLen(zcu)) {
-            0 => unreachable,
-            1 => return block.addBinOp(.array_elem_val, operand, .zero_usize),
-            else => {},
-        }
-        const allow_optimized = switch (vector_ty.childType(zcu).zigTypeTag(zcu)) {
             .float => true,
             .bool, .int => false,
             else => unreachable,
 @@ -36849,7 +36843,7 @@ fn typeOf(sema: *Sema, inst: Air.Inst.Ref) Type {
 pub fn getTmpAir(sema: Sema) Air {
     return .{
         .instructions = sema.air_instructions.slice(),
-        .extra = sema.air_extra.items,
+        .extra = sema.air_extra,
     };
 }

src/Zcu.zig-1

...	@@ -30,7 +30,6 @@ const AstGen = std.zig.AstGen;	...	@@ -30,7 +30,6 @@ const AstGen = std.zig.AstGen;
30	const Sema = @import("Sema.zig");	30	const Sema = @import("Sema.zig");
31	const target_util = @import("target.zig");	31	const target_util = @import("target.zig");
32	const build_options = @import("build_options");	32	const build_options = @import("build_options");
33	const Liveness = @import("Liveness.zig");
34	const isUpDir = @import("introspect.zig").isUpDir;	33	const isUpDir = @import("introspect.zig").isUpDir;
35	const clang = @import("clang.zig");	34	const clang = @import("clang.zig");
36	const InternPool = @import("InternPool.zig");	35	const InternPool = @import("InternPool.zig");

src/Zcu/PerThread.zig+28-24

 @@ -16,7 +16,6 @@ const dev = @import("../dev.zig");
 const InternPool = @import("../InternPool.zig");
 const AnalUnit = InternPool.AnalUnit;
 const introspect = @import("../introspect.zig");
-const Liveness = @import("../Liveness.zig");
 const log = std.log.scoped(.zcu);
 const Module = @import("../Package.zig").Module;
 const Sema = @import("../Sema.zig");
 @@ -1721,34 +1720,43 @@ fn analyzeFuncBody(
 /// Takes ownership of `air`, even on error.
 /// If any types referenced by `air` are unresolved, marks the codegen as failed.
-pub fn linkerUpdateFunc(pt: Zcu.PerThread, func_index: InternPool.Index, air: Air) Allocator.Error!void {
+pub fn linkerUpdateFunc(pt: Zcu.PerThread, func_index: InternPool.Index, air: *Air) Allocator.Error!void {
     const zcu = pt.zcu;
     const gpa = zcu.gpa;
     const ip = &zcu.intern_pool;
     const comp = zcu.comp;
-    defer {
-        var air_mut = air;
-        air_mut.deinit(gpa);
-    }
     const func = zcu.funcInfo(func_index);
     const nav_index = func.owner_nav;
     const nav = ip.getNav(nav_index);
-    var liveness = try Liveness.analyze(gpa, air, ip);
+    const codegen_prog_node = zcu.codegen_prog_node.start(nav.fqn.toSlice(ip), 0);
+    defer codegen_prog_node.end();
+    if (!air.typesFullyResolved(zcu)) {
+        // A type we depend on failed to resolve. This is a transitive failure.
+        // Correcting this failure will involve changing a type this function
+        // depends on, hence triggering re-analysis of this function, so this
+        // interacts correctly with incremental compilation.
+        return;
+    }
+    const backend = target_util.zigBackend(zcu.root_mod.resolved_target.result, zcu.comp.config.use_llvm);
+    try air.legalize(backend, zcu);
+    var liveness = try Air.Liveness.analyze(gpa, air.*, ip);
     defer liveness.deinit(gpa);
     if (build_options.enable_debug_extensions and comp.verbose_air) {
         std.debug.print("# Begin Function AIR: {}:\n", .{nav.fqn.fmt(ip)});
-        @import("../print_air.zig").dump(pt, air, liveness);
+        @import("../print_air.zig").dump(pt, air.*, liveness);
         std.debug.print("# End Function AIR: {}\n\n", .{nav.fqn.fmt(ip)});
     }
     if (std.debug.runtime_safety) {
-        var verify: Liveness.Verify = .{
+        var verify: Air.Liveness.Verify = .{
             .gpa = gpa,
-            .air = air,
+            .air = air.*,
             .liveness = liveness,
             .intern_pool = ip,
         };
 @@ -1768,16 +1776,8 @@ pub fn linkerUpdateFunc(pt: Zcu.PerThread, func_index: InternPool.Index, air: Ai
         };
     }
-    const codegen_prog_node = zcu.codegen_prog_node.start(nav.fqn.toSlice(ip), 0);
+    if (comp.bin_file) |lf| {
-    defer codegen_prog_node.end();
+        lf.updateFunc(pt, func_index, air.*, liveness) catch |err| switch (err) {
-    if (!air.typesFullyResolved(zcu)) {
-        // A type we depend on failed to resolve. This is a transitive failure.
-        // Correcting this failure will involve changing a type this function
-        // depends on, hence triggering re-analysis of this function, so this
-        // interacts correctly with incremental compilation.
-    } else if (comp.bin_file) |lf| {
-        lf.updateFunc(pt, func_index, air, liveness) catch |err| switch (err) {
             error.OutOfMemory => return error.OutOfMemory,
             error.CodegenFail => assert(zcu.failed_codegen.contains(nav_index)),
             error.Overflow, error.RelocationNotByteAligned => {
 @@ -1791,7 +1791,7 @@ pub fn linkerUpdateFunc(pt: Zcu.PerThread, func_index: InternPool.Index, air: Ai
             },
         };
     } else if (zcu.llvm_object) |llvm_object| {
-        llvm_object.updateFunc(pt, func_index, air, liveness) catch |err| switch (err) {
+        llvm_object.updateFunc(pt, func_index, air.*, liveness) catch |err| switch (err) {
             error.OutOfMemory => return error.OutOfMemory,
         };
     }
 @@ -3080,9 +3080,13 @@ fn analyzeFnBodyInner(pt: Zcu.PerThread, func_index: InternPool.Index) Zcu.SemaE
     try sema.flushExports();
+    defer {
+        sema.air_instructions = .empty;
+        sema.air_extra = .empty;
+    }
     return .{
-        .instructions = sema.air_instructions.toOwnedSlice(),
+        .instructions = sema.air_instructions.slice(),
-        .extra = try sema.air_extra.toOwnedSlice(gpa),
+        .extra = sema.air_extra,
     };
 }

src/arch/aarch64/CodeGen.zig+32-33

src/arch/arm/CodeGen.zig+32-33

src/arch/powerpc/CodeGen.zig+1-2

src/arch/riscv64/CodeGen.zig+29-30

src/arch/sparc64/CodeGen.zig+37-38

src/arch/sparc64/Emit.zig-1

...	@@ -7,7 +7,6 @@ const assert = std.debug.assert;	...	@@ -7,7 +7,6 @@ const assert = std.debug.assert;
7	const link = @import("../../link.zig");	7	const link = @import("../../link.zig");
8	const Zcu = @import("../../Zcu.zig");	8	const Zcu = @import("../../Zcu.zig");
9	const ErrorMsg = Zcu.ErrorMsg;	9	const ErrorMsg = Zcu.ErrorMsg;
10	const Liveness = @import("../../Liveness.zig");
11	const log = std.log.scoped(.sparcv9_emit);	10	const log = std.log.scoped(.sparcv9_emit);
12		11
13	const Emit = @This();	12	const Emit = @This();

src/arch/wasm/CodeGen.zig+16-17

src/arch/x86_64/CodeGen.zig+40-36

src/codegen.zig+8-6

 @@ -14,7 +14,6 @@ const Allocator = mem.Allocator;
 const Compilation = @import("Compilation.zig");
 const ErrorMsg = Zcu.ErrorMsg;
 const InternPool = @import("InternPool.zig");
-const Liveness = @import("Liveness.zig");
 const Zcu = @import("Zcu.zig");
 const Type = @import("Type.zig");
 @@ -33,15 +32,18 @@ fn devFeatureForBackend(comptime backend: std.builtin.CompilerBackend) dev.Featu
     return @field(dev.Feature, @tagName(backend)["stage2_".len..] ++ "_backend");
 }
-fn importBackend(comptime backend: std.builtin.CompilerBackend) type {
+pub fn importBackend(comptime backend: std.builtin.CompilerBackend) ?type {
     return switch (backend) {
         .stage2_aarch64 => @import("arch/aarch64/CodeGen.zig"),
         .stage2_arm => @import("arch/arm/CodeGen.zig"),
+        .stage2_c => @import("codegen/c.zig"),
+        .stage2_llvm => @import("codegen/llvm.zig"),
         .stage2_powerpc => @import("arch/powerpc/CodeGen.zig"),
         .stage2_riscv64 => @import("arch/riscv64/CodeGen.zig"),
         .stage2_sparc64 => @import("arch/sparc64/CodeGen.zig"),
+        .stage2_spirv64 => @import("codegen/spirv.zig"),
         .stage2_x86_64 => @import("arch/x86_64/CodeGen.zig"),
-        else => unreachable,
+        else => null,
     };
 }
 @@ -51,7 +53,7 @@ pub fn generateFunction(
     src_loc: Zcu.LazySrcLoc,
     func_index: InternPool.Index,
     air: Air,
-    liveness: Liveness,
+    liveness: Air.Liveness,
     code: *std.ArrayListUnmanaged(u8),
     debug_output: link.File.DebugInfoOutput,
 ) CodeGenError!void {
 @@ -68,7 +70,7 @@ pub fn generateFunction(
         .stage2_x86_64,
         => |backend| {
             dev.check(devFeatureForBackend(backend));
-            return importBackend(backend).generate(lf, pt, src_loc, func_index, air, liveness, code, debug_output);
+            return importBackend(backend).?.generate(lf, pt, src_loc, func_index, air, liveness, code, debug_output);
         },
     }
 }
 @@ -93,7 +95,7 @@ pub fn generateLazyFunction(
         .stage2_x86_64,
         => |backend| {
             dev.check(devFeatureForBackend(backend));
-            return importBackend(backend).generateLazy(lf, pt, src_loc, lazy_sym, code, debug_output);
+            return importBackend(backend).?.generateLazy(lf, pt, src_loc, lazy_sym, code, debug_output);
         },
     }
 }

src/codegen/c.zig+34-35

src/codegen/llvm.zig+20-21

src/codegen/spirv.zig+20-21

src/link.zig+4-4

...	@@ -15,7 +15,6 @@ const Path = std.Build.Cache.Path;	...	@@ -15,7 +15,6 @@ const Path = std.Build.Cache.Path;
15	const Directory = std.Build.Cache.Directory;	15	const Directory = std.Build.Cache.Directory;
16	const Compilation = @import("Compilation.zig");	16	const Compilation = @import("Compilation.zig");
17	const LibCInstallation = std.zig.LibCInstallation;	17	const LibCInstallation = std.zig.LibCInstallation;
18	const Liveness = @import("Liveness.zig");
19	const Zcu = @import("Zcu.zig");	18	const Zcu = @import("Zcu.zig");
20	const InternPool = @import("InternPool.zig");	19	const InternPool = @import("InternPool.zig");
21	const Type = @import("Type.zig");	20	const Type = @import("Type.zig");
...	@@ -738,7 +737,7 @@ pub const File = struct {	...	@@ -738,7 +737,7 @@ pub const File = struct {
738	pt: Zcu.PerThread,	737	pt: Zcu.PerThread,
739	func_index: InternPool.Index,	738	func_index: InternPool.Index,
740	air: Air,	739	air: Air,
741	liveness: Liveness,	740	liveness: Air.Liveness,
742	) UpdateNavError!void {	741	) UpdateNavError!void {
743	switch (base.tag) {	742	switch (base.tag) {
744	inline else => \|tag\| {	743	inline else => \|tag\| {
...	@@ -1601,8 +1600,9 @@ pub fn doTask(comp: *Compilation, tid: usize, task: Task) void {	...	@@ -1601,8 +1600,9 @@ pub fn doTask(comp: *Compilation, tid: usize, task: Task) void {
1601	if (comp.remaining_prelink_tasks == 0) {	1600	if (comp.remaining_prelink_tasks == 0) {
1602	const pt: Zcu.PerThread = .activate(comp.zcu.?, @enumFromInt(tid));	1601	const pt: Zcu.PerThread = .activate(comp.zcu.?, @enumFromInt(tid));
1603	defer pt.deactivate();	1602	defer pt.deactivate();
1604	// This call takes ownership of `func.air`.	1603	var air = func.air;
1605	pt.linkerUpdateFunc(func.func, func.air) catch \|err\| switch (err) {	1604	defer air.deinit(comp.gpa);
		1605	pt.linkerUpdateFunc(func.func, &air) catch \|err\| switch (err) {
1606	error.OutOfMemory => diags.setAllocFailure(),	1606	error.OutOfMemory => diags.setAllocFailure(),
1607	};	1607	};
1608	} else {	1608	} else {

src/link/C.zig+1-2

...	@@ -18,7 +18,6 @@ const trace = @import("../tracy.zig").trace;	...	@@ -18,7 +18,6 @@ const trace = @import("../tracy.zig").trace;
18	const Type = @import("../Type.zig");	18	const Type = @import("../Type.zig");
19	const Value = @import("../Value.zig");	19	const Value = @import("../Value.zig");
20	const Air = @import("../Air.zig");	20	const Air = @import("../Air.zig");
21	const Liveness = @import("../Liveness.zig");
22		21
23	pub const zig_h = "#include \"zig.h\"\n";	22	pub const zig_h = "#include \"zig.h\"\n";
24		23
...	@@ -180,7 +179,7 @@ pub fn updateFunc(	...	@@ -180,7 +179,7 @@ pub fn updateFunc(
180	pt: Zcu.PerThread,	179	pt: Zcu.PerThread,
181	func_index: InternPool.Index,	180	func_index: InternPool.Index,
182	air: Air,	181	air: Air,
183	liveness: Liveness,	182	liveness: Air.Liveness,
184	) link.File.UpdateNavError!void {	183	) link.File.UpdateNavError!void {
185	const zcu = pt.zcu;	184	const zcu = pt.zcu;
186	const gpa = zcu.gpa;	185	const gpa = zcu.gpa;

src/link/Coff.zig+1-2

src/link/Elf.zig+1-2

src/link/Elf/ZigObject.zig+1-2

...	@@ -1416,7 +1416,7 @@ pub fn updateFunc(	...	@@ -1416,7 +1416,7 @@ pub fn updateFunc(
1416	pt: Zcu.PerThread,	1416	pt: Zcu.PerThread,
1417	func_index: InternPool.Index,	1417	func_index: InternPool.Index,
1418	air: Air,	1418	air: Air,
1419	liveness: Liveness,	1419	liveness: Air.Liveness,
1420	) link.File.UpdateNavError!void {	1420	) link.File.UpdateNavError!void {
1421	const tracy = trace(@src());	1421	const tracy = trace(@src());
1422	defer tracy.end();	1422	defer tracy.end();
...	@@ -2367,7 +2367,6 @@ const Dwarf = @import("../Dwarf.zig");	...	@@ -2367,7 +2367,6 @@ const Dwarf = @import("../Dwarf.zig");
2367	const Elf = @import("../Elf.zig");	2367	const Elf = @import("../Elf.zig");
2368	const File = @import("file.zig").File;	2368	const File = @import("file.zig").File;
2369	const InternPool = @import("../../InternPool.zig");	2369	const InternPool = @import("../../InternPool.zig");
2370	const Liveness = @import("../../Liveness.zig");
2371	const Zcu = @import("../../Zcu.zig");	2370	const Zcu = @import("../../Zcu.zig");
2372	const Object = @import("Object.zig");	2371	const Object = @import("Object.zig");
2373	const Symbol = @import("Symbol.zig");	2372	const Symbol = @import("Symbol.zig");

src/link/Goff.zig+1-2

src/link/MachO.zig+1-2

src/link/MachO/ZigObject.zig+1-2

...	@@ -778,7 +778,7 @@ pub fn updateFunc(	...	@@ -778,7 +778,7 @@ pub fn updateFunc(
778	pt: Zcu.PerThread,	778	pt: Zcu.PerThread,
779	func_index: InternPool.Index,	779	func_index: InternPool.Index,
780	air: Air,	780	air: Air,
781	liveness: Liveness,	781	liveness: Air.Liveness,
782	) link.File.UpdateNavError!void {	782	) link.File.UpdateNavError!void {
783	const tracy = trace(@src());	783	const tracy = trace(@src());
784	defer tracy.end();	784	defer tracy.end();
...	@@ -1820,7 +1820,6 @@ const Atom = @import("Atom.zig");	...	@@ -1820,7 +1820,6 @@ const Atom = @import("Atom.zig");
1820	const Dwarf = @import("../Dwarf.zig");	1820	const Dwarf = @import("../Dwarf.zig");
1821	const File = @import("file.zig").File;	1821	const File = @import("file.zig").File;
1822	const InternPool = @import("../../InternPool.zig");	1822	const InternPool = @import("../../InternPool.zig");
1823	const Liveness = @import("../../Liveness.zig");
1824	const MachO = @import("../MachO.zig");	1823	const MachO = @import("../MachO.zig");
1825	const Nlist = Object.Nlist;	1824	const Nlist = Object.Nlist;
1826	const Zcu = @import("../../Zcu.zig");	1825	const Zcu = @import("../../Zcu.zig");

src/link/Plan9.zig+1-2

src/link/SpirV.zig+1-2

src/link/Wasm.zig+1-2

...	@@ -36,7 +36,6 @@ const abi = @import("../arch/wasm/abi.zig");	...	@@ -36,7 +36,6 @@ const abi = @import("../arch/wasm/abi.zig");
36	const Compilation = @import("../Compilation.zig");	36	const Compilation = @import("../Compilation.zig");
37	const Dwarf = @import("Dwarf.zig");	37	const Dwarf = @import("Dwarf.zig");
38	const InternPool = @import("../InternPool.zig");	38	const InternPool = @import("../InternPool.zig");
39	const Liveness = @import("../Liveness.zig");
40	const LlvmObject = @import("../codegen/llvm.zig").Object;	39	const LlvmObject = @import("../codegen/llvm.zig").Object;
41	const Zcu = @import("../Zcu.zig");	40	const Zcu = @import("../Zcu.zig");
42	const codegen = @import("../codegen.zig");	41	const codegen = @import("../codegen.zig");
...	@@ -3193,7 +3192,7 @@ pub fn deinit(wasm: *Wasm) void {	...	@@ -3193,7 +3192,7 @@ pub fn deinit(wasm: *Wasm) void {
3193	wasm.missing_exports.deinit(gpa);	3192	wasm.missing_exports.deinit(gpa);
3194	}	3193	}
3195		3194
3196	pub fn updateFunc(wasm: *Wasm, pt: Zcu.PerThread, func_index: InternPool.Index, air: Air, liveness: Liveness) !void {	3195	pub fn updateFunc(wasm: *Wasm, pt: Zcu.PerThread, func_index: InternPool.Index, air: Air, liveness: Air.Liveness) !void {
3197	if (build_options.skip_non_native and builtin.object_format != .wasm) {	3196	if (build_options.skip_non_native and builtin.object_format != .wasm) {
3198	@panic("Attempted to compile for object format that was disabled by build configuration");	3197	@panic("Attempted to compile for object format that was disabled by build configuration");
3199	}	3198	}

src/link/Xcoff.zig+1-2

src/print_air.zig+34-35

 @@ -6,20 +6,19 @@ const Zcu = @import("Zcu.zig");
 const Value = @import("Value.zig");
 const Type = @import("Type.zig");
 const Air = @import("Air.zig");
-const Liveness = @import("Liveness.zig");
 const InternPool = @import("InternPool.zig");
-pub fn write(stream: anytype, pt: Zcu.PerThread, air: Air, liveness: ?Liveness) void {
+pub fn write(stream: anytype, pt: Zcu.PerThread, air: Air, liveness: ?Air.Liveness) void {
     const instruction_bytes = air.instructions.len *
         // Here we don't use @sizeOf(Air.Inst.Data) because it would include
         // the debug safety tag but we want to measure release size.
         (@sizeOf(Air.Inst.Tag) + 8);
-    const extra_bytes = air.extra.len * @sizeOf(u32);
+    const extra_bytes = air.extra.items.len * @sizeOf(u32);
     const tomb_bytes = if (liveness) |l| l.tomb_bits.len * @sizeOf(usize) else 0;
     const liveness_extra_bytes = if (liveness) |l| l.extra.len * @sizeOf(u32) else 0;
     const liveness_special_bytes = if (liveness) |l| l.special.count() * 8 else 0;
     const total_bytes = @sizeOf(Air) + instruction_bytes + extra_bytes +
-        @sizeOf(Liveness) + liveness_extra_bytes +
+        @sizeOf(Air.Liveness) + liveness_extra_bytes +
         liveness_special_bytes + tomb_bytes;
     // zig fmt: off
 @@ -34,7 +33,7 @@ pub fn write(stream: anytype, pt: Zcu.PerThread, air: Air, liveness: ?Liveness)
     , .{
         fmtIntSizeBin(total_bytes),
         air.instructions.len, fmtIntSizeBin(instruction_bytes),
-        air.extra.len, fmtIntSizeBin(extra_bytes),
+        air.extra.items.len, fmtIntSizeBin(extra_bytes),
         fmtIntSizeBin(tomb_bytes),
         if (liveness) |l| l.extra.len else 0, fmtIntSizeBin(liveness_extra_bytes),
         if (liveness) |l| l.special.count() else 0, fmtIntSizeBin(liveness_special_bytes),
 @@ -57,7 +56,7 @@ pub fn writeInst(
     inst: Air.Inst.Index,
     pt: Zcu.PerThread,
     air: Air,
-    liveness: ?Liveness,
+    liveness: ?Air.Liveness,
 ) void {
     var writer: Writer = .{
         .pt = pt,
 @@ -70,11 +69,11 @@ pub fn writeInst(
     writer.writeInst(stream, inst) catch return;
 }
-pub fn dump(pt: Zcu.PerThread, air: Air, liveness: ?Liveness) void {
+pub fn dump(pt: Zcu.PerThread, air: Air, liveness: ?Air.Liveness) void {
     write(std.io.getStdErr().writer(), pt, air, liveness);
 }
-pub fn dumpInst(inst: Air.Inst.Index, pt: Zcu.PerThread, air: Air, liveness: ?Liveness) void {
+pub fn dumpInst(inst: Air.Inst.Index, pt: Zcu.PerThread, air: Air, liveness: ?Air.Liveness) void {
     writeInst(std.io.getStdErr().writer(), inst, pt, air, liveness);
 }
 @@ -82,7 +81,7 @@ const Writer = struct {
     pt: Zcu.PerThread,
     gpa: Allocator,
     air: Air,
-    liveness: ?Liveness,
+    liveness: ?Air.Liveness,
     indent: usize,
     skip_body: bool,
 @@ -391,15 +390,15 @@ const Writer = struct {
                     },
                     else => unreachable,
                 }
-                break :body w.air.extra[extra.end..][0..extra.data.body_len];
+                break :body w.air.extra.items[extra.end..][0..extra.data.body_len];
             },
             else => unreachable,
         });
         if (w.skip_body) return s.writeAll(", ...");
-        const liveness_block = if (w.liveness) |liveness|
+        const liveness_block: Air.Liveness.BlockSlices = if (w.liveness) |liveness|
             liveness.getBlock(inst)
         else
-            Liveness.BlockSlices{ .deaths = &.{} };
+            .{ .deaths = &.{} };
         try s.writeAll(", {\n");
         const old_indent = w.indent;
 @@ -417,7 +416,7 @@ const Writer = struct {
     fn writeLoop(w: *Writer, s: anytype, inst: Air.Inst.Index) @TypeOf(s).Error!void {
         const ty_pl = w.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
         const extra = w.air.extraData(Air.Block, ty_pl.payload);
-        const body: []const Air.Inst.Index = @ptrCast(w.air.extra[extra.end..][0..extra.data.body_len]);
+        const body: []const Air.Inst.Index = @ptrCast(w.air.extra.items[extra.end..][0..extra.data.body_len]);
         try w.writeType(s, ty_pl.ty.toType());
         if (w.skip_body) return s.writeAll(", ...");
 @@ -435,7 +434,7 @@ const Writer = struct {
         const ty_pl = w.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
         const vector_ty = ty_pl.ty.toType();
         const len = @as(usize, @intCast(vector_ty.arrayLen(zcu)));
-        const elements = @as([]const Air.Inst.Ref, @ptrCast(w.air.extra[ty_pl.payload..][0..len]));
+        const elements = @as([]const Air.Inst.Ref, @ptrCast(w.air.extra.items[ty_pl.payload..][0..len]));
         try w.writeType(s, vector_ty);
         try s.writeAll(", [");
 @@ -622,13 +621,13 @@ const Writer = struct {
             try s.writeAll(", volatile");
         }
-        const outputs = @as([]const Air.Inst.Ref, @ptrCast(w.air.extra[extra_i..][0..extra.data.outputs_len]));
+        const outputs = @as([]const Air.Inst.Ref, @ptrCast(w.air.extra.items[extra_i..][0..extra.data.outputs_len]));
         extra_i += outputs.len;
-        const inputs = @as([]const Air.Inst.Ref, @ptrCast(w.air.extra[extra_i..][0..extra.data.inputs_len]));
+        const inputs = @as([]const Air.Inst.Ref, @ptrCast(w.air.extra.items[extra_i..][0..extra.data.inputs_len]));
         extra_i += inputs.len;
         for (outputs) |output| {
-            const extra_bytes = std.mem.sliceAsBytes(w.air.extra[extra_i..]);
+            const extra_bytes = std.mem.sliceAsBytes(w.air.extra.items[extra_i..]);
             const constraint = std.mem.sliceTo(extra_bytes, 0);
             const name = std.mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0);
 @@ -648,7 +647,7 @@ const Writer = struct {
         }
         for (inputs) |input| {
-            const extra_bytes = std.mem.sliceAsBytes(w.air.extra[extra_i..]);
+            const extra_bytes = std.mem.sliceAsBytes(w.air.extra.items[extra_i..]);
             const constraint = std.mem.sliceTo(extra_bytes, 0);
             const name = std.mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0);
             // This equation accounts for the fact that even if we have exactly 4 bytes
 @@ -665,7 +664,7 @@ const Writer = struct {
         {
             var clobber_i: u32 = 0;
             while (clobber_i < clobbers_len) : (clobber_i += 1) {
-                const extra_bytes = std.mem.sliceAsBytes(w.air.extra[extra_i..]);
+                const extra_bytes = std.mem.sliceAsBytes(w.air.extra.items[extra_i..]);
                 const clobber = std.mem.sliceTo(extra_bytes, 0);
                 // This equation accounts for the fact that even if we have exactly 4 bytes
                 // for the string, we still use the next u32 for the null terminator.
 @@ -676,7 +675,7 @@ const Writer = struct {
                 try s.writeAll("}");
             }
         }
-        const asm_source = std.mem.sliceAsBytes(w.air.extra[extra_i..])[0..extra.data.source_len];
+        const asm_source = std.mem.sliceAsBytes(w.air.extra.items[extra_i..])[0..extra.data.source_len];
         try s.print(", \"{}\"", .{std.zig.fmtEscapes(asm_source)});
     }
 @@ -695,7 +694,7 @@ const Writer = struct {
     fn writeCall(w: *Writer, s: anytype, inst: Air.Inst.Index) @TypeOf(s).Error!void {
         const pl_op = w.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
         const extra = w.air.extraData(Air.Call, pl_op.payload);
-        const args = @as([]const Air.Inst.Ref, @ptrCast(w.air.extra[extra.end..][0..extra.data.args_len]));
+        const args = @as([]const Air.Inst.Ref, @ptrCast(w.air.extra.items[extra.end..][0..extra.data.args_len]));
         try w.writeOperand(s, inst, 0, pl_op.operand);
         try s.writeAll(", [");
         for (args, 0..) |arg, i| {
 @@ -720,11 +719,11 @@ const Writer = struct {
     fn writeTry(w: *Writer, s: anytype, inst: Air.Inst.Index) @TypeOf(s).Error!void {
         const pl_op = w.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
         const extra = w.air.extraData(Air.Try, pl_op.payload);
-        const body: []const Air.Inst.Index = @ptrCast(w.air.extra[extra.end..][0..extra.data.body_len]);
+        const body: []const Air.Inst.Index = @ptrCast(w.air.extra.items[extra.end..][0..extra.data.body_len]);
-        const liveness_condbr = if (w.liveness) |liveness|
+        const liveness_condbr: Air.Liveness.CondBrSlices = if (w.liveness) |liveness|
             liveness.getCondBr(inst)
         else
-            Liveness.CondBrSlices{ .then_deaths = &.{}, .else_deaths = &.{} };
+            .{ .then_deaths = &.{}, .else_deaths = &.{} };
         try w.writeOperand(s, inst, 0, pl_op.operand);
         if (w.skip_body) return s.writeAll(", ...");
 @@ -754,11 +753,11 @@ const Writer = struct {
     fn writeTryPtr(w: *Writer, s: anytype, inst: Air.Inst.Index) @TypeOf(s).Error!void {
         const ty_pl = w.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
         const extra = w.air.extraData(Air.TryPtr, ty_pl.payload);
-        const body: []const Air.Inst.Index = @ptrCast(w.air.extra[extra.end..][0..extra.data.body_len]);
+        const body: []const Air.Inst.Index = @ptrCast(w.air.extra.items[extra.end..][0..extra.data.body_len]);
-        const liveness_condbr = if (w.liveness) |liveness|
+        const liveness_condbr: Air.Liveness.CondBrSlices = if (w.liveness) |liveness|
             liveness.getCondBr(inst)
         else
-            Liveness.CondBrSlices{ .then_deaths = &.{}, .else_deaths = &.{} };
+            .{ .then_deaths = &.{}, .else_deaths = &.{} };
         try w.writeOperand(s, inst, 0, extra.data.ptr);
 @@ -791,12 +790,12 @@ const Writer = struct {
     fn writeCondBr(w: *Writer, s: anytype, inst: Air.Inst.Index) @TypeOf(s).Error!void {
         const pl_op = w.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
         const extra = w.air.extraData(Air.CondBr, pl_op.payload);
-        const then_body: []const Air.Inst.Index = @ptrCast(w.air.extra[extra.end..][0..extra.data.then_body_len]);
+        const then_body: []const Air.Inst.Index = @ptrCast(w.air.extra.items[extra.end..][0..extra.data.then_body_len]);
-        const else_body: []const Air.Inst.Index = @ptrCast(w.air.extra[extra.end + then_body.len ..][0..extra.data.else_body_len]);
+        const else_body: []const Air.Inst.Index = @ptrCast(w.air.extra.items[extra.end + then_body.len ..][0..extra.data.else_body_len]);
-        const liveness_condbr = if (w.liveness) |liveness|
+        const liveness_condbr: Air.Liveness.CondBrSlices = if (w.liveness) |liveness|
             liveness.getCondBr(inst)
         else
-            Liveness.CondBrSlices{ .then_deaths = &.{}, .else_deaths = &.{} };
+            .{ .then_deaths = &.{}, .else_deaths = &.{} };
         try w.writeOperand(s, inst, 0, pl_op.operand);
         if (w.skip_body) return s.writeAll(", ...");
 @@ -850,14 +849,14 @@ const Writer = struct {
     fn writeSwitchBr(w: *Writer, s: anytype, inst: Air.Inst.Index) @TypeOf(s).Error!void {
         const switch_br = w.air.unwrapSwitch(inst);
-        const liveness = if (w.liveness) |liveness|
+        const liveness: Air.Liveness.SwitchBrTable = if (w.liveness) |liveness|
             liveness.getSwitchBr(w.gpa, inst, switch_br.cases_len + 1) catch
                 @panic("out of memory")
         else blk: {
             const slice = w.gpa.alloc([]const Air.Inst.Index, switch_br.cases_len + 1) catch
                 @panic("out of memory");
             @memset(slice, &.{});
-            break :blk Liveness.SwitchBrTable{ .deaths = slice };
+            break :blk .{ .deaths = slice };
         };
         defer w.gpa.free(liveness.deaths);
 @@ -956,10 +955,10 @@ const Writer = struct {
         op_index: usize,
         operand: Air.Inst.Ref,
     ) @TypeOf(s).Error!void {
-        const small_tomb_bits = Liveness.bpi - 1;
+        const small_tomb_bits = Air.Liveness.bpi - 1;
         const dies = if (w.liveness) |liveness| blk: {
             if (op_index < small_tomb_bits)
-                break :blk liveness.operandDies(inst, @as(Liveness.OperandInt, @intCast(op_index)));
+                break :blk liveness.operandDies(inst, @intCast(op_index));
             var extra_index = liveness.special.get(inst).?;
             var tomb_op_index: usize = small_tomb_bits;
             while (true) {

...	@@ -171,7 +171,7 @@ fn checkBody(air: Air, body: []const Air.Inst.Index, zcu: *Zcu) bool {	...	@@ -171,7 +171,7 @@ fn checkBody(air: Air, body: []const Air.Inst.Index, zcu: *Zcu) bool {
171	if (!checkType(data.ty_pl.ty.toType(), zcu)) return false;	171	if (!checkType(data.ty_pl.ty.toType(), zcu)) return false;
172	if (!checkBody(	172	if (!checkBody(
173	air,	173	air,
174	@ptrCast(air.extra[extra.end..][0..extra.data.body_len]),	174	@ptrCast(air.extra.items[extra.end..][0..extra.data.body_len]),
175	zcu,	175	zcu,
176	)) return false;	176	)) return false;
177	},	177	},
...	@@ -181,7 +181,7 @@ fn checkBody(air: Air, body: []const Air.Inst.Index, zcu: *Zcu) bool {	...	@@ -181,7 +181,7 @@ fn checkBody(air: Air, body: []const Air.Inst.Index, zcu: *Zcu) bool {
181	if (!checkType(data.ty_pl.ty.toType(), zcu)) return false;	181	if (!checkType(data.ty_pl.ty.toType(), zcu)) return false;
182	if (!checkBody(	182	if (!checkBody(
183	air,	183	air,
184	@ptrCast(air.extra[extra.end..][0..extra.data.body_len]),	184	@ptrCast(air.extra.items[extra.end..][0..extra.data.body_len]),
185	zcu,	185	zcu,
186	)) return false;	186	)) return false;
187	},	187	},
...	@@ -270,7 +270,7 @@ fn checkBody(air: Air, body: []const Air.Inst.Index, zcu: *Zcu) bool {	...	@@ -270,7 +270,7 @@ fn checkBody(air: Air, body: []const Air.Inst.Index, zcu: *Zcu) bool {
270	.aggregate_init => {	270	.aggregate_init => {
271	const ty = data.ty_pl.ty.toType();	271	const ty = data.ty_pl.ty.toType();
272	const elems_len: usize = @intCast(ty.arrayLen(zcu));	272	const elems_len: usize = @intCast(ty.arrayLen(zcu));
273	const elems: []const Air.Inst.Ref = @ptrCast(air.extra[data.ty_pl.payload..][0..elems_len]);	273	const elems: []const Air.Inst.Ref = @ptrCast(air.extra.items[data.ty_pl.payload..][0..elems_len]);
274	if (!checkType(ty, zcu)) return false;	274	if (!checkType(ty, zcu)) return false;
275	if (ty.zigTypeTag(zcu) == .@"struct") {	275	if (ty.zigTypeTag(zcu) == .@"struct") {
276	for (elems, 0..) \|elem, elem_idx\| {	276	for (elems, 0..) \|elem, elem_idx\| {
...	@@ -336,7 +336,7 @@ fn checkBody(air: Air, body: []const Air.Inst.Index, zcu: *Zcu) bool {	...	@@ -336,7 +336,7 @@ fn checkBody(air: Air, body: []const Air.Inst.Index, zcu: *Zcu) bool {
336	.call_never_inline,	336	.call_never_inline,
337	=> {	337	=> {
338	const extra = air.extraData(Air.Call, data.pl_op.payload);	338	const extra = air.extraData(Air.Call, data.pl_op.payload);
339	const args: []const Air.Inst.Ref = @ptrCast(air.extra[extra.end..][0..extra.data.args_len]);	339	const args: []const Air.Inst.Ref = @ptrCast(air.extra.items[extra.end..][0..extra.data.args_len]);
340	if (!checkRef(data.pl_op.operand, zcu)) return false;	340	if (!checkRef(data.pl_op.operand, zcu)) return false;
341	for (args) \|arg\| if (!checkRef(arg, zcu)) return false;	341	for (args) \|arg\| if (!checkRef(arg, zcu)) return false;
342	},	342	},
...	@@ -353,7 +353,7 @@ fn checkBody(air: Air, body: []const Air.Inst.Index, zcu: *Zcu) bool {	...	@@ -353,7 +353,7 @@ fn checkBody(air: Air, body: []const Air.Inst.Index, zcu: *Zcu) bool {
353	if (!checkRef(data.pl_op.operand, zcu)) return false;	353	if (!checkRef(data.pl_op.operand, zcu)) return false;
354	if (!checkBody(	354	if (!checkBody(
355	air,	355	air,
356	@ptrCast(air.extra[extra.end..][0..extra.data.body_len]),	356	@ptrCast(air.extra.items[extra.end..][0..extra.data.body_len]),
357	zcu,	357	zcu,
358	)) return false;	358	)) return false;
359	},	359	},
...	@@ -364,7 +364,7 @@ fn checkBody(air: Air, body: []const Air.Inst.Index, zcu: *Zcu) bool {	...	@@ -364,7 +364,7 @@ fn checkBody(air: Air, body: []const Air.Inst.Index, zcu: *Zcu) bool {
364	if (!checkRef(extra.data.ptr, zcu)) return false;	364	if (!checkRef(extra.data.ptr, zcu)) return false;
365	if (!checkBody(	365	if (!checkBody(
366	air,	366	air,
367	@ptrCast(air.extra[extra.end..][0..extra.data.body_len]),	367	@ptrCast(air.extra.items[extra.end..][0..extra.data.body_len]),
368	zcu,	368	zcu,
369	)) return false;	369	)) return false;
370	},	370	},
...	@@ -374,12 +374,12 @@ fn checkBody(air: Air, body: []const Air.Inst.Index, zcu: *Zcu) bool {	...	@@ -374,12 +374,12 @@ fn checkBody(air: Air, body: []const Air.Inst.Index, zcu: *Zcu) bool {
374	if (!checkRef(data.pl_op.operand, zcu)) return false;	374	if (!checkRef(data.pl_op.operand, zcu)) return false;
375	if (!checkBody(	375	if (!checkBody(
376	air,	376	air,
377	@ptrCast(air.extra[extra.end..][0..extra.data.then_body_len]),	377	@ptrCast(air.extra.items[extra.end..][0..extra.data.then_body_len]),
378	zcu,	378	zcu,
379	)) return false;	379	)) return false;
380	if (!checkBody(	380	if (!checkBody(
381	air,	381	air,
382	@ptrCast(air.extra[extra.end + extra.data.then_body_len ..][0..extra.data.else_body_len]),	382	@ptrCast(air.extra.items[extra.end + extra.data.then_body_len ..][0..extra.data.else_body_len]),
383	zcu,	383	zcu,
384	)) return false;	384	)) return false;
385	},	385	},
...	@@ -404,8 +404,8 @@ fn checkBody(air: Air, body: []const Air.Inst.Index, zcu: *Zcu) bool {	...	@@ -404,8 +404,8 @@ fn checkBody(air: Air, body: []const Air.Inst.Index, zcu: *Zcu) bool {
404	if (!checkType(data.ty_pl.ty.toType(), zcu)) return false;	404	if (!checkType(data.ty_pl.ty.toType(), zcu)) return false;
405	// Luckily, we only care about the inputs and outputs, so we don't have to do	405	// Luckily, we only care about the inputs and outputs, so we don't have to do
406	// the whole null-terminated string dance.	406	// the whole null-terminated string dance.
407	const outputs: []const Air.Inst.Ref = @ptrCast(air.extra[extra.end..][0..extra.data.outputs_len]);	407	const outputs: []const Air.Inst.Ref = @ptrCast(air.extra.items[extra.end..][0..extra.data.outputs_len]);
408	const inputs: []const Air.Inst.Ref = @ptrCast(air.extra[extra.end + extra.data.outputs_len ..][0..extra.data.inputs_len]);	408	const inputs: []const Air.Inst.Ref = @ptrCast(air.extra.items[extra.end + extra.data.outputs_len ..][0..extra.data.inputs_len]);
409	for (outputs) \|output\| if (output != .none and !checkRef(output, zcu)) return false;	409	for (outputs) \|output\| if (output != .none and !checkRef(output, zcu)) return false;
410	for (inputs) \|input\| if (input != .none and !checkRef(input, zcu)) return false;	410	for (inputs) \|input\| if (input != .none and !checkRef(input, zcu)) return false;
411	},	411	},

...	@@ -5,7 +5,6 @@ const Air = @import("../../Air.zig");	...	@@ -5,7 +5,6 @@ const Air = @import("../../Air.zig");
5	const codegen = @import("../../codegen.zig");	5	const codegen = @import("../../codegen.zig");
6	const InternPool = @import("../../InternPool.zig");	6	const InternPool = @import("../../InternPool.zig");
7	const link = @import("../../link.zig");	7	const link = @import("../../link.zig");
8	const Liveness = @import("../../Liveness.zig");
9	const Zcu = @import("../../Zcu.zig");	8	const Zcu = @import("../../Zcu.zig");
10		9
11	const assert = std.debug.assert;	10	const assert = std.debug.assert;
...	@@ -17,7 +16,7 @@ pub fn generate(	...	@@ -17,7 +16,7 @@ pub fn generate(
17	src_loc: Zcu.LazySrcLoc,	16	src_loc: Zcu.LazySrcLoc,
18	func_index: InternPool.Index,	17	func_index: InternPool.Index,
19	air: Air,	18	air: Air,
20	liveness: Liveness,	19	liveness: Air.Liveness,
21	code: *std.ArrayListUnmanaged(u8),	20	code: *std.ArrayListUnmanaged(u8),
22	debug_output: link.File.DebugInfoOutput,	21	debug_output: link.File.DebugInfoOutput,
23	) codegen.CodeGenError!void {	22	) codegen.CodeGenError!void {

...	@@ -10,7 +10,6 @@ const Allocator = mem.Allocator;	...	@@ -10,7 +10,6 @@ const Allocator = mem.Allocator;
10	const Air = @import("../../Air.zig");	10	const Air = @import("../../Air.zig");
11	const Mir = @import("Mir.zig");	11	const Mir = @import("Mir.zig");
12	const Emit = @import("Emit.zig");	12	const Emit = @import("Emit.zig");
13	const Liveness = @import("../../Liveness.zig");
14	const Type = @import("../../Type.zig");	13	const Type = @import("../../Type.zig");
15	const Value = @import("../../Value.zig");	14	const Value = @import("../../Value.zig");
16	const link = @import("../../link.zig");	15	const link = @import("../../link.zig");
...	@@ -54,7 +53,7 @@ const InnerError = CodeGenError \|\| error{OutOfRegisters};	...	@@ -54,7 +53,7 @@ const InnerError = CodeGenError \|\| error{OutOfRegisters};
54		53
55	pt: Zcu.PerThread,	54	pt: Zcu.PerThread,
56	air: Air,	55	air: Air,
57	liveness: Liveness,	56	liveness: Air.Liveness,
58	bin_file: *link.File,	57	bin_file: *link.File,
59	gpa: Allocator,	58	gpa: Allocator,
60		59
...	@@ -82,7 +81,7 @@ scope_generation: u32,	...	@@ -82,7 +81,7 @@ scope_generation: u32,
82	/// which is a relative jump, based on the address following the reloc.	81	/// which is a relative jump, based on the address following the reloc.
83	exitlude_jump_relocs: std.ArrayListUnmanaged(usize) = .empty,	82	exitlude_jump_relocs: std.ArrayListUnmanaged(usize) = .empty,
84		83
85	reused_operands: std.StaticBitSet(Liveness.bpi - 1) = undefined,	84	reused_operands: std.StaticBitSet(Air.Liveness.bpi - 1) = undefined,
86		85
87	/// Whenever there is a runtime branch, we push a Branch onto this stack,	86	/// Whenever there is a runtime branch, we push a Branch onto this stack,
88	/// and pop it off when the runtime branch joins. This provides an "overlay"	87	/// and pop it off when the runtime branch joins. This provides an "overlay"
...	@@ -739,7 +738,7 @@ pub fn generate(	...	@@ -739,7 +738,7 @@ pub fn generate(
739	src_loc: Zcu.LazySrcLoc,	738	src_loc: Zcu.LazySrcLoc,
740	func_index: InternPool.Index,	739	func_index: InternPool.Index,
741	air: Air,	740	air: Air,
742	liveness: Liveness,	741	liveness: Air.Liveness,
743	code: *std.ArrayListUnmanaged(u8),	742	code: *std.ArrayListUnmanaged(u8),
744	debug_output: link.File.DebugInfoOutput,	743	debug_output: link.File.DebugInfoOutput,
745	) CodeGenError!void {	744	) CodeGenError!void {
...	@@ -1426,7 +1425,7 @@ fn genBody(func: *Func, body: []const Air.Inst.Index) InnerError!void {	...	@@ -1426,7 +1425,7 @@ fn genBody(func: *Func, body: []const Air.Inst.Index) InnerError!void {
1426	verbose_tracking_log.debug("{}", .{func.fmtTracking()});	1425	verbose_tracking_log.debug("{}", .{func.fmtTracking()});
1427		1426
1428	const old_air_bookkeeping = func.air_bookkeeping;	1427	const old_air_bookkeeping = func.air_bookkeeping;
1429	try func.ensureProcessDeathCapacity(Liveness.bpi);	1428	try func.ensureProcessDeathCapacity(Air.Liveness.bpi);
1430		1429
1431	func.reused_operands = @TypeOf(func.reused_operands).initEmpty();	1430	func.reused_operands = @TypeOf(func.reused_operands).initEmpty();
1432	try func.inst_tracking.ensureUnusedCapacity(func.gpa, 1);	1431	try func.inst_tracking.ensureUnusedCapacity(func.gpa, 1);
...	@@ -1731,7 +1730,7 @@ fn freeValue(func: *Func, value: MCValue) !void {	...	@@ -1731,7 +1730,7 @@ fn freeValue(func: *Func, value: MCValue) !void {
1731	}	1730	}
1732	}	1731	}
1733		1732
1734	fn feed(func: Func, bt: Liveness.BigTomb, operand: Air.Inst.Ref) !void {	1733	fn feed(func: Func, bt: Air.Liveness.BigTomb, operand: Air.Inst.Ref) !void {
1735	if (bt.feed()) if (operand.toIndex()) \|inst\| {	1734	if (bt.feed()) if (operand.toIndex()) \|inst\| {
1736	log.debug("feed inst: %{}", .{inst});	1735	log.debug("feed inst: %{}", .{inst});
1737	try func.processDeath(inst);	1736	try func.processDeath(inst);
...	@@ -1776,11 +1775,11 @@ fn finishAir(	...	@@ -1776,11 +1775,11 @@ fn finishAir(
1776	func: *Func,	1775	func: *Func,
1777	inst: Air.Inst.Index,	1776	inst: Air.Inst.Index,
1778	result: MCValue,	1777	result: MCValue,
1779	operands: [Liveness.bpi - 1]Air.Inst.Ref,	1778	operands: [Air.Liveness.bpi - 1]Air.Inst.Ref,
1780	) !void {	1779	) !void {
1781	const tomb_bits = func.liveness.getTombBits(inst);	1780	const tomb_bits = func.liveness.getTombBits(inst);
1782	for (0.., operands) \|op_index, op\| {	1781	for (0.., operands) \|op_index, op\| {
1783	if (tomb_bits & @as(Liveness.Bpi, 1) << @intCast(op_index) == 0) continue;	1782	if (tomb_bits & @as(Air.Liveness.Bpi, 1) << @intCast(op_index) == 0) continue;
1784	if (func.reused_operands.isSet(op_index)) continue;	1783	if (func.reused_operands.isSet(op_index)) continue;
1785	try func.processDeath(op.toIndexAllowNone() orelse continue);	1784	try func.processDeath(op.toIndexAllowNone() orelse continue);
1786	}	1785	}
...	@@ -3651,7 +3650,7 @@ fn airTlvDllimportPtr(func: *Func, inst: Air.Inst.Index) !void {	...	@@ -3651,7 +3650,7 @@ fn airTlvDllimportPtr(func: *Func, inst: Air.Inst.Index) !void {
3651	fn airTry(func: *Func, inst: Air.Inst.Index) !void {	3650	fn airTry(func: *Func, inst: Air.Inst.Index) !void {
3652	const pl_op = func.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;	3651	const pl_op = func.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
3653	const extra = func.air.extraData(Air.Try, pl_op.payload);	3652	const extra = func.air.extraData(Air.Try, pl_op.payload);
3654	const body: []const Air.Inst.Index = @ptrCast(func.air.extra[extra.end..][0..extra.data.body_len]);	3653	const body: []const Air.Inst.Index = @ptrCast(func.air.extra.items[extra.end..][0..extra.data.body_len]);
3655	const operand_ty = func.typeOf(pl_op.operand);	3654	const operand_ty = func.typeOf(pl_op.operand);
3656	const result = try func.genTry(inst, pl_op.operand, body, operand_ty, false);	3655	const result = try func.genTry(inst, pl_op.operand, body, operand_ty, false);
3657	return func.finishAir(inst, result, .{ .none, .none, .none });	3656	return func.finishAir(inst, result, .{ .none, .none, .none });
...	@@ -4419,7 +4418,7 @@ fn reuseOperand(	...	@@ -4419,7 +4418,7 @@ fn reuseOperand(
4419	func: *Func,	4418	func: *Func,
4420	inst: Air.Inst.Index,	4419	inst: Air.Inst.Index,
4421	operand: Air.Inst.Ref,	4420	operand: Air.Inst.Ref,
4422	op_index: Liveness.OperandInt,	4421	op_index: Air.Liveness.OperandInt,
4423	mcv: MCValue,	4422	mcv: MCValue,
4424	) bool {	4423	) bool {
4425	return func.reuseOperandAdvanced(inst, operand, op_index, mcv, inst);	4424	return func.reuseOperandAdvanced(inst, operand, op_index, mcv, inst);
...	@@ -4429,7 +4428,7 @@ fn reuseOperandAdvanced(	...	@@ -4429,7 +4428,7 @@ fn reuseOperandAdvanced(
4429	func: *Func,	4428	func: *Func,
4430	inst: Air.Inst.Index,	4429	inst: Air.Inst.Index,
4431	operand: Air.Inst.Ref,	4430	operand: Air.Inst.Ref,
4432	op_index: Liveness.OperandInt,	4431	op_index: Air.Liveness.OperandInt,
4433	mcv: MCValue,	4432	mcv: MCValue,
4434	maybe_tracked_inst: ?Air.Inst.Index,	4433	maybe_tracked_inst: ?Air.Inst.Index,
4435	) bool {	4434	) bool {
...	@@ -4816,7 +4815,7 @@ fn airCall(func: *Func, inst: Air.Inst.Index, modifier: std.builtin.CallModifier	...	@@ -4816,7 +4815,7 @@ fn airCall(func: *Func, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
4816	const pl_op = func.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;	4815	const pl_op = func.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
4817	const callee = pl_op.operand;	4816	const callee = pl_op.operand;
4818	const extra = func.air.extraData(Air.Call, pl_op.payload);	4817	const extra = func.air.extraData(Air.Call, pl_op.payload);
4819	const arg_refs: []const Air.Inst.Ref = @ptrCast(func.air.extra[extra.end..][0..extra.data.args_len]);	4818	const arg_refs: []const Air.Inst.Ref = @ptrCast(func.air.extra.items[extra.end..][0..extra.data.args_len]);
4820		4819
4821	const expected_num_args = 8;	4820	const expected_num_args = 8;
4822	const ExpectedContents = extern struct {	4821	const ExpectedContents = extern struct {
...	@@ -5232,7 +5231,7 @@ fn airDbgStmt(func: *Func, inst: Air.Inst.Index) !void {	...	@@ -5232,7 +5231,7 @@ fn airDbgStmt(func: *Func, inst: Air.Inst.Index) !void {
5232	fn airDbgInlineBlock(func: *Func, inst: Air.Inst.Index) !void {	5231	fn airDbgInlineBlock(func: *Func, inst: Air.Inst.Index) !void {
5233	const ty_pl = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;	5232	const ty_pl = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
5234	const extra = func.air.extraData(Air.DbgInlineBlock, ty_pl.payload);	5233	const extra = func.air.extraData(Air.DbgInlineBlock, ty_pl.payload);
5235	try func.lowerBlock(inst, @ptrCast(func.air.extra[extra.end..][0..extra.data.body_len]));	5234	try func.lowerBlock(inst, @ptrCast(func.air.extra.items[extra.end..][0..extra.data.body_len]));
5236	}	5235	}
5237		5236
5238	fn airDbgVar(func: *Func, inst: Air.Inst.Index) InnerError!void {	5237	fn airDbgVar(func: *Func, inst: Air.Inst.Index) InnerError!void {
...	@@ -5284,8 +5283,8 @@ fn airCondBr(func: *Func, inst: Air.Inst.Index) !void {	...	@@ -5284,8 +5283,8 @@ fn airCondBr(func: *Func, inst: Air.Inst.Index) !void {
5284	const cond = try func.resolveInst(pl_op.operand);	5283	const cond = try func.resolveInst(pl_op.operand);
5285	const cond_ty = func.typeOf(pl_op.operand);	5284	const cond_ty = func.typeOf(pl_op.operand);
5286	const extra = func.air.extraData(Air.CondBr, pl_op.payload);	5285	const extra = func.air.extraData(Air.CondBr, pl_op.payload);
5287	const then_body: []const Air.Inst.Index = @ptrCast(func.air.extra[extra.end..][0..extra.data.then_body_len]);	5286	const then_body: []const Air.Inst.Index = @ptrCast(func.air.extra.items[extra.end..][0..extra.data.then_body_len]);
5288	const else_body: []const Air.Inst.Index = @ptrCast(func.air.extra[extra.end + then_body.len ..][0..extra.data.else_body_len]);	5287	const else_body: []const Air.Inst.Index = @ptrCast(func.air.extra.items[extra.end + then_body.len ..][0..extra.data.else_body_len]);
5289	const liveness_cond_br = func.liveness.getCondBr(inst);	5288	const liveness_cond_br = func.liveness.getCondBr(inst);
5290		5289
5291	// If the condition dies here in this condbr instruction, process	5290	// If the condition dies here in this condbr instruction, process
...	@@ -5644,7 +5643,7 @@ fn airLoop(func: *Func, inst: Air.Inst.Index) !void {	...	@@ -5644,7 +5643,7 @@ fn airLoop(func: *Func, inst: Air.Inst.Index) !void {
5644	// A loop is a setup to be able to jump back to the beginning.	5643	// A loop is a setup to be able to jump back to the beginning.
5645	const ty_pl = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;	5644	const ty_pl = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
5646	const loop = func.air.extraData(Air.Block, ty_pl.payload);	5645	const loop = func.air.extraData(Air.Block, ty_pl.payload);
5647	const body: []const Air.Inst.Index = @ptrCast(func.air.extra[loop.end..][0..loop.data.body_len]);	5646	const body: []const Air.Inst.Index = @ptrCast(func.air.extra.items[loop.end..][0..loop.data.body_len]);
5648		5647
5649	func.scope_generation += 1;	5648	func.scope_generation += 1;
5650	const state = try func.saveState();	5649	const state = try func.saveState();
...	@@ -5674,7 +5673,7 @@ fn jump(func: *Func, index: Mir.Inst.Index) !Mir.Inst.Index {	...	@@ -5674,7 +5673,7 @@ fn jump(func: *Func, index: Mir.Inst.Index) !Mir.Inst.Index {
5674	fn airBlock(func: *Func, inst: Air.Inst.Index) !void {	5673	fn airBlock(func: *Func, inst: Air.Inst.Index) !void {
5675	const ty_pl = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;	5674	const ty_pl = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
5676	const extra = func.air.extraData(Air.Block, ty_pl.payload);	5675	const extra = func.air.extraData(Air.Block, ty_pl.payload);
5677	try func.lowerBlock(inst, @ptrCast(func.air.extra[extra.end..][0..extra.data.body_len]));	5676	try func.lowerBlock(inst, @ptrCast(func.air.extra.items[extra.end..][0..extra.data.body_len]));
5678	}	5677	}
5679		5678
5680	fn lowerBlock(func: *Func, inst: Air.Inst.Index, body: []const Air.Inst.Index) !void {	5679	fn lowerBlock(func: *Func, inst: Air.Inst.Index, body: []const Air.Inst.Index) !void {
...	@@ -6063,9 +6062,9 @@ fn airAsm(func: *Func, inst: Air.Inst.Index) !void {	...	@@ -6063,9 +6062,9 @@ fn airAsm(func: *Func, inst: Air.Inst.Index) !void {
6063	const clobbers_len: u31 = @truncate(extra.data.flags);	6062	const clobbers_len: u31 = @truncate(extra.data.flags);
6064	var extra_i: usize = extra.end;	6063	var extra_i: usize = extra.end;
6065	const outputs: []const Air.Inst.Ref =	6064	const outputs: []const Air.Inst.Ref =
6066	@ptrCast(func.air.extra[extra_i..][0..extra.data.outputs_len]);	6065	@ptrCast(func.air.extra.items[extra_i..][0..extra.data.outputs_len]);
6067	extra_i += outputs.len;	6066	extra_i += outputs.len;
6068	const inputs: []const Air.Inst.Ref = @ptrCast(func.air.extra[extra_i..][0..extra.data.inputs_len]);	6067	const inputs: []const Air.Inst.Ref = @ptrCast(func.air.extra.items[extra_i..][0..extra.data.inputs_len]);
6069	extra_i += inputs.len;	6068	extra_i += inputs.len;
6070		6069
6071	var result: MCValue = .none;	6070	var result: MCValue = .none;
...	@@ -6083,8 +6082,8 @@ fn airAsm(func: *Func, inst: Air.Inst.Index) !void {	...	@@ -6083,8 +6082,8 @@ fn airAsm(func: *Func, inst: Air.Inst.Index) !void {
6083		6082
6084	var outputs_extra_i = extra_i;	6083	var outputs_extra_i = extra_i;
6085	for (outputs) \|output\| {	6084	for (outputs) \|output\| {
6086	const extra_bytes = mem.sliceAsBytes(func.air.extra[extra_i..]);	6085	const extra_bytes = mem.sliceAsBytes(func.air.extra.items[extra_i..]);
6087	const constraint = mem.sliceTo(mem.sliceAsBytes(func.air.extra[extra_i..]), 0);	6086	const constraint = mem.sliceTo(mem.sliceAsBytes(func.air.extra.items[extra_i..]), 0);
6088	const name = mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0);	6087	const name = mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0);
6089	// This equation accounts for the fact that even if we have exactly 4 bytes	6088	// This equation accounts for the fact that even if we have exactly 4 bytes
6090	// for the string, we still use the next u32 for the null terminator.	6089	// for the string, we still use the next u32 for the null terminator.
...	@@ -6141,7 +6140,7 @@ fn airAsm(func: *Func, inst: Air.Inst.Index) !void {	...	@@ -6141,7 +6140,7 @@ fn airAsm(func: *Func, inst: Air.Inst.Index) !void {
6141	}	6140	}
6142		6141
6143	for (inputs) \|input\| {	6142	for (inputs) \|input\| {
6144	const input_bytes = mem.sliceAsBytes(func.air.extra[extra_i..]);	6143	const input_bytes = mem.sliceAsBytes(func.air.extra.items[extra_i..]);
6145	const constraint = mem.sliceTo(input_bytes, 0);	6144	const constraint = mem.sliceTo(input_bytes, 0);
6146	const name = mem.sliceTo(input_bytes[constraint.len + 1 ..], 0);	6145	const name = mem.sliceTo(input_bytes[constraint.len + 1 ..], 0);
6147	// This equation accounts for the fact that even if we have exactly 4 bytes	6146	// This equation accounts for the fact that even if we have exactly 4 bytes
...	@@ -6177,7 +6176,7 @@ fn airAsm(func: *Func, inst: Air.Inst.Index) !void {	...	@@ -6177,7 +6176,7 @@ fn airAsm(func: *Func, inst: Air.Inst.Index) !void {
6177	{	6176	{
6178	var clobber_i: u32 = 0;	6177	var clobber_i: u32 = 0;
6179	while (clobber_i < clobbers_len) : (clobber_i += 1) {	6178	while (clobber_i < clobbers_len) : (clobber_i += 1) {
6180	const clobber = std.mem.sliceTo(std.mem.sliceAsBytes(func.air.extra[extra_i..]), 0);	6179	const clobber = std.mem.sliceTo(std.mem.sliceAsBytes(func.air.extra.items[extra_i..]), 0);
6181	// This equation accounts for the fact that even if we have exactly 4 bytes	6180	// This equation accounts for the fact that even if we have exactly 4 bytes
6182	// for the string, we still use the next u32 for the null terminator.	6181	// for the string, we still use the next u32 for the null terminator.
6183	extra_i += clobber.len / 4 + 1;	6182	extra_i += clobber.len / 4 + 1;
...	@@ -6224,7 +6223,7 @@ fn airAsm(func: *Func, inst: Air.Inst.Index) !void {	...	@@ -6224,7 +6223,7 @@ fn airAsm(func: *Func, inst: Air.Inst.Index) !void {
6224	labels.deinit(func.gpa);	6223	labels.deinit(func.gpa);
6225	}	6224	}
6226		6225
6227	const asm_source = std.mem.sliceAsBytes(func.air.extra[extra_i..])[0..extra.data.source_len];	6226	const asm_source = std.mem.sliceAsBytes(func.air.extra.items[extra_i..])[0..extra.data.source_len];
6228	var line_it = mem.tokenizeAny(u8, asm_source, "\n\r;");	6227	var line_it = mem.tokenizeAny(u8, asm_source, "\n\r;");
6229	next_line: while (line_it.next()) \|line\| {	6228	next_line: while (line_it.next()) \|line\| {
6230	var mnem_it = mem.tokenizeAny(u8, line, " \t");	6229	var mnem_it = mem.tokenizeAny(u8, line, " \t");
...	@@ -6493,9 +6492,9 @@ fn airAsm(func: *Func, inst: Air.Inst.Index) !void {	...	@@ -6493,9 +6492,9 @@ fn airAsm(func: *Func, inst: Air.Inst.Index) !void {
6493	return func.fail("undefined label: '{s}'", .{label.key_ptr.*});	6492	return func.fail("undefined label: '{s}'", .{label.key_ptr.*});
6494		6493
6495	for (outputs, args.items[0..outputs.len]) \|output, arg_mcv\| {	6494	for (outputs, args.items[0..outputs.len]) \|output, arg_mcv\| {
6496	const extra_bytes = mem.sliceAsBytes(func.air.extra[outputs_extra_i..]);	6495	const extra_bytes = mem.sliceAsBytes(func.air.extra.items[outputs_extra_i..]);
6497	const constraint =	6496	const constraint =
6498	mem.sliceTo(mem.sliceAsBytes(func.air.extra[outputs_extra_i..]), 0);	6497	mem.sliceTo(mem.sliceAsBytes(func.air.extra.items[outputs_extra_i..]), 0);
6499	const name = mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0);	6498	const name = mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0);
6500	// This equation accounts for the fact that even if we have exactly 4 bytes	6499	// This equation accounts for the fact that even if we have exactly 4 bytes
6501	// for the string, we still use the next u32 for the null terminator.	6500	// for the string, we still use the next u32 for the null terminator.
...	@@ -6508,7 +6507,7 @@ fn airAsm(func: *Func, inst: Air.Inst.Index) !void {	...	@@ -6508,7 +6507,7 @@ fn airAsm(func: *Func, inst: Air.Inst.Index) !void {
6508	}	6507	}
6509		6508
6510	simple: {	6509	simple: {
6511	var buf = [1]Air.Inst.Ref{.none} ** (Liveness.bpi - 1);	6510	var buf = [1]Air.Inst.Ref{.none} ** (Air.Liveness.bpi - 1);
6512	var buf_index: usize = 0;	6511	var buf_index: usize = 0;
6513	for (outputs) \|output\| {	6512	for (outputs) \|output\| {
6514	if (output == .none) continue;	6513	if (output == .none) continue;
...	@@ -8027,7 +8026,7 @@ fn airAggregateInit(func: *Func, inst: Air.Inst.Index) !void {	...	@@ -8027,7 +8026,7 @@ fn airAggregateInit(func: *Func, inst: Air.Inst.Index) !void {
8027	const result_ty = func.typeOfIndex(inst);	8026	const result_ty = func.typeOfIndex(inst);
8028	const len: usize = @intCast(result_ty.arrayLen(zcu));	8027	const len: usize = @intCast(result_ty.arrayLen(zcu));
8029	const ty_pl = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;	8028	const ty_pl = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
8030	const elements: []const Air.Inst.Ref = @ptrCast(func.air.extra[ty_pl.payload..][0..len]);	8029	const elements: []const Air.Inst.Ref = @ptrCast(func.air.extra.items[ty_pl.payload..][0..len]);
8031		8030
8032	const result: MCValue = result: {	8031	const result: MCValue = result: {
8033	switch (result_ty.zigTypeTag(zcu)) {	8032	switch (result_ty.zigTypeTag(zcu)) {
...	@@ -8113,8 +8112,8 @@ fn airAggregateInit(func: *Func, inst: Air.Inst.Index) !void {	...	@@ -8113,8 +8112,8 @@ fn airAggregateInit(func: *Func, inst: Air.Inst.Index) !void {
8113	}	8112	}
8114	};	8113	};
8115		8114
8116	if (elements.len <= Liveness.bpi - 1) {	8115	if (elements.len <= Air.Liveness.bpi - 1) {
8117	var buf = [1]Air.Inst.Ref{.none} ** (Liveness.bpi - 1);	8116	var buf = [1]Air.Inst.Ref{.none} ** (Air.Liveness.bpi - 1);
8118	@memcpy(buf[0..elements.len], elements);	8117	@memcpy(buf[0..elements.len], elements);
8119	return func.finishAir(inst, result, buf);	8118	return func.finishAir(inst, result, buf);
8120	}	8119	}

...	@@ -17,7 +17,6 @@ const Value = @import("../../Value.zig");	...	@@ -17,7 +17,6 @@ const Value = @import("../../Value.zig");
17	const Compilation = @import("../../Compilation.zig");	17	const Compilation = @import("../../Compilation.zig");
18	const link = @import("../../link.zig");	18	const link = @import("../../link.zig");
19	const Air = @import("../../Air.zig");	19	const Air = @import("../../Air.zig");
20	const Liveness = @import("../../Liveness.zig");
21	const Mir = @import("Mir.zig");	20	const Mir = @import("Mir.zig");
22	const Emit = @import("Emit.zig");	21	const Emit = @import("Emit.zig");
23	const abi = @import("abi.zig");	22	const abi = @import("abi.zig");
...	@@ -39,7 +38,7 @@ owner_nav: InternPool.Nav.Index,	...	@@ -39,7 +38,7 @@ owner_nav: InternPool.Nav.Index,
39	/// and block	38	/// and block
40	block_depth: u32 = 0,	39	block_depth: u32 = 0,
41	air: Air,	40	air: Air,
42	liveness: Liveness,	41	liveness: Air.Liveness,
43	gpa: mem.Allocator,	42	gpa: mem.Allocator,
44	func_index: InternPool.Index,	43	func_index: InternPool.Index,
45	/// Contains a list of current branches.	44	/// Contains a list of current branches.
...	@@ -771,7 +770,7 @@ fn resolveValue(cg: *CodeGen, val: Value) InnerError!WValue {	...	@@ -771,7 +770,7 @@ fn resolveValue(cg: *CodeGen, val: Value) InnerError!WValue {
771		770
772	/// NOTE: if result == .stack, it will be stored in .local	771	/// NOTE: if result == .stack, it will be stored in .local
773	fn finishAir(cg: *CodeGen, inst: Air.Inst.Index, result: WValue, operands: []const Air.Inst.Ref) InnerError!void {	772	fn finishAir(cg: *CodeGen, inst: Air.Inst.Index, result: WValue, operands: []const Air.Inst.Ref) InnerError!void {
774	assert(operands.len <= Liveness.bpi - 1);	773	assert(operands.len <= Air.Liveness.bpi - 1);
775	var tomb_bits = cg.liveness.getTombBits(inst);	774	var tomb_bits = cg.liveness.getTombBits(inst);
776	for (operands) \|operand\| {	775	for (operands) \|operand\| {
777	const dies = @as(u1, @truncate(tomb_bits)) != 0;	776	const dies = @as(u1, @truncate(tomb_bits)) != 0;
...	@@ -811,7 +810,7 @@ inline fn currentBranch(cg: CodeGen) Branch {	...	@@ -811,7 +810,7 @@ inline fn currentBranch(cg: CodeGen) Branch {
811	const BigTomb = struct {	810	const BigTomb = struct {
812	gen: *CodeGen,	811	gen: *CodeGen,
813	inst: Air.Inst.Index,	812	inst: Air.Inst.Index,
814	lbt: Liveness.BigTomb,	813	lbt: Air.Liveness.BigTomb,
815		814
816	fn feed(bt: *BigTomb, op_ref: Air.Inst.Ref) void {	815	fn feed(bt: *BigTomb, op_ref: Air.Inst.Ref) void {
817	const dies = bt.lbt.feed();	816	const dies = bt.lbt.feed();
...	@@ -1262,7 +1261,7 @@ pub fn function(	...	@@ -1262,7 +1261,7 @@ pub fn function(
1262	pt: Zcu.PerThread,	1261	pt: Zcu.PerThread,
1263	func_index: InternPool.Index,	1262	func_index: InternPool.Index,
1264	air: Air,	1263	air: Air,
1265	liveness: Liveness,	1264	liveness: Air.Liveness,
1266	) Error!Function {	1265	) Error!Function {
1267	const zcu = pt.zcu;	1266	const zcu = pt.zcu;
1268	const gpa = zcu.gpa;	1267	const gpa = zcu.gpa;
...	@@ -2123,7 +2122,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {	...	@@ -2123,7 +2122,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2123	continue;	2122	continue;
2124	}	2123	}
2125	const old_bookkeeping_value = cg.air_bookkeeping;	2124	const old_bookkeeping_value = cg.air_bookkeeping;
2126	try cg.currentBranch().values.ensureUnusedCapacity(cg.gpa, Liveness.bpi);	2125	try cg.currentBranch().values.ensureUnusedCapacity(cg.gpa, Air.Liveness.bpi);
2127	try cg.genInst(inst);	2126	try cg.genInst(inst);
2128		2127
2129	if (std.debug.runtime_safety and cg.air_bookkeeping < old_bookkeeping_value + 1) {	2128	if (std.debug.runtime_safety and cg.air_bookkeeping < old_bookkeeping_value + 1) {
...	@@ -2217,7 +2216,7 @@ fn airCall(cg: *CodeGen, inst: Air.Inst.Index, modifier: std.builtin.CallModifie	...	@@ -2217,7 +2216,7 @@ fn airCall(cg: *CodeGen, inst: Air.Inst.Index, modifier: std.builtin.CallModifie
2217	if (modifier == .always_tail) return cg.fail("TODO implement tail calls for wasm", .{});	2216	if (modifier == .always_tail) return cg.fail("TODO implement tail calls for wasm", .{});
2218	const pl_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;	2217	const pl_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
2219	const extra = cg.air.extraData(Air.Call, pl_op.payload);	2218	const extra = cg.air.extraData(Air.Call, pl_op.payload);
2220	const args: []const Air.Inst.Ref = @ptrCast(cg.air.extra[extra.end..][0..extra.data.args_len]);	2219	const args: []const Air.Inst.Ref = @ptrCast(cg.air.extra.items[extra.end..][0..extra.data.args_len]);
2221	const ty = cg.typeOf(pl_op.operand);	2220	const ty = cg.typeOf(pl_op.operand);
2222		2221
2223	const pt = cg.pt;	2222	const pt = cg.pt;
...	@@ -3410,7 +3409,7 @@ fn emitUndefined(cg: *CodeGen, ty: Type) InnerError!WValue {	...	@@ -3410,7 +3409,7 @@ fn emitUndefined(cg: *CodeGen, ty: Type) InnerError!WValue {
3410	fn airBlock(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {	3409	fn airBlock(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
3411	const ty_pl = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;	3410	const ty_pl = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
3412	const extra = cg.air.extraData(Air.Block, ty_pl.payload);	3411	const extra = cg.air.extraData(Air.Block, ty_pl.payload);
3413	try cg.lowerBlock(inst, ty_pl.ty.toType(), @ptrCast(cg.air.extra[extra.end..][0..extra.data.body_len]));	3412	try cg.lowerBlock(inst, ty_pl.ty.toType(), @ptrCast(cg.air.extra.items[extra.end..][0..extra.data.body_len]));
3414	}	3413	}
3415		3414
3416	fn lowerBlock(cg: *CodeGen, inst: Air.Inst.Index, block_ty: Type, body: []const Air.Inst.Index) InnerError!void {	3415	fn lowerBlock(cg: *CodeGen, inst: Air.Inst.Index, block_ty: Type, body: []const Air.Inst.Index) InnerError!void {
...	@@ -3456,7 +3455,7 @@ fn endBlock(cg: *CodeGen) !void {	...	@@ -3456,7 +3455,7 @@ fn endBlock(cg: *CodeGen) !void {
3456	fn airLoop(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {	3455	fn airLoop(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
3457	const ty_pl = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;	3456	const ty_pl = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
3458	const loop = cg.air.extraData(Air.Block, ty_pl.payload);	3457	const loop = cg.air.extraData(Air.Block, ty_pl.payload);
3459	const body: []const Air.Inst.Index = @ptrCast(cg.air.extra[loop.end..][0..loop.data.body_len]);	3458	const body: []const Air.Inst.Index = @ptrCast(cg.air.extra.items[loop.end..][0..loop.data.body_len]);
3460		3459
3461	// result type of loop is always 'noreturn', meaning we can always	3460	// result type of loop is always 'noreturn', meaning we can always
3462	// emit the wasm type 'block_empty'.	3461	// emit the wasm type 'block_empty'.
...	@@ -3475,8 +3474,8 @@ fn airCondBr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {	...	@@ -3475,8 +3474,8 @@ fn airCondBr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
3475	const pl_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;	3474	const pl_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
3476	const condition = try cg.resolveInst(pl_op.operand);	3475	const condition = try cg.resolveInst(pl_op.operand);
3477	const extra = cg.air.extraData(Air.CondBr, pl_op.payload);	3476	const extra = cg.air.extraData(Air.CondBr, pl_op.payload);
3478	const then_body: []const Air.Inst.Index = @ptrCast(cg.air.extra[extra.end..][0..extra.data.then_body_len]);	3477	const then_body: []const Air.Inst.Index = @ptrCast(cg.air.extra.items[extra.end..][0..extra.data.then_body_len]);
3479	const else_body: []const Air.Inst.Index = @ptrCast(cg.air.extra[extra.end + then_body.len ..][0..extra.data.else_body_len]);	3478	const else_body: []const Air.Inst.Index = @ptrCast(cg.air.extra.items[extra.end + then_body.len ..][0..extra.data.else_body_len]);
3480	const liveness_condbr = cg.liveness.getCondBr(inst);	3479	const liveness_condbr = cg.liveness.getCondBr(inst);
3481		3480
3482	// result type is always noreturn, so use `block_empty` as type.	3481	// result type is always noreturn, so use `block_empty` as type.
...	@@ -5238,7 +5237,7 @@ fn airAggregateInit(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {	...	@@ -5238,7 +5237,7 @@ fn airAggregateInit(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
5238	const ty_pl = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;	5237	const ty_pl = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
5239	const result_ty = cg.typeOfIndex(inst);	5238	const result_ty = cg.typeOfIndex(inst);
5240	const len = @as(usize, @intCast(result_ty.arrayLen(zcu)));	5239	const len = @as(usize, @intCast(result_ty.arrayLen(zcu)));
5241	const elements = @as([]const Air.Inst.Ref, @ptrCast(cg.air.extra[ty_pl.payload..][0..len]));	5240	const elements: []const Air.Inst.Ref = @ptrCast(cg.air.extra.items[ty_pl.payload..][0..len]);
5242		5241
5243	const result: WValue = result_value: {	5242	const result: WValue = result_value: {
5244	switch (result_ty.zigTypeTag(zcu)) {	5243	switch (result_ty.zigTypeTag(zcu)) {
...	@@ -5352,8 +5351,8 @@ fn airAggregateInit(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {	...	@@ -5352,8 +5351,8 @@ fn airAggregateInit(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
5352	}	5351	}
5353	};	5352	};
5354		5353
5355	if (elements.len <= Liveness.bpi - 1) {	5354	if (elements.len <= Air.Liveness.bpi - 1) {
5356	var buf = [1]Air.Inst.Ref{.none} ** (Liveness.bpi - 1);	5355	var buf = [1]Air.Inst.Ref{.none} ** (Air.Liveness.bpi - 1);
5357	@memcpy(buf[0..elements.len], elements);	5356	@memcpy(buf[0..elements.len], elements);
5358	return cg.finishAir(inst, result, &buf);	5357	return cg.finishAir(inst, result, &buf);
5359	}	5358	}
...	@@ -6454,7 +6453,7 @@ fn airDbgInlineBlock(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {	...	@@ -6454,7 +6453,7 @@ fn airDbgInlineBlock(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6454	const ty_pl = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;	6453	const ty_pl = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
6455	const extra = cg.air.extraData(Air.DbgInlineBlock, ty_pl.payload);	6454	const extra = cg.air.extraData(Air.DbgInlineBlock, ty_pl.payload);
6456	// TODO	6455	// TODO
6457	try cg.lowerBlock(inst, ty_pl.ty.toType(), @ptrCast(cg.air.extra[extra.end..][0..extra.data.body_len]));	6456	try cg.lowerBlock(inst, ty_pl.ty.toType(), @ptrCast(cg.air.extra.items[extra.end..][0..extra.data.body_len]));
6458	}	6457	}
6459		6458
6460	fn airDbgVar(	6459	fn airDbgVar(
...	@@ -6472,7 +6471,7 @@ fn airTry(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {	...	@@ -6472,7 +6471,7 @@ fn airTry(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6472	const pl_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;	6471	const pl_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
6473	const err_union = try cg.resolveInst(pl_op.operand);	6472	const err_union = try cg.resolveInst(pl_op.operand);
6474	const extra = cg.air.extraData(Air.Try, pl_op.payload);	6473	const extra = cg.air.extraData(Air.Try, pl_op.payload);
6475	const body: []const Air.Inst.Index = @ptrCast(cg.air.extra[extra.end..][0..extra.data.body_len]);	6474	const body: []const Air.Inst.Index = @ptrCast(cg.air.extra.items[extra.end..][0..extra.data.body_len]);
6476	const err_union_ty = cg.typeOf(pl_op.operand);	6475	const err_union_ty = cg.typeOf(pl_op.operand);
6477	const result = try lowerTry(cg, inst, err_union, body, err_union_ty, false);	6476	const result = try lowerTry(cg, inst, err_union, body, err_union_ty, false);
6478	return cg.finishAir(inst, result, &.{pl_op.operand});	6477	return cg.finishAir(inst, result, &.{pl_op.operand});
...	@@ -6483,7 +6482,7 @@ fn airTryPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {	...	@@ -6483,7 +6482,7 @@ fn airTryPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6483	const ty_pl = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;	6482	const ty_pl = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
6484	const extra = cg.air.extraData(Air.TryPtr, ty_pl.payload);	6483	const extra = cg.air.extraData(Air.TryPtr, ty_pl.payload);
6485	const err_union_ptr = try cg.resolveInst(extra.data.ptr);	6484	const err_union_ptr = try cg.resolveInst(extra.data.ptr);
6486	const body: []const Air.Inst.Index = @ptrCast(cg.air.extra[extra.end..][0..extra.data.body_len]);	6485	const body: []const Air.Inst.Index = @ptrCast(cg.air.extra.items[extra.end..][0..extra.data.body_len]);
6487	const err_union_ty = cg.typeOf(extra.data.ptr).childType(zcu);	6486	const err_union_ty = cg.typeOf(extra.data.ptr).childType(zcu);
6488	const result = try lowerTry(cg, inst, err_union_ptr, body, err_union_ty, true);	6487	const result = try lowerTry(cg, inst, err_union_ptr, body, err_union_ty, true);
6489	return cg.finishAir(inst, result, &.{extra.data.ptr});	6488	return cg.finishAir(inst, result, &.{extra.data.ptr});

...	@@ -18,7 +18,6 @@ const Zcu = @import("../Zcu.zig");	...	@@ -18,7 +18,6 @@ const Zcu = @import("../Zcu.zig");
18	const InternPool = @import("../InternPool.zig");	18	const InternPool = @import("../InternPool.zig");
19	const Package = @import("../Package.zig");	19	const Package = @import("../Package.zig");
20	const Air = @import("../Air.zig");	20	const Air = @import("../Air.zig");
21	const Liveness = @import("../Liveness.zig");
22	const Value = @import("../Value.zig");	21	const Value = @import("../Value.zig");
23	const Type = @import("../Type.zig");	22	const Type = @import("../Type.zig");
24	const x86_64_abi = @import("../arch/x86_64/abi.zig");	23	const x86_64_abi = @import("../arch/x86_64/abi.zig");
...	@@ -1121,7 +1120,7 @@ pub const Object = struct {	...	@@ -1121,7 +1120,7 @@ pub const Object = struct {
1121	pt: Zcu.PerThread,	1120	pt: Zcu.PerThread,
1122	func_index: InternPool.Index,	1121	func_index: InternPool.Index,
1123	air: Air,	1122	air: Air,
1124	liveness: Liveness,	1123	liveness: Air.Liveness,
1125	) !void {	1124	) !void {
1126	assert(std.meta.eql(pt, o.pt));	1125	assert(std.meta.eql(pt, o.pt));
1127	const zcu = pt.zcu;	1126	const zcu = pt.zcu;
...	@@ -4616,7 +4615,7 @@ pub const FuncGen = struct {	...	@@ -4616,7 +4615,7 @@ pub const FuncGen = struct {
4616	gpa: Allocator,	4615	gpa: Allocator,
4617	ng: *NavGen,	4616	ng: *NavGen,
4618	air: Air,	4617	air: Air,
4619	liveness: Liveness,	4618	liveness: Air.Liveness,
4620	wip: Builder.WipFunction,	4619	wip: Builder.WipFunction,
4621	is_naked: bool,	4620	is_naked: bool,
4622	fuzz: ?Fuzz,	4621	fuzz: ?Fuzz,
...	@@ -5183,7 +5182,7 @@ pub const FuncGen = struct {	...	@@ -5183,7 +5182,7 @@ pub const FuncGen = struct {
5183	fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.builtin.CallModifier) !Builder.Value {	5182	fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.builtin.CallModifier) !Builder.Value {
5184	const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;	5183	const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
5185	const extra = self.air.extraData(Air.Call, pl_op.payload);	5184	const extra = self.air.extraData(Air.Call, pl_op.payload);
5186	const args: []const Air.Inst.Ref = @ptrCast(self.air.extra[extra.end..][0..extra.data.args_len]);	5185	const args: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[extra.end..][0..extra.data.args_len]);
5187	const o = self.ng.object;	5186	const o = self.ng.object;
5188	const pt = o.pt;	5187	const pt = o.pt;
5189	const zcu = pt.zcu;	5188	const zcu = pt.zcu;
...	@@ -5856,7 +5855,7 @@ pub const FuncGen = struct {	...	@@ -5856,7 +5855,7 @@ pub const FuncGen = struct {
5856	fn airBlock(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {	5855	fn airBlock(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
5857	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;	5856	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
5858	const extra = self.air.extraData(Air.Block, ty_pl.payload);	5857	const extra = self.air.extraData(Air.Block, ty_pl.payload);
5859	return self.lowerBlock(inst, null, @ptrCast(self.air.extra[extra.end..][0..extra.data.body_len]));	5858	return self.lowerBlock(inst, null, @ptrCast(self.air.extra.items[extra.end..][0..extra.data.body_len]));
5860	}	5859	}
5861		5860
5862	fn lowerBlock(	5861	fn lowerBlock(
...	@@ -6140,8 +6139,8 @@ pub const FuncGen = struct {	...	@@ -6140,8 +6139,8 @@ pub const FuncGen = struct {
6140	const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;	6139	const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
6141	const cond = try self.resolveInst(pl_op.operand);	6140	const cond = try self.resolveInst(pl_op.operand);
6142	const extra = self.air.extraData(Air.CondBr, pl_op.payload);	6141	const extra = self.air.extraData(Air.CondBr, pl_op.payload);
6143	const then_body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra.end..][0..extra.data.then_body_len]);	6142	const then_body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[extra.end..][0..extra.data.then_body_len]);
6144	const else_body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra.end + then_body.len ..][0..extra.data.else_body_len]);	6143	const else_body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[extra.end + then_body.len ..][0..extra.data.else_body_len]);
6145		6144
6146	const Hint = enum {	6145	const Hint = enum {
6147	none,	6146	none,
...	@@ -6205,7 +6204,7 @@ pub const FuncGen = struct {	...	@@ -6205,7 +6204,7 @@ pub const FuncGen = struct {
6205	const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;	6204	const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
6206	const err_union = try self.resolveInst(pl_op.operand);	6205	const err_union = try self.resolveInst(pl_op.operand);
6207	const extra = self.air.extraData(Air.Try, pl_op.payload);	6206	const extra = self.air.extraData(Air.Try, pl_op.payload);
6208	const body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra.end..][0..extra.data.body_len]);	6207	const body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[extra.end..][0..extra.data.body_len]);
6209	const err_union_ty = self.typeOf(pl_op.operand);	6208	const err_union_ty = self.typeOf(pl_op.operand);
6210	const payload_ty = self.typeOfIndex(inst);	6209	const payload_ty = self.typeOfIndex(inst);
6211	const can_elide_load = if (isByRef(payload_ty, zcu)) self.canElideLoad(body_tail) else false;	6210	const can_elide_load = if (isByRef(payload_ty, zcu)) self.canElideLoad(body_tail) else false;
...	@@ -6219,7 +6218,7 @@ pub const FuncGen = struct {	...	@@ -6219,7 +6218,7 @@ pub const FuncGen = struct {
6219	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;	6218	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
6220	const extra = self.air.extraData(Air.TryPtr, ty_pl.payload);	6219	const extra = self.air.extraData(Air.TryPtr, ty_pl.payload);
6221	const err_union_ptr = try self.resolveInst(extra.data.ptr);	6220	const err_union_ptr = try self.resolveInst(extra.data.ptr);
6222	const body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra.end..][0..extra.data.body_len]);	6221	const body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[extra.end..][0..extra.data.body_len]);
6223	const err_union_ty = self.typeOf(extra.data.ptr).childType(zcu);	6222	const err_union_ty = self.typeOf(extra.data.ptr).childType(zcu);
6224	const is_unused = self.liveness.isUnused(inst);	6223	const is_unused = self.liveness.isUnused(inst);
6225		6224
...	@@ -6550,7 +6549,7 @@ pub const FuncGen = struct {	...	@@ -6550,7 +6549,7 @@ pub const FuncGen = struct {
6550	fn airLoop(self: *FuncGen, inst: Air.Inst.Index) !void {	6549	fn airLoop(self: *FuncGen, inst: Air.Inst.Index) !void {
6551	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;	6550	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
6552	const loop = self.air.extraData(Air.Block, ty_pl.payload);	6551	const loop = self.air.extraData(Air.Block, ty_pl.payload);
6553	const body: []const Air.Inst.Index = @ptrCast(self.air.extra[loop.end..][0..loop.data.body_len]);	6552	const body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[loop.end..][0..loop.data.body_len]);
6554	const loop_block = try self.wip.block(1, "Loop"); // `airRepeat` will increment incoming each time	6553	const loop_block = try self.wip.block(1, "Loop"); // `airRepeat` will increment incoming each time
6555	_ = try self.wip.br(loop_block);	6554	_ = try self.wip.br(loop_block);
6556		6555
...	@@ -7076,7 +7075,7 @@ pub const FuncGen = struct {	...	@@ -7076,7 +7075,7 @@ pub const FuncGen = struct {
7076	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;	7075	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
7077	const extra = self.air.extraData(Air.DbgInlineBlock, ty_pl.payload);	7076	const extra = self.air.extraData(Air.DbgInlineBlock, ty_pl.payload);
7078	self.arg_inline_index = 0;	7077	self.arg_inline_index = 0;
7079	return self.lowerBlock(inst, extra.data.func, @ptrCast(self.air.extra[extra.end..][0..extra.data.body_len]));	7078	return self.lowerBlock(inst, extra.data.func, @ptrCast(self.air.extra.items[extra.end..][0..extra.data.body_len]));
7080	}	7079	}
7081		7080
7082	fn airDbgVarPtr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {	7081	fn airDbgVarPtr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
...	@@ -7201,9 +7200,9 @@ pub const FuncGen = struct {	...	@@ -7201,9 +7200,9 @@ pub const FuncGen = struct {
7201	const clobbers_len: u31 = @truncate(extra.data.flags);	7200	const clobbers_len: u31 = @truncate(extra.data.flags);
7202	var extra_i: usize = extra.end;	7201	var extra_i: usize = extra.end;
7203		7202
7204	const outputs: []const Air.Inst.Ref = @ptrCast(self.air.extra[extra_i..][0..extra.data.outputs_len]);	7203	const outputs: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[extra_i..][0..extra.data.outputs_len]);
7205	extra_i += outputs.len;	7204	extra_i += outputs.len;
7206	const inputs: []const Air.Inst.Ref = @ptrCast(self.air.extra[extra_i..][0..extra.data.inputs_len]);	7205	const inputs: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[extra_i..][0..extra.data.inputs_len]);
7207	extra_i += inputs.len;	7206	extra_i += inputs.len;
7208		7207
7209	var llvm_constraints: std.ArrayListUnmanaged(u8) = .empty;	7208	var llvm_constraints: std.ArrayListUnmanaged(u8) = .empty;
...	@@ -7239,8 +7238,8 @@ pub const FuncGen = struct {	...	@@ -7239,8 +7238,8 @@ pub const FuncGen = struct {
7239		7238
7240	var rw_extra_i = extra_i;	7239	var rw_extra_i = extra_i;
7241	for (outputs, llvm_ret_indirect, llvm_rw_vals) \|output, is_indirect, llvm_rw_val\| {	7240	for (outputs, llvm_ret_indirect, llvm_rw_vals) \|output, is_indirect, llvm_rw_val\| {
7242	const extra_bytes = std.mem.sliceAsBytes(self.air.extra[extra_i..]);	7241	const extra_bytes = std.mem.sliceAsBytes(self.air.extra.items[extra_i..]);
7243	const constraint = std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra[extra_i..]), 0);	7242	const constraint = std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra.items[extra_i..]), 0);
7244	const name = std.mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0);	7243	const name = std.mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0);
7245	// This equation accounts for the fact that even if we have exactly 4 bytes	7244	// This equation accounts for the fact that even if we have exactly 4 bytes
7246	// for the string, we still use the next u32 for the null terminator.	7245	// for the string, we still use the next u32 for the null terminator.
...	@@ -7320,7 +7319,7 @@ pub const FuncGen = struct {	...	@@ -7320,7 +7319,7 @@ pub const FuncGen = struct {
7320	}	7319	}
7321		7320
7322	for (inputs) \|input\| {	7321	for (inputs) \|input\| {
7323	const extra_bytes = std.mem.sliceAsBytes(self.air.extra[extra_i..]);	7322	const extra_bytes = std.mem.sliceAsBytes(self.air.extra.items[extra_i..]);
7324	const constraint = std.mem.sliceTo(extra_bytes, 0);	7323	const constraint = std.mem.sliceTo(extra_bytes, 0);
7325	const name = std.mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0);	7324	const name = std.mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0);
7326	// This equation accounts for the fact that even if we have exactly 4 bytes	7325	// This equation accounts for the fact that even if we have exactly 4 bytes
...	@@ -7385,8 +7384,8 @@ pub const FuncGen = struct {	...	@@ -7385,8 +7384,8 @@ pub const FuncGen = struct {
7385	}	7384	}
7386		7385
7387	for (outputs, llvm_ret_indirect, llvm_rw_vals, 0..) \|output, is_indirect, llvm_rw_val, output_index\| {	7386	for (outputs, llvm_ret_indirect, llvm_rw_vals, 0..) \|output, is_indirect, llvm_rw_val, output_index\| {
7388	const extra_bytes = std.mem.sliceAsBytes(self.air.extra[rw_extra_i..]);	7387	const extra_bytes = std.mem.sliceAsBytes(self.air.extra.items[rw_extra_i..]);
7389	const constraint = std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra[rw_extra_i..]), 0);	7388	const constraint = std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra.items[rw_extra_i..]), 0);
7390	const name = std.mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0);	7389	const name = std.mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0);
7391	// This equation accounts for the fact that even if we have exactly 4 bytes	7390	// This equation accounts for the fact that even if we have exactly 4 bytes
7392	// for the string, we still use the next u32 for the null terminator.	7391	// for the string, we still use the next u32 for the null terminator.
...	@@ -7425,7 +7424,7 @@ pub const FuncGen = struct {	...	@@ -7425,7 +7424,7 @@ pub const FuncGen = struct {
7425	{	7424	{
7426	var clobber_i: u32 = 0;	7425	var clobber_i: u32 = 0;
7427	while (clobber_i < clobbers_len) : (clobber_i += 1) {	7426	while (clobber_i < clobbers_len) : (clobber_i += 1) {
7428	const clobber = std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra[extra_i..]), 0);	7427	const clobber = std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra.items[extra_i..]), 0);
7429	// This equation accounts for the fact that even if we have exactly 4 bytes	7428	// This equation accounts for the fact that even if we have exactly 4 bytes
7430	// for the string, we still use the next u32 for the null terminator.	7429	// for the string, we still use the next u32 for the null terminator.
7431	extra_i += clobber.len / 4 + 1;	7430	extra_i += clobber.len / 4 + 1;
...	@@ -7465,7 +7464,7 @@ pub const FuncGen = struct {	...	@@ -7465,7 +7464,7 @@ pub const FuncGen = struct {
7465	else => {},	7464	else => {},
7466	}	7465	}
7467		7466
7468	const asm_source = std.mem.sliceAsBytes(self.air.extra[extra_i..])[0..extra.data.source_len];	7467	const asm_source = std.mem.sliceAsBytes(self.air.extra.items[extra_i..])[0..extra.data.source_len];
7469		7468
7470	// hackety hacks until stage2 has proper inline asm in the frontend.	7469	// hackety hacks until stage2 has proper inline asm in the frontend.
7471	var rendered_template = std.ArrayList(u8).init(self.gpa);	7470	var rendered_template = std.ArrayList(u8).init(self.gpa);
...	@@ -10628,7 +10627,7 @@ pub const FuncGen = struct {	...	@@ -10628,7 +10627,7 @@ pub const FuncGen = struct {
10628	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;	10627	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
10629	const result_ty = self.typeOfIndex(inst);	10628	const result_ty = self.typeOfIndex(inst);
10630	const len: usize = @intCast(result_ty.arrayLen(zcu));	10629	const len: usize = @intCast(result_ty.arrayLen(zcu));
10631	const elements: []const Air.Inst.Ref = @ptrCast(self.air.extra[ty_pl.payload..][0..len]);	10630	const elements: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[ty_pl.payload..][0..len]);
10632	const llvm_result_ty = try o.lowerType(result_ty);	10631	const llvm_result_ty = try o.lowerType(result_ty);
10633		10632
10634	switch (result_ty.zigTypeTag(zcu)) {	10633	switch (result_ty.zigTypeTag(zcu)) {

...	@@ -10,7 +10,6 @@ const Decl = Zcu.Decl;	...	@@ -10,7 +10,6 @@ const Decl = Zcu.Decl;
10	const Type = @import("../Type.zig");	10	const Type = @import("../Type.zig");
11	const Value = @import("../Value.zig");	11	const Value = @import("../Value.zig");
12	const Air = @import("../Air.zig");	12	const Air = @import("../Air.zig");
13	const Liveness = @import("../Liveness.zig");
14	const InternPool = @import("../InternPool.zig");	13	const InternPool = @import("../InternPool.zig");
15		14
16	const spec = @import("spirv/spec.zig");	15	const spec = @import("spirv/spec.zig");
...	@@ -195,7 +194,7 @@ pub const Object = struct {	...	@@ -195,7 +194,7 @@ pub const Object = struct {
195	pt: Zcu.PerThread,	194	pt: Zcu.PerThread,
196	nav_index: InternPool.Nav.Index,	195	nav_index: InternPool.Nav.Index,
197	air: Air,	196	air: Air,
198	liveness: Liveness,	197	liveness: Air.Liveness,
199	do_codegen: bool,	198	do_codegen: bool,
200	) !void {	199	) !void {
201	const zcu = pt.zcu;	200	const zcu = pt.zcu;
...	@@ -242,7 +241,7 @@ pub const Object = struct {	...	@@ -242,7 +241,7 @@ pub const Object = struct {
242	pt: Zcu.PerThread,	241	pt: Zcu.PerThread,
243	func_index: InternPool.Index,	242	func_index: InternPool.Index,
244	air: Air,	243	air: Air,
245	liveness: Liveness,	244	liveness: Air.Liveness,
246	) !void {	245	) !void {
247	const nav = pt.zcu.funcInfo(func_index).owner_nav;	246	const nav = pt.zcu.funcInfo(func_index).owner_nav;
248	// TODO: Separate types for generating decls and functions?	247	// TODO: Separate types for generating decls and functions?
...	@@ -303,7 +302,7 @@ const NavGen = struct {	...	@@ -303,7 +302,7 @@ const NavGen = struct {
303		302
304	/// The liveness analysis of the intermediate code for the declaration we are currently generating.	303	/// The liveness analysis of the intermediate code for the declaration we are currently generating.
305	/// Note: If the declaration is not a function, this value will be undefined!	304	/// Note: If the declaration is not a function, this value will be undefined!
306	liveness: Liveness,	305	liveness: Air.Liveness,
307		306
308	/// An array of function argument result-ids. Each index corresponds with the	307	/// An array of function argument result-ids. Each index corresponds with the
309	/// function argument of the same index.	308	/// function argument of the same index.
...	@@ -4627,7 +4626,7 @@ const NavGen = struct {	...	@@ -4627,7 +4626,7 @@ const NavGen = struct {
4627	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;	4626	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
4628	const result_ty = self.typeOfIndex(inst);	4627	const result_ty = self.typeOfIndex(inst);
4629	const len: usize = @intCast(result_ty.arrayLen(zcu));	4628	const len: usize = @intCast(result_ty.arrayLen(zcu));
4630	const elements: []const Air.Inst.Ref = @ptrCast(self.air.extra[ty_pl.payload..][0..len]);	4629	const elements: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[ty_pl.payload..][0..len]);
4631		4630
4632	switch (result_ty.zigTypeTag(zcu)) {	4631	switch (result_ty.zigTypeTag(zcu)) {
4633	.@"struct" => {	4632	.@"struct" => {
...	@@ -5474,7 +5473,7 @@ const NavGen = struct {	...	@@ -5474,7 +5473,7 @@ const NavGen = struct {
5474	fn airBlock(self: *NavGen, inst: Air.Inst.Index) !?IdRef {	5473	fn airBlock(self: *NavGen, inst: Air.Inst.Index) !?IdRef {
5475	const inst_datas = self.air.instructions.items(.data);	5474	const inst_datas = self.air.instructions.items(.data);
5476	const extra = self.air.extraData(Air.Block, inst_datas[@intFromEnum(inst)].ty_pl.payload);	5475	const extra = self.air.extraData(Air.Block, inst_datas[@intFromEnum(inst)].ty_pl.payload);
5477	return self.lowerBlock(inst, @ptrCast(self.air.extra[extra.end..][0..extra.data.body_len]));	5476	return self.lowerBlock(inst, @ptrCast(self.air.extra.items[extra.end..][0..extra.data.body_len]));
5478	}	5477	}
5479		5478
5480	fn lowerBlock(self: *NavGen, inst: Air.Inst.Index, body: []const Air.Inst.Index) !?IdRef {	5479	fn lowerBlock(self: *NavGen, inst: Air.Inst.Index, body: []const Air.Inst.Index) !?IdRef {
...	@@ -5657,8 +5656,8 @@ const NavGen = struct {	...	@@ -5657,8 +5656,8 @@ const NavGen = struct {
5657	fn airCondBr(self: *NavGen, inst: Air.Inst.Index) !void {	5656	fn airCondBr(self: *NavGen, inst: Air.Inst.Index) !void {
5658	const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;	5657	const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
5659	const cond_br = self.air.extraData(Air.CondBr, pl_op.payload);	5658	const cond_br = self.air.extraData(Air.CondBr, pl_op.payload);
5660	const then_body: []const Air.Inst.Index = @ptrCast(self.air.extra[cond_br.end..][0..cond_br.data.then_body_len]);	5659	const then_body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[cond_br.end..][0..cond_br.data.then_body_len]);
5661	const else_body: []const Air.Inst.Index = @ptrCast(self.air.extra[cond_br.end + then_body.len ..][0..cond_br.data.else_body_len]);	5660	const else_body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[cond_br.end + then_body.len ..][0..cond_br.data.else_body_len]);
5662	const condition_id = try self.resolve(pl_op.operand);	5661	const condition_id = try self.resolve(pl_op.operand);
5663		5662
5664	const then_label = self.spv.allocId();	5663	const then_label = self.spv.allocId();
...	@@ -5717,7 +5716,7 @@ const NavGen = struct {	...	@@ -5717,7 +5716,7 @@ const NavGen = struct {
5717	fn airLoop(self: *NavGen, inst: Air.Inst.Index) !void {	5716	fn airLoop(self: *NavGen, inst: Air.Inst.Index) !void {
5718	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;	5717	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
5719	const loop = self.air.extraData(Air.Block, ty_pl.payload);	5718	const loop = self.air.extraData(Air.Block, ty_pl.payload);
5720	const body: []const Air.Inst.Index = @ptrCast(self.air.extra[loop.end..][0..loop.data.body_len]);	5719	const body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[loop.end..][0..loop.data.body_len]);
5721		5720
5722	const body_label = self.spv.allocId();	5721	const body_label = self.spv.allocId();
5723		5722
...	@@ -5837,7 +5836,7 @@ const NavGen = struct {	...	@@ -5837,7 +5836,7 @@ const NavGen = struct {
5837	const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;	5836	const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
5838	const err_union_id = try self.resolve(pl_op.operand);	5837	const err_union_id = try self.resolve(pl_op.operand);
5839	const extra = self.air.extraData(Air.Try, pl_op.payload);	5838	const extra = self.air.extraData(Air.Try, pl_op.payload);
5840	const body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra.end..][0..extra.data.body_len]);	5839	const body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[extra.end..][0..extra.data.body_len]);
5841		5840
5842	const err_union_ty = self.typeOf(pl_op.operand);	5841	const err_union_ty = self.typeOf(pl_op.operand);
5843	const payload_ty = self.typeOfIndex(inst);	5842	const payload_ty = self.typeOfIndex(inst);
...	@@ -6344,7 +6343,7 @@ const NavGen = struct {	...	@@ -6344,7 +6343,7 @@ const NavGen = struct {
6344	const old_base_line = self.base_line;	6343	const old_base_line = self.base_line;
6345	defer self.base_line = old_base_line;	6344	defer self.base_line = old_base_line;
6346	self.base_line = zcu.navSrcLine(zcu.funcInfo(extra.data.func).owner_nav);	6345	self.base_line = zcu.navSrcLine(zcu.funcInfo(extra.data.func).owner_nav);
6347	return self.lowerBlock(inst, @ptrCast(self.air.extra[extra.end..][0..extra.data.body_len]));	6346	return self.lowerBlock(inst, @ptrCast(self.air.extra.items[extra.end..][0..extra.data.body_len]));
6348	}	6347	}
6349		6348
6350	fn airDbgVar(self: *NavGen, inst: Air.Inst.Index) !void {	6349	fn airDbgVar(self: *NavGen, inst: Air.Inst.Index) !void {
...	@@ -6365,9 +6364,9 @@ const NavGen = struct {	...	@@ -6365,9 +6364,9 @@ const NavGen = struct {
6365	if (!is_volatile and self.liveness.isUnused(inst)) return null;	6364	if (!is_volatile and self.liveness.isUnused(inst)) return null;
6366		6365
6367	var extra_i: usize = extra.end;	6366	var extra_i: usize = extra.end;
6368	const outputs: []const Air.Inst.Ref = @ptrCast(self.air.extra[extra_i..][0..extra.data.outputs_len]);	6367	const outputs: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[extra_i..][0..extra.data.outputs_len]);
6369	extra_i += outputs.len;	6368	extra_i += outputs.len;
6370	const inputs: []const Air.Inst.Ref = @ptrCast(self.air.extra[extra_i..][0..extra.data.inputs_len]);	6369	const inputs: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[extra_i..][0..extra.data.inputs_len]);
6371	extra_i += inputs.len;	6370	extra_i += inputs.len;
6372		6371
6373	if (outputs.len > 1) {	6372	if (outputs.len > 1) {
...	@@ -6386,15 +6385,15 @@ const NavGen = struct {	...	@@ -6386,15 +6385,15 @@ const NavGen = struct {
6386	if (output != .none) {	6385	if (output != .none) {
6387	return self.todo("implement inline asm with non-returned output", .{});	6386	return self.todo("implement inline asm with non-returned output", .{});
6388	}	6387	}
6389	const extra_bytes = std.mem.sliceAsBytes(self.air.extra[extra_i..]);	6388	const extra_bytes = std.mem.sliceAsBytes(self.air.extra.items[extra_i..]);
6390	const constraint = std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra[extra_i..]), 0);	6389	const constraint = std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra.items[extra_i..]), 0);
6391	const name = std.mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0);	6390	const name = std.mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0);
6392	extra_i += (constraint.len + name.len + (2 + 3)) / 4;	6391	extra_i += (constraint.len + name.len + (2 + 3)) / 4;
6393	// TODO: Record output and use it somewhere.	6392	// TODO: Record output and use it somewhere.
6394	}	6393	}
6395		6394
6396	for (inputs) \|input\| {	6395	for (inputs) \|input\| {
6397	const extra_bytes = std.mem.sliceAsBytes(self.air.extra[extra_i..]);	6396	const extra_bytes = std.mem.sliceAsBytes(self.air.extra.items[extra_i..]);
6398	const constraint = std.mem.sliceTo(extra_bytes, 0);	6397	const constraint = std.mem.sliceTo(extra_bytes, 0);
6399	const name = std.mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0);	6398	const name = std.mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0);
6400	// This equation accounts for the fact that even if we have exactly 4 bytes	6399	// This equation accounts for the fact that even if we have exactly 4 bytes
...	@@ -6461,13 +6460,13 @@ const NavGen = struct {	...	@@ -6461,13 +6460,13 @@ const NavGen = struct {
6461	{	6460	{
6462	var clobber_i: u32 = 0;	6461	var clobber_i: u32 = 0;
6463	while (clobber_i < clobbers_len) : (clobber_i += 1) {	6462	while (clobber_i < clobbers_len) : (clobber_i += 1) {
6464	const clobber = std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra[extra_i..]), 0);	6463	const clobber = std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra.items[extra_i..]), 0);
6465	extra_i += clobber.len / 4 + 1;	6464	extra_i += clobber.len / 4 + 1;
6466	// TODO: Record clobber and use it somewhere.	6465	// TODO: Record clobber and use it somewhere.
6467	}	6466	}
6468	}	6467	}
6469		6468
6470	const asm_source = std.mem.sliceAsBytes(self.air.extra[extra_i..])[0..extra.data.source_len];	6469	const asm_source = std.mem.sliceAsBytes(self.air.extra.items[extra_i..])[0..extra.data.source_len];
6471		6470
6472	as.assemble(asm_source) catch \|err\| switch (err) {	6471	as.assemble(asm_source) catch \|err\| switch (err) {
6473	error.AssembleFail => {	6472	error.AssembleFail => {
...	@@ -6501,8 +6500,8 @@ const NavGen = struct {	...	@@ -6501,8 +6500,8 @@ const NavGen = struct {
6501		6500
6502	for (outputs) \|output\| {	6501	for (outputs) \|output\| {
6503	_ = output;	6502	_ = output;
6504	const extra_bytes = std.mem.sliceAsBytes(self.air.extra[output_extra_i..]);	6503	const extra_bytes = std.mem.sliceAsBytes(self.air.extra.items[output_extra_i..]);
6505	const constraint = std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra[output_extra_i..]), 0);	6504	const constraint = std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra.items[output_extra_i..]), 0);
6506	const name = std.mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0);	6505	const name = std.mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0);
6507	output_extra_i += (constraint.len + name.len + (2 + 3)) / 4;	6506	output_extra_i += (constraint.len + name.len + (2 + 3)) / 4;
6508		6507
...	@@ -6531,7 +6530,7 @@ const NavGen = struct {	...	@@ -6531,7 +6530,7 @@ const NavGen = struct {
6531	const zcu = pt.zcu;	6530	const zcu = pt.zcu;
6532	const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;	6531	const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
6533	const extra = self.air.extraData(Air.Call, pl_op.payload);	6532	const extra = self.air.extraData(Air.Call, pl_op.payload);
6534	const args: []const Air.Inst.Ref = @ptrCast(self.air.extra[extra.end..][0..extra.data.args_len]);	6533	const args: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[extra.end..][0..extra.data.args_len]);
6535	const callee_ty = self.typeOf(pl_op.operand);	6534	const callee_ty = self.typeOf(pl_op.operand);
6536	const zig_fn_ty = switch (callee_ty.zigTypeTag(zcu)) {	6535	const zig_fn_ty = switch (callee_ty.zigTypeTag(zcu)) {
6537	.@"fn" => callee_ty,	6536	.@"fn" => callee_ty,

...	@@ -1098,7 +1098,7 @@ pub fn updateFunc(	...	@@ -1098,7 +1098,7 @@ pub fn updateFunc(
1098	pt: Zcu.PerThread,	1098	pt: Zcu.PerThread,
1099	func_index: InternPool.Index,	1099	func_index: InternPool.Index,
1100	air: Air,	1100	air: Air,
1101	liveness: Liveness,	1101	liveness: Air.Liveness,
1102	) link.File.UpdateNavError!void {	1102	) link.File.UpdateNavError!void {
1103	if (build_options.skip_non_native and builtin.object_format != .coff) {	1103	if (build_options.skip_non_native and builtin.object_format != .coff) {
1104	@panic("Attempted to compile for object format that was disabled by build configuration");	1104	@panic("Attempted to compile for object format that was disabled by build configuration");
...	@@ -3802,7 +3802,6 @@ const trace = @import("../tracy.zig").trace;	...	@@ -3802,7 +3802,6 @@ const trace = @import("../tracy.zig").trace;
3802		3802
3803	const Air = @import("../Air.zig");	3803	const Air = @import("../Air.zig");
3804	const Compilation = @import("../Compilation.zig");	3804	const Compilation = @import("../Compilation.zig");
3805	const Liveness = @import("../Liveness.zig");
3806	const LlvmObject = @import("../codegen/llvm.zig").Object;	3805	const LlvmObject = @import("../codegen/llvm.zig").Object;
3807	const Zcu = @import("../Zcu.zig");	3806	const Zcu = @import("../Zcu.zig");
3808	const InternPool = @import("../InternPool.zig");	3807	const InternPool = @import("../InternPool.zig");

...	@@ -2385,7 +2385,7 @@ pub fn updateFunc(	...	@@ -2385,7 +2385,7 @@ pub fn updateFunc(
2385	pt: Zcu.PerThread,	2385	pt: Zcu.PerThread,
2386	func_index: InternPool.Index,	2386	func_index: InternPool.Index,
2387	air: Air,	2387	air: Air,
2388	liveness: Liveness,	2388	liveness: Air.Liveness,
2389	) link.File.UpdateNavError!void {	2389	) link.File.UpdateNavError!void {
2390	if (build_options.skip_non_native and builtin.object_format != .elf) {	2390	if (build_options.skip_non_native and builtin.object_format != .elf) {
2391	@panic("Attempted to compile for object format that was disabled by build configuration");	2391	@panic("Attempted to compile for object format that was disabled by build configuration");
...	@@ -5323,7 +5323,6 @@ const GotSection = synthetic_sections.GotSection;	...	@@ -5323,7 +5323,6 @@ const GotSection = synthetic_sections.GotSection;
5323	const GotPltSection = synthetic_sections.GotPltSection;	5323	const GotPltSection = synthetic_sections.GotPltSection;
5324	const HashSection = synthetic_sections.HashSection;	5324	const HashSection = synthetic_sections.HashSection;
5325	const LinkerDefined = @import("Elf/LinkerDefined.zig");	5325	const LinkerDefined = @import("Elf/LinkerDefined.zig");
5326	const Liveness = @import("../Liveness.zig");
5327	const LlvmObject = @import("../codegen/llvm.zig").Object;	5326	const LlvmObject = @import("../codegen/llvm.zig").Object;
5328	const Zcu = @import("../Zcu.zig");	5327	const Zcu = @import("../Zcu.zig");
5329	const Object = @import("Elf/Object.zig");	5328	const Object = @import("Elf/Object.zig");

...	@@ -17,7 +17,6 @@ const link = @import("../link.zig");	...	@@ -17,7 +17,6 @@ const link = @import("../link.zig");
17	const trace = @import("../tracy.zig").trace;	17	const trace = @import("../tracy.zig").trace;
18	const build_options = @import("build_options");	18	const build_options = @import("build_options");
19	const Air = @import("../Air.zig");	19	const Air = @import("../Air.zig");
20	const Liveness = @import("../Liveness.zig");
21	const LlvmObject = @import("../codegen/llvm.zig").Object;	20	const LlvmObject = @import("../codegen/llvm.zig").Object;
22		21
23	base: link.File,	22	base: link.File,
...	@@ -79,7 +78,7 @@ pub fn updateFunc(	...	@@ -79,7 +78,7 @@ pub fn updateFunc(
79	pt: Zcu.PerThread,	78	pt: Zcu.PerThread,
80	func_index: InternPool.Index,	79	func_index: InternPool.Index,
81	air: Air,	80	air: Air,
82	liveness: Liveness,	81	liveness: Air.Liveness,
83	) link.File.UpdateNavError!void {	82	) link.File.UpdateNavError!void {
84	if (build_options.skip_non_native and builtin.object_format != .goff)	83	if (build_options.skip_non_native and builtin.object_format != .goff)
85	@panic("Attempted to compile for object format that was disabled by build configuration");	84	@panic("Attempted to compile for object format that was disabled by build configuration");

...	@@ -3074,7 +3074,7 @@ pub fn updateFunc(	...	@@ -3074,7 +3074,7 @@ pub fn updateFunc(
3074	pt: Zcu.PerThread,	3074	pt: Zcu.PerThread,
3075	func_index: InternPool.Index,	3075	func_index: InternPool.Index,
3076	air: Air,	3076	air: Air,
3077	liveness: Liveness,	3077	liveness: Air.Liveness,
3078	) link.File.UpdateNavError!void {	3078	) link.File.UpdateNavError!void {
3079	if (build_options.skip_non_native and builtin.object_format != .macho) {	3079	if (build_options.skip_non_native and builtin.object_format != .macho) {
3080	@panic("Attempted to compile for object format that was disabled by build configuration");	3080	@panic("Attempted to compile for object format that was disabled by build configuration");
...	@@ -5496,7 +5496,6 @@ const ObjcStubsSection = synthetic.ObjcStubsSection;	...	@@ -5496,7 +5496,6 @@ const ObjcStubsSection = synthetic.ObjcStubsSection;
5496	const Object = @import("MachO/Object.zig");	5496	const Object = @import("MachO/Object.zig");
5497	const LazyBind = bind.LazyBind;	5497	const LazyBind = bind.LazyBind;
5498	const LaSymbolPtrSection = synthetic.LaSymbolPtrSection;	5498	const LaSymbolPtrSection = synthetic.LaSymbolPtrSection;
5499	const Liveness = @import("../Liveness.zig");
5500	const LlvmObject = @import("../codegen/llvm.zig").Object;	5499	const LlvmObject = @import("../codegen/llvm.zig").Object;
5501	const Md5 = std.crypto.hash.Md5;	5500	const Md5 = std.crypto.hash.Md5;
5502	const Zcu = @import("../Zcu.zig");	5501	const Zcu = @import("../Zcu.zig");

...	@@ -12,7 +12,6 @@ const trace = @import("../tracy.zig").trace;	...	@@ -12,7 +12,6 @@ const trace = @import("../tracy.zig").trace;
12	const File = link.File;	12	const File = link.File;
13	const build_options = @import("build_options");	13	const build_options = @import("build_options");
14	const Air = @import("../Air.zig");	14	const Air = @import("../Air.zig");
15	const Liveness = @import("../Liveness.zig");
16	const Type = @import("../Type.zig");	15	const Type = @import("../Type.zig");
17	const Value = @import("../Value.zig");	16	const Value = @import("../Value.zig");
18	const AnalUnit = InternPool.AnalUnit;	17	const AnalUnit = InternPool.AnalUnit;
...	@@ -389,7 +388,7 @@ pub fn updateFunc(	...	@@ -389,7 +388,7 @@ pub fn updateFunc(
389	pt: Zcu.PerThread,	388	pt: Zcu.PerThread,
390	func_index: InternPool.Index,	389	func_index: InternPool.Index,
391	air: Air,	390	air: Air,
392	liveness: Liveness,	391	liveness: Air.Liveness,
393	) link.File.UpdateNavError!void {	392	) link.File.UpdateNavError!void {
394	if (build_options.skip_non_native and builtin.object_format != .plan9) {	393	if (build_options.skip_non_native and builtin.object_format != .plan9) {
395	@panic("Attempted to compile for object format that was disabled by build configuration");	394	@panic("Attempted to compile for object format that was disabled by build configuration");

...	@@ -36,7 +36,6 @@ const codegen = @import("../codegen/spirv.zig");	...	@@ -36,7 +36,6 @@ const codegen = @import("../codegen/spirv.zig");
36	const trace = @import("../tracy.zig").trace;	36	const trace = @import("../tracy.zig").trace;
37	const build_options = @import("build_options");	37	const build_options = @import("build_options");
38	const Air = @import("../Air.zig");	38	const Air = @import("../Air.zig");
39	const Liveness = @import("../Liveness.zig");
40	const Type = @import("../Type.zig");	39	const Type = @import("../Type.zig");
41	const Value = @import("../Value.zig");	40	const Value = @import("../Value.zig");
42		41
...	@@ -118,7 +117,7 @@ pub fn updateFunc(	...	@@ -118,7 +117,7 @@ pub fn updateFunc(
118	pt: Zcu.PerThread,	117	pt: Zcu.PerThread,
119	func_index: InternPool.Index,	118	func_index: InternPool.Index,
120	air: Air,	119	air: Air,
121	liveness: Liveness,	120	liveness: Air.Liveness,
122	) link.File.UpdateNavError!void {	121	) link.File.UpdateNavError!void {
123	if (build_options.skip_non_native) {	122	if (build_options.skip_non_native) {
124	@panic("Attempted to compile for architecture that was disabled by build configuration");	123	@panic("Attempted to compile for architecture that was disabled by build configuration");