diff --git a/CMakeLists.txt b/CMakeLists.txt index cfba9f257972029fe576ab214bf15d53c76f97c8..6b8d1ba4a5b9457b1db4f86b2f3edf6a6f40a1b2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -512,13 +512,15 @@ set(ZIG_STAGE2_SOURCES lib/std/zig/llvm/bitcode_writer.zig lib/std/zig/llvm/ir.zig src/Air.zig + src/Air/Legalize.zig + src/Air/Liveness.zig + src/Air/Liveness/Verify.zig + src/Air/types_resolved.zig src/Builtin.zig src/Compilation.zig src/Compilation/Config.zig src/DarwinPosixSpawn.zig src/InternPool.zig - src/Liveness.zig - src/Liveness/Verify.zig src/Package.zig src/Package/Fetch.zig src/Package/Fetch/git.zig diff --git a/src/Air.zig b/src/Air.zig index 409a040f1c24ddc841b6bb8ef273944b265d0b5e..b468edbb4718efba956191139f30a0e24fd34f17 100644 --- a/src/Air.zig +++ b/src/Air.zig @@ -9,16 +9,19 @@ const builtin = @import("builtin"); const assert = std.debug.assert; const Air = @This(); -const Value = @import("Value.zig"); -const Type = @import("Type.zig"); const InternPool = @import("InternPool.zig"); +const Type = @import("Type.zig"); +const Value = @import("Value.zig"); const Zcu = @import("Zcu.zig"); const types_resolved = @import("Air/types_resolved.zig"); +pub const Legalize = @import("Air/Legalize.zig"); +pub const Liveness = @import("Air/Liveness.zig"); + instructions: std.MultiArrayList(Inst).Slice, /// The meaning of this data is determined by `Inst.Tag` value. /// The first few indexes are reserved. See `ExtraIndex` for the values. -extra: []const u32, +extra: std.ArrayListUnmanaged(u32), pub const ExtraIndex = enum(u32) { /// Payload index of the main `Block` in the `extra` array. @@ -244,22 +247,27 @@ pub const Inst = struct { /// Uses the `bin_op` field. bit_or, /// Shift right. `>>` + /// The rhs type may be a scalar version of the lhs type. /// Uses the `bin_op` field. shr, /// Shift right. The shift produces a poison value if it shifts out any non-zero bits. + /// The rhs type may be a scalar version of the lhs type. /// Uses the `bin_op` field. shr_exact, /// Shift left. `<<` + /// The rhs type may be a scalar version of the lhs type. /// Uses the `bin_op` field. shl, /// Shift left; For unsigned integers, the shift produces a poison value if it shifts /// out any non-zero bits. For signed integers, the shift produces a poison value if /// it shifts out any bits that disagree with the resultant sign bit. + /// The rhs type may be a scalar version of the lhs type. /// Uses the `bin_op` field. shl_exact, /// Saturating integer shift left. `<<|`. The result is the same type as the `lhs`. /// The `rhs` must have the same vector shape as the `lhs`, but with any unsigned /// integer as the scalar type. + /// The rhs type may be a scalar version of the lhs type. /// Uses the `bin_op` field. shl_sat, /// Bitwise XOR. `^` @@ -1378,9 +1386,9 @@ pub const UnionInit = struct { }; pub fn getMainBody(air: Air) []const Air.Inst.Index { - const body_index = air.extra[@intFromEnum(ExtraIndex.main_block)]; + const body_index = air.extra.items[@intFromEnum(ExtraIndex.main_block)]; const extra = air.extraData(Block, body_index); - return @ptrCast(air.extra[extra.end..][0..extra.data.body_len]); + return @ptrCast(air.extra.items[extra.end..][0..extra.data.body_len]); } pub fn typeOf(air: *const Air, inst: Air.Inst.Ref, ip: *const InternPool) Type { @@ -1656,9 +1664,9 @@ pub fn extraData(air: Air, comptime T: type, index: usize) struct { data: T, end var result: T = undefined; inline for (fields) |field| { @field(result, field.name) = switch (field.type) { - u32 => air.extra[i], - InternPool.Index, Inst.Ref => @enumFromInt(air.extra[i]), - i32, CondBr.BranchHints => @bitCast(air.extra[i]), + u32 => air.extra.items[i], + InternPool.Index, Inst.Ref => @enumFromInt(air.extra.items[i]), + i32, CondBr.BranchHints => @bitCast(air.extra.items[i]), else => @compileError("bad field type: " ++ @typeName(field.type)), }; i += 1; @@ -1671,7 +1679,7 @@ pub fn extraData(air: Air, comptime T: type, index: usize) struct { data: T, end pub fn deinit(air: *Air, gpa: std.mem.Allocator) void { air.instructions.deinit(gpa); - gpa.free(air.extra); + air.extra.deinit(gpa); air.* = undefined; } @@ -1700,7 +1708,7 @@ pub const NullTerminatedString = enum(u32) { pub fn toSlice(nts: NullTerminatedString, air: Air) [:0]const u8 { if (nts == .none) return ""; - const bytes = std.mem.sliceAsBytes(air.extra[@intFromEnum(nts)..]); + const bytes = std.mem.sliceAsBytes(air.extra.items[@intFromEnum(nts)..]); return bytes[0..std.mem.indexOfScalar(u8, bytes, 0).? :0]; } }; @@ -1943,7 +1951,7 @@ pub const UnwrappedSwitch = struct { return us.getHintInner(us.cases_len); } fn getHintInner(us: UnwrappedSwitch, idx: u32) std.builtin.BranchHint { - const bag = us.air.extra[us.branch_hints_start..][idx / 10]; + const bag = us.air.extra.items[us.branch_hints_start..][idx / 10]; const bits: u3 = @truncate(bag >> @intCast(3 * (idx % 10))); return @enumFromInt(bits); } @@ -1971,13 +1979,13 @@ pub const UnwrappedSwitch = struct { const extra = it.air.extraData(SwitchBr.Case, it.extra_index); var extra_index = extra.end; - const items: []const Inst.Ref = @ptrCast(it.air.extra[extra_index..][0..extra.data.items_len]); + const items: []const Inst.Ref = @ptrCast(it.air.extra.items[extra_index..][0..extra.data.items_len]); extra_index += items.len; // TODO: ptrcast from []const Inst.Ref to []const [2]Inst.Ref when supported - const ranges_ptr: [*]const [2]Inst.Ref = @ptrCast(it.air.extra[extra_index..]); + const ranges_ptr: [*]const [2]Inst.Ref = @ptrCast(it.air.extra.items[extra_index..]); const ranges: []const [2]Inst.Ref = ranges_ptr[0..extra.data.ranges_len]; extra_index += ranges.len * 2; - const body: []const Inst.Index = @ptrCast(it.air.extra[extra_index..][0..extra.data.body_len]); + const body: []const Inst.Index = @ptrCast(it.air.extra.items[extra_index..][0..extra.data.body_len]); extra_index += body.len; it.extra_index = @intCast(extra_index); @@ -1992,7 +2000,7 @@ pub const UnwrappedSwitch = struct { /// Returns the body of the "default" (`else`) case. pub fn elseBody(it: *CaseIterator) []const Inst.Index { assert(it.next_case == it.cases_len); - return @ptrCast(it.air.extra[it.extra_index..][0..it.else_body_len]); + return @ptrCast(it.air.extra.items[it.extra_index..][0..it.else_body_len]); } pub const Case = struct { idx: u32, @@ -2025,6 +2033,7 @@ pub fn unwrapSwitch(air: *const Air, switch_inst: Inst.Index) UnwrappedSwitch { pub const typesFullyResolved = types_resolved.typesFullyResolved; pub const typeFullyResolved = types_resolved.checkType; pub const valFullyResolved = types_resolved.checkVal; +pub const legalize = Legalize.legalize; pub const CoveragePoint = enum(u1) { /// Indicates the block is not a place of interest corresponding to diff --git a/src/Air/Legalize.zig b/src/Air/Legalize.zig new file mode 100644 index 0000000000000000000000000000000000000000..3d38798aa3477ff2a7ef690c4cef4a48cb286265 --- /dev/null +++ b/src/Air/Legalize.zig @@ -0,0 +1,147 @@ +zcu: *const Zcu, +air: Air, +features: std.enums.EnumSet(Feature), + +pub const Feature = enum { + /// Legalize (shift lhs, (splat rhs)) -> (shift lhs, rhs) + remove_shift_vector_rhs_splat, + /// Legalize reduce of a one element vector to a bitcast + reduce_one_elem_to_bitcast, +}; + +pub const Features = std.enums.EnumFieldStruct(Feature, bool, false); + +pub fn legalize(air: *Air, backend: std.builtin.CompilerBackend, zcu: *const Zcu) std.mem.Allocator.Error!void { + var l: Legalize = .{ + .zcu = zcu, + .air = air.*, + .features = features: switch (backend) { + .other, .stage1 => unreachable, + inline .stage2_llvm, + .stage2_c, + .stage2_wasm, + .stage2_arm, + .stage2_x86_64, + .stage2_aarch64, + .stage2_x86, + .stage2_riscv64, + .stage2_sparc64, + .stage2_spirv64, + .stage2_powerpc, + => |ct_backend| { + const Backend = codegen.importBackend(ct_backend) orelse break :features .initEmpty(); + break :features if (@hasDecl(Backend, "legalize_features")) + .init(Backend.legalize_features) + else + .initEmpty(); + }, + _ => unreachable, + }, + }; + defer air.* = l.air; + if (!l.features.bits.eql(.initEmpty())) try l.legalizeBody(l.air.getMainBody()); +} + +fn legalizeBody(l: *Legalize, body: []const Air.Inst.Index) std.mem.Allocator.Error!void { + const zcu = l.zcu; + const ip = &zcu.intern_pool; + const tags = l.air.instructions.items(.tag); + const data = l.air.instructions.items(.data); + for (body) |inst| inst: switch (tags[@intFromEnum(inst)]) { + else => {}, + + .shl, + .shl_exact, + .shl_sat, + .shr, + .shr_exact, + => |air_tag| if (l.features.contains(.remove_shift_vector_rhs_splat)) done: { + const bin_op = data[@intFromEnum(inst)].bin_op; + const ty = l.air.typeOf(bin_op.rhs, ip); + if (!ty.isVector(zcu)) break :done; + if (bin_op.rhs.toInterned()) |rhs_ip_index| switch (ip.indexToKey(rhs_ip_index)) { + else => {}, + .aggregate => |aggregate| switch (aggregate.storage) { + else => {}, + .repeated_elem => |splat| continue :inst l.replaceInst(inst, air_tag, .{ .bin_op = .{ + .lhs = bin_op.lhs, + .rhs = Air.internedToRef(splat), + } }), + }, + } else { + const rhs_inst = bin_op.rhs.toIndex().?; + switch (tags[@intFromEnum(rhs_inst)]) { + else => {}, + .splat => continue :inst l.replaceInst(inst, air_tag, .{ .bin_op = .{ + .lhs = bin_op.lhs, + .rhs = data[@intFromEnum(rhs_inst)].ty_op.operand, + } }), + } + } + }, + + .reduce, + .reduce_optimized, + => if (l.features.contains(.reduce_one_elem_to_bitcast)) done: { + const reduce = data[@intFromEnum(inst)].reduce; + const vector_ty = l.air.typeOf(reduce.operand, ip); + switch (vector_ty.vectorLen(zcu)) { + 0 => unreachable, + 1 => continue :inst l.replaceInst(inst, .bitcast, .{ .ty_op = .{ + .ty = Air.internedToRef(vector_ty.scalarType(zcu).toIntern()), + .operand = reduce.operand, + } }), + else => break :done, + } + }, + + .@"try", .try_cold => { + const pl_op = data[@intFromEnum(inst)].pl_op; + const extra = l.air.extraData(Air.Try, pl_op.payload); + try l.legalizeBody(@ptrCast(l.air.extra.items[extra.end..][0..extra.data.body_len])); + }, + .try_ptr, .try_ptr_cold => { + const ty_pl = data[@intFromEnum(inst)].ty_pl; + const extra = l.air.extraData(Air.TryPtr, ty_pl.payload); + try l.legalizeBody(@ptrCast(l.air.extra.items[extra.end..][0..extra.data.body_len])); + }, + .block, .loop => { + const ty_pl = data[@intFromEnum(inst)].ty_pl; + const extra = l.air.extraData(Air.Block, ty_pl.payload); + try l.legalizeBody(@ptrCast(l.air.extra.items[extra.end..][0..extra.data.body_len])); + }, + .dbg_inline_block => { + const ty_pl = data[@intFromEnum(inst)].ty_pl; + const extra = l.air.extraData(Air.DbgInlineBlock, ty_pl.payload); + try l.legalizeBody(@ptrCast(l.air.extra.items[extra.end..][0..extra.data.body_len])); + }, + .cond_br => { + const pl_op = data[@intFromEnum(inst)].pl_op; + const extra = l.air.extraData(Air.CondBr, pl_op.payload); + try l.legalizeBody(@ptrCast(l.air.extra.items[extra.end..][0..extra.data.then_body_len])); + try l.legalizeBody(@ptrCast(l.air.extra.items[extra.end + extra.data.then_body_len ..][0..extra.data.else_body_len])); + }, + .switch_br, .loop_switch_br => { + const switch_br = l.air.unwrapSwitch(inst); + var it = switch_br.iterateCases(); + while (it.next()) |case| try l.legalizeBody(case.body); + try l.legalizeBody(it.elseBody()); + }, + }; +} + +// inline to propagate comptime `tag`s +inline fn replaceInst(l: *Legalize, inst: Air.Inst.Index, tag: Air.Inst.Tag, data: Air.Inst.Data) Air.Inst.Tag { + const ip = &l.zcu.intern_pool; + const orig_ty = if (std.debug.runtime_safety) l.air.typeOfIndex(inst, ip) else {}; + l.air.instructions.items(.tag)[@intFromEnum(inst)] = tag; + l.air.instructions.items(.data)[@intFromEnum(inst)] = data; + if (std.debug.runtime_safety) std.debug.assert(l.air.typeOfIndex(inst, ip).toIntern() == orig_ty.toIntern()); + return tag; +} + +const Air = @import("../Air.zig"); +const codegen = @import("../codegen.zig"); +const Legalize = @This(); +const std = @import("std"); +const Zcu = @import("../Zcu.zig"); diff --git a/src/Air/Liveness.zig b/src/Air/Liveness.zig new file mode 100644 index 0000000000000000000000000000000000000000..34ecde26e2fe5720919b403a19cb6995ef7bca66 --- /dev/null +++ b/src/Air/Liveness.zig @@ -0,0 +1,2050 @@ +//! For each AIR instruction, we want to know: +//! * Is the instruction unreferenced (e.g. dies immediately)? +//! * For each of its operands, does the operand die with this instruction (e.g. is +//! this the last reference to it)? +//! Some instructions are special, such as: +//! * Conditional Branches +//! * Switch Branches +const std = @import("std"); +const log = std.log.scoped(.liveness); +const assert = std.debug.assert; +const Allocator = std.mem.Allocator; +const Log2Int = std.math.Log2Int; + +const Liveness = @This(); +const trace = @import("../tracy.zig").trace; +const Air = @import("../Air.zig"); +const InternPool = @import("../InternPool.zig"); + +pub const Verify = @import("Liveness/Verify.zig"); + +/// This array is split into sets of 4 bits per AIR instruction. +/// The MSB (0bX000) is whether the instruction is unreferenced. +/// The LSB (0b000X) is the first operand, and so on, up to 3 operands. A set bit means the +/// operand dies after this instruction. +/// Instructions which need more data to track liveness have special handling via the +/// `special` table. +tomb_bits: []usize, +/// Sparse table of specially handled instructions. The value is an index into the `extra` +/// array. The meaning of the data depends on the AIR tag. +/// * `cond_br` - points to a `CondBr` in `extra` at this index. +/// * `try`, `try_ptr` - points to a `CondBr` in `extra` at this index. The error path (the block +/// in the instruction) is considered the "else" path, and the rest of the block the "then". +/// * `switch_br` - points to a `SwitchBr` in `extra` at this index. +/// * `loop_switch_br` - points to a `SwitchBr` in `extra` at this index. +/// * `block` - points to a `Block` in `extra` at this index. +/// * `asm`, `call`, `aggregate_init` - the value is a set of bits which are the extra tomb +/// bits of operands. +/// The main tomb bits are still used and the extra ones are starting with the lsb of the +/// value here. +special: std.AutoHashMapUnmanaged(Air.Inst.Index, u32), +/// Auxiliary data. The way this data is interpreted is determined contextually. +extra: []const u32, + +/// Trailing is the set of instructions whose lifetimes end at the start of the then branch, +/// followed by the set of instructions whose lifetimes end at the start of the else branch. +pub const CondBr = struct { + then_death_count: u32, + else_death_count: u32, +}; + +/// Trailing is: +/// * For each case in the same order as in the AIR: +/// - case_death_count: u32 +/// - Air.Inst.Index for each `case_death_count`: set of instructions whose lifetimes +/// end at the start of this case. +/// * Air.Inst.Index for each `else_death_count`: set of instructions whose lifetimes +/// end at the start of the else case. +pub const SwitchBr = struct { + else_death_count: u32, +}; + +/// Trailing is the set of instructions which die in the block. Note that these are not additional +/// deaths (they are all recorded as normal within the block), but backends may use this information +/// as a more efficient way to track which instructions are still alive after a block. +pub const Block = struct { + death_count: u32, +}; + +/// Liveness analysis runs in several passes. Each pass iterates backwards over instructions in +/// bodies, and recurses into bodies. +const LivenessPass = enum { + /// In this pass, we perform some basic analysis of loops to gain information the main pass needs. + /// In particular, for every `loop` and `loop_switch_br`, we track the following information: + /// * Every outer block which the loop body contains a `br` to. + /// * Every outer loop which the loop body contains a `repeat` to. + /// * Every operand referenced within the loop body but created outside the loop. + /// This gives the main analysis pass enough information to determine the full set of + /// instructions which need to be alive when a loop repeats. This data is TEMPORARILY stored in + /// `a.extra`. It is not re-added to `extra` by the main pass, since it is not useful to + /// backends. + loop_analysis, + + /// This pass performs the main liveness analysis, setting up tombs and extra data while + /// considering control flow etc. + main_analysis, +}; + +/// Each analysis pass may wish to pass data through calls. A pointer to a `LivenessPassData(pass)` +/// stored on the stack is passed through calls to `analyzeInst` etc. +fn LivenessPassData(comptime pass: LivenessPass) type { + return switch (pass) { + .loop_analysis => struct { + /// The set of blocks which are exited with a `br` instruction at some point within this + /// body and which we are currently within. Also includes `loop`s which are the target + /// of a `repeat` instruction, and `loop_switch_br`s which are the target of a + /// `switch_dispatch` instruction. + breaks: std.AutoHashMapUnmanaged(Air.Inst.Index, void) = .empty, + + /// The set of operands for which we have seen at least one usage but not their birth. + live_set: std.AutoHashMapUnmanaged(Air.Inst.Index, void) = .empty, + + fn deinit(self: *@This(), gpa: Allocator) void { + self.breaks.deinit(gpa); + self.live_set.deinit(gpa); + } + }, + + .main_analysis => struct { + /// Every `block` and `loop` currently under analysis. + block_scopes: std.AutoHashMapUnmanaged(Air.Inst.Index, BlockScope) = .empty, + + /// The set of instructions currently alive in the current control + /// flow branch. + live_set: std.AutoHashMapUnmanaged(Air.Inst.Index, void) = .empty, + + /// The extra data initialized by the `loop_analysis` pass for this pass to consume. + /// Owned by this struct during this pass. + old_extra: std.ArrayListUnmanaged(u32) = .empty, + + const BlockScope = struct { + /// If this is a `block`, these instructions are alive upon a `br` to this block. + /// If this is a `loop`, these instructions are alive upon a `repeat` to this block. + live_set: std.AutoHashMapUnmanaged(Air.Inst.Index, void), + }; + + fn deinit(self: *@This(), gpa: Allocator) void { + var it = self.block_scopes.valueIterator(); + while (it.next()) |block| { + block.live_set.deinit(gpa); + } + self.block_scopes.deinit(gpa); + self.live_set.deinit(gpa); + self.old_extra.deinit(gpa); + } + }, + }; +} + +pub fn analyze(gpa: Allocator, air: Air, intern_pool: *InternPool) Allocator.Error!Liveness { + const tracy = trace(@src()); + defer tracy.end(); + + var a: Analysis = .{ + .gpa = gpa, + .air = air, + .tomb_bits = try gpa.alloc( + usize, + (air.instructions.len * bpi + @bitSizeOf(usize) - 1) / @bitSizeOf(usize), + ), + .extra = .{}, + .special = .{}, + .intern_pool = intern_pool, + }; + errdefer gpa.free(a.tomb_bits); + errdefer a.special.deinit(gpa); + defer a.extra.deinit(gpa); + + @memset(a.tomb_bits, 0); + + const main_body = air.getMainBody(); + + { + var data: LivenessPassData(.loop_analysis) = .{}; + defer data.deinit(gpa); + try analyzeBody(&a, .loop_analysis, &data, main_body); + } + + { + var data: LivenessPassData(.main_analysis) = .{}; + defer data.deinit(gpa); + data.old_extra = a.extra; + a.extra = .{}; + try analyzeBody(&a, .main_analysis, &data, main_body); + assert(data.live_set.count() == 0); + } + + return .{ + .tomb_bits = a.tomb_bits, + .special = a.special, + .extra = try a.extra.toOwnedSlice(gpa), + }; +} + +pub fn getTombBits(l: Liveness, inst: Air.Inst.Index) Bpi { + const usize_index = (@intFromEnum(inst) * bpi) / @bitSizeOf(usize); + return @as(Bpi, @truncate(l.tomb_bits[usize_index] >> + @as(Log2Int(usize), @intCast((@intFromEnum(inst) % (@bitSizeOf(usize) / bpi)) * bpi)))); +} + +pub fn isUnused(l: Liveness, inst: Air.Inst.Index) bool { + const usize_index = (@intFromEnum(inst) * bpi) / @bitSizeOf(usize); + const mask = @as(usize, 1) << + @as(Log2Int(usize), @intCast((@intFromEnum(inst) % (@bitSizeOf(usize) / bpi)) * bpi + (bpi - 1))); + return (l.tomb_bits[usize_index] & mask) != 0; +} + +pub fn operandDies(l: Liveness, inst: Air.Inst.Index, operand: OperandInt) bool { + assert(operand < bpi - 1); + const usize_index = (@intFromEnum(inst) * bpi) / @bitSizeOf(usize); + const mask = @as(usize, 1) << + @as(Log2Int(usize), @intCast((@intFromEnum(inst) % (@bitSizeOf(usize) / bpi)) * bpi + operand)); + return (l.tomb_bits[usize_index] & mask) != 0; +} + +const OperandCategory = enum { + /// The operand lives on, but this instruction cannot possibly mutate memory. + none, + /// The operand lives on and this instruction can mutate memory. + write, + /// The operand dies at this instruction. + tomb, + /// The operand lives on, and this instruction is noreturn. + noret, + /// This instruction is too complicated for analysis, no information is available. + complex, +}; + +/// Given an instruction that we are examining, and an operand that we are looking for, +/// returns a classification. +pub fn categorizeOperand( + l: Liveness, + air: Air, + inst: Air.Inst.Index, + operand: Air.Inst.Index, + ip: *const InternPool, +) OperandCategory { + const air_tags = air.instructions.items(.tag); + const air_datas = air.instructions.items(.data); + const operand_ref = operand.toRef(); + switch (air_tags[@intFromEnum(inst)]) { + .add, + .add_safe, + .add_wrap, + .add_sat, + .add_optimized, + .sub, + .sub_safe, + .sub_wrap, + .sub_sat, + .sub_optimized, + .mul, + .mul_safe, + .mul_wrap, + .mul_sat, + .mul_optimized, + .div_float, + .div_trunc, + .div_floor, + .div_exact, + .rem, + .mod, + .bit_and, + .bit_or, + .xor, + .cmp_lt, + .cmp_lte, + .cmp_eq, + .cmp_gte, + .cmp_gt, + .cmp_neq, + .bool_and, + .bool_or, + .array_elem_val, + .slice_elem_val, + .ptr_elem_val, + .shl, + .shl_exact, + .shl_sat, + .shr, + .shr_exact, + .min, + .max, + .div_float_optimized, + .div_trunc_optimized, + .div_floor_optimized, + .div_exact_optimized, + .rem_optimized, + .mod_optimized, + .neg_optimized, + .cmp_lt_optimized, + .cmp_lte_optimized, + .cmp_eq_optimized, + .cmp_gte_optimized, + .cmp_gt_optimized, + .cmp_neq_optimized, + => { + const o = air_datas[@intFromEnum(inst)].bin_op; + if (o.lhs == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none); + if (o.rhs == operand_ref) return matchOperandSmallIndex(l, inst, 1, .none); + return .none; + }, + + .store, + .store_safe, + .atomic_store_unordered, + .atomic_store_monotonic, + .atomic_store_release, + .atomic_store_seq_cst, + .set_union_tag, + .memset, + .memset_safe, + .memcpy, + .memmove, + => { + const o = air_datas[@intFromEnum(inst)].bin_op; + if (o.lhs == operand_ref) return matchOperandSmallIndex(l, inst, 0, .write); + if (o.rhs == operand_ref) return matchOperandSmallIndex(l, inst, 1, .write); + return .write; + }, + + .vector_store_elem => { + const o = air_datas[@intFromEnum(inst)].vector_store_elem; + const extra = air.extraData(Air.Bin, o.payload).data; + if (o.vector_ptr == operand_ref) return matchOperandSmallIndex(l, inst, 0, .write); + if (extra.lhs == operand_ref) return matchOperandSmallIndex(l, inst, 1, .none); + if (extra.rhs == operand_ref) return matchOperandSmallIndex(l, inst, 2, .none); + return .write; + }, + + .arg, + .alloc, + .inferred_alloc, + .inferred_alloc_comptime, + .ret_ptr, + .trap, + .breakpoint, + .repeat, + .switch_dispatch, + .dbg_stmt, + .dbg_empty_stmt, + .unreach, + .ret_addr, + .frame_addr, + .wasm_memory_size, + .err_return_trace, + .save_err_return_trace_index, + .tlv_dllimport_ptr, + .c_va_start, + .work_item_id, + .work_group_size, + .work_group_id, + => return .none, + + .not, + .bitcast, + .load, + .fpext, + .fptrunc, + .intcast, + .intcast_safe, + .trunc, + .optional_payload, + .optional_payload_ptr, + .wrap_optional, + .unwrap_errunion_payload, + .unwrap_errunion_err, + .unwrap_errunion_payload_ptr, + .unwrap_errunion_err_ptr, + .wrap_errunion_payload, + .wrap_errunion_err, + .slice_ptr, + .slice_len, + .ptr_slice_len_ptr, + .ptr_slice_ptr_ptr, + .struct_field_ptr_index_0, + .struct_field_ptr_index_1, + .struct_field_ptr_index_2, + .struct_field_ptr_index_3, + .array_to_slice, + .int_from_float, + .int_from_float_optimized, + .float_from_int, + .get_union_tag, + .clz, + .ctz, + .popcount, + .byte_swap, + .bit_reverse, + .splat, + .error_set_has_value, + .addrspace_cast, + .c_va_arg, + .c_va_copy, + .abs, + => { + const o = air_datas[@intFromEnum(inst)].ty_op; + if (o.operand == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none); + return .none; + }, + + .optional_payload_ptr_set, + .errunion_payload_ptr_set, + => { + const o = air_datas[@intFromEnum(inst)].ty_op; + if (o.operand == operand_ref) return matchOperandSmallIndex(l, inst, 0, .write); + return .write; + }, + + .is_null, + .is_non_null, + .is_null_ptr, + .is_non_null_ptr, + .is_err, + .is_non_err, + .is_err_ptr, + .is_non_err_ptr, + .is_named_enum_value, + .tag_name, + .error_name, + .sqrt, + .sin, + .cos, + .tan, + .exp, + .exp2, + .log, + .log2, + .log10, + .floor, + .ceil, + .round, + .trunc_float, + .neg, + .cmp_lt_errors_len, + .c_va_end, + => { + const o = air_datas[@intFromEnum(inst)].un_op; + if (o == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none); + return .none; + }, + + .ret, + .ret_safe, + .ret_load, + => { + const o = air_datas[@intFromEnum(inst)].un_op; + if (o == operand_ref) return matchOperandSmallIndex(l, inst, 0, .noret); + return .noret; + }, + + .set_err_return_trace => { + const o = air_datas[@intFromEnum(inst)].un_op; + if (o == operand_ref) return matchOperandSmallIndex(l, inst, 0, .write); + return .write; + }, + + .add_with_overflow, + .sub_with_overflow, + .mul_with_overflow, + .shl_with_overflow, + .ptr_add, + .ptr_sub, + .ptr_elem_ptr, + .slice_elem_ptr, + .slice, + => { + const ty_pl = air_datas[@intFromEnum(inst)].ty_pl; + const extra = air.extraData(Air.Bin, ty_pl.payload).data; + if (extra.lhs == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none); + if (extra.rhs == operand_ref) return matchOperandSmallIndex(l, inst, 1, .none); + return .none; + }, + + .dbg_var_ptr, + .dbg_var_val, + .dbg_arg_inline, + => { + const o = air_datas[@intFromEnum(inst)].pl_op.operand; + if (o == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none); + return .none; + }, + + .prefetch => { + const prefetch = air_datas[@intFromEnum(inst)].prefetch; + if (prefetch.ptr == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none); + return .none; + }, + + .call, .call_always_tail, .call_never_tail, .call_never_inline => { + const inst_data = air_datas[@intFromEnum(inst)].pl_op; + const callee = inst_data.operand; + const extra = air.extraData(Air.Call, inst_data.payload); + const args = @as([]const Air.Inst.Ref, @ptrCast(air.extra.items[extra.end..][0..extra.data.args_len])); + if (args.len + 1 <= bpi - 1) { + if (callee == operand_ref) return matchOperandSmallIndex(l, inst, 0, .write); + for (args, 0..) |arg, i| { + if (arg == operand_ref) return matchOperandSmallIndex(l, inst, @as(OperandInt, @intCast(i + 1)), .write); + } + return .write; + } + var bt = l.iterateBigTomb(inst); + if (bt.feed()) { + if (callee == operand_ref) return .tomb; + } else { + if (callee == operand_ref) return .write; + } + for (args) |arg| { + if (bt.feed()) { + if (arg == operand_ref) return .tomb; + } else { + if (arg == operand_ref) return .write; + } + } + return .write; + }, + .select => { + const pl_op = air_datas[@intFromEnum(inst)].pl_op; + const extra = air.extraData(Air.Bin, pl_op.payload).data; + if (pl_op.operand == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none); + if (extra.lhs == operand_ref) return matchOperandSmallIndex(l, inst, 1, .none); + if (extra.rhs == operand_ref) return matchOperandSmallIndex(l, inst, 2, .none); + return .none; + }, + .shuffle => { + const extra = air.extraData(Air.Shuffle, air_datas[@intFromEnum(inst)].ty_pl.payload).data; + if (extra.a == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none); + if (extra.b == operand_ref) return matchOperandSmallIndex(l, inst, 1, .none); + return .none; + }, + .reduce, .reduce_optimized => { + const reduce = air_datas[@intFromEnum(inst)].reduce; + if (reduce.operand == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none); + return .none; + }, + .cmp_vector, .cmp_vector_optimized => { + const extra = air.extraData(Air.VectorCmp, air_datas[@intFromEnum(inst)].ty_pl.payload).data; + if (extra.lhs == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none); + if (extra.rhs == operand_ref) return matchOperandSmallIndex(l, inst, 1, .none); + return .none; + }, + .aggregate_init => { + const ty_pl = air_datas[@intFromEnum(inst)].ty_pl; + const aggregate_ty = ty_pl.ty.toType(); + const len = @as(usize, @intCast(aggregate_ty.arrayLenIp(ip))); + const elements = @as([]const Air.Inst.Ref, @ptrCast(air.extra.items[ty_pl.payload..][0..len])); + + if (elements.len <= bpi - 1) { + for (elements, 0..) |elem, i| { + if (elem == operand_ref) return matchOperandSmallIndex(l, inst, @as(OperandInt, @intCast(i)), .none); + } + return .none; + } + + var bt = l.iterateBigTomb(inst); + for (elements) |elem| { + if (bt.feed()) { + if (elem == operand_ref) return .tomb; + } else { + if (elem == operand_ref) return .write; + } + } + return .write; + }, + .union_init => { + const extra = air.extraData(Air.UnionInit, air_datas[@intFromEnum(inst)].ty_pl.payload).data; + if (extra.init == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none); + return .none; + }, + .struct_field_ptr, .struct_field_val => { + const extra = air.extraData(Air.StructField, air_datas[@intFromEnum(inst)].ty_pl.payload).data; + if (extra.struct_operand == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none); + return .none; + }, + .field_parent_ptr => { + const extra = air.extraData(Air.FieldParentPtr, air_datas[@intFromEnum(inst)].ty_pl.payload).data; + if (extra.field_ptr == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none); + return .none; + }, + .cmpxchg_strong, .cmpxchg_weak => { + const extra = air.extraData(Air.Cmpxchg, air_datas[@intFromEnum(inst)].ty_pl.payload).data; + if (extra.ptr == operand_ref) return matchOperandSmallIndex(l, inst, 0, .write); + if (extra.expected_value == operand_ref) return matchOperandSmallIndex(l, inst, 1, .write); + if (extra.new_value == operand_ref) return matchOperandSmallIndex(l, inst, 2, .write); + return .write; + }, + .mul_add => { + const pl_op = air_datas[@intFromEnum(inst)].pl_op; + const extra = air.extraData(Air.Bin, pl_op.payload).data; + if (extra.lhs == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none); + if (extra.rhs == operand_ref) return matchOperandSmallIndex(l, inst, 1, .none); + if (pl_op.operand == operand_ref) return matchOperandSmallIndex(l, inst, 2, .none); + return .none; + }, + .atomic_load => { + const ptr = air_datas[@intFromEnum(inst)].atomic_load.ptr; + if (ptr == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none); + return .none; + }, + .atomic_rmw => { + const pl_op = air_datas[@intFromEnum(inst)].pl_op; + const extra = air.extraData(Air.AtomicRmw, pl_op.payload).data; + if (pl_op.operand == operand_ref) return matchOperandSmallIndex(l, inst, 0, .write); + if (extra.operand == operand_ref) return matchOperandSmallIndex(l, inst, 1, .write); + return .write; + }, + + .br => { + const br = air_datas[@intFromEnum(inst)].br; + if (br.operand == operand_ref) return matchOperandSmallIndex(l, operand, 0, .noret); + return .noret; + }, + .assembly => { + return .complex; + }, + .block, .dbg_inline_block => |tag| { + const ty_pl = air_datas[@intFromEnum(inst)].ty_pl; + const body: []const Air.Inst.Index = @ptrCast(switch (tag) { + inline .block, .dbg_inline_block => |comptime_tag| body: { + const extra = air.extraData(switch (comptime_tag) { + .block => Air.Block, + .dbg_inline_block => Air.DbgInlineBlock, + else => unreachable, + }, ty_pl.payload); + break :body air.extra.items[extra.end..][0..extra.data.body_len]; + }, + else => unreachable, + }); + + if (body.len == 1 and air_tags[@intFromEnum(body[0])] == .cond_br) { + // Peephole optimization for "panic-like" conditionals, which have + // one empty branch and another which calls a `noreturn` function. + // This allows us to infer that safety checks do not modify memory, + // as far as control flow successors are concerned. + + const inst_data = air_datas[@intFromEnum(body[0])].pl_op; + const cond_extra = air.extraData(Air.CondBr, inst_data.payload); + if (inst_data.operand == operand_ref and operandDies(l, body[0], 0)) + return .tomb; + + if (cond_extra.data.then_body_len > 2 or cond_extra.data.else_body_len > 2) + return .complex; + + const then_body: []const Air.Inst.Index = @ptrCast(air.extra.items[cond_extra.end..][0..cond_extra.data.then_body_len]); + const else_body: []const Air.Inst.Index = @ptrCast(air.extra.items[cond_extra.end + cond_extra.data.then_body_len ..][0..cond_extra.data.else_body_len]); + if (then_body.len > 1 and air_tags[@intFromEnum(then_body[1])] != .unreach) + return .complex; + if (else_body.len > 1 and air_tags[@intFromEnum(else_body[1])] != .unreach) + return .complex; + + var operand_live: bool = true; + for (&[_]Air.Inst.Index{ then_body[0], else_body[0] }) |cond_inst| { + if (l.categorizeOperand(air, cond_inst, operand, ip) == .tomb) + operand_live = false; + + switch (air_tags[@intFromEnum(cond_inst)]) { + .br => { // Breaks immediately back to block + const br = air_datas[@intFromEnum(cond_inst)].br; + if (br.block_inst != inst) + return .complex; + }, + .call => {}, // Calls a noreturn function + else => return .complex, + } + } + return if (operand_live) .none else .tomb; + } + + return .complex; + }, + + .@"try", + .try_cold, + .try_ptr, + .try_ptr_cold, + .loop, + .cond_br, + .switch_br, + .loop_switch_br, + => return .complex, + + .wasm_memory_grow => { + const pl_op = air_datas[@intFromEnum(inst)].pl_op; + if (pl_op.operand == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none); + return .none; + }, + } +} + +fn matchOperandSmallIndex( + l: Liveness, + inst: Air.Inst.Index, + operand: OperandInt, + default: OperandCategory, +) OperandCategory { + if (operandDies(l, inst, operand)) { + return .tomb; + } else { + return default; + } +} + +/// Higher level API. +pub const CondBrSlices = struct { + then_deaths: []const Air.Inst.Index, + else_deaths: []const Air.Inst.Index, +}; + +pub fn getCondBr(l: Liveness, inst: Air.Inst.Index) CondBrSlices { + var index: usize = l.special.get(inst) orelse return .{ + .then_deaths = &.{}, + .else_deaths = &.{}, + }; + const then_death_count = l.extra[index]; + index += 1; + const else_death_count = l.extra[index]; + index += 1; + const then_deaths: []const Air.Inst.Index = @ptrCast(l.extra[index..][0..then_death_count]); + index += then_death_count; + return .{ + .then_deaths = then_deaths, + .else_deaths = @ptrCast(l.extra[index..][0..else_death_count]), + }; +} + +/// Indexed by case number as they appear in AIR. +/// Else is the last element. +pub const SwitchBrTable = struct { + deaths: []const []const Air.Inst.Index, +}; + +/// Caller owns the memory. +pub fn getSwitchBr(l: Liveness, gpa: Allocator, inst: Air.Inst.Index, cases_len: u32) Allocator.Error!SwitchBrTable { + var index: usize = l.special.get(inst) orelse return .{ .deaths = &.{} }; + const else_death_count = l.extra[index]; + index += 1; + + var deaths = try gpa.alloc([]const Air.Inst.Index, cases_len); + errdefer gpa.free(deaths); + + var case_i: u32 = 0; + while (case_i < cases_len - 1) : (case_i += 1) { + const case_death_count: u32 = l.extra[index]; + index += 1; + deaths[case_i] = @ptrCast(l.extra[index..][0..case_death_count]); + index += case_death_count; + } + { + // Else + deaths[case_i] = @ptrCast(l.extra[index..][0..else_death_count]); + } + return .{ .deaths = deaths }; +} + +/// Note that this information is technically redundant, but is useful for +/// backends nonetheless: see `Block`. +pub const BlockSlices = struct { + deaths: []const Air.Inst.Index, +}; + +pub fn getBlock(l: Liveness, inst: Air.Inst.Index) BlockSlices { + const index: usize = l.special.get(inst) orelse return .{ + .deaths = &.{}, + }; + const death_count = l.extra[index]; + const deaths: []const Air.Inst.Index = @ptrCast(l.extra[index + 1 ..][0..death_count]); + return .{ + .deaths = deaths, + }; +} + +pub const LoopSlice = struct { + deaths: []const Air.Inst.Index, +}; + +pub fn deinit(l: *Liveness, gpa: Allocator) void { + gpa.free(l.tomb_bits); + gpa.free(l.extra); + l.special.deinit(gpa); + l.* = undefined; +} + +pub fn iterateBigTomb(l: Liveness, inst: Air.Inst.Index) BigTomb { + return .{ + .tomb_bits = l.getTombBits(inst), + .extra_start = l.special.get(inst) orelse 0, + .extra_offset = 0, + .extra = l.extra, + .bit_index = 0, + .reached_end = false, + }; +} + +/// How many tomb bits per AIR instruction. +pub const bpi = 4; +pub const Bpi = std.meta.Int(.unsigned, bpi); +pub const OperandInt = std.math.Log2Int(Bpi); + +/// Useful for decoders of Liveness information. +pub const BigTomb = struct { + tomb_bits: Liveness.Bpi, + bit_index: u32, + extra_start: u32, + extra_offset: u32, + extra: []const u32, + reached_end: bool, + + /// Returns whether the next operand dies. + pub fn feed(bt: *BigTomb) bool { + if (bt.reached_end) return false; + + const this_bit_index = bt.bit_index; + bt.bit_index += 1; + + const small_tombs = bpi - 1; + if (this_bit_index < small_tombs) { + const dies = @as(u1, @truncate(bt.tomb_bits >> @as(Liveness.OperandInt, @intCast(this_bit_index)))) != 0; + return dies; + } + + const big_bit_index = this_bit_index - small_tombs; + while (big_bit_index - bt.extra_offset * 31 >= 31) { + if (@as(u1, @truncate(bt.extra[bt.extra_start + bt.extra_offset] >> 31)) != 0) { + bt.reached_end = true; + return false; + } + bt.extra_offset += 1; + } + const dies = @as(u1, @truncate(bt.extra[bt.extra_start + bt.extra_offset] >> + @as(u5, @intCast(big_bit_index - bt.extra_offset * 31)))) != 0; + return dies; + } +}; + +/// In-progress data; on successful analysis converted into `Liveness`. +const Analysis = struct { + gpa: Allocator, + air: Air, + intern_pool: *InternPool, + tomb_bits: []usize, + special: std.AutoHashMapUnmanaged(Air.Inst.Index, u32), + extra: std.ArrayListUnmanaged(u32), + + fn addExtra(a: *Analysis, extra: anytype) Allocator.Error!u32 { + const fields = std.meta.fields(@TypeOf(extra)); + try a.extra.ensureUnusedCapacity(a.gpa, fields.len); + return addExtraAssumeCapacity(a, extra); + } + + fn addExtraAssumeCapacity(a: *Analysis, extra: anytype) u32 { + const fields = std.meta.fields(@TypeOf(extra)); + const result = @as(u32, @intCast(a.extra.items.len)); + inline for (fields) |field| { + a.extra.appendAssumeCapacity(switch (field.type) { + u32 => @field(extra, field.name), + else => @compileError("bad field type"), + }); + } + return result; + } +}; + +fn analyzeBody( + a: *Analysis, + comptime pass: LivenessPass, + data: *LivenessPassData(pass), + body: []const Air.Inst.Index, +) Allocator.Error!void { + var i: usize = body.len; + while (i != 0) { + i -= 1; + const inst = body[i]; + try analyzeInst(a, pass, data, inst); + } +} + +fn analyzeInst( + a: *Analysis, + comptime pass: LivenessPass, + data: *LivenessPassData(pass), + inst: Air.Inst.Index, +) Allocator.Error!void { + const ip = a.intern_pool; + const inst_tags = a.air.instructions.items(.tag); + const inst_datas = a.air.instructions.items(.data); + + switch (inst_tags[@intFromEnum(inst)]) { + .add, + .add_safe, + .add_optimized, + .add_wrap, + .add_sat, + .sub, + .sub_safe, + .sub_optimized, + .sub_wrap, + .sub_sat, + .mul, + .mul_safe, + .mul_optimized, + .mul_wrap, + .mul_sat, + .div_float, + .div_float_optimized, + .div_trunc, + .div_trunc_optimized, + .div_floor, + .div_floor_optimized, + .div_exact, + .div_exact_optimized, + .rem, + .rem_optimized, + .mod, + .mod_optimized, + .bit_and, + .bit_or, + .xor, + .cmp_lt, + .cmp_lt_optimized, + .cmp_lte, + .cmp_lte_optimized, + .cmp_eq, + .cmp_eq_optimized, + .cmp_gte, + .cmp_gte_optimized, + .cmp_gt, + .cmp_gt_optimized, + .cmp_neq, + .cmp_neq_optimized, + .bool_and, + .bool_or, + .store, + .store_safe, + .array_elem_val, + .slice_elem_val, + .ptr_elem_val, + .shl, + .shl_exact, + .shl_sat, + .shr, + .shr_exact, + .atomic_store_unordered, + .atomic_store_monotonic, + .atomic_store_release, + .atomic_store_seq_cst, + .set_union_tag, + .min, + .max, + .memset, + .memset_safe, + .memcpy, + .memmove, + => { + const o = inst_datas[@intFromEnum(inst)].bin_op; + return analyzeOperands(a, pass, data, inst, .{ o.lhs, o.rhs, .none }); + }, + + .vector_store_elem => { + const o = inst_datas[@intFromEnum(inst)].vector_store_elem; + const extra = a.air.extraData(Air.Bin, o.payload).data; + return analyzeOperands(a, pass, data, inst, .{ o.vector_ptr, extra.lhs, extra.rhs }); + }, + + .arg, + .alloc, + .ret_ptr, + .breakpoint, + .dbg_stmt, + .dbg_empty_stmt, + .ret_addr, + .frame_addr, + .wasm_memory_size, + .err_return_trace, + .save_err_return_trace_index, + .tlv_dllimport_ptr, + .c_va_start, + .work_item_id, + .work_group_size, + .work_group_id, + => return analyzeOperands(a, pass, data, inst, .{ .none, .none, .none }), + + .inferred_alloc, .inferred_alloc_comptime => unreachable, + + .trap, + .unreach, + => return analyzeFuncEnd(a, pass, data, inst, .{ .none, .none, .none }), + + .not, + .bitcast, + .load, + .fpext, + .fptrunc, + .intcast, + .intcast_safe, + .trunc, + .optional_payload, + .optional_payload_ptr, + .optional_payload_ptr_set, + .errunion_payload_ptr_set, + .wrap_optional, + .unwrap_errunion_payload, + .unwrap_errunion_err, + .unwrap_errunion_payload_ptr, + .unwrap_errunion_err_ptr, + .wrap_errunion_payload, + .wrap_errunion_err, + .slice_ptr, + .slice_len, + .ptr_slice_len_ptr, + .ptr_slice_ptr_ptr, + .struct_field_ptr_index_0, + .struct_field_ptr_index_1, + .struct_field_ptr_index_2, + .struct_field_ptr_index_3, + .array_to_slice, + .int_from_float, + .int_from_float_optimized, + .float_from_int, + .get_union_tag, + .clz, + .ctz, + .popcount, + .byte_swap, + .bit_reverse, + .splat, + .error_set_has_value, + .addrspace_cast, + .c_va_arg, + .c_va_copy, + .abs, + => { + const o = inst_datas[@intFromEnum(inst)].ty_op; + return analyzeOperands(a, pass, data, inst, .{ o.operand, .none, .none }); + }, + + .is_null, + .is_non_null, + .is_null_ptr, + .is_non_null_ptr, + .is_err, + .is_non_err, + .is_err_ptr, + .is_non_err_ptr, + .is_named_enum_value, + .tag_name, + .error_name, + .sqrt, + .sin, + .cos, + .tan, + .exp, + .exp2, + .log, + .log2, + .log10, + .floor, + .ceil, + .round, + .trunc_float, + .neg, + .neg_optimized, + .cmp_lt_errors_len, + .set_err_return_trace, + .c_va_end, + => { + const operand = inst_datas[@intFromEnum(inst)].un_op; + return analyzeOperands(a, pass, data, inst, .{ operand, .none, .none }); + }, + + .ret, + .ret_safe, + .ret_load, + => { + const operand = inst_datas[@intFromEnum(inst)].un_op; + return analyzeFuncEnd(a, pass, data, inst, .{ operand, .none, .none }); + }, + + .add_with_overflow, + .sub_with_overflow, + .mul_with_overflow, + .shl_with_overflow, + .ptr_add, + .ptr_sub, + .ptr_elem_ptr, + .slice_elem_ptr, + .slice, + => { + const ty_pl = inst_datas[@intFromEnum(inst)].ty_pl; + const extra = a.air.extraData(Air.Bin, ty_pl.payload).data; + return analyzeOperands(a, pass, data, inst, .{ extra.lhs, extra.rhs, .none }); + }, + + .dbg_var_ptr, + .dbg_var_val, + .dbg_arg_inline, + => { + const operand = inst_datas[@intFromEnum(inst)].pl_op.operand; + return analyzeOperands(a, pass, data, inst, .{ operand, .none, .none }); + }, + + .prefetch => { + const prefetch = inst_datas[@intFromEnum(inst)].prefetch; + return analyzeOperands(a, pass, data, inst, .{ prefetch.ptr, .none, .none }); + }, + + .call, .call_always_tail, .call_never_tail, .call_never_inline => { + const inst_data = inst_datas[@intFromEnum(inst)].pl_op; + const callee = inst_data.operand; + const extra = a.air.extraData(Air.Call, inst_data.payload); + const args = @as([]const Air.Inst.Ref, @ptrCast(a.air.extra.items[extra.end..][0..extra.data.args_len])); + if (args.len + 1 <= bpi - 1) { + var buf = [1]Air.Inst.Ref{.none} ** (bpi - 1); + buf[0] = callee; + @memcpy(buf[1..][0..args.len], args); + return analyzeOperands(a, pass, data, inst, buf); + } + + var big = try AnalyzeBigOperands(pass).init(a, data, inst, args.len + 1); + defer big.deinit(); + var i: usize = args.len; + while (i > 0) { + i -= 1; + try big.feed(args[i]); + } + try big.feed(callee); + return big.finish(); + }, + .select => { + const pl_op = inst_datas[@intFromEnum(inst)].pl_op; + const extra = a.air.extraData(Air.Bin, pl_op.payload).data; + return analyzeOperands(a, pass, data, inst, .{ pl_op.operand, extra.lhs, extra.rhs }); + }, + .shuffle => { + const extra = a.air.extraData(Air.Shuffle, inst_datas[@intFromEnum(inst)].ty_pl.payload).data; + return analyzeOperands(a, pass, data, inst, .{ extra.a, extra.b, .none }); + }, + .reduce, .reduce_optimized => { + const reduce = inst_datas[@intFromEnum(inst)].reduce; + return analyzeOperands(a, pass, data, inst, .{ reduce.operand, .none, .none }); + }, + .cmp_vector, .cmp_vector_optimized => { + const extra = a.air.extraData(Air.VectorCmp, inst_datas[@intFromEnum(inst)].ty_pl.payload).data; + return analyzeOperands(a, pass, data, inst, .{ extra.lhs, extra.rhs, .none }); + }, + .aggregate_init => { + const ty_pl = inst_datas[@intFromEnum(inst)].ty_pl; + const aggregate_ty = ty_pl.ty.toType(); + const len = @as(usize, @intCast(aggregate_ty.arrayLenIp(ip))); + const elements = @as([]const Air.Inst.Ref, @ptrCast(a.air.extra.items[ty_pl.payload..][0..len])); + + if (elements.len <= bpi - 1) { + var buf = [1]Air.Inst.Ref{.none} ** (bpi - 1); + @memcpy(buf[0..elements.len], elements); + return analyzeOperands(a, pass, data, inst, buf); + } + + var big = try AnalyzeBigOperands(pass).init(a, data, inst, elements.len); + defer big.deinit(); + var i: usize = elements.len; + while (i > 0) { + i -= 1; + try big.feed(elements[i]); + } + return big.finish(); + }, + .union_init => { + const extra = a.air.extraData(Air.UnionInit, inst_datas[@intFromEnum(inst)].ty_pl.payload).data; + return analyzeOperands(a, pass, data, inst, .{ extra.init, .none, .none }); + }, + .struct_field_ptr, .struct_field_val => { + const extra = a.air.extraData(Air.StructField, inst_datas[@intFromEnum(inst)].ty_pl.payload).data; + return analyzeOperands(a, pass, data, inst, .{ extra.struct_operand, .none, .none }); + }, + .field_parent_ptr => { + const extra = a.air.extraData(Air.FieldParentPtr, inst_datas[@intFromEnum(inst)].ty_pl.payload).data; + return analyzeOperands(a, pass, data, inst, .{ extra.field_ptr, .none, .none }); + }, + .cmpxchg_strong, .cmpxchg_weak => { + const extra = a.air.extraData(Air.Cmpxchg, inst_datas[@intFromEnum(inst)].ty_pl.payload).data; + return analyzeOperands(a, pass, data, inst, .{ extra.ptr, extra.expected_value, extra.new_value }); + }, + .mul_add => { + const pl_op = inst_datas[@intFromEnum(inst)].pl_op; + const extra = a.air.extraData(Air.Bin, pl_op.payload).data; + return analyzeOperands(a, pass, data, inst, .{ extra.lhs, extra.rhs, pl_op.operand }); + }, + .atomic_load => { + const ptr = inst_datas[@intFromEnum(inst)].atomic_load.ptr; + return analyzeOperands(a, pass, data, inst, .{ ptr, .none, .none }); + }, + .atomic_rmw => { + const pl_op = inst_datas[@intFromEnum(inst)].pl_op; + const extra = a.air.extraData(Air.AtomicRmw, pl_op.payload).data; + return analyzeOperands(a, pass, data, inst, .{ pl_op.operand, extra.operand, .none }); + }, + + .br => return analyzeInstBr(a, pass, data, inst), + .repeat => return analyzeInstRepeat(a, pass, data, inst), + .switch_dispatch => return analyzeInstSwitchDispatch(a, pass, data, inst), + + .assembly => { + const extra = a.air.extraData(Air.Asm, inst_datas[@intFromEnum(inst)].ty_pl.payload); + var extra_i: usize = extra.end; + const outputs = @as([]const Air.Inst.Ref, @ptrCast(a.air.extra.items[extra_i..][0..extra.data.outputs_len])); + extra_i += outputs.len; + const inputs = @as([]const Air.Inst.Ref, @ptrCast(a.air.extra.items[extra_i..][0..extra.data.inputs_len])); + extra_i += inputs.len; + + const num_operands = simple: { + var buf = [1]Air.Inst.Ref{.none} ** (bpi - 1); + var buf_index: usize = 0; + for (outputs) |output| { + if (output != .none) { + if (buf_index < buf.len) buf[buf_index] = output; + buf_index += 1; + } + } + if (buf_index + inputs.len > buf.len) { + break :simple buf_index + inputs.len; + } + @memcpy(buf[buf_index..][0..inputs.len], inputs); + return analyzeOperands(a, pass, data, inst, buf); + }; + + var big = try AnalyzeBigOperands(pass).init(a, data, inst, num_operands); + defer big.deinit(); + var i: usize = inputs.len; + while (i > 0) { + i -= 1; + try big.feed(inputs[i]); + } + i = outputs.len; + while (i > 0) { + i -= 1; + if (outputs[i] != .none) { + try big.feed(outputs[i]); + } + } + return big.finish(); + }, + + inline .block, .dbg_inline_block => |comptime_tag| { + const ty_pl = inst_datas[@intFromEnum(inst)].ty_pl; + const extra = a.air.extraData(switch (comptime_tag) { + .block => Air.Block, + .dbg_inline_block => Air.DbgInlineBlock, + else => unreachable, + }, ty_pl.payload); + return analyzeInstBlock(a, pass, data, inst, ty_pl.ty, @ptrCast(a.air.extra.items[extra.end..][0..extra.data.body_len])); + }, + .loop => return analyzeInstLoop(a, pass, data, inst), + + .@"try", .try_cold => return analyzeInstCondBr(a, pass, data, inst, .@"try"), + .try_ptr, .try_ptr_cold => return analyzeInstCondBr(a, pass, data, inst, .try_ptr), + .cond_br => return analyzeInstCondBr(a, pass, data, inst, .cond_br), + .switch_br => return analyzeInstSwitchBr(a, pass, data, inst, false), + .loop_switch_br => return analyzeInstSwitchBr(a, pass, data, inst, true), + + .wasm_memory_grow => { + const pl_op = inst_datas[@intFromEnum(inst)].pl_op; + return analyzeOperands(a, pass, data, inst, .{ pl_op.operand, .none, .none }); + }, + } +} + +/// Every instruction should hit this (after handling any nested bodies), in every pass. In the +/// initial pass, it is responsible for marking deaths of the (first three) operands and noticing +/// immediate deaths. +fn analyzeOperands( + a: *Analysis, + comptime pass: LivenessPass, + data: *LivenessPassData(pass), + inst: Air.Inst.Index, + operands: [bpi - 1]Air.Inst.Ref, +) Allocator.Error!void { + const gpa = a.gpa; + const ip = a.intern_pool; + + switch (pass) { + .loop_analysis => { + _ = data.live_set.remove(inst); + + for (operands) |op_ref| { + const operand = op_ref.toIndexAllowNone() orelse continue; + _ = try data.live_set.put(gpa, operand, {}); + } + }, + + .main_analysis => { + const usize_index = (@intFromEnum(inst) * bpi) / @bitSizeOf(usize); + + // This logic must synchronize with `will_die_immediately` in `AnalyzeBigOperands.init`. + const immediate_death = if (data.live_set.remove(inst)) blk: { + log.debug("[{}] %{}: removed from live set", .{ pass, @intFromEnum(inst) }); + break :blk false; + } else blk: { + log.debug("[{}] %{}: immediate death", .{ pass, @intFromEnum(inst) }); + break :blk true; + }; + + var tomb_bits: Bpi = @as(Bpi, @intFromBool(immediate_death)) << (bpi - 1); + + // If our result is unused and the instruction doesn't need to be lowered, backends will + // skip the lowering of this instruction, so we don't want to record uses of operands. + // That way, we can mark as many instructions as possible unused. + if (!immediate_death or a.air.mustLower(inst, ip)) { + // Note that it's important we iterate over the operands backwards, so that if a dying + // operand is used multiple times we mark its last use as its death. + var i = operands.len; + while (i > 0) { + i -= 1; + const op_ref = operands[i]; + const operand = op_ref.toIndexAllowNone() orelse continue; + + const mask = @as(Bpi, 1) << @as(OperandInt, @intCast(i)); + + if ((try data.live_set.fetchPut(gpa, operand, {})) == null) { + log.debug("[{}] %{}: added %{} to live set (operand dies here)", .{ pass, @intFromEnum(inst), operand }); + tomb_bits |= mask; + } + } + } + + a.tomb_bits[usize_index] |= @as(usize, tomb_bits) << + @as(Log2Int(usize), @intCast((@intFromEnum(inst) % (@bitSizeOf(usize) / bpi)) * bpi)); + }, + } +} + +/// Like `analyzeOperands`, but for an instruction which returns from a function, so should +/// effectively kill every remaining live value other than its operands. +fn analyzeFuncEnd( + a: *Analysis, + comptime pass: LivenessPass, + data: *LivenessPassData(pass), + inst: Air.Inst.Index, + operands: [bpi - 1]Air.Inst.Ref, +) Allocator.Error!void { + switch (pass) { + .loop_analysis => { + // No operands need to be alive if we're returning from the function, so we don't need + // to touch `breaks` here even though this is sort of like a break to the top level. + }, + + .main_analysis => { + data.live_set.clearRetainingCapacity(); + }, + } + + return analyzeOperands(a, pass, data, inst, operands); +} + +fn analyzeInstBr( + a: *Analysis, + comptime pass: LivenessPass, + data: *LivenessPassData(pass), + inst: Air.Inst.Index, +) !void { + const inst_datas = a.air.instructions.items(.data); + const br = inst_datas[@intFromEnum(inst)].br; + const gpa = a.gpa; + + switch (pass) { + .loop_analysis => { + try data.breaks.put(gpa, br.block_inst, {}); + }, + + .main_analysis => { + const block_scope = data.block_scopes.get(br.block_inst).?; // we should always be breaking from an enclosing block + + const new_live_set = try block_scope.live_set.clone(gpa); + data.live_set.deinit(gpa); + data.live_set = new_live_set; + }, + } + + return analyzeOperands(a, pass, data, inst, .{ br.operand, .none, .none }); +} + +fn analyzeInstRepeat( + a: *Analysis, + comptime pass: LivenessPass, + data: *LivenessPassData(pass), + inst: Air.Inst.Index, +) !void { + const inst_datas = a.air.instructions.items(.data); + const repeat = inst_datas[@intFromEnum(inst)].repeat; + const gpa = a.gpa; + + switch (pass) { + .loop_analysis => { + try data.breaks.put(gpa, repeat.loop_inst, {}); + }, + + .main_analysis => { + const block_scope = data.block_scopes.get(repeat.loop_inst).?; // we should always be repeating an enclosing loop + + const new_live_set = try block_scope.live_set.clone(gpa); + data.live_set.deinit(gpa); + data.live_set = new_live_set; + }, + } + + return analyzeOperands(a, pass, data, inst, .{ .none, .none, .none }); +} + +fn analyzeInstSwitchDispatch( + a: *Analysis, + comptime pass: LivenessPass, + data: *LivenessPassData(pass), + inst: Air.Inst.Index, +) !void { + // This happens to be identical to `analyzeInstBr`, but is separated anyway for clarity. + + const inst_datas = a.air.instructions.items(.data); + const br = inst_datas[@intFromEnum(inst)].br; + const gpa = a.gpa; + + switch (pass) { + .loop_analysis => { + try data.breaks.put(gpa, br.block_inst, {}); + }, + + .main_analysis => { + const block_scope = data.block_scopes.get(br.block_inst).?; // we should always be repeating an enclosing loop + + const new_live_set = try block_scope.live_set.clone(gpa); + data.live_set.deinit(gpa); + data.live_set = new_live_set; + }, + } + + return analyzeOperands(a, pass, data, inst, .{ br.operand, .none, .none }); +} + +fn analyzeInstBlock( + a: *Analysis, + comptime pass: LivenessPass, + data: *LivenessPassData(pass), + inst: Air.Inst.Index, + ty: Air.Inst.Ref, + body: []const Air.Inst.Index, +) !void { + const gpa = a.gpa; + + // We actually want to do `analyzeOperands` *first*, since our result logically doesn't + // exist until the block body ends (and we're iterating backwards) + try analyzeOperands(a, pass, data, inst, .{ .none, .none, .none }); + + switch (pass) { + .loop_analysis => { + try analyzeBody(a, pass, data, body); + _ = data.breaks.remove(inst); + }, + + .main_analysis => { + log.debug("[{}] %{}: block live set is {}", .{ pass, inst, fmtInstSet(&data.live_set) }); + // We can move the live set because the body should have a noreturn + // instruction which overrides the set. + try data.block_scopes.put(gpa, inst, .{ + .live_set = data.live_set.move(), + }); + defer { + log.debug("[{}] %{}: popped block scope", .{ pass, inst }); + var scope = data.block_scopes.fetchRemove(inst).?.value; + scope.live_set.deinit(gpa); + } + + log.debug("[{}] %{}: pushed new block scope", .{ pass, inst }); + try analyzeBody(a, pass, data, body); + + // If the block is noreturn, block deaths not only aren't useful, they're impossible to + // find: there could be more stuff alive after the block than before it! + if (!a.intern_pool.isNoReturn(ty.toType().toIntern())) { + // The block kills the difference in the live sets + const block_scope = data.block_scopes.get(inst).?; + const num_deaths = data.live_set.count() - block_scope.live_set.count(); + + try a.extra.ensureUnusedCapacity(gpa, num_deaths + std.meta.fields(Block).len); + const extra_index = a.addExtraAssumeCapacity(Block{ + .death_count = num_deaths, + }); + + var measured_num: u32 = 0; + var it = data.live_set.keyIterator(); + while (it.next()) |key| { + const alive = key.*; + if (!block_scope.live_set.contains(alive)) { + // Dies in block + a.extra.appendAssumeCapacity(@intFromEnum(alive)); + measured_num += 1; + } + } + assert(measured_num == num_deaths); // post-live-set should be a subset of pre-live-set + try a.special.put(gpa, inst, extra_index); + log.debug("[{}] %{}: block deaths are {}", .{ + pass, + inst, + fmtInstList(@ptrCast(a.extra.items[extra_index + 1 ..][0..num_deaths])), + }); + } + }, + } +} + +fn writeLoopInfo( + a: *Analysis, + data: *LivenessPassData(.loop_analysis), + inst: Air.Inst.Index, + old_breaks: std.AutoHashMapUnmanaged(Air.Inst.Index, void), + old_live: std.AutoHashMapUnmanaged(Air.Inst.Index, void), +) !void { + const gpa = a.gpa; + + // `loop`s are guaranteed to have at least one matching `repeat`. + // Similarly, `loop_switch_br`s have a matching `switch_dispatch`. + // However, we no longer care about repeats of this loop for resolving + // which operands must live within it. + assert(data.breaks.remove(inst)); + + const extra_index: u32 = @intCast(a.extra.items.len); + + const num_breaks = data.breaks.count(); + try a.extra.ensureUnusedCapacity(gpa, 1 + num_breaks); + + a.extra.appendAssumeCapacity(num_breaks); + + var it = data.breaks.keyIterator(); + while (it.next()) |key| { + const block_inst = key.*; + a.extra.appendAssumeCapacity(@intFromEnum(block_inst)); + } + log.debug("[{}] %{}: includes breaks to {}", .{ LivenessPass.loop_analysis, inst, fmtInstSet(&data.breaks) }); + + // Now we put the live operands from the loop body in too + const num_live = data.live_set.count(); + try a.extra.ensureUnusedCapacity(gpa, 1 + num_live); + + a.extra.appendAssumeCapacity(num_live); + it = data.live_set.keyIterator(); + while (it.next()) |key| { + const alive = key.*; + a.extra.appendAssumeCapacity(@intFromEnum(alive)); + } + log.debug("[{}] %{}: maintain liveness of {}", .{ LivenessPass.loop_analysis, inst, fmtInstSet(&data.live_set) }); + + try a.special.put(gpa, inst, extra_index); + + // Add back operands which were previously alive + it = old_live.keyIterator(); + while (it.next()) |key| { + const alive = key.*; + try data.live_set.put(gpa, alive, {}); + } + + // And the same for breaks + it = old_breaks.keyIterator(); + while (it.next()) |key| { + const block_inst = key.*; + try data.breaks.put(gpa, block_inst, {}); + } +} + +/// When analyzing a loop in the main pass, sets up `data.live_set` to be the set +/// of operands known to be alive when the loop repeats. +fn resolveLoopLiveSet( + a: *Analysis, + data: *LivenessPassData(.main_analysis), + inst: Air.Inst.Index, +) !void { + const gpa = a.gpa; + + const extra_idx = a.special.fetchRemove(inst).?.value; + const num_breaks = data.old_extra.items[extra_idx]; + const breaks: []const Air.Inst.Index = @ptrCast(data.old_extra.items[extra_idx + 1 ..][0..num_breaks]); + + const num_loop_live = data.old_extra.items[extra_idx + num_breaks + 1]; + const loop_live: []const Air.Inst.Index = @ptrCast(data.old_extra.items[extra_idx + num_breaks + 2 ..][0..num_loop_live]); + + // This is necessarily not in the same control flow branch, because loops are noreturn + data.live_set.clearRetainingCapacity(); + + try data.live_set.ensureUnusedCapacity(gpa, @intCast(loop_live.len)); + for (loop_live) |alive| data.live_set.putAssumeCapacity(alive, {}); + + log.debug("[{}] %{}: block live set is {}", .{ LivenessPass.main_analysis, inst, fmtInstSet(&data.live_set) }); + + for (breaks) |block_inst| { + // We might break to this block, so include every operand that the block needs alive + const block_scope = data.block_scopes.get(block_inst).?; + + var it = block_scope.live_set.keyIterator(); + while (it.next()) |key| { + const alive = key.*; + try data.live_set.put(gpa, alive, {}); + } + } + + log.debug("[{}] %{}: loop live set is {}", .{ LivenessPass.main_analysis, inst, fmtInstSet(&data.live_set) }); +} + +fn analyzeInstLoop( + a: *Analysis, + comptime pass: LivenessPass, + data: *LivenessPassData(pass), + inst: Air.Inst.Index, +) !void { + const inst_datas = a.air.instructions.items(.data); + const extra = a.air.extraData(Air.Block, inst_datas[@intFromEnum(inst)].ty_pl.payload); + const body: []const Air.Inst.Index = @ptrCast(a.air.extra.items[extra.end..][0..extra.data.body_len]); + const gpa = a.gpa; + + try analyzeOperands(a, pass, data, inst, .{ .none, .none, .none }); + + switch (pass) { + .loop_analysis => { + var old_breaks = data.breaks.move(); + defer old_breaks.deinit(gpa); + + var old_live = data.live_set.move(); + defer old_live.deinit(gpa); + + try analyzeBody(a, pass, data, body); + + try writeLoopInfo(a, data, inst, old_breaks, old_live); + }, + + .main_analysis => { + try resolveLoopLiveSet(a, data, inst); + + // Now, `data.live_set` is the operands which must be alive when the loop repeats. + // Move them into a block scope for corresponding `repeat` instructions to notice. + try data.block_scopes.putNoClobber(gpa, inst, .{ + .live_set = data.live_set.move(), + }); + defer { + log.debug("[{}] %{}: popped loop block scop", .{ pass, inst }); + var scope = data.block_scopes.fetchRemove(inst).?.value; + scope.live_set.deinit(gpa); + } + try analyzeBody(a, pass, data, body); + }, + } +} + +/// Despite its name, this function is used for analysis of not only `cond_br` instructions, but +/// also `try` and `try_ptr`, which are highly related. The `inst_type` parameter indicates which +/// type of instruction `inst` points to. +fn analyzeInstCondBr( + a: *Analysis, + comptime pass: LivenessPass, + data: *LivenessPassData(pass), + inst: Air.Inst.Index, + comptime inst_type: enum { cond_br, @"try", try_ptr }, +) !void { + const inst_datas = a.air.instructions.items(.data); + const gpa = a.gpa; + + const extra = switch (inst_type) { + .cond_br => a.air.extraData(Air.CondBr, inst_datas[@intFromEnum(inst)].pl_op.payload), + .@"try" => a.air.extraData(Air.Try, inst_datas[@intFromEnum(inst)].pl_op.payload), + .try_ptr => a.air.extraData(Air.TryPtr, inst_datas[@intFromEnum(inst)].ty_pl.payload), + }; + + const condition = switch (inst_type) { + .cond_br, .@"try" => inst_datas[@intFromEnum(inst)].pl_op.operand, + .try_ptr => extra.data.ptr, + }; + + const then_body: []const Air.Inst.Index = switch (inst_type) { + .cond_br => @ptrCast(a.air.extra.items[extra.end..][0..extra.data.then_body_len]), + else => &.{}, // we won't use this + }; + + const else_body: []const Air.Inst.Index = @ptrCast(switch (inst_type) { + .cond_br => a.air.extra.items[extra.end + then_body.len ..][0..extra.data.else_body_len], + .@"try", .try_ptr => a.air.extra.items[extra.end..][0..extra.data.body_len], + }); + + switch (pass) { + .loop_analysis => { + switch (inst_type) { + .cond_br => try analyzeBody(a, pass, data, then_body), + .@"try", .try_ptr => {}, + } + try analyzeBody(a, pass, data, else_body); + }, + + .main_analysis => { + switch (inst_type) { + .cond_br => try analyzeBody(a, pass, data, then_body), + .@"try", .try_ptr => {}, // The "then body" is just the remainder of this block + } + var then_live = data.live_set.move(); + defer then_live.deinit(gpa); + + try analyzeBody(a, pass, data, else_body); + var else_live = data.live_set.move(); + defer else_live.deinit(gpa); + + // Operands which are alive in one branch but not the other need to die at the start of + // the peer branch. + + var then_mirrored_deaths: std.ArrayListUnmanaged(Air.Inst.Index) = .empty; + defer then_mirrored_deaths.deinit(gpa); + + var else_mirrored_deaths: std.ArrayListUnmanaged(Air.Inst.Index) = .empty; + defer else_mirrored_deaths.deinit(gpa); + + // Note: this invalidates `else_live`, but expands `then_live` to be their union + { + var it = then_live.keyIterator(); + while (it.next()) |key| { + const death = key.*; + if (else_live.remove(death)) continue; // removing makes the loop below faster + + // If this is a `try`, the "then body" (rest of the branch) might have + // referenced our result. We want to avoid killing this value in the else branch + // if that's the case, since it only exists in the (fake) then branch. + switch (inst_type) { + .cond_br => {}, + .@"try", .try_ptr => if (death == inst) continue, + } + + try else_mirrored_deaths.append(gpa, death); + } + // Since we removed common stuff above, `else_live` is now only operands + // which are *only* alive in the else branch + it = else_live.keyIterator(); + while (it.next()) |key| { + const death = key.*; + try then_mirrored_deaths.append(gpa, death); + // Make `then_live` contain the full live set (i.e. union of both) + try then_live.put(gpa, death, {}); + } + } + + log.debug("[{}] %{}: 'then' branch mirrored deaths are {}", .{ pass, inst, fmtInstList(then_mirrored_deaths.items) }); + log.debug("[{}] %{}: 'else' branch mirrored deaths are {}", .{ pass, inst, fmtInstList(else_mirrored_deaths.items) }); + + data.live_set.deinit(gpa); + data.live_set = then_live.move(); // Really the union of both live sets + + log.debug("[{}] %{}: new live set is {}", .{ pass, inst, fmtInstSet(&data.live_set) }); + + // Write the mirrored deaths to `extra` + const then_death_count = @as(u32, @intCast(then_mirrored_deaths.items.len)); + const else_death_count = @as(u32, @intCast(else_mirrored_deaths.items.len)); + try a.extra.ensureUnusedCapacity(gpa, std.meta.fields(CondBr).len + then_death_count + else_death_count); + const extra_index = a.addExtraAssumeCapacity(CondBr{ + .then_death_count = then_death_count, + .else_death_count = else_death_count, + }); + a.extra.appendSliceAssumeCapacity(@ptrCast(then_mirrored_deaths.items)); + a.extra.appendSliceAssumeCapacity(@ptrCast(else_mirrored_deaths.items)); + try a.special.put(gpa, inst, extra_index); + }, + } + + try analyzeOperands(a, pass, data, inst, .{ condition, .none, .none }); +} + +fn analyzeInstSwitchBr( + a: *Analysis, + comptime pass: LivenessPass, + data: *LivenessPassData(pass), + inst: Air.Inst.Index, + is_dispatch_loop: bool, +) !void { + const inst_datas = a.air.instructions.items(.data); + const pl_op = inst_datas[@intFromEnum(inst)].pl_op; + const condition = pl_op.operand; + const switch_br = a.air.unwrapSwitch(inst); + const gpa = a.gpa; + const ncases = switch_br.cases_len; + + switch (pass) { + .loop_analysis => { + var old_breaks: std.AutoHashMapUnmanaged(Air.Inst.Index, void) = .empty; + defer old_breaks.deinit(gpa); + + var old_live: std.AutoHashMapUnmanaged(Air.Inst.Index, void) = .empty; + defer old_live.deinit(gpa); + + if (is_dispatch_loop) { + old_breaks = data.breaks.move(); + old_live = data.live_set.move(); + } + + var it = switch_br.iterateCases(); + while (it.next()) |case| { + try analyzeBody(a, pass, data, case.body); + } + { // else + const else_body = it.elseBody(); + try analyzeBody(a, pass, data, else_body); + } + + if (is_dispatch_loop) { + try writeLoopInfo(a, data, inst, old_breaks, old_live); + } + }, + + .main_analysis => { + if (is_dispatch_loop) { + try resolveLoopLiveSet(a, data, inst); + try data.block_scopes.putNoClobber(gpa, inst, .{ + .live_set = data.live_set.move(), + }); + } + defer if (is_dispatch_loop) { + log.debug("[{}] %{}: popped loop block scop", .{ pass, inst }); + var scope = data.block_scopes.fetchRemove(inst).?.value; + scope.live_set.deinit(gpa); + }; + // This is, all in all, just a messier version of the `cond_br` logic. If you're trying + // to understand it, I encourage looking at `analyzeInstCondBr` first. + + const DeathSet = std.AutoHashMapUnmanaged(Air.Inst.Index, void); + const DeathList = std.ArrayListUnmanaged(Air.Inst.Index); + + var case_live_sets = try gpa.alloc(std.AutoHashMapUnmanaged(Air.Inst.Index, void), ncases + 1); // +1 for else + defer gpa.free(case_live_sets); + + @memset(case_live_sets, .{}); + defer for (case_live_sets) |*live_set| live_set.deinit(gpa); + + var case_it = switch_br.iterateCases(); + while (case_it.next()) |case| { + try analyzeBody(a, pass, data, case.body); + case_live_sets[case.idx] = data.live_set.move(); + } + { // else + const else_body = case_it.elseBody(); + try analyzeBody(a, pass, data, else_body); + case_live_sets[ncases] = data.live_set.move(); + } + + const mirrored_deaths = try gpa.alloc(DeathList, ncases + 1); + defer gpa.free(mirrored_deaths); + + @memset(mirrored_deaths, .{}); + defer for (mirrored_deaths) |*md| md.deinit(gpa); + + { + var all_alive: DeathSet = .{}; + defer all_alive.deinit(gpa); + + for (case_live_sets) |*live_set| { + try all_alive.ensureUnusedCapacity(gpa, live_set.count()); + var it = live_set.keyIterator(); + while (it.next()) |key| { + const alive = key.*; + all_alive.putAssumeCapacity(alive, {}); + } + } + + for (mirrored_deaths, case_live_sets) |*mirrored, *live_set| { + var it = all_alive.keyIterator(); + while (it.next()) |key| { + const alive = key.*; + if (!live_set.contains(alive)) { + // Should die at the start of this branch + try mirrored.append(gpa, alive); + } + } + } + + for (mirrored_deaths, 0..) |mirrored, i| { + log.debug("[{}] %{}: case {} mirrored deaths are {}", .{ pass, inst, i, fmtInstList(mirrored.items) }); + } + + data.live_set.deinit(gpa); + data.live_set = all_alive.move(); + + log.debug("[{}] %{}: new live set is {}", .{ pass, inst, fmtInstSet(&data.live_set) }); + } + + const else_death_count = @as(u32, @intCast(mirrored_deaths[ncases].items.len)); + const extra_index = try a.addExtra(SwitchBr{ + .else_death_count = else_death_count, + }); + for (mirrored_deaths[0..ncases]) |mirrored| { + const num = @as(u32, @intCast(mirrored.items.len)); + try a.extra.ensureUnusedCapacity(gpa, num + 1); + a.extra.appendAssumeCapacity(num); + a.extra.appendSliceAssumeCapacity(@ptrCast(mirrored.items)); + } + try a.extra.ensureUnusedCapacity(gpa, else_death_count); + a.extra.appendSliceAssumeCapacity(@ptrCast(mirrored_deaths[ncases].items)); + try a.special.put(gpa, inst, extra_index); + }, + } + + try analyzeOperands(a, pass, data, inst, .{ condition, .none, .none }); +} + +fn AnalyzeBigOperands(comptime pass: LivenessPass) type { + return struct { + a: *Analysis, + data: *LivenessPassData(pass), + inst: Air.Inst.Index, + + operands_remaining: u32, + small: [bpi - 1]Air.Inst.Ref = .{.none} ** (bpi - 1), + extra_tombs: []u32, + + // Only used in `LivenessPass.main_analysis` + will_die_immediately: bool, + + const Self = @This(); + + fn init( + a: *Analysis, + data: *LivenessPassData(pass), + inst: Air.Inst.Index, + total_operands: usize, + ) !Self { + const extra_operands = @as(u32, @intCast(total_operands)) -| (bpi - 1); + const max_extra_tombs = (extra_operands + 30) / 31; + + const extra_tombs: []u32 = switch (pass) { + .loop_analysis => &.{}, + .main_analysis => try a.gpa.alloc(u32, max_extra_tombs), + }; + errdefer a.gpa.free(extra_tombs); + + @memset(extra_tombs, 0); + + const will_die_immediately: bool = switch (pass) { + .loop_analysis => false, // track everything, since we don't have full liveness information yet + .main_analysis => !data.live_set.contains(inst), + }; + + return .{ + .a = a, + .data = data, + .inst = inst, + .operands_remaining = @as(u32, @intCast(total_operands)), + .extra_tombs = extra_tombs, + .will_die_immediately = will_die_immediately, + }; + } + + /// Must be called with operands in reverse order. + fn feed(big: *Self, op_ref: Air.Inst.Ref) !void { + const ip = big.a.intern_pool; + // Note that after this, `operands_remaining` becomes the index of the current operand + big.operands_remaining -= 1; + + if (big.operands_remaining < bpi - 1) { + big.small[big.operands_remaining] = op_ref; + return; + } + + const operand = op_ref.toIndex() orelse return; + + // If our result is unused and the instruction doesn't need to be lowered, backends will + // skip the lowering of this instruction, so we don't want to record uses of operands. + // That way, we can mark as many instructions as possible unused. + if (big.will_die_immediately and !big.a.air.mustLower(big.inst, ip)) return; + + const extra_byte = (big.operands_remaining - (bpi - 1)) / 31; + const extra_bit = @as(u5, @intCast(big.operands_remaining - (bpi - 1) - extra_byte * 31)); + + const gpa = big.a.gpa; + + switch (pass) { + .loop_analysis => { + _ = try big.data.live_set.put(gpa, operand, {}); + }, + + .main_analysis => { + if ((try big.data.live_set.fetchPut(gpa, operand, {})) == null) { + log.debug("[{}] %{}: added %{} to live set (operand dies here)", .{ pass, big.inst, operand }); + big.extra_tombs[extra_byte] |= @as(u32, 1) << extra_bit; + } + }, + } + } + + fn finish(big: *Self) !void { + const gpa = big.a.gpa; + + std.debug.assert(big.operands_remaining == 0); + + switch (pass) { + .loop_analysis => {}, + + .main_analysis => { + // Note that the MSB is set on the final tomb to indicate the terminal element. This + // allows for an optimisation where we only add as many extra tombs as are needed to + // represent the dying operands. Each pass modifies operand bits and so needs to write + // back, so let's figure out how many extra tombs we really need. Note that we always + // keep at least one. + var num: usize = big.extra_tombs.len; + while (num > 1) { + if (@as(u31, @truncate(big.extra_tombs[num - 1])) != 0) { + // Some operand dies here + break; + } + num -= 1; + } + // Mark final tomb + big.extra_tombs[num - 1] |= @as(u32, 1) << 31; + + const extra_tombs = big.extra_tombs[0..num]; + + const extra_index = @as(u32, @intCast(big.a.extra.items.len)); + try big.a.extra.appendSlice(gpa, extra_tombs); + try big.a.special.put(gpa, big.inst, extra_index); + }, + } + + try analyzeOperands(big.a, pass, big.data, big.inst, big.small); + } + + fn deinit(big: *Self) void { + big.a.gpa.free(big.extra_tombs); + } + }; +} + +fn fmtInstSet(set: *const std.AutoHashMapUnmanaged(Air.Inst.Index, void)) FmtInstSet { + return .{ .set = set }; +} + +const FmtInstSet = struct { + set: *const std.AutoHashMapUnmanaged(Air.Inst.Index, void), + + pub fn format(val: FmtInstSet, comptime _: []const u8, _: std.fmt.FormatOptions, w: anytype) !void { + if (val.set.count() == 0) { + try w.writeAll("[no instructions]"); + return; + } + var it = val.set.keyIterator(); + try w.print("%{}", .{it.next().?.*}); + while (it.next()) |key| { + try w.print(" %{}", .{key.*}); + } + } +}; + +fn fmtInstList(list: []const Air.Inst.Index) FmtInstList { + return .{ .list = list }; +} + +const FmtInstList = struct { + list: []const Air.Inst.Index, + + pub fn format(val: FmtInstList, comptime _: []const u8, _: std.fmt.FormatOptions, w: anytype) !void { + if (val.list.len == 0) { + try w.writeAll("[no instructions]"); + return; + } + try w.print("%{}", .{val.list[0]}); + for (val.list[1..]) |inst| { + try w.print(" %{}", .{inst}); + } + } +}; diff --git a/src/Air/Liveness/Verify.zig b/src/Air/Liveness/Verify.zig new file mode 100644 index 0000000000000000000000000000000000000000..e7ed37956d16f4b969be03153dddb8c6d11c86e9 --- /dev/null +++ b/src/Air/Liveness/Verify.zig @@ -0,0 +1,642 @@ +//! Verifies that Liveness information is valid. + +gpa: std.mem.Allocator, +air: Air, +liveness: Liveness, +live: LiveMap = .{}, +blocks: std.AutoHashMapUnmanaged(Air.Inst.Index, LiveMap) = .empty, +loops: std.AutoHashMapUnmanaged(Air.Inst.Index, LiveMap) = .empty, +intern_pool: *const InternPool, + +pub const Error = error{ LivenessInvalid, OutOfMemory }; + +pub fn deinit(self: *Verify) void { + self.live.deinit(self.gpa); + { + var it = self.blocks.valueIterator(); + while (it.next()) |block| block.deinit(self.gpa); + self.blocks.deinit(self.gpa); + } + { + var it = self.loops.valueIterator(); + while (it.next()) |block| block.deinit(self.gpa); + self.loops.deinit(self.gpa); + } + self.* = undefined; +} + +pub fn verify(self: *Verify) Error!void { + self.live.clearRetainingCapacity(); + self.blocks.clearRetainingCapacity(); + self.loops.clearRetainingCapacity(); + try self.verifyBody(self.air.getMainBody()); + // We don't care about `self.live` now, because the loop body was noreturn - everything being dead was checked on `ret` etc + assert(self.blocks.count() == 0); + assert(self.loops.count() == 0); +} + +const LiveMap = std.AutoHashMapUnmanaged(Air.Inst.Index, void); + +fn verifyBody(self: *Verify, body: []const Air.Inst.Index) Error!void { + const ip = self.intern_pool; + const tags = self.air.instructions.items(.tag); + const data = self.air.instructions.items(.data); + for (body) |inst| { + if (self.liveness.isUnused(inst) and !self.air.mustLower(inst, ip)) { + // This instruction will not be lowered and should be ignored. + continue; + } + + switch (tags[@intFromEnum(inst)]) { + // no operands + .arg, + .alloc, + .inferred_alloc, + .inferred_alloc_comptime, + .ret_ptr, + .breakpoint, + .dbg_stmt, + .dbg_empty_stmt, + .ret_addr, + .frame_addr, + .wasm_memory_size, + .err_return_trace, + .save_err_return_trace_index, + .tlv_dllimport_ptr, + .c_va_start, + .work_item_id, + .work_group_size, + .work_group_id, + => try self.verifyInstOperands(inst, .{ .none, .none, .none }), + + .trap, .unreach => { + try self.verifyInstOperands(inst, .{ .none, .none, .none }); + // This instruction terminates the function, so everything should be dead + if (self.live.count() > 0) return invalid("%{}: instructions still alive", .{inst}); + }, + + // unary + .not, + .bitcast, + .load, + .fpext, + .fptrunc, + .intcast, + .intcast_safe, + .trunc, + .optional_payload, + .optional_payload_ptr, + .optional_payload_ptr_set, + .errunion_payload_ptr_set, + .wrap_optional, + .unwrap_errunion_payload, + .unwrap_errunion_err, + .unwrap_errunion_payload_ptr, + .unwrap_errunion_err_ptr, + .wrap_errunion_payload, + .wrap_errunion_err, + .slice_ptr, + .slice_len, + .ptr_slice_len_ptr, + .ptr_slice_ptr_ptr, + .struct_field_ptr_index_0, + .struct_field_ptr_index_1, + .struct_field_ptr_index_2, + .struct_field_ptr_index_3, + .array_to_slice, + .int_from_float, + .int_from_float_optimized, + .float_from_int, + .get_union_tag, + .clz, + .ctz, + .popcount, + .byte_swap, + .bit_reverse, + .splat, + .error_set_has_value, + .addrspace_cast, + .c_va_arg, + .c_va_copy, + .abs, + => { + const ty_op = data[@intFromEnum(inst)].ty_op; + try self.verifyInstOperands(inst, .{ ty_op.operand, .none, .none }); + }, + .is_null, + .is_non_null, + .is_null_ptr, + .is_non_null_ptr, + .is_err, + .is_non_err, + .is_err_ptr, + .is_non_err_ptr, + .is_named_enum_value, + .tag_name, + .error_name, + .sqrt, + .sin, + .cos, + .tan, + .exp, + .exp2, + .log, + .log2, + .log10, + .floor, + .ceil, + .round, + .trunc_float, + .neg, + .neg_optimized, + .cmp_lt_errors_len, + .set_err_return_trace, + .c_va_end, + => { + const un_op = data[@intFromEnum(inst)].un_op; + try self.verifyInstOperands(inst, .{ un_op, .none, .none }); + }, + .ret, + .ret_safe, + .ret_load, + => { + const un_op = data[@intFromEnum(inst)].un_op; + try self.verifyInstOperands(inst, .{ un_op, .none, .none }); + // This instruction terminates the function, so everything should be dead + if (self.live.count() > 0) return invalid("%{}: instructions still alive", .{inst}); + }, + .dbg_var_ptr, + .dbg_var_val, + .dbg_arg_inline, + .wasm_memory_grow, + => { + const pl_op = data[@intFromEnum(inst)].pl_op; + try self.verifyInstOperands(inst, .{ pl_op.operand, .none, .none }); + }, + .prefetch => { + const prefetch = data[@intFromEnum(inst)].prefetch; + try self.verifyInstOperands(inst, .{ prefetch.ptr, .none, .none }); + }, + .reduce, + .reduce_optimized, + => { + const reduce = data[@intFromEnum(inst)].reduce; + try self.verifyInstOperands(inst, .{ reduce.operand, .none, .none }); + }, + .union_init => { + const ty_pl = data[@intFromEnum(inst)].ty_pl; + const extra = self.air.extraData(Air.UnionInit, ty_pl.payload).data; + try self.verifyInstOperands(inst, .{ extra.init, .none, .none }); + }, + .struct_field_ptr, .struct_field_val => { + const ty_pl = data[@intFromEnum(inst)].ty_pl; + const extra = self.air.extraData(Air.StructField, ty_pl.payload).data; + try self.verifyInstOperands(inst, .{ extra.struct_operand, .none, .none }); + }, + .field_parent_ptr => { + const ty_pl = data[@intFromEnum(inst)].ty_pl; + const extra = self.air.extraData(Air.FieldParentPtr, ty_pl.payload).data; + try self.verifyInstOperands(inst, .{ extra.field_ptr, .none, .none }); + }, + .atomic_load => { + const atomic_load = data[@intFromEnum(inst)].atomic_load; + try self.verifyInstOperands(inst, .{ atomic_load.ptr, .none, .none }); + }, + + // binary + .add, + .add_safe, + .add_optimized, + .add_wrap, + .add_sat, + .sub, + .sub_safe, + .sub_optimized, + .sub_wrap, + .sub_sat, + .mul, + .mul_safe, + .mul_optimized, + .mul_wrap, + .mul_sat, + .div_float, + .div_float_optimized, + .div_trunc, + .div_trunc_optimized, + .div_floor, + .div_floor_optimized, + .div_exact, + .div_exact_optimized, + .rem, + .rem_optimized, + .mod, + .mod_optimized, + .bit_and, + .bit_or, + .xor, + .cmp_lt, + .cmp_lt_optimized, + .cmp_lte, + .cmp_lte_optimized, + .cmp_eq, + .cmp_eq_optimized, + .cmp_gte, + .cmp_gte_optimized, + .cmp_gt, + .cmp_gt_optimized, + .cmp_neq, + .cmp_neq_optimized, + .bool_and, + .bool_or, + .store, + .store_safe, + .array_elem_val, + .slice_elem_val, + .ptr_elem_val, + .shl, + .shl_exact, + .shl_sat, + .shr, + .shr_exact, + .atomic_store_unordered, + .atomic_store_monotonic, + .atomic_store_release, + .atomic_store_seq_cst, + .set_union_tag, + .min, + .max, + .memset, + .memset_safe, + .memcpy, + .memmove, + => { + const bin_op = data[@intFromEnum(inst)].bin_op; + try self.verifyInstOperands(inst, .{ bin_op.lhs, bin_op.rhs, .none }); + }, + .add_with_overflow, + .sub_with_overflow, + .mul_with_overflow, + .shl_with_overflow, + .ptr_add, + .ptr_sub, + .ptr_elem_ptr, + .slice_elem_ptr, + .slice, + => { + const ty_pl = data[@intFromEnum(inst)].ty_pl; + const extra = self.air.extraData(Air.Bin, ty_pl.payload).data; + try self.verifyInstOperands(inst, .{ extra.lhs, extra.rhs, .none }); + }, + .shuffle => { + const ty_pl = data[@intFromEnum(inst)].ty_pl; + const extra = self.air.extraData(Air.Shuffle, ty_pl.payload).data; + try self.verifyInstOperands(inst, .{ extra.a, extra.b, .none }); + }, + .cmp_vector, + .cmp_vector_optimized, + => { + const ty_pl = data[@intFromEnum(inst)].ty_pl; + const extra = self.air.extraData(Air.VectorCmp, ty_pl.payload).data; + try self.verifyInstOperands(inst, .{ extra.lhs, extra.rhs, .none }); + }, + .atomic_rmw => { + const pl_op = data[@intFromEnum(inst)].pl_op; + const extra = self.air.extraData(Air.AtomicRmw, pl_op.payload).data; + try self.verifyInstOperands(inst, .{ pl_op.operand, extra.operand, .none }); + }, + + // ternary + .select => { + const pl_op = data[@intFromEnum(inst)].pl_op; + const extra = self.air.extraData(Air.Bin, pl_op.payload).data; + try self.verifyInstOperands(inst, .{ pl_op.operand, extra.lhs, extra.rhs }); + }, + .mul_add => { + const pl_op = data[@intFromEnum(inst)].pl_op; + const extra = self.air.extraData(Air.Bin, pl_op.payload).data; + try self.verifyInstOperands(inst, .{ extra.lhs, extra.rhs, pl_op.operand }); + }, + .vector_store_elem => { + const vector_store_elem = data[@intFromEnum(inst)].vector_store_elem; + const extra = self.air.extraData(Air.Bin, vector_store_elem.payload).data; + try self.verifyInstOperands(inst, .{ vector_store_elem.vector_ptr, extra.lhs, extra.rhs }); + }, + .cmpxchg_strong, + .cmpxchg_weak, + => { + const ty_pl = data[@intFromEnum(inst)].ty_pl; + const extra = self.air.extraData(Air.Cmpxchg, ty_pl.payload).data; + try self.verifyInstOperands(inst, .{ extra.ptr, extra.expected_value, extra.new_value }); + }, + + // big tombs + .aggregate_init => { + const ty_pl = data[@intFromEnum(inst)].ty_pl; + const aggregate_ty = ty_pl.ty.toType(); + const len = @as(usize, @intCast(aggregate_ty.arrayLenIp(ip))); + const elements = @as([]const Air.Inst.Ref, @ptrCast(self.air.extra.items[ty_pl.payload..][0..len])); + + var bt = self.liveness.iterateBigTomb(inst); + for (elements) |element| { + try self.verifyOperand(inst, element, bt.feed()); + } + try self.verifyInst(inst); + }, + .call, .call_always_tail, .call_never_tail, .call_never_inline => { + const pl_op = data[@intFromEnum(inst)].pl_op; + const extra = self.air.extraData(Air.Call, pl_op.payload); + const args = @as( + []const Air.Inst.Ref, + @ptrCast(self.air.extra.items[extra.end..][0..extra.data.args_len]), + ); + + var bt = self.liveness.iterateBigTomb(inst); + try self.verifyOperand(inst, pl_op.operand, bt.feed()); + for (args) |arg| { + try self.verifyOperand(inst, arg, bt.feed()); + } + try self.verifyInst(inst); + }, + .assembly => { + const ty_pl = data[@intFromEnum(inst)].ty_pl; + const extra = self.air.extraData(Air.Asm, ty_pl.payload); + var extra_i = extra.end; + const outputs = @as( + []const Air.Inst.Ref, + @ptrCast(self.air.extra.items[extra_i..][0..extra.data.outputs_len]), + ); + extra_i += outputs.len; + const inputs = @as( + []const Air.Inst.Ref, + @ptrCast(self.air.extra.items[extra_i..][0..extra.data.inputs_len]), + ); + extra_i += inputs.len; + + var bt = self.liveness.iterateBigTomb(inst); + for (outputs) |output| { + if (output != .none) { + try self.verifyOperand(inst, output, bt.feed()); + } + } + for (inputs) |input| { + try self.verifyOperand(inst, input, bt.feed()); + } + try self.verifyInst(inst); + }, + + // control flow + .@"try", .try_cold => { + const pl_op = data[@intFromEnum(inst)].pl_op; + const extra = self.air.extraData(Air.Try, pl_op.payload); + const try_body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[extra.end..][0..extra.data.body_len]); + + const cond_br_liveness = self.liveness.getCondBr(inst); + + try self.verifyOperand(inst, pl_op.operand, self.liveness.operandDies(inst, 0)); + + var live = try self.live.clone(self.gpa); + defer live.deinit(self.gpa); + + for (cond_br_liveness.else_deaths) |death| try self.verifyDeath(inst, death); + try self.verifyBody(try_body); + + self.live.deinit(self.gpa); + self.live = live.move(); + + for (cond_br_liveness.then_deaths) |death| try self.verifyDeath(inst, death); + + try self.verifyInst(inst); + }, + .try_ptr, .try_ptr_cold => { + const ty_pl = data[@intFromEnum(inst)].ty_pl; + const extra = self.air.extraData(Air.TryPtr, ty_pl.payload); + const try_body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[extra.end..][0..extra.data.body_len]); + + const cond_br_liveness = self.liveness.getCondBr(inst); + + try self.verifyOperand(inst, extra.data.ptr, self.liveness.operandDies(inst, 0)); + + var live = try self.live.clone(self.gpa); + defer live.deinit(self.gpa); + + for (cond_br_liveness.else_deaths) |death| try self.verifyDeath(inst, death); + try self.verifyBody(try_body); + + self.live.deinit(self.gpa); + self.live = live.move(); + + for (cond_br_liveness.then_deaths) |death| try self.verifyDeath(inst, death); + + try self.verifyInst(inst); + }, + .br => { + const br = data[@intFromEnum(inst)].br; + const gop = try self.blocks.getOrPut(self.gpa, br.block_inst); + + try self.verifyOperand(inst, br.operand, self.liveness.operandDies(inst, 0)); + if (gop.found_existing) { + try self.verifyMatchingLiveness(br.block_inst, gop.value_ptr.*); + } else { + gop.value_ptr.* = try self.live.clone(self.gpa); + } + try self.verifyInst(inst); + }, + .repeat => { + const repeat = data[@intFromEnum(inst)].repeat; + const expected_live = self.loops.get(repeat.loop_inst) orelse + return invalid("%{}: loop %{} not in scope", .{ @intFromEnum(inst), @intFromEnum(repeat.loop_inst) }); + + try self.verifyMatchingLiveness(repeat.loop_inst, expected_live); + }, + .switch_dispatch => { + const br = data[@intFromEnum(inst)].br; + + try self.verifyOperand(inst, br.operand, self.liveness.operandDies(inst, 0)); + + const expected_live = self.loops.get(br.block_inst) orelse + return invalid("%{}: loop %{} not in scope", .{ @intFromEnum(inst), @intFromEnum(br.block_inst) }); + + try self.verifyMatchingLiveness(br.block_inst, expected_live); + }, + .block, .dbg_inline_block => |tag| { + const ty_pl = data[@intFromEnum(inst)].ty_pl; + const block_ty = ty_pl.ty.toType(); + const block_body: []const Air.Inst.Index = @ptrCast(switch (tag) { + inline .block, .dbg_inline_block => |comptime_tag| body: { + const extra = self.air.extraData(switch (comptime_tag) { + .block => Air.Block, + .dbg_inline_block => Air.DbgInlineBlock, + else => unreachable, + }, ty_pl.payload); + break :body self.air.extra.items[extra.end..][0..extra.data.body_len]; + }, + else => unreachable, + }); + const block_liveness = self.liveness.getBlock(inst); + + var orig_live = try self.live.clone(self.gpa); + defer orig_live.deinit(self.gpa); + + assert(!self.blocks.contains(inst)); + try self.verifyBody(block_body); + + // Liveness data after the block body is garbage, but we want to + // restore it to verify deaths + self.live.deinit(self.gpa); + self.live = orig_live.move(); + + for (block_liveness.deaths) |death| try self.verifyDeath(inst, death); + + if (ip.isNoReturn(block_ty.toIntern())) { + assert(!self.blocks.contains(inst)); + } else { + var live = self.blocks.fetchRemove(inst).?.value; + defer live.deinit(self.gpa); + + try self.verifyMatchingLiveness(inst, live); + } + + try self.verifyInstOperands(inst, .{ .none, .none, .none }); + }, + .loop => { + const ty_pl = data[@intFromEnum(inst)].ty_pl; + const extra = self.air.extraData(Air.Block, ty_pl.payload); + const loop_body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[extra.end..][0..extra.data.body_len]); + + // The same stuff should be alive after the loop as before it. + const gop = try self.loops.getOrPut(self.gpa, inst); + if (gop.found_existing) return invalid("%{}: loop already exists", .{@intFromEnum(inst)}); + defer { + var live = self.loops.fetchRemove(inst).?; + live.value.deinit(self.gpa); + } + gop.value_ptr.* = try self.live.clone(self.gpa); + + try self.verifyBody(loop_body); + + try self.verifyInstOperands(inst, .{ .none, .none, .none }); + }, + .cond_br => { + const pl_op = data[@intFromEnum(inst)].pl_op; + const extra = self.air.extraData(Air.CondBr, pl_op.payload); + const then_body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[extra.end..][0..extra.data.then_body_len]); + const else_body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[extra.end + then_body.len ..][0..extra.data.else_body_len]); + const cond_br_liveness = self.liveness.getCondBr(inst); + + try self.verifyOperand(inst, pl_op.operand, self.liveness.operandDies(inst, 0)); + + var live = try self.live.clone(self.gpa); + defer live.deinit(self.gpa); + + for (cond_br_liveness.then_deaths) |death| try self.verifyDeath(inst, death); + try self.verifyBody(then_body); + + self.live.deinit(self.gpa); + self.live = live.move(); + + for (cond_br_liveness.else_deaths) |death| try self.verifyDeath(inst, death); + try self.verifyBody(else_body); + + try self.verifyInst(inst); + }, + .switch_br, .loop_switch_br => { + const switch_br = self.air.unwrapSwitch(inst); + const switch_br_liveness = try self.liveness.getSwitchBr( + self.gpa, + inst, + switch_br.cases_len + 1, + ); + defer self.gpa.free(switch_br_liveness.deaths); + + try self.verifyOperand(inst, switch_br.operand, self.liveness.operandDies(inst, 0)); + + // Excluding the operand (which we just handled), the same stuff should be alive + // after the loop as before it. + { + const gop = try self.loops.getOrPut(self.gpa, inst); + if (gop.found_existing) return invalid("%{}: loop already exists", .{@intFromEnum(inst)}); + gop.value_ptr.* = self.live.move(); + } + defer { + var live = self.loops.fetchRemove(inst).?; + live.value.deinit(self.gpa); + } + + var it = switch_br.iterateCases(); + while (it.next()) |case| { + self.live.deinit(self.gpa); + self.live = try self.loops.get(inst).?.clone(self.gpa); + + for (switch_br_liveness.deaths[case.idx]) |death| try self.verifyDeath(inst, death); + try self.verifyBody(case.body); + } + + const else_body = it.elseBody(); + if (else_body.len > 0) { + self.live.deinit(self.gpa); + self.live = try self.loops.get(inst).?.clone(self.gpa); + for (switch_br_liveness.deaths[switch_br.cases_len]) |death| try self.verifyDeath(inst, death); + try self.verifyBody(else_body); + } + + try self.verifyInst(inst); + }, + } + } +} + +fn verifyDeath(self: *Verify, inst: Air.Inst.Index, operand: Air.Inst.Index) Error!void { + try self.verifyOperand(inst, operand.toRef(), true); +} + +fn verifyOperand(self: *Verify, inst: Air.Inst.Index, op_ref: Air.Inst.Ref, dies: bool) Error!void { + const operand = op_ref.toIndexAllowNone() orelse { + assert(!dies); + return; + }; + if (dies) { + if (!self.live.remove(operand)) return invalid("%{}: dead operand %{} reused and killed again", .{ inst, operand }); + } else { + if (!self.live.contains(operand)) return invalid("%{}: dead operand %{} reused", .{ inst, operand }); + } +} + +fn verifyInstOperands( + self: *Verify, + inst: Air.Inst.Index, + operands: [Liveness.bpi - 1]Air.Inst.Ref, +) Error!void { + for (operands, 0..) |operand, operand_index| { + const dies = self.liveness.operandDies(inst, @as(Liveness.OperandInt, @intCast(operand_index))); + try self.verifyOperand(inst, operand, dies); + } + try self.verifyInst(inst); +} + +fn verifyInst(self: *Verify, inst: Air.Inst.Index) Error!void { + if (self.liveness.isUnused(inst)) { + assert(!self.live.contains(inst)); + } else { + try self.live.putNoClobber(self.gpa, inst, {}); + } +} + +fn verifyMatchingLiveness(self: *Verify, block: Air.Inst.Index, live: LiveMap) Error!void { + if (self.live.count() != live.count()) return invalid("%{}: different deaths across branches", .{block}); + var live_it = self.live.keyIterator(); + while (live_it.next()) |live_inst| if (!live.contains(live_inst.*)) return invalid("%{}: different deaths across branches", .{block}); +} + +fn invalid(comptime fmt: []const u8, args: anytype) error{LivenessInvalid} { + log.err(fmt, args); + return error.LivenessInvalid; +} + +const std = @import("std"); +const assert = std.debug.assert; +const log = std.log.scoped(.liveness_verify); + +const Air = @import("../../Air.zig"); +const Liveness = @import("../Liveness.zig"); +const InternPool = @import("../../InternPool.zig"); +const Verify = @This(); diff --git a/src/Air/types_resolved.zig b/src/Air/types_resolved.zig index fcc2667a29d6c8b724405a8b426e5d2b958c17f4..873f70ec50a6811e0f46262e47d2cbeebf4b3063 100644 --- a/src/Air/types_resolved.zig +++ b/src/Air/types_resolved.zig @@ -171,7 +171,7 @@ fn checkBody(air: Air, body: []const Air.Inst.Index, zcu: *Zcu) bool { if (!checkType(data.ty_pl.ty.toType(), zcu)) return false; if (!checkBody( air, - @ptrCast(air.extra[extra.end..][0..extra.data.body_len]), + @ptrCast(air.extra.items[extra.end..][0..extra.data.body_len]), zcu, )) return false; }, @@ -181,7 +181,7 @@ fn checkBody(air: Air, body: []const Air.Inst.Index, zcu: *Zcu) bool { if (!checkType(data.ty_pl.ty.toType(), zcu)) return false; if (!checkBody( air, - @ptrCast(air.extra[extra.end..][0..extra.data.body_len]), + @ptrCast(air.extra.items[extra.end..][0..extra.data.body_len]), zcu, )) return false; }, @@ -270,7 +270,7 @@ fn checkBody(air: Air, body: []const Air.Inst.Index, zcu: *Zcu) bool { .aggregate_init => { const ty = data.ty_pl.ty.toType(); const elems_len: usize = @intCast(ty.arrayLen(zcu)); - const elems: []const Air.Inst.Ref = @ptrCast(air.extra[data.ty_pl.payload..][0..elems_len]); + const elems: []const Air.Inst.Ref = @ptrCast(air.extra.items[data.ty_pl.payload..][0..elems_len]); if (!checkType(ty, zcu)) return false; if (ty.zigTypeTag(zcu) == .@"struct") { for (elems, 0..) |elem, elem_idx| { @@ -336,7 +336,7 @@ fn checkBody(air: Air, body: []const Air.Inst.Index, zcu: *Zcu) bool { .call_never_inline, => { const extra = air.extraData(Air.Call, data.pl_op.payload); - const args: []const Air.Inst.Ref = @ptrCast(air.extra[extra.end..][0..extra.data.args_len]); + const args: []const Air.Inst.Ref = @ptrCast(air.extra.items[extra.end..][0..extra.data.args_len]); if (!checkRef(data.pl_op.operand, zcu)) return false; for (args) |arg| if (!checkRef(arg, zcu)) return false; }, @@ -353,7 +353,7 @@ fn checkBody(air: Air, body: []const Air.Inst.Index, zcu: *Zcu) bool { if (!checkRef(data.pl_op.operand, zcu)) return false; if (!checkBody( air, - @ptrCast(air.extra[extra.end..][0..extra.data.body_len]), + @ptrCast(air.extra.items[extra.end..][0..extra.data.body_len]), zcu, )) return false; }, @@ -364,7 +364,7 @@ fn checkBody(air: Air, body: []const Air.Inst.Index, zcu: *Zcu) bool { if (!checkRef(extra.data.ptr, zcu)) return false; if (!checkBody( air, - @ptrCast(air.extra[extra.end..][0..extra.data.body_len]), + @ptrCast(air.extra.items[extra.end..][0..extra.data.body_len]), zcu, )) return false; }, @@ -374,12 +374,12 @@ fn checkBody(air: Air, body: []const Air.Inst.Index, zcu: *Zcu) bool { if (!checkRef(data.pl_op.operand, zcu)) return false; if (!checkBody( air, - @ptrCast(air.extra[extra.end..][0..extra.data.then_body_len]), + @ptrCast(air.extra.items[extra.end..][0..extra.data.then_body_len]), zcu, )) return false; if (!checkBody( air, - @ptrCast(air.extra[extra.end + extra.data.then_body_len ..][0..extra.data.else_body_len]), + @ptrCast(air.extra.items[extra.end + extra.data.then_body_len ..][0..extra.data.else_body_len]), zcu, )) return false; }, @@ -404,8 +404,8 @@ fn checkBody(air: Air, body: []const Air.Inst.Index, zcu: *Zcu) bool { if (!checkType(data.ty_pl.ty.toType(), zcu)) return false; // Luckily, we only care about the inputs and outputs, so we don't have to do // the whole null-terminated string dance. - const outputs: []const Air.Inst.Ref = @ptrCast(air.extra[extra.end..][0..extra.data.outputs_len]); - const inputs: []const Air.Inst.Ref = @ptrCast(air.extra[extra.end + extra.data.outputs_len ..][0..extra.data.inputs_len]); + const outputs: []const Air.Inst.Ref = @ptrCast(air.extra.items[extra.end..][0..extra.data.outputs_len]); + const inputs: []const Air.Inst.Ref = @ptrCast(air.extra.items[extra.end + extra.data.outputs_len ..][0..extra.data.inputs_len]); for (outputs) |output| if (output != .none and !checkRef(output, zcu)) return false; for (inputs) |input| if (input != .none and !checkRef(input, zcu)) return false; }, diff --git a/src/Liveness.zig b/src/Liveness.zig deleted file mode 100644 index 9c47e12679963676f575a66e31c360d43ee7e762..0000000000000000000000000000000000000000 --- a/src/Liveness.zig +++ /dev/null @@ -1,2050 +0,0 @@ -//! For each AIR instruction, we want to know: -//! * Is the instruction unreferenced (e.g. dies immediately)? -//! * For each of its operands, does the operand die with this instruction (e.g. is -//! this the last reference to it)? -//! Some instructions are special, such as: -//! * Conditional Branches -//! * Switch Branches -const std = @import("std"); -const log = std.log.scoped(.liveness); -const assert = std.debug.assert; -const Allocator = std.mem.Allocator; -const Log2Int = std.math.Log2Int; - -const Liveness = @This(); -const trace = @import("tracy.zig").trace; -const Air = @import("Air.zig"); -const InternPool = @import("InternPool.zig"); - -pub const Verify = @import("Liveness/Verify.zig"); - -/// This array is split into sets of 4 bits per AIR instruction. -/// The MSB (0bX000) is whether the instruction is unreferenced. -/// The LSB (0b000X) is the first operand, and so on, up to 3 operands. A set bit means the -/// operand dies after this instruction. -/// Instructions which need more data to track liveness have special handling via the -/// `special` table. -tomb_bits: []usize, -/// Sparse table of specially handled instructions. The value is an index into the `extra` -/// array. The meaning of the data depends on the AIR tag. -/// * `cond_br` - points to a `CondBr` in `extra` at this index. -/// * `try`, `try_ptr` - points to a `CondBr` in `extra` at this index. The error path (the block -/// in the instruction) is considered the "else" path, and the rest of the block the "then". -/// * `switch_br` - points to a `SwitchBr` in `extra` at this index. -/// * `loop_switch_br` - points to a `SwitchBr` in `extra` at this index. -/// * `block` - points to a `Block` in `extra` at this index. -/// * `asm`, `call`, `aggregate_init` - the value is a set of bits which are the extra tomb -/// bits of operands. -/// The main tomb bits are still used and the extra ones are starting with the lsb of the -/// value here. -special: std.AutoHashMapUnmanaged(Air.Inst.Index, u32), -/// Auxiliary data. The way this data is interpreted is determined contextually. -extra: []const u32, - -/// Trailing is the set of instructions whose lifetimes end at the start of the then branch, -/// followed by the set of instructions whose lifetimes end at the start of the else branch. -pub const CondBr = struct { - then_death_count: u32, - else_death_count: u32, -}; - -/// Trailing is: -/// * For each case in the same order as in the AIR: -/// - case_death_count: u32 -/// - Air.Inst.Index for each `case_death_count`: set of instructions whose lifetimes -/// end at the start of this case. -/// * Air.Inst.Index for each `else_death_count`: set of instructions whose lifetimes -/// end at the start of the else case. -pub const SwitchBr = struct { - else_death_count: u32, -}; - -/// Trailing is the set of instructions which die in the block. Note that these are not additional -/// deaths (they are all recorded as normal within the block), but backends may use this information -/// as a more efficient way to track which instructions are still alive after a block. -pub const Block = struct { - death_count: u32, -}; - -/// Liveness analysis runs in several passes. Each pass iterates backwards over instructions in -/// bodies, and recurses into bodies. -const LivenessPass = enum { - /// In this pass, we perform some basic analysis of loops to gain information the main pass needs. - /// In particular, for every `loop` and `loop_switch_br`, we track the following information: - /// * Every outer block which the loop body contains a `br` to. - /// * Every outer loop which the loop body contains a `repeat` to. - /// * Every operand referenced within the loop body but created outside the loop. - /// This gives the main analysis pass enough information to determine the full set of - /// instructions which need to be alive when a loop repeats. This data is TEMPORARILY stored in - /// `a.extra`. It is not re-added to `extra` by the main pass, since it is not useful to - /// backends. - loop_analysis, - - /// This pass performs the main liveness analysis, setting up tombs and extra data while - /// considering control flow etc. - main_analysis, -}; - -/// Each analysis pass may wish to pass data through calls. A pointer to a `LivenessPassData(pass)` -/// stored on the stack is passed through calls to `analyzeInst` etc. -fn LivenessPassData(comptime pass: LivenessPass) type { - return switch (pass) { - .loop_analysis => struct { - /// The set of blocks which are exited with a `br` instruction at some point within this - /// body and which we are currently within. Also includes `loop`s which are the target - /// of a `repeat` instruction, and `loop_switch_br`s which are the target of a - /// `switch_dispatch` instruction. - breaks: std.AutoHashMapUnmanaged(Air.Inst.Index, void) = .empty, - - /// The set of operands for which we have seen at least one usage but not their birth. - live_set: std.AutoHashMapUnmanaged(Air.Inst.Index, void) = .empty, - - fn deinit(self: *@This(), gpa: Allocator) void { - self.breaks.deinit(gpa); - self.live_set.deinit(gpa); - } - }, - - .main_analysis => struct { - /// Every `block` and `loop` currently under analysis. - block_scopes: std.AutoHashMapUnmanaged(Air.Inst.Index, BlockScope) = .empty, - - /// The set of instructions currently alive in the current control - /// flow branch. - live_set: std.AutoHashMapUnmanaged(Air.Inst.Index, void) = .empty, - - /// The extra data initialized by the `loop_analysis` pass for this pass to consume. - /// Owned by this struct during this pass. - old_extra: std.ArrayListUnmanaged(u32) = .empty, - - const BlockScope = struct { - /// If this is a `block`, these instructions are alive upon a `br` to this block. - /// If this is a `loop`, these instructions are alive upon a `repeat` to this block. - live_set: std.AutoHashMapUnmanaged(Air.Inst.Index, void), - }; - - fn deinit(self: *@This(), gpa: Allocator) void { - var it = self.block_scopes.valueIterator(); - while (it.next()) |block| { - block.live_set.deinit(gpa); - } - self.block_scopes.deinit(gpa); - self.live_set.deinit(gpa); - self.old_extra.deinit(gpa); - } - }, - }; -} - -pub fn analyze(gpa: Allocator, air: Air, intern_pool: *InternPool) Allocator.Error!Liveness { - const tracy = trace(@src()); - defer tracy.end(); - - var a: Analysis = .{ - .gpa = gpa, - .air = air, - .tomb_bits = try gpa.alloc( - usize, - (air.instructions.len * bpi + @bitSizeOf(usize) - 1) / @bitSizeOf(usize), - ), - .extra = .{}, - .special = .{}, - .intern_pool = intern_pool, - }; - errdefer gpa.free(a.tomb_bits); - errdefer a.special.deinit(gpa); - defer a.extra.deinit(gpa); - - @memset(a.tomb_bits, 0); - - const main_body = air.getMainBody(); - - { - var data: LivenessPassData(.loop_analysis) = .{}; - defer data.deinit(gpa); - try analyzeBody(&a, .loop_analysis, &data, main_body); - } - - { - var data: LivenessPassData(.main_analysis) = .{}; - defer data.deinit(gpa); - data.old_extra = a.extra; - a.extra = .{}; - try analyzeBody(&a, .main_analysis, &data, main_body); - assert(data.live_set.count() == 0); - } - - return .{ - .tomb_bits = a.tomb_bits, - .special = a.special, - .extra = try a.extra.toOwnedSlice(gpa), - }; -} - -pub fn getTombBits(l: Liveness, inst: Air.Inst.Index) Bpi { - const usize_index = (@intFromEnum(inst) * bpi) / @bitSizeOf(usize); - return @as(Bpi, @truncate(l.tomb_bits[usize_index] >> - @as(Log2Int(usize), @intCast((@intFromEnum(inst) % (@bitSizeOf(usize) / bpi)) * bpi)))); -} - -pub fn isUnused(l: Liveness, inst: Air.Inst.Index) bool { - const usize_index = (@intFromEnum(inst) * bpi) / @bitSizeOf(usize); - const mask = @as(usize, 1) << - @as(Log2Int(usize), @intCast((@intFromEnum(inst) % (@bitSizeOf(usize) / bpi)) * bpi + (bpi - 1))); - return (l.tomb_bits[usize_index] & mask) != 0; -} - -pub fn operandDies(l: Liveness, inst: Air.Inst.Index, operand: OperandInt) bool { - assert(operand < bpi - 1); - const usize_index = (@intFromEnum(inst) * bpi) / @bitSizeOf(usize); - const mask = @as(usize, 1) << - @as(Log2Int(usize), @intCast((@intFromEnum(inst) % (@bitSizeOf(usize) / bpi)) * bpi + operand)); - return (l.tomb_bits[usize_index] & mask) != 0; -} - -const OperandCategory = enum { - /// The operand lives on, but this instruction cannot possibly mutate memory. - none, - /// The operand lives on and this instruction can mutate memory. - write, - /// The operand dies at this instruction. - tomb, - /// The operand lives on, and this instruction is noreturn. - noret, - /// This instruction is too complicated for analysis, no information is available. - complex, -}; - -/// Given an instruction that we are examining, and an operand that we are looking for, -/// returns a classification. -pub fn categorizeOperand( - l: Liveness, - air: Air, - inst: Air.Inst.Index, - operand: Air.Inst.Index, - ip: *const InternPool, -) OperandCategory { - const air_tags = air.instructions.items(.tag); - const air_datas = air.instructions.items(.data); - const operand_ref = operand.toRef(); - switch (air_tags[@intFromEnum(inst)]) { - .add, - .add_safe, - .add_wrap, - .add_sat, - .add_optimized, - .sub, - .sub_safe, - .sub_wrap, - .sub_sat, - .sub_optimized, - .mul, - .mul_safe, - .mul_wrap, - .mul_sat, - .mul_optimized, - .div_float, - .div_trunc, - .div_floor, - .div_exact, - .rem, - .mod, - .bit_and, - .bit_or, - .xor, - .cmp_lt, - .cmp_lte, - .cmp_eq, - .cmp_gte, - .cmp_gt, - .cmp_neq, - .bool_and, - .bool_or, - .array_elem_val, - .slice_elem_val, - .ptr_elem_val, - .shl, - .shl_exact, - .shl_sat, - .shr, - .shr_exact, - .min, - .max, - .div_float_optimized, - .div_trunc_optimized, - .div_floor_optimized, - .div_exact_optimized, - .rem_optimized, - .mod_optimized, - .neg_optimized, - .cmp_lt_optimized, - .cmp_lte_optimized, - .cmp_eq_optimized, - .cmp_gte_optimized, - .cmp_gt_optimized, - .cmp_neq_optimized, - => { - const o = air_datas[@intFromEnum(inst)].bin_op; - if (o.lhs == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none); - if (o.rhs == operand_ref) return matchOperandSmallIndex(l, inst, 1, .none); - return .none; - }, - - .store, - .store_safe, - .atomic_store_unordered, - .atomic_store_monotonic, - .atomic_store_release, - .atomic_store_seq_cst, - .set_union_tag, - .memset, - .memset_safe, - .memcpy, - .memmove, - => { - const o = air_datas[@intFromEnum(inst)].bin_op; - if (o.lhs == operand_ref) return matchOperandSmallIndex(l, inst, 0, .write); - if (o.rhs == operand_ref) return matchOperandSmallIndex(l, inst, 1, .write); - return .write; - }, - - .vector_store_elem => { - const o = air_datas[@intFromEnum(inst)].vector_store_elem; - const extra = air.extraData(Air.Bin, o.payload).data; - if (o.vector_ptr == operand_ref) return matchOperandSmallIndex(l, inst, 0, .write); - if (extra.lhs == operand_ref) return matchOperandSmallIndex(l, inst, 1, .none); - if (extra.rhs == operand_ref) return matchOperandSmallIndex(l, inst, 2, .none); - return .write; - }, - - .arg, - .alloc, - .inferred_alloc, - .inferred_alloc_comptime, - .ret_ptr, - .trap, - .breakpoint, - .repeat, - .switch_dispatch, - .dbg_stmt, - .dbg_empty_stmt, - .unreach, - .ret_addr, - .frame_addr, - .wasm_memory_size, - .err_return_trace, - .save_err_return_trace_index, - .tlv_dllimport_ptr, - .c_va_start, - .work_item_id, - .work_group_size, - .work_group_id, - => return .none, - - .not, - .bitcast, - .load, - .fpext, - .fptrunc, - .intcast, - .intcast_safe, - .trunc, - .optional_payload, - .optional_payload_ptr, - .wrap_optional, - .unwrap_errunion_payload, - .unwrap_errunion_err, - .unwrap_errunion_payload_ptr, - .unwrap_errunion_err_ptr, - .wrap_errunion_payload, - .wrap_errunion_err, - .slice_ptr, - .slice_len, - .ptr_slice_len_ptr, - .ptr_slice_ptr_ptr, - .struct_field_ptr_index_0, - .struct_field_ptr_index_1, - .struct_field_ptr_index_2, - .struct_field_ptr_index_3, - .array_to_slice, - .int_from_float, - .int_from_float_optimized, - .float_from_int, - .get_union_tag, - .clz, - .ctz, - .popcount, - .byte_swap, - .bit_reverse, - .splat, - .error_set_has_value, - .addrspace_cast, - .c_va_arg, - .c_va_copy, - .abs, - => { - const o = air_datas[@intFromEnum(inst)].ty_op; - if (o.operand == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none); - return .none; - }, - - .optional_payload_ptr_set, - .errunion_payload_ptr_set, - => { - const o = air_datas[@intFromEnum(inst)].ty_op; - if (o.operand == operand_ref) return matchOperandSmallIndex(l, inst, 0, .write); - return .write; - }, - - .is_null, - .is_non_null, - .is_null_ptr, - .is_non_null_ptr, - .is_err, - .is_non_err, - .is_err_ptr, - .is_non_err_ptr, - .is_named_enum_value, - .tag_name, - .error_name, - .sqrt, - .sin, - .cos, - .tan, - .exp, - .exp2, - .log, - .log2, - .log10, - .floor, - .ceil, - .round, - .trunc_float, - .neg, - .cmp_lt_errors_len, - .c_va_end, - => { - const o = air_datas[@intFromEnum(inst)].un_op; - if (o == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none); - return .none; - }, - - .ret, - .ret_safe, - .ret_load, - => { - const o = air_datas[@intFromEnum(inst)].un_op; - if (o == operand_ref) return matchOperandSmallIndex(l, inst, 0, .noret); - return .noret; - }, - - .set_err_return_trace => { - const o = air_datas[@intFromEnum(inst)].un_op; - if (o == operand_ref) return matchOperandSmallIndex(l, inst, 0, .write); - return .write; - }, - - .add_with_overflow, - .sub_with_overflow, - .mul_with_overflow, - .shl_with_overflow, - .ptr_add, - .ptr_sub, - .ptr_elem_ptr, - .slice_elem_ptr, - .slice, - => { - const ty_pl = air_datas[@intFromEnum(inst)].ty_pl; - const extra = air.extraData(Air.Bin, ty_pl.payload).data; - if (extra.lhs == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none); - if (extra.rhs == operand_ref) return matchOperandSmallIndex(l, inst, 1, .none); - return .none; - }, - - .dbg_var_ptr, - .dbg_var_val, - .dbg_arg_inline, - => { - const o = air_datas[@intFromEnum(inst)].pl_op.operand; - if (o == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none); - return .none; - }, - - .prefetch => { - const prefetch = air_datas[@intFromEnum(inst)].prefetch; - if (prefetch.ptr == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none); - return .none; - }, - - .call, .call_always_tail, .call_never_tail, .call_never_inline => { - const inst_data = air_datas[@intFromEnum(inst)].pl_op; - const callee = inst_data.operand; - const extra = air.extraData(Air.Call, inst_data.payload); - const args = @as([]const Air.Inst.Ref, @ptrCast(air.extra[extra.end..][0..extra.data.args_len])); - if (args.len + 1 <= bpi - 1) { - if (callee == operand_ref) return matchOperandSmallIndex(l, inst, 0, .write); - for (args, 0..) |arg, i| { - if (arg == operand_ref) return matchOperandSmallIndex(l, inst, @as(OperandInt, @intCast(i + 1)), .write); - } - return .write; - } - var bt = l.iterateBigTomb(inst); - if (bt.feed()) { - if (callee == operand_ref) return .tomb; - } else { - if (callee == operand_ref) return .write; - } - for (args) |arg| { - if (bt.feed()) { - if (arg == operand_ref) return .tomb; - } else { - if (arg == operand_ref) return .write; - } - } - return .write; - }, - .select => { - const pl_op = air_datas[@intFromEnum(inst)].pl_op; - const extra = air.extraData(Air.Bin, pl_op.payload).data; - if (pl_op.operand == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none); - if (extra.lhs == operand_ref) return matchOperandSmallIndex(l, inst, 1, .none); - if (extra.rhs == operand_ref) return matchOperandSmallIndex(l, inst, 2, .none); - return .none; - }, - .shuffle => { - const extra = air.extraData(Air.Shuffle, air_datas[@intFromEnum(inst)].ty_pl.payload).data; - if (extra.a == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none); - if (extra.b == operand_ref) return matchOperandSmallIndex(l, inst, 1, .none); - return .none; - }, - .reduce, .reduce_optimized => { - const reduce = air_datas[@intFromEnum(inst)].reduce; - if (reduce.operand == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none); - return .none; - }, - .cmp_vector, .cmp_vector_optimized => { - const extra = air.extraData(Air.VectorCmp, air_datas[@intFromEnum(inst)].ty_pl.payload).data; - if (extra.lhs == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none); - if (extra.rhs == operand_ref) return matchOperandSmallIndex(l, inst, 1, .none); - return .none; - }, - .aggregate_init => { - const ty_pl = air_datas[@intFromEnum(inst)].ty_pl; - const aggregate_ty = ty_pl.ty.toType(); - const len = @as(usize, @intCast(aggregate_ty.arrayLenIp(ip))); - const elements = @as([]const Air.Inst.Ref, @ptrCast(air.extra[ty_pl.payload..][0..len])); - - if (elements.len <= bpi - 1) { - for (elements, 0..) |elem, i| { - if (elem == operand_ref) return matchOperandSmallIndex(l, inst, @as(OperandInt, @intCast(i)), .none); - } - return .none; - } - - var bt = l.iterateBigTomb(inst); - for (elements) |elem| { - if (bt.feed()) { - if (elem == operand_ref) return .tomb; - } else { - if (elem == operand_ref) return .write; - } - } - return .write; - }, - .union_init => { - const extra = air.extraData(Air.UnionInit, air_datas[@intFromEnum(inst)].ty_pl.payload).data; - if (extra.init == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none); - return .none; - }, - .struct_field_ptr, .struct_field_val => { - const extra = air.extraData(Air.StructField, air_datas[@intFromEnum(inst)].ty_pl.payload).data; - if (extra.struct_operand == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none); - return .none; - }, - .field_parent_ptr => { - const extra = air.extraData(Air.FieldParentPtr, air_datas[@intFromEnum(inst)].ty_pl.payload).data; - if (extra.field_ptr == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none); - return .none; - }, - .cmpxchg_strong, .cmpxchg_weak => { - const extra = air.extraData(Air.Cmpxchg, air_datas[@intFromEnum(inst)].ty_pl.payload).data; - if (extra.ptr == operand_ref) return matchOperandSmallIndex(l, inst, 0, .write); - if (extra.expected_value == operand_ref) return matchOperandSmallIndex(l, inst, 1, .write); - if (extra.new_value == operand_ref) return matchOperandSmallIndex(l, inst, 2, .write); - return .write; - }, - .mul_add => { - const pl_op = air_datas[@intFromEnum(inst)].pl_op; - const extra = air.extraData(Air.Bin, pl_op.payload).data; - if (extra.lhs == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none); - if (extra.rhs == operand_ref) return matchOperandSmallIndex(l, inst, 1, .none); - if (pl_op.operand == operand_ref) return matchOperandSmallIndex(l, inst, 2, .none); - return .none; - }, - .atomic_load => { - const ptr = air_datas[@intFromEnum(inst)].atomic_load.ptr; - if (ptr == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none); - return .none; - }, - .atomic_rmw => { - const pl_op = air_datas[@intFromEnum(inst)].pl_op; - const extra = air.extraData(Air.AtomicRmw, pl_op.payload).data; - if (pl_op.operand == operand_ref) return matchOperandSmallIndex(l, inst, 0, .write); - if (extra.operand == operand_ref) return matchOperandSmallIndex(l, inst, 1, .write); - return .write; - }, - - .br => { - const br = air_datas[@intFromEnum(inst)].br; - if (br.operand == operand_ref) return matchOperandSmallIndex(l, operand, 0, .noret); - return .noret; - }, - .assembly => { - return .complex; - }, - .block, .dbg_inline_block => |tag| { - const ty_pl = air_datas[@intFromEnum(inst)].ty_pl; - const body: []const Air.Inst.Index = @ptrCast(switch (tag) { - inline .block, .dbg_inline_block => |comptime_tag| body: { - const extra = air.extraData(switch (comptime_tag) { - .block => Air.Block, - .dbg_inline_block => Air.DbgInlineBlock, - else => unreachable, - }, ty_pl.payload); - break :body air.extra[extra.end..][0..extra.data.body_len]; - }, - else => unreachable, - }); - - if (body.len == 1 and air_tags[@intFromEnum(body[0])] == .cond_br) { - // Peephole optimization for "panic-like" conditionals, which have - // one empty branch and another which calls a `noreturn` function. - // This allows us to infer that safety checks do not modify memory, - // as far as control flow successors are concerned. - - const inst_data = air_datas[@intFromEnum(body[0])].pl_op; - const cond_extra = air.extraData(Air.CondBr, inst_data.payload); - if (inst_data.operand == operand_ref and operandDies(l, body[0], 0)) - return .tomb; - - if (cond_extra.data.then_body_len > 2 or cond_extra.data.else_body_len > 2) - return .complex; - - const then_body: []const Air.Inst.Index = @ptrCast(air.extra[cond_extra.end..][0..cond_extra.data.then_body_len]); - const else_body: []const Air.Inst.Index = @ptrCast(air.extra[cond_extra.end + cond_extra.data.then_body_len ..][0..cond_extra.data.else_body_len]); - if (then_body.len > 1 and air_tags[@intFromEnum(then_body[1])] != .unreach) - return .complex; - if (else_body.len > 1 and air_tags[@intFromEnum(else_body[1])] != .unreach) - return .complex; - - var operand_live: bool = true; - for (&[_]Air.Inst.Index{ then_body[0], else_body[0] }) |cond_inst| { - if (l.categorizeOperand(air, cond_inst, operand, ip) == .tomb) - operand_live = false; - - switch (air_tags[@intFromEnum(cond_inst)]) { - .br => { // Breaks immediately back to block - const br = air_datas[@intFromEnum(cond_inst)].br; - if (br.block_inst != inst) - return .complex; - }, - .call => {}, // Calls a noreturn function - else => return .complex, - } - } - return if (operand_live) .none else .tomb; - } - - return .complex; - }, - - .@"try", - .try_cold, - .try_ptr, - .try_ptr_cold, - .loop, - .cond_br, - .switch_br, - .loop_switch_br, - => return .complex, - - .wasm_memory_grow => { - const pl_op = air_datas[@intFromEnum(inst)].pl_op; - if (pl_op.operand == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none); - return .none; - }, - } -} - -fn matchOperandSmallIndex( - l: Liveness, - inst: Air.Inst.Index, - operand: OperandInt, - default: OperandCategory, -) OperandCategory { - if (operandDies(l, inst, operand)) { - return .tomb; - } else { - return default; - } -} - -/// Higher level API. -pub const CondBrSlices = struct { - then_deaths: []const Air.Inst.Index, - else_deaths: []const Air.Inst.Index, -}; - -pub fn getCondBr(l: Liveness, inst: Air.Inst.Index) CondBrSlices { - var index: usize = l.special.get(inst) orelse return .{ - .then_deaths = &.{}, - .else_deaths = &.{}, - }; - const then_death_count = l.extra[index]; - index += 1; - const else_death_count = l.extra[index]; - index += 1; - const then_deaths: []const Air.Inst.Index = @ptrCast(l.extra[index..][0..then_death_count]); - index += then_death_count; - return .{ - .then_deaths = then_deaths, - .else_deaths = @ptrCast(l.extra[index..][0..else_death_count]), - }; -} - -/// Indexed by case number as they appear in AIR. -/// Else is the last element. -pub const SwitchBrTable = struct { - deaths: []const []const Air.Inst.Index, -}; - -/// Caller owns the memory. -pub fn getSwitchBr(l: Liveness, gpa: Allocator, inst: Air.Inst.Index, cases_len: u32) Allocator.Error!SwitchBrTable { - var index: usize = l.special.get(inst) orelse return .{ .deaths = &.{} }; - const else_death_count = l.extra[index]; - index += 1; - - var deaths = try gpa.alloc([]const Air.Inst.Index, cases_len); - errdefer gpa.free(deaths); - - var case_i: u32 = 0; - while (case_i < cases_len - 1) : (case_i += 1) { - const case_death_count: u32 = l.extra[index]; - index += 1; - deaths[case_i] = @ptrCast(l.extra[index..][0..case_death_count]); - index += case_death_count; - } - { - // Else - deaths[case_i] = @ptrCast(l.extra[index..][0..else_death_count]); - } - return .{ .deaths = deaths }; -} - -/// Note that this information is technically redundant, but is useful for -/// backends nonetheless: see `Block`. -pub const BlockSlices = struct { - deaths: []const Air.Inst.Index, -}; - -pub fn getBlock(l: Liveness, inst: Air.Inst.Index) BlockSlices { - const index: usize = l.special.get(inst) orelse return .{ - .deaths = &.{}, - }; - const death_count = l.extra[index]; - const deaths: []const Air.Inst.Index = @ptrCast(l.extra[index + 1 ..][0..death_count]); - return .{ - .deaths = deaths, - }; -} - -pub const LoopSlice = struct { - deaths: []const Air.Inst.Index, -}; - -pub fn deinit(l: *Liveness, gpa: Allocator) void { - gpa.free(l.tomb_bits); - gpa.free(l.extra); - l.special.deinit(gpa); - l.* = undefined; -} - -pub fn iterateBigTomb(l: Liveness, inst: Air.Inst.Index) BigTomb { - return .{ - .tomb_bits = l.getTombBits(inst), - .extra_start = l.special.get(inst) orelse 0, - .extra_offset = 0, - .extra = l.extra, - .bit_index = 0, - .reached_end = false, - }; -} - -/// How many tomb bits per AIR instruction. -pub const bpi = 4; -pub const Bpi = std.meta.Int(.unsigned, bpi); -pub const OperandInt = std.math.Log2Int(Bpi); - -/// Useful for decoders of Liveness information. -pub const BigTomb = struct { - tomb_bits: Liveness.Bpi, - bit_index: u32, - extra_start: u32, - extra_offset: u32, - extra: []const u32, - reached_end: bool, - - /// Returns whether the next operand dies. - pub fn feed(bt: *BigTomb) bool { - if (bt.reached_end) return false; - - const this_bit_index = bt.bit_index; - bt.bit_index += 1; - - const small_tombs = bpi - 1; - if (this_bit_index < small_tombs) { - const dies = @as(u1, @truncate(bt.tomb_bits >> @as(Liveness.OperandInt, @intCast(this_bit_index)))) != 0; - return dies; - } - - const big_bit_index = this_bit_index - small_tombs; - while (big_bit_index - bt.extra_offset * 31 >= 31) { - if (@as(u1, @truncate(bt.extra[bt.extra_start + bt.extra_offset] >> 31)) != 0) { - bt.reached_end = true; - return false; - } - bt.extra_offset += 1; - } - const dies = @as(u1, @truncate(bt.extra[bt.extra_start + bt.extra_offset] >> - @as(u5, @intCast(big_bit_index - bt.extra_offset * 31)))) != 0; - return dies; - } -}; - -/// In-progress data; on successful analysis converted into `Liveness`. -const Analysis = struct { - gpa: Allocator, - air: Air, - intern_pool: *InternPool, - tomb_bits: []usize, - special: std.AutoHashMapUnmanaged(Air.Inst.Index, u32), - extra: std.ArrayListUnmanaged(u32), - - fn addExtra(a: *Analysis, extra: anytype) Allocator.Error!u32 { - const fields = std.meta.fields(@TypeOf(extra)); - try a.extra.ensureUnusedCapacity(a.gpa, fields.len); - return addExtraAssumeCapacity(a, extra); - } - - fn addExtraAssumeCapacity(a: *Analysis, extra: anytype) u32 { - const fields = std.meta.fields(@TypeOf(extra)); - const result = @as(u32, @intCast(a.extra.items.len)); - inline for (fields) |field| { - a.extra.appendAssumeCapacity(switch (field.type) { - u32 => @field(extra, field.name), - else => @compileError("bad field type"), - }); - } - return result; - } -}; - -fn analyzeBody( - a: *Analysis, - comptime pass: LivenessPass, - data: *LivenessPassData(pass), - body: []const Air.Inst.Index, -) Allocator.Error!void { - var i: usize = body.len; - while (i != 0) { - i -= 1; - const inst = body[i]; - try analyzeInst(a, pass, data, inst); - } -} - -fn analyzeInst( - a: *Analysis, - comptime pass: LivenessPass, - data: *LivenessPassData(pass), - inst: Air.Inst.Index, -) Allocator.Error!void { - const ip = a.intern_pool; - const inst_tags = a.air.instructions.items(.tag); - const inst_datas = a.air.instructions.items(.data); - - switch (inst_tags[@intFromEnum(inst)]) { - .add, - .add_safe, - .add_optimized, - .add_wrap, - .add_sat, - .sub, - .sub_safe, - .sub_optimized, - .sub_wrap, - .sub_sat, - .mul, - .mul_safe, - .mul_optimized, - .mul_wrap, - .mul_sat, - .div_float, - .div_float_optimized, - .div_trunc, - .div_trunc_optimized, - .div_floor, - .div_floor_optimized, - .div_exact, - .div_exact_optimized, - .rem, - .rem_optimized, - .mod, - .mod_optimized, - .bit_and, - .bit_or, - .xor, - .cmp_lt, - .cmp_lt_optimized, - .cmp_lte, - .cmp_lte_optimized, - .cmp_eq, - .cmp_eq_optimized, - .cmp_gte, - .cmp_gte_optimized, - .cmp_gt, - .cmp_gt_optimized, - .cmp_neq, - .cmp_neq_optimized, - .bool_and, - .bool_or, - .store, - .store_safe, - .array_elem_val, - .slice_elem_val, - .ptr_elem_val, - .shl, - .shl_exact, - .shl_sat, - .shr, - .shr_exact, - .atomic_store_unordered, - .atomic_store_monotonic, - .atomic_store_release, - .atomic_store_seq_cst, - .set_union_tag, - .min, - .max, - .memset, - .memset_safe, - .memcpy, - .memmove, - => { - const o = inst_datas[@intFromEnum(inst)].bin_op; - return analyzeOperands(a, pass, data, inst, .{ o.lhs, o.rhs, .none }); - }, - - .vector_store_elem => { - const o = inst_datas[@intFromEnum(inst)].vector_store_elem; - const extra = a.air.extraData(Air.Bin, o.payload).data; - return analyzeOperands(a, pass, data, inst, .{ o.vector_ptr, extra.lhs, extra.rhs }); - }, - - .arg, - .alloc, - .ret_ptr, - .breakpoint, - .dbg_stmt, - .dbg_empty_stmt, - .ret_addr, - .frame_addr, - .wasm_memory_size, - .err_return_trace, - .save_err_return_trace_index, - .tlv_dllimport_ptr, - .c_va_start, - .work_item_id, - .work_group_size, - .work_group_id, - => return analyzeOperands(a, pass, data, inst, .{ .none, .none, .none }), - - .inferred_alloc, .inferred_alloc_comptime => unreachable, - - .trap, - .unreach, - => return analyzeFuncEnd(a, pass, data, inst, .{ .none, .none, .none }), - - .not, - .bitcast, - .load, - .fpext, - .fptrunc, - .intcast, - .intcast_safe, - .trunc, - .optional_payload, - .optional_payload_ptr, - .optional_payload_ptr_set, - .errunion_payload_ptr_set, - .wrap_optional, - .unwrap_errunion_payload, - .unwrap_errunion_err, - .unwrap_errunion_payload_ptr, - .unwrap_errunion_err_ptr, - .wrap_errunion_payload, - .wrap_errunion_err, - .slice_ptr, - .slice_len, - .ptr_slice_len_ptr, - .ptr_slice_ptr_ptr, - .struct_field_ptr_index_0, - .struct_field_ptr_index_1, - .struct_field_ptr_index_2, - .struct_field_ptr_index_3, - .array_to_slice, - .int_from_float, - .int_from_float_optimized, - .float_from_int, - .get_union_tag, - .clz, - .ctz, - .popcount, - .byte_swap, - .bit_reverse, - .splat, - .error_set_has_value, - .addrspace_cast, - .c_va_arg, - .c_va_copy, - .abs, - => { - const o = inst_datas[@intFromEnum(inst)].ty_op; - return analyzeOperands(a, pass, data, inst, .{ o.operand, .none, .none }); - }, - - .is_null, - .is_non_null, - .is_null_ptr, - .is_non_null_ptr, - .is_err, - .is_non_err, - .is_err_ptr, - .is_non_err_ptr, - .is_named_enum_value, - .tag_name, - .error_name, - .sqrt, - .sin, - .cos, - .tan, - .exp, - .exp2, - .log, - .log2, - .log10, - .floor, - .ceil, - .round, - .trunc_float, - .neg, - .neg_optimized, - .cmp_lt_errors_len, - .set_err_return_trace, - .c_va_end, - => { - const operand = inst_datas[@intFromEnum(inst)].un_op; - return analyzeOperands(a, pass, data, inst, .{ operand, .none, .none }); - }, - - .ret, - .ret_safe, - .ret_load, - => { - const operand = inst_datas[@intFromEnum(inst)].un_op; - return analyzeFuncEnd(a, pass, data, inst, .{ operand, .none, .none }); - }, - - .add_with_overflow, - .sub_with_overflow, - .mul_with_overflow, - .shl_with_overflow, - .ptr_add, - .ptr_sub, - .ptr_elem_ptr, - .slice_elem_ptr, - .slice, - => { - const ty_pl = inst_datas[@intFromEnum(inst)].ty_pl; - const extra = a.air.extraData(Air.Bin, ty_pl.payload).data; - return analyzeOperands(a, pass, data, inst, .{ extra.lhs, extra.rhs, .none }); - }, - - .dbg_var_ptr, - .dbg_var_val, - .dbg_arg_inline, - => { - const operand = inst_datas[@intFromEnum(inst)].pl_op.operand; - return analyzeOperands(a, pass, data, inst, .{ operand, .none, .none }); - }, - - .prefetch => { - const prefetch = inst_datas[@intFromEnum(inst)].prefetch; - return analyzeOperands(a, pass, data, inst, .{ prefetch.ptr, .none, .none }); - }, - - .call, .call_always_tail, .call_never_tail, .call_never_inline => { - const inst_data = inst_datas[@intFromEnum(inst)].pl_op; - const callee = inst_data.operand; - const extra = a.air.extraData(Air.Call, inst_data.payload); - const args = @as([]const Air.Inst.Ref, @ptrCast(a.air.extra[extra.end..][0..extra.data.args_len])); - if (args.len + 1 <= bpi - 1) { - var buf = [1]Air.Inst.Ref{.none} ** (bpi - 1); - buf[0] = callee; - @memcpy(buf[1..][0..args.len], args); - return analyzeOperands(a, pass, data, inst, buf); - } - - var big = try AnalyzeBigOperands(pass).init(a, data, inst, args.len + 1); - defer big.deinit(); - var i: usize = args.len; - while (i > 0) { - i -= 1; - try big.feed(args[i]); - } - try big.feed(callee); - return big.finish(); - }, - .select => { - const pl_op = inst_datas[@intFromEnum(inst)].pl_op; - const extra = a.air.extraData(Air.Bin, pl_op.payload).data; - return analyzeOperands(a, pass, data, inst, .{ pl_op.operand, extra.lhs, extra.rhs }); - }, - .shuffle => { - const extra = a.air.extraData(Air.Shuffle, inst_datas[@intFromEnum(inst)].ty_pl.payload).data; - return analyzeOperands(a, pass, data, inst, .{ extra.a, extra.b, .none }); - }, - .reduce, .reduce_optimized => { - const reduce = inst_datas[@intFromEnum(inst)].reduce; - return analyzeOperands(a, pass, data, inst, .{ reduce.operand, .none, .none }); - }, - .cmp_vector, .cmp_vector_optimized => { - const extra = a.air.extraData(Air.VectorCmp, inst_datas[@intFromEnum(inst)].ty_pl.payload).data; - return analyzeOperands(a, pass, data, inst, .{ extra.lhs, extra.rhs, .none }); - }, - .aggregate_init => { - const ty_pl = inst_datas[@intFromEnum(inst)].ty_pl; - const aggregate_ty = ty_pl.ty.toType(); - const len = @as(usize, @intCast(aggregate_ty.arrayLenIp(ip))); - const elements = @as([]const Air.Inst.Ref, @ptrCast(a.air.extra[ty_pl.payload..][0..len])); - - if (elements.len <= bpi - 1) { - var buf = [1]Air.Inst.Ref{.none} ** (bpi - 1); - @memcpy(buf[0..elements.len], elements); - return analyzeOperands(a, pass, data, inst, buf); - } - - var big = try AnalyzeBigOperands(pass).init(a, data, inst, elements.len); - defer big.deinit(); - var i: usize = elements.len; - while (i > 0) { - i -= 1; - try big.feed(elements[i]); - } - return big.finish(); - }, - .union_init => { - const extra = a.air.extraData(Air.UnionInit, inst_datas[@intFromEnum(inst)].ty_pl.payload).data; - return analyzeOperands(a, pass, data, inst, .{ extra.init, .none, .none }); - }, - .struct_field_ptr, .struct_field_val => { - const extra = a.air.extraData(Air.StructField, inst_datas[@intFromEnum(inst)].ty_pl.payload).data; - return analyzeOperands(a, pass, data, inst, .{ extra.struct_operand, .none, .none }); - }, - .field_parent_ptr => { - const extra = a.air.extraData(Air.FieldParentPtr, inst_datas[@intFromEnum(inst)].ty_pl.payload).data; - return analyzeOperands(a, pass, data, inst, .{ extra.field_ptr, .none, .none }); - }, - .cmpxchg_strong, .cmpxchg_weak => { - const extra = a.air.extraData(Air.Cmpxchg, inst_datas[@intFromEnum(inst)].ty_pl.payload).data; - return analyzeOperands(a, pass, data, inst, .{ extra.ptr, extra.expected_value, extra.new_value }); - }, - .mul_add => { - const pl_op = inst_datas[@intFromEnum(inst)].pl_op; - const extra = a.air.extraData(Air.Bin, pl_op.payload).data; - return analyzeOperands(a, pass, data, inst, .{ extra.lhs, extra.rhs, pl_op.operand }); - }, - .atomic_load => { - const ptr = inst_datas[@intFromEnum(inst)].atomic_load.ptr; - return analyzeOperands(a, pass, data, inst, .{ ptr, .none, .none }); - }, - .atomic_rmw => { - const pl_op = inst_datas[@intFromEnum(inst)].pl_op; - const extra = a.air.extraData(Air.AtomicRmw, pl_op.payload).data; - return analyzeOperands(a, pass, data, inst, .{ pl_op.operand, extra.operand, .none }); - }, - - .br => return analyzeInstBr(a, pass, data, inst), - .repeat => return analyzeInstRepeat(a, pass, data, inst), - .switch_dispatch => return analyzeInstSwitchDispatch(a, pass, data, inst), - - .assembly => { - const extra = a.air.extraData(Air.Asm, inst_datas[@intFromEnum(inst)].ty_pl.payload); - var extra_i: usize = extra.end; - const outputs = @as([]const Air.Inst.Ref, @ptrCast(a.air.extra[extra_i..][0..extra.data.outputs_len])); - extra_i += outputs.len; - const inputs = @as([]const Air.Inst.Ref, @ptrCast(a.air.extra[extra_i..][0..extra.data.inputs_len])); - extra_i += inputs.len; - - const num_operands = simple: { - var buf = [1]Air.Inst.Ref{.none} ** (bpi - 1); - var buf_index: usize = 0; - for (outputs) |output| { - if (output != .none) { - if (buf_index < buf.len) buf[buf_index] = output; - buf_index += 1; - } - } - if (buf_index + inputs.len > buf.len) { - break :simple buf_index + inputs.len; - } - @memcpy(buf[buf_index..][0..inputs.len], inputs); - return analyzeOperands(a, pass, data, inst, buf); - }; - - var big = try AnalyzeBigOperands(pass).init(a, data, inst, num_operands); - defer big.deinit(); - var i: usize = inputs.len; - while (i > 0) { - i -= 1; - try big.feed(inputs[i]); - } - i = outputs.len; - while (i > 0) { - i -= 1; - if (outputs[i] != .none) { - try big.feed(outputs[i]); - } - } - return big.finish(); - }, - - inline .block, .dbg_inline_block => |comptime_tag| { - const ty_pl = inst_datas[@intFromEnum(inst)].ty_pl; - const extra = a.air.extraData(switch (comptime_tag) { - .block => Air.Block, - .dbg_inline_block => Air.DbgInlineBlock, - else => unreachable, - }, ty_pl.payload); - return analyzeInstBlock(a, pass, data, inst, ty_pl.ty, @ptrCast(a.air.extra[extra.end..][0..extra.data.body_len])); - }, - .loop => return analyzeInstLoop(a, pass, data, inst), - - .@"try", .try_cold => return analyzeInstCondBr(a, pass, data, inst, .@"try"), - .try_ptr, .try_ptr_cold => return analyzeInstCondBr(a, pass, data, inst, .try_ptr), - .cond_br => return analyzeInstCondBr(a, pass, data, inst, .cond_br), - .switch_br => return analyzeInstSwitchBr(a, pass, data, inst, false), - .loop_switch_br => return analyzeInstSwitchBr(a, pass, data, inst, true), - - .wasm_memory_grow => { - const pl_op = inst_datas[@intFromEnum(inst)].pl_op; - return analyzeOperands(a, pass, data, inst, .{ pl_op.operand, .none, .none }); - }, - } -} - -/// Every instruction should hit this (after handling any nested bodies), in every pass. In the -/// initial pass, it is responsible for marking deaths of the (first three) operands and noticing -/// immediate deaths. -fn analyzeOperands( - a: *Analysis, - comptime pass: LivenessPass, - data: *LivenessPassData(pass), - inst: Air.Inst.Index, - operands: [bpi - 1]Air.Inst.Ref, -) Allocator.Error!void { - const gpa = a.gpa; - const ip = a.intern_pool; - - switch (pass) { - .loop_analysis => { - _ = data.live_set.remove(inst); - - for (operands) |op_ref| { - const operand = op_ref.toIndexAllowNone() orelse continue; - _ = try data.live_set.put(gpa, operand, {}); - } - }, - - .main_analysis => { - const usize_index = (@intFromEnum(inst) * bpi) / @bitSizeOf(usize); - - // This logic must synchronize with `will_die_immediately` in `AnalyzeBigOperands.init`. - const immediate_death = if (data.live_set.remove(inst)) blk: { - log.debug("[{}] %{}: removed from live set", .{ pass, @intFromEnum(inst) }); - break :blk false; - } else blk: { - log.debug("[{}] %{}: immediate death", .{ pass, @intFromEnum(inst) }); - break :blk true; - }; - - var tomb_bits: Bpi = @as(Bpi, @intFromBool(immediate_death)) << (bpi - 1); - - // If our result is unused and the instruction doesn't need to be lowered, backends will - // skip the lowering of this instruction, so we don't want to record uses of operands. - // That way, we can mark as many instructions as possible unused. - if (!immediate_death or a.air.mustLower(inst, ip)) { - // Note that it's important we iterate over the operands backwards, so that if a dying - // operand is used multiple times we mark its last use as its death. - var i = operands.len; - while (i > 0) { - i -= 1; - const op_ref = operands[i]; - const operand = op_ref.toIndexAllowNone() orelse continue; - - const mask = @as(Bpi, 1) << @as(OperandInt, @intCast(i)); - - if ((try data.live_set.fetchPut(gpa, operand, {})) == null) { - log.debug("[{}] %{}: added %{} to live set (operand dies here)", .{ pass, @intFromEnum(inst), operand }); - tomb_bits |= mask; - } - } - } - - a.tomb_bits[usize_index] |= @as(usize, tomb_bits) << - @as(Log2Int(usize), @intCast((@intFromEnum(inst) % (@bitSizeOf(usize) / bpi)) * bpi)); - }, - } -} - -/// Like `analyzeOperands`, but for an instruction which returns from a function, so should -/// effectively kill every remaining live value other than its operands. -fn analyzeFuncEnd( - a: *Analysis, - comptime pass: LivenessPass, - data: *LivenessPassData(pass), - inst: Air.Inst.Index, - operands: [bpi - 1]Air.Inst.Ref, -) Allocator.Error!void { - switch (pass) { - .loop_analysis => { - // No operands need to be alive if we're returning from the function, so we don't need - // to touch `breaks` here even though this is sort of like a break to the top level. - }, - - .main_analysis => { - data.live_set.clearRetainingCapacity(); - }, - } - - return analyzeOperands(a, pass, data, inst, operands); -} - -fn analyzeInstBr( - a: *Analysis, - comptime pass: LivenessPass, - data: *LivenessPassData(pass), - inst: Air.Inst.Index, -) !void { - const inst_datas = a.air.instructions.items(.data); - const br = inst_datas[@intFromEnum(inst)].br; - const gpa = a.gpa; - - switch (pass) { - .loop_analysis => { - try data.breaks.put(gpa, br.block_inst, {}); - }, - - .main_analysis => { - const block_scope = data.block_scopes.get(br.block_inst).?; // we should always be breaking from an enclosing block - - const new_live_set = try block_scope.live_set.clone(gpa); - data.live_set.deinit(gpa); - data.live_set = new_live_set; - }, - } - - return analyzeOperands(a, pass, data, inst, .{ br.operand, .none, .none }); -} - -fn analyzeInstRepeat( - a: *Analysis, - comptime pass: LivenessPass, - data: *LivenessPassData(pass), - inst: Air.Inst.Index, -) !void { - const inst_datas = a.air.instructions.items(.data); - const repeat = inst_datas[@intFromEnum(inst)].repeat; - const gpa = a.gpa; - - switch (pass) { - .loop_analysis => { - try data.breaks.put(gpa, repeat.loop_inst, {}); - }, - - .main_analysis => { - const block_scope = data.block_scopes.get(repeat.loop_inst).?; // we should always be repeating an enclosing loop - - const new_live_set = try block_scope.live_set.clone(gpa); - data.live_set.deinit(gpa); - data.live_set = new_live_set; - }, - } - - return analyzeOperands(a, pass, data, inst, .{ .none, .none, .none }); -} - -fn analyzeInstSwitchDispatch( - a: *Analysis, - comptime pass: LivenessPass, - data: *LivenessPassData(pass), - inst: Air.Inst.Index, -) !void { - // This happens to be identical to `analyzeInstBr`, but is separated anyway for clarity. - - const inst_datas = a.air.instructions.items(.data); - const br = inst_datas[@intFromEnum(inst)].br; - const gpa = a.gpa; - - switch (pass) { - .loop_analysis => { - try data.breaks.put(gpa, br.block_inst, {}); - }, - - .main_analysis => { - const block_scope = data.block_scopes.get(br.block_inst).?; // we should always be repeating an enclosing loop - - const new_live_set = try block_scope.live_set.clone(gpa); - data.live_set.deinit(gpa); - data.live_set = new_live_set; - }, - } - - return analyzeOperands(a, pass, data, inst, .{ br.operand, .none, .none }); -} - -fn analyzeInstBlock( - a: *Analysis, - comptime pass: LivenessPass, - data: *LivenessPassData(pass), - inst: Air.Inst.Index, - ty: Air.Inst.Ref, - body: []const Air.Inst.Index, -) !void { - const gpa = a.gpa; - - // We actually want to do `analyzeOperands` *first*, since our result logically doesn't - // exist until the block body ends (and we're iterating backwards) - try analyzeOperands(a, pass, data, inst, .{ .none, .none, .none }); - - switch (pass) { - .loop_analysis => { - try analyzeBody(a, pass, data, body); - _ = data.breaks.remove(inst); - }, - - .main_analysis => { - log.debug("[{}] %{}: block live set is {}", .{ pass, inst, fmtInstSet(&data.live_set) }); - // We can move the live set because the body should have a noreturn - // instruction which overrides the set. - try data.block_scopes.put(gpa, inst, .{ - .live_set = data.live_set.move(), - }); - defer { - log.debug("[{}] %{}: popped block scope", .{ pass, inst }); - var scope = data.block_scopes.fetchRemove(inst).?.value; - scope.live_set.deinit(gpa); - } - - log.debug("[{}] %{}: pushed new block scope", .{ pass, inst }); - try analyzeBody(a, pass, data, body); - - // If the block is noreturn, block deaths not only aren't useful, they're impossible to - // find: there could be more stuff alive after the block than before it! - if (!a.intern_pool.isNoReturn(ty.toType().toIntern())) { - // The block kills the difference in the live sets - const block_scope = data.block_scopes.get(inst).?; - const num_deaths = data.live_set.count() - block_scope.live_set.count(); - - try a.extra.ensureUnusedCapacity(gpa, num_deaths + std.meta.fields(Block).len); - const extra_index = a.addExtraAssumeCapacity(Block{ - .death_count = num_deaths, - }); - - var measured_num: u32 = 0; - var it = data.live_set.keyIterator(); - while (it.next()) |key| { - const alive = key.*; - if (!block_scope.live_set.contains(alive)) { - // Dies in block - a.extra.appendAssumeCapacity(@intFromEnum(alive)); - measured_num += 1; - } - } - assert(measured_num == num_deaths); // post-live-set should be a subset of pre-live-set - try a.special.put(gpa, inst, extra_index); - log.debug("[{}] %{}: block deaths are {}", .{ - pass, - inst, - fmtInstList(@ptrCast(a.extra.items[extra_index + 1 ..][0..num_deaths])), - }); - } - }, - } -} - -fn writeLoopInfo( - a: *Analysis, - data: *LivenessPassData(.loop_analysis), - inst: Air.Inst.Index, - old_breaks: std.AutoHashMapUnmanaged(Air.Inst.Index, void), - old_live: std.AutoHashMapUnmanaged(Air.Inst.Index, void), -) !void { - const gpa = a.gpa; - - // `loop`s are guaranteed to have at least one matching `repeat`. - // Similarly, `loop_switch_br`s have a matching `switch_dispatch`. - // However, we no longer care about repeats of this loop for resolving - // which operands must live within it. - assert(data.breaks.remove(inst)); - - const extra_index: u32 = @intCast(a.extra.items.len); - - const num_breaks = data.breaks.count(); - try a.extra.ensureUnusedCapacity(gpa, 1 + num_breaks); - - a.extra.appendAssumeCapacity(num_breaks); - - var it = data.breaks.keyIterator(); - while (it.next()) |key| { - const block_inst = key.*; - a.extra.appendAssumeCapacity(@intFromEnum(block_inst)); - } - log.debug("[{}] %{}: includes breaks to {}", .{ LivenessPass.loop_analysis, inst, fmtInstSet(&data.breaks) }); - - // Now we put the live operands from the loop body in too - const num_live = data.live_set.count(); - try a.extra.ensureUnusedCapacity(gpa, 1 + num_live); - - a.extra.appendAssumeCapacity(num_live); - it = data.live_set.keyIterator(); - while (it.next()) |key| { - const alive = key.*; - a.extra.appendAssumeCapacity(@intFromEnum(alive)); - } - log.debug("[{}] %{}: maintain liveness of {}", .{ LivenessPass.loop_analysis, inst, fmtInstSet(&data.live_set) }); - - try a.special.put(gpa, inst, extra_index); - - // Add back operands which were previously alive - it = old_live.keyIterator(); - while (it.next()) |key| { - const alive = key.*; - try data.live_set.put(gpa, alive, {}); - } - - // And the same for breaks - it = old_breaks.keyIterator(); - while (it.next()) |key| { - const block_inst = key.*; - try data.breaks.put(gpa, block_inst, {}); - } -} - -/// When analyzing a loop in the main pass, sets up `data.live_set` to be the set -/// of operands known to be alive when the loop repeats. -fn resolveLoopLiveSet( - a: *Analysis, - data: *LivenessPassData(.main_analysis), - inst: Air.Inst.Index, -) !void { - const gpa = a.gpa; - - const extra_idx = a.special.fetchRemove(inst).?.value; - const num_breaks = data.old_extra.items[extra_idx]; - const breaks: []const Air.Inst.Index = @ptrCast(data.old_extra.items[extra_idx + 1 ..][0..num_breaks]); - - const num_loop_live = data.old_extra.items[extra_idx + num_breaks + 1]; - const loop_live: []const Air.Inst.Index = @ptrCast(data.old_extra.items[extra_idx + num_breaks + 2 ..][0..num_loop_live]); - - // This is necessarily not in the same control flow branch, because loops are noreturn - data.live_set.clearRetainingCapacity(); - - try data.live_set.ensureUnusedCapacity(gpa, @intCast(loop_live.len)); - for (loop_live) |alive| data.live_set.putAssumeCapacity(alive, {}); - - log.debug("[{}] %{}: block live set is {}", .{ LivenessPass.main_analysis, inst, fmtInstSet(&data.live_set) }); - - for (breaks) |block_inst| { - // We might break to this block, so include every operand that the block needs alive - const block_scope = data.block_scopes.get(block_inst).?; - - var it = block_scope.live_set.keyIterator(); - while (it.next()) |key| { - const alive = key.*; - try data.live_set.put(gpa, alive, {}); - } - } - - log.debug("[{}] %{}: loop live set is {}", .{ LivenessPass.main_analysis, inst, fmtInstSet(&data.live_set) }); -} - -fn analyzeInstLoop( - a: *Analysis, - comptime pass: LivenessPass, - data: *LivenessPassData(pass), - inst: Air.Inst.Index, -) !void { - const inst_datas = a.air.instructions.items(.data); - const extra = a.air.extraData(Air.Block, inst_datas[@intFromEnum(inst)].ty_pl.payload); - const body: []const Air.Inst.Index = @ptrCast(a.air.extra[extra.end..][0..extra.data.body_len]); - const gpa = a.gpa; - - try analyzeOperands(a, pass, data, inst, .{ .none, .none, .none }); - - switch (pass) { - .loop_analysis => { - var old_breaks = data.breaks.move(); - defer old_breaks.deinit(gpa); - - var old_live = data.live_set.move(); - defer old_live.deinit(gpa); - - try analyzeBody(a, pass, data, body); - - try writeLoopInfo(a, data, inst, old_breaks, old_live); - }, - - .main_analysis => { - try resolveLoopLiveSet(a, data, inst); - - // Now, `data.live_set` is the operands which must be alive when the loop repeats. - // Move them into a block scope for corresponding `repeat` instructions to notice. - try data.block_scopes.putNoClobber(gpa, inst, .{ - .live_set = data.live_set.move(), - }); - defer { - log.debug("[{}] %{}: popped loop block scop", .{ pass, inst }); - var scope = data.block_scopes.fetchRemove(inst).?.value; - scope.live_set.deinit(gpa); - } - try analyzeBody(a, pass, data, body); - }, - } -} - -/// Despite its name, this function is used for analysis of not only `cond_br` instructions, but -/// also `try` and `try_ptr`, which are highly related. The `inst_type` parameter indicates which -/// type of instruction `inst` points to. -fn analyzeInstCondBr( - a: *Analysis, - comptime pass: LivenessPass, - data: *LivenessPassData(pass), - inst: Air.Inst.Index, - comptime inst_type: enum { cond_br, @"try", try_ptr }, -) !void { - const inst_datas = a.air.instructions.items(.data); - const gpa = a.gpa; - - const extra = switch (inst_type) { - .cond_br => a.air.extraData(Air.CondBr, inst_datas[@intFromEnum(inst)].pl_op.payload), - .@"try" => a.air.extraData(Air.Try, inst_datas[@intFromEnum(inst)].pl_op.payload), - .try_ptr => a.air.extraData(Air.TryPtr, inst_datas[@intFromEnum(inst)].ty_pl.payload), - }; - - const condition = switch (inst_type) { - .cond_br, .@"try" => inst_datas[@intFromEnum(inst)].pl_op.operand, - .try_ptr => extra.data.ptr, - }; - - const then_body: []const Air.Inst.Index = switch (inst_type) { - .cond_br => @ptrCast(a.air.extra[extra.end..][0..extra.data.then_body_len]), - else => &.{}, // we won't use this - }; - - const else_body: []const Air.Inst.Index = @ptrCast(switch (inst_type) { - .cond_br => a.air.extra[extra.end + then_body.len ..][0..extra.data.else_body_len], - .@"try", .try_ptr => a.air.extra[extra.end..][0..extra.data.body_len], - }); - - switch (pass) { - .loop_analysis => { - switch (inst_type) { - .cond_br => try analyzeBody(a, pass, data, then_body), - .@"try", .try_ptr => {}, - } - try analyzeBody(a, pass, data, else_body); - }, - - .main_analysis => { - switch (inst_type) { - .cond_br => try analyzeBody(a, pass, data, then_body), - .@"try", .try_ptr => {}, // The "then body" is just the remainder of this block - } - var then_live = data.live_set.move(); - defer then_live.deinit(gpa); - - try analyzeBody(a, pass, data, else_body); - var else_live = data.live_set.move(); - defer else_live.deinit(gpa); - - // Operands which are alive in one branch but not the other need to die at the start of - // the peer branch. - - var then_mirrored_deaths: std.ArrayListUnmanaged(Air.Inst.Index) = .empty; - defer then_mirrored_deaths.deinit(gpa); - - var else_mirrored_deaths: std.ArrayListUnmanaged(Air.Inst.Index) = .empty; - defer else_mirrored_deaths.deinit(gpa); - - // Note: this invalidates `else_live`, but expands `then_live` to be their union - { - var it = then_live.keyIterator(); - while (it.next()) |key| { - const death = key.*; - if (else_live.remove(death)) continue; // removing makes the loop below faster - - // If this is a `try`, the "then body" (rest of the branch) might have - // referenced our result. We want to avoid killing this value in the else branch - // if that's the case, since it only exists in the (fake) then branch. - switch (inst_type) { - .cond_br => {}, - .@"try", .try_ptr => if (death == inst) continue, - } - - try else_mirrored_deaths.append(gpa, death); - } - // Since we removed common stuff above, `else_live` is now only operands - // which are *only* alive in the else branch - it = else_live.keyIterator(); - while (it.next()) |key| { - const death = key.*; - try then_mirrored_deaths.append(gpa, death); - // Make `then_live` contain the full live set (i.e. union of both) - try then_live.put(gpa, death, {}); - } - } - - log.debug("[{}] %{}: 'then' branch mirrored deaths are {}", .{ pass, inst, fmtInstList(then_mirrored_deaths.items) }); - log.debug("[{}] %{}: 'else' branch mirrored deaths are {}", .{ pass, inst, fmtInstList(else_mirrored_deaths.items) }); - - data.live_set.deinit(gpa); - data.live_set = then_live.move(); // Really the union of both live sets - - log.debug("[{}] %{}: new live set is {}", .{ pass, inst, fmtInstSet(&data.live_set) }); - - // Write the mirrored deaths to `extra` - const then_death_count = @as(u32, @intCast(then_mirrored_deaths.items.len)); - const else_death_count = @as(u32, @intCast(else_mirrored_deaths.items.len)); - try a.extra.ensureUnusedCapacity(gpa, std.meta.fields(CondBr).len + then_death_count + else_death_count); - const extra_index = a.addExtraAssumeCapacity(CondBr{ - .then_death_count = then_death_count, - .else_death_count = else_death_count, - }); - a.extra.appendSliceAssumeCapacity(@ptrCast(then_mirrored_deaths.items)); - a.extra.appendSliceAssumeCapacity(@ptrCast(else_mirrored_deaths.items)); - try a.special.put(gpa, inst, extra_index); - }, - } - - try analyzeOperands(a, pass, data, inst, .{ condition, .none, .none }); -} - -fn analyzeInstSwitchBr( - a: *Analysis, - comptime pass: LivenessPass, - data: *LivenessPassData(pass), - inst: Air.Inst.Index, - is_dispatch_loop: bool, -) !void { - const inst_datas = a.air.instructions.items(.data); - const pl_op = inst_datas[@intFromEnum(inst)].pl_op; - const condition = pl_op.operand; - const switch_br = a.air.unwrapSwitch(inst); - const gpa = a.gpa; - const ncases = switch_br.cases_len; - - switch (pass) { - .loop_analysis => { - var old_breaks: std.AutoHashMapUnmanaged(Air.Inst.Index, void) = .empty; - defer old_breaks.deinit(gpa); - - var old_live: std.AutoHashMapUnmanaged(Air.Inst.Index, void) = .empty; - defer old_live.deinit(gpa); - - if (is_dispatch_loop) { - old_breaks = data.breaks.move(); - old_live = data.live_set.move(); - } - - var it = switch_br.iterateCases(); - while (it.next()) |case| { - try analyzeBody(a, pass, data, case.body); - } - { // else - const else_body = it.elseBody(); - try analyzeBody(a, pass, data, else_body); - } - - if (is_dispatch_loop) { - try writeLoopInfo(a, data, inst, old_breaks, old_live); - } - }, - - .main_analysis => { - if (is_dispatch_loop) { - try resolveLoopLiveSet(a, data, inst); - try data.block_scopes.putNoClobber(gpa, inst, .{ - .live_set = data.live_set.move(), - }); - } - defer if (is_dispatch_loop) { - log.debug("[{}] %{}: popped loop block scop", .{ pass, inst }); - var scope = data.block_scopes.fetchRemove(inst).?.value; - scope.live_set.deinit(gpa); - }; - // This is, all in all, just a messier version of the `cond_br` logic. If you're trying - // to understand it, I encourage looking at `analyzeInstCondBr` first. - - const DeathSet = std.AutoHashMapUnmanaged(Air.Inst.Index, void); - const DeathList = std.ArrayListUnmanaged(Air.Inst.Index); - - var case_live_sets = try gpa.alloc(std.AutoHashMapUnmanaged(Air.Inst.Index, void), ncases + 1); // +1 for else - defer gpa.free(case_live_sets); - - @memset(case_live_sets, .{}); - defer for (case_live_sets) |*live_set| live_set.deinit(gpa); - - var case_it = switch_br.iterateCases(); - while (case_it.next()) |case| { - try analyzeBody(a, pass, data, case.body); - case_live_sets[case.idx] = data.live_set.move(); - } - { // else - const else_body = case_it.elseBody(); - try analyzeBody(a, pass, data, else_body); - case_live_sets[ncases] = data.live_set.move(); - } - - const mirrored_deaths = try gpa.alloc(DeathList, ncases + 1); - defer gpa.free(mirrored_deaths); - - @memset(mirrored_deaths, .{}); - defer for (mirrored_deaths) |*md| md.deinit(gpa); - - { - var all_alive: DeathSet = .{}; - defer all_alive.deinit(gpa); - - for (case_live_sets) |*live_set| { - try all_alive.ensureUnusedCapacity(gpa, live_set.count()); - var it = live_set.keyIterator(); - while (it.next()) |key| { - const alive = key.*; - all_alive.putAssumeCapacity(alive, {}); - } - } - - for (mirrored_deaths, case_live_sets) |*mirrored, *live_set| { - var it = all_alive.keyIterator(); - while (it.next()) |key| { - const alive = key.*; - if (!live_set.contains(alive)) { - // Should die at the start of this branch - try mirrored.append(gpa, alive); - } - } - } - - for (mirrored_deaths, 0..) |mirrored, i| { - log.debug("[{}] %{}: case {} mirrored deaths are {}", .{ pass, inst, i, fmtInstList(mirrored.items) }); - } - - data.live_set.deinit(gpa); - data.live_set = all_alive.move(); - - log.debug("[{}] %{}: new live set is {}", .{ pass, inst, fmtInstSet(&data.live_set) }); - } - - const else_death_count = @as(u32, @intCast(mirrored_deaths[ncases].items.len)); - const extra_index = try a.addExtra(SwitchBr{ - .else_death_count = else_death_count, - }); - for (mirrored_deaths[0..ncases]) |mirrored| { - const num = @as(u32, @intCast(mirrored.items.len)); - try a.extra.ensureUnusedCapacity(gpa, num + 1); - a.extra.appendAssumeCapacity(num); - a.extra.appendSliceAssumeCapacity(@ptrCast(mirrored.items)); - } - try a.extra.ensureUnusedCapacity(gpa, else_death_count); - a.extra.appendSliceAssumeCapacity(@ptrCast(mirrored_deaths[ncases].items)); - try a.special.put(gpa, inst, extra_index); - }, - } - - try analyzeOperands(a, pass, data, inst, .{ condition, .none, .none }); -} - -fn AnalyzeBigOperands(comptime pass: LivenessPass) type { - return struct { - a: *Analysis, - data: *LivenessPassData(pass), - inst: Air.Inst.Index, - - operands_remaining: u32, - small: [bpi - 1]Air.Inst.Ref = .{.none} ** (bpi - 1), - extra_tombs: []u32, - - // Only used in `LivenessPass.main_analysis` - will_die_immediately: bool, - - const Self = @This(); - - fn init( - a: *Analysis, - data: *LivenessPassData(pass), - inst: Air.Inst.Index, - total_operands: usize, - ) !Self { - const extra_operands = @as(u32, @intCast(total_operands)) -| (bpi - 1); - const max_extra_tombs = (extra_operands + 30) / 31; - - const extra_tombs: []u32 = switch (pass) { - .loop_analysis => &.{}, - .main_analysis => try a.gpa.alloc(u32, max_extra_tombs), - }; - errdefer a.gpa.free(extra_tombs); - - @memset(extra_tombs, 0); - - const will_die_immediately: bool = switch (pass) { - .loop_analysis => false, // track everything, since we don't have full liveness information yet - .main_analysis => !data.live_set.contains(inst), - }; - - return .{ - .a = a, - .data = data, - .inst = inst, - .operands_remaining = @as(u32, @intCast(total_operands)), - .extra_tombs = extra_tombs, - .will_die_immediately = will_die_immediately, - }; - } - - /// Must be called with operands in reverse order. - fn feed(big: *Self, op_ref: Air.Inst.Ref) !void { - const ip = big.a.intern_pool; - // Note that after this, `operands_remaining` becomes the index of the current operand - big.operands_remaining -= 1; - - if (big.operands_remaining < bpi - 1) { - big.small[big.operands_remaining] = op_ref; - return; - } - - const operand = op_ref.toIndex() orelse return; - - // If our result is unused and the instruction doesn't need to be lowered, backends will - // skip the lowering of this instruction, so we don't want to record uses of operands. - // That way, we can mark as many instructions as possible unused. - if (big.will_die_immediately and !big.a.air.mustLower(big.inst, ip)) return; - - const extra_byte = (big.operands_remaining - (bpi - 1)) / 31; - const extra_bit = @as(u5, @intCast(big.operands_remaining - (bpi - 1) - extra_byte * 31)); - - const gpa = big.a.gpa; - - switch (pass) { - .loop_analysis => { - _ = try big.data.live_set.put(gpa, operand, {}); - }, - - .main_analysis => { - if ((try big.data.live_set.fetchPut(gpa, operand, {})) == null) { - log.debug("[{}] %{}: added %{} to live set (operand dies here)", .{ pass, big.inst, operand }); - big.extra_tombs[extra_byte] |= @as(u32, 1) << extra_bit; - } - }, - } - } - - fn finish(big: *Self) !void { - const gpa = big.a.gpa; - - std.debug.assert(big.operands_remaining == 0); - - switch (pass) { - .loop_analysis => {}, - - .main_analysis => { - // Note that the MSB is set on the final tomb to indicate the terminal element. This - // allows for an optimisation where we only add as many extra tombs as are needed to - // represent the dying operands. Each pass modifies operand bits and so needs to write - // back, so let's figure out how many extra tombs we really need. Note that we always - // keep at least one. - var num: usize = big.extra_tombs.len; - while (num > 1) { - if (@as(u31, @truncate(big.extra_tombs[num - 1])) != 0) { - // Some operand dies here - break; - } - num -= 1; - } - // Mark final tomb - big.extra_tombs[num - 1] |= @as(u32, 1) << 31; - - const extra_tombs = big.extra_tombs[0..num]; - - const extra_index = @as(u32, @intCast(big.a.extra.items.len)); - try big.a.extra.appendSlice(gpa, extra_tombs); - try big.a.special.put(gpa, big.inst, extra_index); - }, - } - - try analyzeOperands(big.a, pass, big.data, big.inst, big.small); - } - - fn deinit(big: *Self) void { - big.a.gpa.free(big.extra_tombs); - } - }; -} - -fn fmtInstSet(set: *const std.AutoHashMapUnmanaged(Air.Inst.Index, void)) FmtInstSet { - return .{ .set = set }; -} - -const FmtInstSet = struct { - set: *const std.AutoHashMapUnmanaged(Air.Inst.Index, void), - - pub fn format(val: FmtInstSet, comptime _: []const u8, _: std.fmt.FormatOptions, w: anytype) !void { - if (val.set.count() == 0) { - try w.writeAll("[no instructions]"); - return; - } - var it = val.set.keyIterator(); - try w.print("%{}", .{it.next().?.*}); - while (it.next()) |key| { - try w.print(" %{}", .{key.*}); - } - } -}; - -fn fmtInstList(list: []const Air.Inst.Index) FmtInstList { - return .{ .list = list }; -} - -const FmtInstList = struct { - list: []const Air.Inst.Index, - - pub fn format(val: FmtInstList, comptime _: []const u8, _: std.fmt.FormatOptions, w: anytype) !void { - if (val.list.len == 0) { - try w.writeAll("[no instructions]"); - return; - } - try w.print("%{}", .{val.list[0]}); - for (val.list[1..]) |inst| { - try w.print(" %{}", .{inst}); - } - } -}; diff --git a/src/Liveness/Verify.zig b/src/Liveness/Verify.zig deleted file mode 100644 index 1d61145b3c542646a6707612be9d058578987622..0000000000000000000000000000000000000000 --- a/src/Liveness/Verify.zig +++ /dev/null @@ -1,642 +0,0 @@ -//! Verifies that Liveness information is valid. - -gpa: std.mem.Allocator, -air: Air, -liveness: Liveness, -live: LiveMap = .{}, -blocks: std.AutoHashMapUnmanaged(Air.Inst.Index, LiveMap) = .empty, -loops: std.AutoHashMapUnmanaged(Air.Inst.Index, LiveMap) = .empty, -intern_pool: *const InternPool, - -pub const Error = error{ LivenessInvalid, OutOfMemory }; - -pub fn deinit(self: *Verify) void { - self.live.deinit(self.gpa); - { - var it = self.blocks.valueIterator(); - while (it.next()) |block| block.deinit(self.gpa); - self.blocks.deinit(self.gpa); - } - { - var it = self.loops.valueIterator(); - while (it.next()) |block| block.deinit(self.gpa); - self.loops.deinit(self.gpa); - } - self.* = undefined; -} - -pub fn verify(self: *Verify) Error!void { - self.live.clearRetainingCapacity(); - self.blocks.clearRetainingCapacity(); - self.loops.clearRetainingCapacity(); - try self.verifyBody(self.air.getMainBody()); - // We don't care about `self.live` now, because the loop body was noreturn - everything being dead was checked on `ret` etc - assert(self.blocks.count() == 0); - assert(self.loops.count() == 0); -} - -const LiveMap = std.AutoHashMapUnmanaged(Air.Inst.Index, void); - -fn verifyBody(self: *Verify, body: []const Air.Inst.Index) Error!void { - const ip = self.intern_pool; - const tags = self.air.instructions.items(.tag); - const data = self.air.instructions.items(.data); - for (body) |inst| { - if (self.liveness.isUnused(inst) and !self.air.mustLower(inst, ip)) { - // This instruction will not be lowered and should be ignored. - continue; - } - - switch (tags[@intFromEnum(inst)]) { - // no operands - .arg, - .alloc, - .inferred_alloc, - .inferred_alloc_comptime, - .ret_ptr, - .breakpoint, - .dbg_stmt, - .dbg_empty_stmt, - .ret_addr, - .frame_addr, - .wasm_memory_size, - .err_return_trace, - .save_err_return_trace_index, - .tlv_dllimport_ptr, - .c_va_start, - .work_item_id, - .work_group_size, - .work_group_id, - => try self.verifyInstOperands(inst, .{ .none, .none, .none }), - - .trap, .unreach => { - try self.verifyInstOperands(inst, .{ .none, .none, .none }); - // This instruction terminates the function, so everything should be dead - if (self.live.count() > 0) return invalid("%{}: instructions still alive", .{inst}); - }, - - // unary - .not, - .bitcast, - .load, - .fpext, - .fptrunc, - .intcast, - .intcast_safe, - .trunc, - .optional_payload, - .optional_payload_ptr, - .optional_payload_ptr_set, - .errunion_payload_ptr_set, - .wrap_optional, - .unwrap_errunion_payload, - .unwrap_errunion_err, - .unwrap_errunion_payload_ptr, - .unwrap_errunion_err_ptr, - .wrap_errunion_payload, - .wrap_errunion_err, - .slice_ptr, - .slice_len, - .ptr_slice_len_ptr, - .ptr_slice_ptr_ptr, - .struct_field_ptr_index_0, - .struct_field_ptr_index_1, - .struct_field_ptr_index_2, - .struct_field_ptr_index_3, - .array_to_slice, - .int_from_float, - .int_from_float_optimized, - .float_from_int, - .get_union_tag, - .clz, - .ctz, - .popcount, - .byte_swap, - .bit_reverse, - .splat, - .error_set_has_value, - .addrspace_cast, - .c_va_arg, - .c_va_copy, - .abs, - => { - const ty_op = data[@intFromEnum(inst)].ty_op; - try self.verifyInstOperands(inst, .{ ty_op.operand, .none, .none }); - }, - .is_null, - .is_non_null, - .is_null_ptr, - .is_non_null_ptr, - .is_err, - .is_non_err, - .is_err_ptr, - .is_non_err_ptr, - .is_named_enum_value, - .tag_name, - .error_name, - .sqrt, - .sin, - .cos, - .tan, - .exp, - .exp2, - .log, - .log2, - .log10, - .floor, - .ceil, - .round, - .trunc_float, - .neg, - .neg_optimized, - .cmp_lt_errors_len, - .set_err_return_trace, - .c_va_end, - => { - const un_op = data[@intFromEnum(inst)].un_op; - try self.verifyInstOperands(inst, .{ un_op, .none, .none }); - }, - .ret, - .ret_safe, - .ret_load, - => { - const un_op = data[@intFromEnum(inst)].un_op; - try self.verifyInstOperands(inst, .{ un_op, .none, .none }); - // This instruction terminates the function, so everything should be dead - if (self.live.count() > 0) return invalid("%{}: instructions still alive", .{inst}); - }, - .dbg_var_ptr, - .dbg_var_val, - .dbg_arg_inline, - .wasm_memory_grow, - => { - const pl_op = data[@intFromEnum(inst)].pl_op; - try self.verifyInstOperands(inst, .{ pl_op.operand, .none, .none }); - }, - .prefetch => { - const prefetch = data[@intFromEnum(inst)].prefetch; - try self.verifyInstOperands(inst, .{ prefetch.ptr, .none, .none }); - }, - .reduce, - .reduce_optimized, - => { - const reduce = data[@intFromEnum(inst)].reduce; - try self.verifyInstOperands(inst, .{ reduce.operand, .none, .none }); - }, - .union_init => { - const ty_pl = data[@intFromEnum(inst)].ty_pl; - const extra = self.air.extraData(Air.UnionInit, ty_pl.payload).data; - try self.verifyInstOperands(inst, .{ extra.init, .none, .none }); - }, - .struct_field_ptr, .struct_field_val => { - const ty_pl = data[@intFromEnum(inst)].ty_pl; - const extra = self.air.extraData(Air.StructField, ty_pl.payload).data; - try self.verifyInstOperands(inst, .{ extra.struct_operand, .none, .none }); - }, - .field_parent_ptr => { - const ty_pl = data[@intFromEnum(inst)].ty_pl; - const extra = self.air.extraData(Air.FieldParentPtr, ty_pl.payload).data; - try self.verifyInstOperands(inst, .{ extra.field_ptr, .none, .none }); - }, - .atomic_load => { - const atomic_load = data[@intFromEnum(inst)].atomic_load; - try self.verifyInstOperands(inst, .{ atomic_load.ptr, .none, .none }); - }, - - // binary - .add, - .add_safe, - .add_optimized, - .add_wrap, - .add_sat, - .sub, - .sub_safe, - .sub_optimized, - .sub_wrap, - .sub_sat, - .mul, - .mul_safe, - .mul_optimized, - .mul_wrap, - .mul_sat, - .div_float, - .div_float_optimized, - .div_trunc, - .div_trunc_optimized, - .div_floor, - .div_floor_optimized, - .div_exact, - .div_exact_optimized, - .rem, - .rem_optimized, - .mod, - .mod_optimized, - .bit_and, - .bit_or, - .xor, - .cmp_lt, - .cmp_lt_optimized, - .cmp_lte, - .cmp_lte_optimized, - .cmp_eq, - .cmp_eq_optimized, - .cmp_gte, - .cmp_gte_optimized, - .cmp_gt, - .cmp_gt_optimized, - .cmp_neq, - .cmp_neq_optimized, - .bool_and, - .bool_or, - .store, - .store_safe, - .array_elem_val, - .slice_elem_val, - .ptr_elem_val, - .shl, - .shl_exact, - .shl_sat, - .shr, - .shr_exact, - .atomic_store_unordered, - .atomic_store_monotonic, - .atomic_store_release, - .atomic_store_seq_cst, - .set_union_tag, - .min, - .max, - .memset, - .memset_safe, - .memcpy, - .memmove, - => { - const bin_op = data[@intFromEnum(inst)].bin_op; - try self.verifyInstOperands(inst, .{ bin_op.lhs, bin_op.rhs, .none }); - }, - .add_with_overflow, - .sub_with_overflow, - .mul_with_overflow, - .shl_with_overflow, - .ptr_add, - .ptr_sub, - .ptr_elem_ptr, - .slice_elem_ptr, - .slice, - => { - const ty_pl = data[@intFromEnum(inst)].ty_pl; - const extra = self.air.extraData(Air.Bin, ty_pl.payload).data; - try self.verifyInstOperands(inst, .{ extra.lhs, extra.rhs, .none }); - }, - .shuffle => { - const ty_pl = data[@intFromEnum(inst)].ty_pl; - const extra = self.air.extraData(Air.Shuffle, ty_pl.payload).data; - try self.verifyInstOperands(inst, .{ extra.a, extra.b, .none }); - }, - .cmp_vector, - .cmp_vector_optimized, - => { - const ty_pl = data[@intFromEnum(inst)].ty_pl; - const extra = self.air.extraData(Air.VectorCmp, ty_pl.payload).data; - try self.verifyInstOperands(inst, .{ extra.lhs, extra.rhs, .none }); - }, - .atomic_rmw => { - const pl_op = data[@intFromEnum(inst)].pl_op; - const extra = self.air.extraData(Air.AtomicRmw, pl_op.payload).data; - try self.verifyInstOperands(inst, .{ pl_op.operand, extra.operand, .none }); - }, - - // ternary - .select => { - const pl_op = data[@intFromEnum(inst)].pl_op; - const extra = self.air.extraData(Air.Bin, pl_op.payload).data; - try self.verifyInstOperands(inst, .{ pl_op.operand, extra.lhs, extra.rhs }); - }, - .mul_add => { - const pl_op = data[@intFromEnum(inst)].pl_op; - const extra = self.air.extraData(Air.Bin, pl_op.payload).data; - try self.verifyInstOperands(inst, .{ extra.lhs, extra.rhs, pl_op.operand }); - }, - .vector_store_elem => { - const vector_store_elem = data[@intFromEnum(inst)].vector_store_elem; - const extra = self.air.extraData(Air.Bin, vector_store_elem.payload).data; - try self.verifyInstOperands(inst, .{ vector_store_elem.vector_ptr, extra.lhs, extra.rhs }); - }, - .cmpxchg_strong, - .cmpxchg_weak, - => { - const ty_pl = data[@intFromEnum(inst)].ty_pl; - const extra = self.air.extraData(Air.Cmpxchg, ty_pl.payload).data; - try self.verifyInstOperands(inst, .{ extra.ptr, extra.expected_value, extra.new_value }); - }, - - // big tombs - .aggregate_init => { - const ty_pl = data[@intFromEnum(inst)].ty_pl; - const aggregate_ty = ty_pl.ty.toType(); - const len = @as(usize, @intCast(aggregate_ty.arrayLenIp(ip))); - const elements = @as([]const Air.Inst.Ref, @ptrCast(self.air.extra[ty_pl.payload..][0..len])); - - var bt = self.liveness.iterateBigTomb(inst); - for (elements) |element| { - try self.verifyOperand(inst, element, bt.feed()); - } - try self.verifyInst(inst); - }, - .call, .call_always_tail, .call_never_tail, .call_never_inline => { - const pl_op = data[@intFromEnum(inst)].pl_op; - const extra = self.air.extraData(Air.Call, pl_op.payload); - const args = @as( - []const Air.Inst.Ref, - @ptrCast(self.air.extra[extra.end..][0..extra.data.args_len]), - ); - - var bt = self.liveness.iterateBigTomb(inst); - try self.verifyOperand(inst, pl_op.operand, bt.feed()); - for (args) |arg| { - try self.verifyOperand(inst, arg, bt.feed()); - } - try self.verifyInst(inst); - }, - .assembly => { - const ty_pl = data[@intFromEnum(inst)].ty_pl; - const extra = self.air.extraData(Air.Asm, ty_pl.payload); - var extra_i = extra.end; - const outputs = @as( - []const Air.Inst.Ref, - @ptrCast(self.air.extra[extra_i..][0..extra.data.outputs_len]), - ); - extra_i += outputs.len; - const inputs = @as( - []const Air.Inst.Ref, - @ptrCast(self.air.extra[extra_i..][0..extra.data.inputs_len]), - ); - extra_i += inputs.len; - - var bt = self.liveness.iterateBigTomb(inst); - for (outputs) |output| { - if (output != .none) { - try self.verifyOperand(inst, output, bt.feed()); - } - } - for (inputs) |input| { - try self.verifyOperand(inst, input, bt.feed()); - } - try self.verifyInst(inst); - }, - - // control flow - .@"try", .try_cold => { - const pl_op = data[@intFromEnum(inst)].pl_op; - const extra = self.air.extraData(Air.Try, pl_op.payload); - const try_body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra.end..][0..extra.data.body_len]); - - const cond_br_liveness = self.liveness.getCondBr(inst); - - try self.verifyOperand(inst, pl_op.operand, self.liveness.operandDies(inst, 0)); - - var live = try self.live.clone(self.gpa); - defer live.deinit(self.gpa); - - for (cond_br_liveness.else_deaths) |death| try self.verifyDeath(inst, death); - try self.verifyBody(try_body); - - self.live.deinit(self.gpa); - self.live = live.move(); - - for (cond_br_liveness.then_deaths) |death| try self.verifyDeath(inst, death); - - try self.verifyInst(inst); - }, - .try_ptr, .try_ptr_cold => { - const ty_pl = data[@intFromEnum(inst)].ty_pl; - const extra = self.air.extraData(Air.TryPtr, ty_pl.payload); - const try_body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra.end..][0..extra.data.body_len]); - - const cond_br_liveness = self.liveness.getCondBr(inst); - - try self.verifyOperand(inst, extra.data.ptr, self.liveness.operandDies(inst, 0)); - - var live = try self.live.clone(self.gpa); - defer live.deinit(self.gpa); - - for (cond_br_liveness.else_deaths) |death| try self.verifyDeath(inst, death); - try self.verifyBody(try_body); - - self.live.deinit(self.gpa); - self.live = live.move(); - - for (cond_br_liveness.then_deaths) |death| try self.verifyDeath(inst, death); - - try self.verifyInst(inst); - }, - .br => { - const br = data[@intFromEnum(inst)].br; - const gop = try self.blocks.getOrPut(self.gpa, br.block_inst); - - try self.verifyOperand(inst, br.operand, self.liveness.operandDies(inst, 0)); - if (gop.found_existing) { - try self.verifyMatchingLiveness(br.block_inst, gop.value_ptr.*); - } else { - gop.value_ptr.* = try self.live.clone(self.gpa); - } - try self.verifyInst(inst); - }, - .repeat => { - const repeat = data[@intFromEnum(inst)].repeat; - const expected_live = self.loops.get(repeat.loop_inst) orelse - return invalid("%{}: loop %{} not in scope", .{ @intFromEnum(inst), @intFromEnum(repeat.loop_inst) }); - - try self.verifyMatchingLiveness(repeat.loop_inst, expected_live); - }, - .switch_dispatch => { - const br = data[@intFromEnum(inst)].br; - - try self.verifyOperand(inst, br.operand, self.liveness.operandDies(inst, 0)); - - const expected_live = self.loops.get(br.block_inst) orelse - return invalid("%{}: loop %{} not in scope", .{ @intFromEnum(inst), @intFromEnum(br.block_inst) }); - - try self.verifyMatchingLiveness(br.block_inst, expected_live); - }, - .block, .dbg_inline_block => |tag| { - const ty_pl = data[@intFromEnum(inst)].ty_pl; - const block_ty = ty_pl.ty.toType(); - const block_body: []const Air.Inst.Index = @ptrCast(switch (tag) { - inline .block, .dbg_inline_block => |comptime_tag| body: { - const extra = self.air.extraData(switch (comptime_tag) { - .block => Air.Block, - .dbg_inline_block => Air.DbgInlineBlock, - else => unreachable, - }, ty_pl.payload); - break :body self.air.extra[extra.end..][0..extra.data.body_len]; - }, - else => unreachable, - }); - const block_liveness = self.liveness.getBlock(inst); - - var orig_live = try self.live.clone(self.gpa); - defer orig_live.deinit(self.gpa); - - assert(!self.blocks.contains(inst)); - try self.verifyBody(block_body); - - // Liveness data after the block body is garbage, but we want to - // restore it to verify deaths - self.live.deinit(self.gpa); - self.live = orig_live.move(); - - for (block_liveness.deaths) |death| try self.verifyDeath(inst, death); - - if (ip.isNoReturn(block_ty.toIntern())) { - assert(!self.blocks.contains(inst)); - } else { - var live = self.blocks.fetchRemove(inst).?.value; - defer live.deinit(self.gpa); - - try self.verifyMatchingLiveness(inst, live); - } - - try self.verifyInstOperands(inst, .{ .none, .none, .none }); - }, - .loop => { - const ty_pl = data[@intFromEnum(inst)].ty_pl; - const extra = self.air.extraData(Air.Block, ty_pl.payload); - const loop_body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra.end..][0..extra.data.body_len]); - - // The same stuff should be alive after the loop as before it. - const gop = try self.loops.getOrPut(self.gpa, inst); - if (gop.found_existing) return invalid("%{}: loop already exists", .{@intFromEnum(inst)}); - defer { - var live = self.loops.fetchRemove(inst).?; - live.value.deinit(self.gpa); - } - gop.value_ptr.* = try self.live.clone(self.gpa); - - try self.verifyBody(loop_body); - - try self.verifyInstOperands(inst, .{ .none, .none, .none }); - }, - .cond_br => { - const pl_op = data[@intFromEnum(inst)].pl_op; - const extra = self.air.extraData(Air.CondBr, pl_op.payload); - const then_body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra.end..][0..extra.data.then_body_len]); - const else_body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra.end + then_body.len ..][0..extra.data.else_body_len]); - const cond_br_liveness = self.liveness.getCondBr(inst); - - try self.verifyOperand(inst, pl_op.operand, self.liveness.operandDies(inst, 0)); - - var live = try self.live.clone(self.gpa); - defer live.deinit(self.gpa); - - for (cond_br_liveness.then_deaths) |death| try self.verifyDeath(inst, death); - try self.verifyBody(then_body); - - self.live.deinit(self.gpa); - self.live = live.move(); - - for (cond_br_liveness.else_deaths) |death| try self.verifyDeath(inst, death); - try self.verifyBody(else_body); - - try self.verifyInst(inst); - }, - .switch_br, .loop_switch_br => { - const switch_br = self.air.unwrapSwitch(inst); - const switch_br_liveness = try self.liveness.getSwitchBr( - self.gpa, - inst, - switch_br.cases_len + 1, - ); - defer self.gpa.free(switch_br_liveness.deaths); - - try self.verifyOperand(inst, switch_br.operand, self.liveness.operandDies(inst, 0)); - - // Excluding the operand (which we just handled), the same stuff should be alive - // after the loop as before it. - { - const gop = try self.loops.getOrPut(self.gpa, inst); - if (gop.found_existing) return invalid("%{}: loop already exists", .{@intFromEnum(inst)}); - gop.value_ptr.* = self.live.move(); - } - defer { - var live = self.loops.fetchRemove(inst).?; - live.value.deinit(self.gpa); - } - - var it = switch_br.iterateCases(); - while (it.next()) |case| { - self.live.deinit(self.gpa); - self.live = try self.loops.get(inst).?.clone(self.gpa); - - for (switch_br_liveness.deaths[case.idx]) |death| try self.verifyDeath(inst, death); - try self.verifyBody(case.body); - } - - const else_body = it.elseBody(); - if (else_body.len > 0) { - self.live.deinit(self.gpa); - self.live = try self.loops.get(inst).?.clone(self.gpa); - for (switch_br_liveness.deaths[switch_br.cases_len]) |death| try self.verifyDeath(inst, death); - try self.verifyBody(else_body); - } - - try self.verifyInst(inst); - }, - } - } -} - -fn verifyDeath(self: *Verify, inst: Air.Inst.Index, operand: Air.Inst.Index) Error!void { - try self.verifyOperand(inst, operand.toRef(), true); -} - -fn verifyOperand(self: *Verify, inst: Air.Inst.Index, op_ref: Air.Inst.Ref, dies: bool) Error!void { - const operand = op_ref.toIndexAllowNone() orelse { - assert(!dies); - return; - }; - if (dies) { - if (!self.live.remove(operand)) return invalid("%{}: dead operand %{} reused and killed again", .{ inst, operand }); - } else { - if (!self.live.contains(operand)) return invalid("%{}: dead operand %{} reused", .{ inst, operand }); - } -} - -fn verifyInstOperands( - self: *Verify, - inst: Air.Inst.Index, - operands: [Liveness.bpi - 1]Air.Inst.Ref, -) Error!void { - for (operands, 0..) |operand, operand_index| { - const dies = self.liveness.operandDies(inst, @as(Liveness.OperandInt, @intCast(operand_index))); - try self.verifyOperand(inst, operand, dies); - } - try self.verifyInst(inst); -} - -fn verifyInst(self: *Verify, inst: Air.Inst.Index) Error!void { - if (self.liveness.isUnused(inst)) { - assert(!self.live.contains(inst)); - } else { - try self.live.putNoClobber(self.gpa, inst, {}); - } -} - -fn verifyMatchingLiveness(self: *Verify, block: Air.Inst.Index, live: LiveMap) Error!void { - if (self.live.count() != live.count()) return invalid("%{}: different deaths across branches", .{block}); - var live_it = self.live.keyIterator(); - while (live_it.next()) |live_inst| if (!live.contains(live_inst.*)) return invalid("%{}: different deaths across branches", .{block}); -} - -fn invalid(comptime fmt: []const u8, args: anytype) error{LivenessInvalid} { - log.err(fmt, args); - return error.LivenessInvalid; -} - -const std = @import("std"); -const assert = std.debug.assert; -const log = std.log.scoped(.liveness_verify); - -const Air = @import("../Air.zig"); -const Liveness = @import("../Liveness.zig"); -const InternPool = @import("../InternPool.zig"); -const Verify = @This(); diff --git a/src/Sema.zig b/src/Sema.zig index 967fa3a23ef4755066e125e659aaa70286874289..3fa264be489a5c87c058e32384555f9746942d39 100644 --- a/src/Sema.zig +++ b/src/Sema.zig @@ -756,13 +756,7 @@ pub const Block = struct { fn addReduce(block: *Block, operand: Air.Inst.Ref, operation: std.builtin.ReduceOp) !Air.Inst.Ref { const sema = block.sema; const zcu = sema.pt.zcu; - const vector_ty = sema.typeOf(operand); - switch (vector_ty.vectorLen(zcu)) { - 0 => unreachable, - 1 => return block.addBinOp(.array_elem_val, operand, .zero_usize), - else => {}, - } - const allow_optimized = switch (vector_ty.childType(zcu).zigTypeTag(zcu)) { + const allow_optimized = switch (sema.typeOf(operand).childType(zcu).zigTypeTag(zcu)) { .float => true, .bool, .int => false, else => unreachable, @@ -36849,7 +36843,7 @@ fn typeOf(sema: *Sema, inst: Air.Inst.Ref) Type { pub fn getTmpAir(sema: Sema) Air { return .{ .instructions = sema.air_instructions.slice(), - .extra = sema.air_extra.items, + .extra = sema.air_extra, }; } diff --git a/src/Zcu.zig b/src/Zcu.zig index 7617085f8ef225c6ef4ae73034dde87f377dae75..c49a1d46b187899e12b1817ceb331c24095d6c2a 100644 --- a/src/Zcu.zig +++ b/src/Zcu.zig @@ -30,7 +30,6 @@ const AstGen = std.zig.AstGen; const Sema = @import("Sema.zig"); const target_util = @import("target.zig"); const build_options = @import("build_options"); -const Liveness = @import("Liveness.zig"); const isUpDir = @import("introspect.zig").isUpDir; const clang = @import("clang.zig"); const InternPool = @import("InternPool.zig"); diff --git a/src/Zcu/PerThread.zig b/src/Zcu/PerThread.zig index 65ccfbb1908966799764832a7b47ce75a951eda2..44abb3cbf3b397c24260e5fa2b3a12734e9777b0 100644 --- a/src/Zcu/PerThread.zig +++ b/src/Zcu/PerThread.zig @@ -16,7 +16,6 @@ const dev = @import("../dev.zig"); const InternPool = @import("../InternPool.zig"); const AnalUnit = InternPool.AnalUnit; const introspect = @import("../introspect.zig"); -const Liveness = @import("../Liveness.zig"); const log = std.log.scoped(.zcu); const Module = @import("../Package.zig").Module; const Sema = @import("../Sema.zig"); @@ -1721,34 +1720,43 @@ fn analyzeFuncBody( /// Takes ownership of `air`, even on error. /// If any types referenced by `air` are unresolved, marks the codegen as failed. -pub fn linkerUpdateFunc(pt: Zcu.PerThread, func_index: InternPool.Index, air: Air) Allocator.Error!void { +pub fn linkerUpdateFunc(pt: Zcu.PerThread, func_index: InternPool.Index, air: *Air) Allocator.Error!void { const zcu = pt.zcu; const gpa = zcu.gpa; const ip = &zcu.intern_pool; const comp = zcu.comp; - defer { - var air_mut = air; - air_mut.deinit(gpa); - } - const func = zcu.funcInfo(func_index); const nav_index = func.owner_nav; const nav = ip.getNav(nav_index); - var liveness = try Liveness.analyze(gpa, air, ip); + const codegen_prog_node = zcu.codegen_prog_node.start(nav.fqn.toSlice(ip), 0); + defer codegen_prog_node.end(); + + if (!air.typesFullyResolved(zcu)) { + // A type we depend on failed to resolve. This is a transitive failure. + // Correcting this failure will involve changing a type this function + // depends on, hence triggering re-analysis of this function, so this + // interacts correctly with incremental compilation. + return; + } + + const backend = target_util.zigBackend(zcu.root_mod.resolved_target.result, zcu.comp.config.use_llvm); + try air.legalize(backend, zcu); + + var liveness = try Air.Liveness.analyze(gpa, air.*, ip); defer liveness.deinit(gpa); if (build_options.enable_debug_extensions and comp.verbose_air) { std.debug.print("# Begin Function AIR: {}:\n", .{nav.fqn.fmt(ip)}); - @import("../print_air.zig").dump(pt, air, liveness); + @import("../print_air.zig").dump(pt, air.*, liveness); std.debug.print("# End Function AIR: {}\n\n", .{nav.fqn.fmt(ip)}); } if (std.debug.runtime_safety) { - var verify: Liveness.Verify = .{ + var verify: Air.Liveness.Verify = .{ .gpa = gpa, - .air = air, + .air = air.*, .liveness = liveness, .intern_pool = ip, }; @@ -1768,16 +1776,8 @@ pub fn linkerUpdateFunc(pt: Zcu.PerThread, func_index: InternPool.Index, air: Ai }; } - const codegen_prog_node = zcu.codegen_prog_node.start(nav.fqn.toSlice(ip), 0); - defer codegen_prog_node.end(); - - if (!air.typesFullyResolved(zcu)) { - // A type we depend on failed to resolve. This is a transitive failure. - // Correcting this failure will involve changing a type this function - // depends on, hence triggering re-analysis of this function, so this - // interacts correctly with incremental compilation. - } else if (comp.bin_file) |lf| { - lf.updateFunc(pt, func_index, air, liveness) catch |err| switch (err) { + if (comp.bin_file) |lf| { + lf.updateFunc(pt, func_index, air.*, liveness) catch |err| switch (err) { error.OutOfMemory => return error.OutOfMemory, error.CodegenFail => assert(zcu.failed_codegen.contains(nav_index)), error.Overflow, error.RelocationNotByteAligned => { @@ -1791,7 +1791,7 @@ pub fn linkerUpdateFunc(pt: Zcu.PerThread, func_index: InternPool.Index, air: Ai }, }; } else if (zcu.llvm_object) |llvm_object| { - llvm_object.updateFunc(pt, func_index, air, liveness) catch |err| switch (err) { + llvm_object.updateFunc(pt, func_index, air.*, liveness) catch |err| switch (err) { error.OutOfMemory => return error.OutOfMemory, }; } @@ -3080,9 +3080,13 @@ fn analyzeFnBodyInner(pt: Zcu.PerThread, func_index: InternPool.Index) Zcu.SemaE try sema.flushExports(); + defer { + sema.air_instructions = .empty; + sema.air_extra = .empty; + } return .{ - .instructions = sema.air_instructions.toOwnedSlice(), - .extra = try sema.air_extra.toOwnedSlice(gpa), + .instructions = sema.air_instructions.slice(), + .extra = sema.air_extra, }; } diff --git a/src/arch/aarch64/CodeGen.zig b/src/arch/aarch64/CodeGen.zig index 5c5ccfa6ae37c2c1e1c19e5b5dbe2ebcb34c1a94..6fd23cfd182435b4d3e38b2b7c56e1396a9d8e68 100644 --- a/src/arch/aarch64/CodeGen.zig +++ b/src/arch/aarch64/CodeGen.zig @@ -7,7 +7,6 @@ const codegen = @import("../../codegen.zig"); const Air = @import("../../Air.zig"); const Mir = @import("Mir.zig"); const Emit = @import("Emit.zig"); -const Liveness = @import("../../Liveness.zig"); const Type = @import("../../Type.zig"); const Value = @import("../../Value.zig"); const link = @import("../../link.zig"); @@ -44,7 +43,7 @@ const InnerError = CodeGenError || error{OutOfRegisters}; gpa: Allocator, pt: Zcu.PerThread, air: Air, -liveness: Liveness, +liveness: Air.Liveness, bin_file: *link.File, debug_output: link.File.DebugInfoOutput, target: *const std.Target, @@ -71,7 +70,7 @@ end_di_column: u32, /// which is a relative jump, based on the address following the reloc. exitlude_jump_relocs: std.ArrayListUnmanaged(usize) = .empty, -reused_operands: std.StaticBitSet(Liveness.bpi - 1) = undefined, +reused_operands: std.StaticBitSet(Air.Liveness.bpi - 1) = undefined, /// We postpone the creation of debug info for function args and locals /// until after all Mir instructions have been generated. Only then we @@ -273,7 +272,7 @@ const BlockData = struct { const BigTomb = struct { function: *Self, inst: Air.Inst.Index, - lbt: Liveness.BigTomb, + lbt: Air.Liveness.BigTomb, fn feed(bt: *BigTomb, op_ref: Air.Inst.Ref) void { const dies = bt.lbt.feed(); @@ -324,7 +323,7 @@ pub fn generate( src_loc: Zcu.LazySrcLoc, func_index: InternPool.Index, air: Air, - liveness: Liveness, + liveness: Air.Liveness, code: *std.ArrayListUnmanaged(u8), debug_output: link.File.DebugInfoOutput, ) CodeGenError!void { @@ -646,7 +645,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void { continue; const old_air_bookkeeping = self.air_bookkeeping; - try self.ensureProcessDeathCapacity(Liveness.bpi); + try self.ensureProcessDeathCapacity(Air.Liveness.bpi); self.reused_operands = @TypeOf(self.reused_operands).initEmpty(); switch (air_tags[@intFromEnum(inst)]) { @@ -930,14 +929,14 @@ fn finishAirBookkeeping(self: *Self) void { } } -fn finishAir(self: *Self, inst: Air.Inst.Index, result: MCValue, operands: [Liveness.bpi - 1]Air.Inst.Ref) void { +fn finishAir(self: *Self, inst: Air.Inst.Index, result: MCValue, operands: [Air.Liveness.bpi - 1]Air.Inst.Ref) void { const tomb_bits = self.liveness.getTombBits(inst); for (0.., operands) |op_index, op| { - if (tomb_bits & @as(Liveness.Bpi, 1) << @intCast(op_index) == 0) continue; + if (tomb_bits & @as(Air.Liveness.Bpi, 1) << @intCast(op_index) == 0) continue; if (self.reused_operands.isSet(op_index)) continue; self.processDeath(op.toIndexAllowNone() orelse continue); } - if (tomb_bits & 1 << (Liveness.bpi - 1) == 0) { + if (tomb_bits & 1 << (Air.Liveness.bpi - 1) == 0) { log.debug("%{d} => {}", .{ inst, result }); const branch = &self.branch_stack.items[self.branch_stack.items.len - 1]; branch.inst_table.putAssumeCapacityNoClobber(inst, result); @@ -1568,7 +1567,7 @@ const ReuseMetadata = struct { /// inputs to the Air instruction are omitted (e.g. when they can /// be represented as immediates to the Mir instruction), /// operand_mapping should reflect that fact. - operand_mapping: []const Liveness.OperandInt, + operand_mapping: []const Air.Liveness.OperandInt, }; /// Allocate a set of registers for use as arguments for a Mir @@ -1835,7 +1834,7 @@ fn binOpImmediate( const write_args = [_]WriteArg{ .{ .ty = lhs_ty, .bind = .none, .class = gp, .reg = &dest_reg }, }; - const operand_mapping: []const Liveness.OperandInt = if (lhs_and_rhs_swapped) &.{1} else &.{0}; + const operand_mapping: []const Air.Liveness.OperandInt = if (lhs_and_rhs_swapped) &.{1} else &.{0}; try self.allocRegs( &read_args, &write_args, @@ -3584,7 +3583,7 @@ fn reuseOperand( self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, - op_index: Liveness.OperandInt, + op_index: Air.Liveness.OperandInt, mcv: MCValue, ) bool { if (!self.liveness.operandDies(inst, op_index)) @@ -4250,7 +4249,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; const callee = pl_op.operand; const extra = self.air.extraData(Air.Call, pl_op.payload); - const args = @as([]const Air.Inst.Ref, @ptrCast(self.air.extra[extra.end..][0..extra.data.args_len])); + const args: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[extra.end..][0..extra.data.args_len]); const ty = self.typeOf(callee); const pt = self.pt; const zcu = pt.zcu; @@ -4389,8 +4388,8 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier break :result info.return_value; }; - if (args.len + 1 <= Liveness.bpi - 1) { - var buf = [1]Air.Inst.Ref{.none} ** (Liveness.bpi - 1); + if (args.len + 1 <= Air.Liveness.bpi - 1) { + var buf = [1]Air.Inst.Ref{.none} ** (Air.Liveness.bpi - 1); buf[0] = callee; @memcpy(buf[1..][0..args.len], args); return self.finishAir(inst, result, buf); @@ -4613,7 +4612,7 @@ fn airDbgInlineBlock(self: *Self, inst: Air.Inst.Index) InnerError!void { const func = zcu.funcInfo(extra.data.func); // TODO emit debug info for function change _ = func; - try self.lowerBlock(inst, @ptrCast(self.air.extra[extra.end..][0..extra.data.body_len])); + try self.lowerBlock(inst, @ptrCast(self.air.extra.items[extra.end..][0..extra.data.body_len])); } fn airDbgVar(self: *Self, inst: Air.Inst.Index) InnerError!void { @@ -4671,8 +4670,8 @@ fn airCondBr(self: *Self, inst: Air.Inst.Index) InnerError!void { const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; const cond = try self.resolveInst(pl_op.operand); const extra = self.air.extraData(Air.CondBr, pl_op.payload); - const then_body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra.end..][0..extra.data.then_body_len]); - const else_body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra.end + then_body.len ..][0..extra.data.else_body_len]); + const then_body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[extra.end..][0..extra.data.then_body_len]); + const else_body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[extra.end + then_body.len ..][0..extra.data.else_body_len]); const liveness_condbr = self.liveness.getCondBr(inst); const reloc = try self.condBr(cond); @@ -5016,7 +5015,7 @@ fn airLoop(self: *Self, inst: Air.Inst.Index) InnerError!void { // A loop is a setup to be able to jump back to the beginning. const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; const loop = self.air.extraData(Air.Block, ty_pl.payload); - const body: []const Air.Inst.Index = @ptrCast(self.air.extra[loop.end..][0..loop.data.body_len]); + const body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[loop.end..][0..loop.data.body_len]); const start_index = @as(u32, @intCast(self.mir_instructions.len)); try self.genBody(body); @@ -5036,7 +5035,7 @@ fn jump(self: *Self, inst: Mir.Inst.Index) !void { fn airBlock(self: *Self, inst: Air.Inst.Index) InnerError!void { const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; const extra = self.air.extraData(Air.Block, ty_pl.payload); - try self.lowerBlock(inst, @ptrCast(self.air.extra[extra.end..][0..extra.data.body_len])); + try self.lowerBlock(inst, @ptrCast(self.air.extra.items[extra.end..][0..extra.data.body_len])); } fn lowerBlock(self: *Self, inst: Air.Inst.Index, body: []const Air.Inst.Index) !void { @@ -5255,9 +5254,9 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) InnerError!void { const is_volatile = @as(u1, @truncate(extra.data.flags >> 31)) != 0; const clobbers_len = @as(u31, @truncate(extra.data.flags)); var extra_i: usize = extra.end; - const outputs = @as([]const Air.Inst.Ref, @ptrCast(self.air.extra[extra_i..][0..extra.data.outputs_len])); + const outputs: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[extra_i..][0..extra.data.outputs_len]); extra_i += outputs.len; - const inputs = @as([]const Air.Inst.Ref, @ptrCast(self.air.extra[extra_i..][0..extra.data.inputs_len])); + const inputs: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[extra_i..][0..extra.data.inputs_len]); extra_i += inputs.len; const dead = !is_volatile and self.liveness.isUnused(inst); @@ -5270,8 +5269,8 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) InnerError!void { if (output != .none) { return self.fail("TODO implement codegen for non-expr asm", .{}); } - const extra_bytes = std.mem.sliceAsBytes(self.air.extra[extra_i..]); - const constraint = std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra[extra_i..]), 0); + const extra_bytes = std.mem.sliceAsBytes(self.air.extra.items[extra_i..]); + const constraint = std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra.items[extra_i..]), 0); const name = std.mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0); // This equation accounts for the fact that even if we have exactly 4 bytes // for the string, we still use the next u32 for the null terminator. @@ -5281,7 +5280,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) InnerError!void { } else null; for (inputs) |input| { - const input_bytes = std.mem.sliceAsBytes(self.air.extra[extra_i..]); + const input_bytes = std.mem.sliceAsBytes(self.air.extra.items[extra_i..]); const constraint = std.mem.sliceTo(input_bytes, 0); const name = std.mem.sliceTo(input_bytes[constraint.len + 1 ..], 0); // This equation accounts for the fact that even if we have exactly 4 bytes @@ -5303,7 +5302,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) InnerError!void { { var clobber_i: u32 = 0; while (clobber_i < clobbers_len) : (clobber_i += 1) { - const clobber = std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra[extra_i..]), 0); + const clobber = std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra.items[extra_i..]), 0); // This equation accounts for the fact that even if we have exactly 4 bytes // for the string, we still use the next u32 for the null terminator. extra_i += clobber.len / 4 + 1; @@ -5312,7 +5311,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) InnerError!void { } } - const asm_source = std.mem.sliceAsBytes(self.air.extra[extra_i..])[0..extra.data.source_len]; + const asm_source = std.mem.sliceAsBytes(self.air.extra.items[extra_i..])[0..extra.data.source_len]; if (mem.eql(u8, asm_source, "svc #0")) { _ = try self.addInst(.{ @@ -5342,7 +5341,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) InnerError!void { }; simple: { - var buf = [1]Air.Inst.Ref{.none} ** (Liveness.bpi - 1); + var buf = [1]Air.Inst.Ref{.none} ** (Air.Liveness.bpi - 1); var buf_index: usize = 0; for (outputs) |output| { if (output == .none) continue; @@ -6052,14 +6051,14 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) InnerError!void { const vector_ty = self.typeOfIndex(inst); const len = vector_ty.vectorLen(zcu); const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; - const elements = @as([]const Air.Inst.Ref, @ptrCast(self.air.extra[ty_pl.payload..][0..len])); + const elements: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[ty_pl.payload..][0..len]); const result: MCValue = res: { if (self.liveness.isUnused(inst)) break :res MCValue.dead; return self.fail("TODO implement airAggregateInit for {}", .{self.target.cpu.arch}); }; - if (elements.len <= Liveness.bpi - 1) { - var buf = [1]Air.Inst.Ref{.none} ** (Liveness.bpi - 1); + if (elements.len <= Air.Liveness.bpi - 1) { + var buf = [1]Air.Inst.Ref{.none} ** (Air.Liveness.bpi - 1); @memcpy(buf[0..elements.len], elements); return self.finishAir(inst, result, buf); } @@ -6095,7 +6094,7 @@ fn airTry(self: *Self, inst: Air.Inst.Index) InnerError!void { const pt = self.pt; const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; const extra = self.air.extraData(Air.Try, pl_op.payload); - const body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra.end..][0..extra.data.body_len]); + const body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[extra.end..][0..extra.data.body_len]); const result: MCValue = result: { const error_union_bind: ReadArg.Bind = .{ .inst = pl_op.operand }; const error_union_ty = self.typeOf(pl_op.operand); @@ -6122,7 +6121,7 @@ fn airTry(self: *Self, inst: Air.Inst.Index) InnerError!void { fn airTryPtr(self: *Self, inst: Air.Inst.Index) InnerError!void { const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; const extra = self.air.extraData(Air.TryPtr, ty_pl.payload); - const body = self.air.extra[extra.end..][0..extra.data.body_len]; + const body = self.air.extra.items[extra.end..][0..extra.data.body_len]; _ = body; return self.fail("TODO implement airTryPtr for arm", .{}); // return self.finishAir(inst, result, .{ extra.data.ptr, .none, .none }); diff --git a/src/arch/arm/CodeGen.zig b/src/arch/arm/CodeGen.zig index c665e89a183b55d0eef5ffa42def529a4df8de82..e9d0e91db151d9ea9f5bc5c977af642b502f92a8 100644 --- a/src/arch/arm/CodeGen.zig +++ b/src/arch/arm/CodeGen.zig @@ -7,7 +7,6 @@ const codegen = @import("../../codegen.zig"); const Air = @import("../../Air.zig"); const Mir = @import("Mir.zig"); const Emit = @import("Emit.zig"); -const Liveness = @import("../../Liveness.zig"); const Type = @import("../../Type.zig"); const Value = @import("../../Value.zig"); const link = @import("../../link.zig"); @@ -45,7 +44,7 @@ const InnerError = CodeGenError || error{OutOfRegisters}; gpa: Allocator, pt: Zcu.PerThread, air: Air, -liveness: Liveness, +liveness: Air.Liveness, bin_file: *link.File, debug_output: link.File.DebugInfoOutput, target: *const std.Target, @@ -72,7 +71,7 @@ end_di_column: u32, /// which is a relative jump, based on the address following the reloc. exitlude_jump_relocs: std.ArrayListUnmanaged(usize) = .empty, -reused_operands: std.StaticBitSet(Liveness.bpi - 1) = undefined, +reused_operands: std.StaticBitSet(Air.Liveness.bpi - 1) = undefined, /// We postpone the creation of debug info for function args and locals /// until after all Mir instructions have been generated. Only then we @@ -195,7 +194,7 @@ const BlockData = struct { const BigTomb = struct { function: *Self, inst: Air.Inst.Index, - lbt: Liveness.BigTomb, + lbt: Air.Liveness.BigTomb, fn feed(bt: *BigTomb, op_ref: Air.Inst.Ref) void { const dies = bt.lbt.feed(); @@ -333,7 +332,7 @@ pub fn generate( src_loc: Zcu.LazySrcLoc, func_index: InternPool.Index, air: Air, - liveness: Liveness, + liveness: Air.Liveness, code: *std.ArrayListUnmanaged(u8), debug_output: link.File.DebugInfoOutput, ) CodeGenError!void { @@ -635,7 +634,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void { continue; const old_air_bookkeeping = self.air_bookkeeping; - try self.ensureProcessDeathCapacity(Liveness.bpi); + try self.ensureProcessDeathCapacity(Air.Liveness.bpi); self.reused_operands = @TypeOf(self.reused_operands).initEmpty(); switch (air_tags[@intFromEnum(inst)]) { @@ -921,14 +920,14 @@ fn finishAirBookkeeping(self: *Self) void { } } -fn finishAir(self: *Self, inst: Air.Inst.Index, result: MCValue, operands: [Liveness.bpi - 1]Air.Inst.Ref) void { +fn finishAir(self: *Self, inst: Air.Inst.Index, result: MCValue, operands: [Air.Liveness.bpi - 1]Air.Inst.Ref) void { const tomb_bits = self.liveness.getTombBits(inst); for (0.., operands) |op_index, op| { - if (tomb_bits & @as(Liveness.Bpi, 1) << @intCast(op_index) == 0) continue; + if (tomb_bits & @as(Air.Liveness.Bpi, 1) << @intCast(op_index) == 0) continue; if (self.reused_operands.isSet(op_index)) continue; self.processDeath(op.toIndexAllowNone() orelse continue); } - if (tomb_bits & 1 << (Liveness.bpi - 1) == 0) { + if (tomb_bits & 1 << (Air.Liveness.bpi - 1) == 0) { log.debug("%{d} => {}", .{ inst, result }); const branch = &self.branch_stack.items[self.branch_stack.items.len - 1]; branch.inst_table.putAssumeCapacityNoClobber(inst, result); @@ -2617,7 +2616,7 @@ fn reuseOperand( self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, - op_index: Liveness.OperandInt, + op_index: Air.Liveness.OperandInt, mcv: MCValue, ) bool { if (!self.liveness.operandDies(inst, op_index)) @@ -3094,7 +3093,7 @@ const ReuseMetadata = struct { /// inputs to the Air instruction are omitted (e.g. when they can /// be represented as immediates to the Mir instruction), /// operand_mapping should reflect that fact. - operand_mapping: []const Liveness.OperandInt, + operand_mapping: []const Air.Liveness.OperandInt, }; /// Allocate a set of registers for use as arguments for a Mir @@ -3342,7 +3341,7 @@ fn binOpImmediate( const write_args = [_]WriteArg{ .{ .ty = lhs_ty, .bind = .none, .class = gp, .reg = &dest_reg }, }; - const operand_mapping: []const Liveness.OperandInt = if (lhs_and_rhs_swapped) &.{1} else &.{0}; + const operand_mapping: []const Air.Liveness.OperandInt = if (lhs_and_rhs_swapped) &.{1} else &.{0}; try self.allocRegs( &read_args, &write_args, @@ -4232,7 +4231,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; const callee = pl_op.operand; const extra = self.air.extraData(Air.Call, pl_op.payload); - const args: []const Air.Inst.Ref = @ptrCast(self.air.extra[extra.end..][0..extra.data.args_len]); + const args: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[extra.end..][0..extra.data.args_len]); const ty = self.typeOf(callee); const pt = self.pt; const zcu = pt.zcu; @@ -4361,8 +4360,8 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier break :result info.return_value; }; - if (args.len <= Liveness.bpi - 2) { - var buf = [1]Air.Inst.Ref{.none} ** (Liveness.bpi - 1); + if (args.len <= Air.Liveness.bpi - 2) { + var buf = [1]Air.Inst.Ref{.none} ** (Air.Liveness.bpi - 1); buf[0] = callee; @memcpy(buf[1..][0..args.len], args); return self.finishAir(inst, result, buf); @@ -4585,7 +4584,7 @@ fn airDbgInlineBlock(self: *Self, inst: Air.Inst.Index) !void { const func = zcu.funcInfo(extra.data.func); // TODO emit debug info for function change _ = func; - try self.lowerBlock(inst, @ptrCast(self.air.extra[extra.end..][0..extra.data.body_len])); + try self.lowerBlock(inst, @ptrCast(self.air.extra.items[extra.end..][0..extra.data.body_len])); } fn airDbgVar(self: *Self, inst: Air.Inst.Index) !void { @@ -4646,8 +4645,8 @@ fn airCondBr(self: *Self, inst: Air.Inst.Index) !void { const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; const cond_inst = try self.resolveInst(pl_op.operand); const extra = self.air.extraData(Air.CondBr, pl_op.payload); - const then_body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra.end..][0..extra.data.then_body_len]); - const else_body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra.end + then_body.len ..][0..extra.data.else_body_len]); + const then_body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[extra.end..][0..extra.data.then_body_len]); + const else_body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[extra.end + then_body.len ..][0..extra.data.else_body_len]); const liveness_condbr = self.liveness.getCondBr(inst); const reloc: Mir.Inst.Index = try self.condBr(cond_inst); @@ -4966,7 +4965,7 @@ fn airLoop(self: *Self, inst: Air.Inst.Index) !void { // A loop is a setup to be able to jump back to the beginning. const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; const loop = self.air.extraData(Air.Block, ty_pl.payload); - const body: []const Air.Inst.Index = @ptrCast(self.air.extra[loop.end..][0..loop.data.body_len]); + const body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[loop.end..][0..loop.data.body_len]); const start_index: Mir.Inst.Index = @intCast(self.mir_instructions.len); try self.genBody(body); @@ -4986,7 +4985,7 @@ fn jump(self: *Self, inst: Mir.Inst.Index) !void { fn airBlock(self: *Self, inst: Air.Inst.Index) !void { const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; const extra = self.air.extraData(Air.Block, ty_pl.payload); - try self.lowerBlock(inst, @ptrCast(self.air.extra[extra.end..][0..extra.data.body_len])); + try self.lowerBlock(inst, @ptrCast(self.air.extra.items[extra.end..][0..extra.data.body_len])); } fn lowerBlock(self: *Self, inst: Air.Inst.Index, body: []const Air.Inst.Index) !void { @@ -5199,9 +5198,9 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void { const is_volatile = @as(u1, @truncate(extra.data.flags >> 31)) != 0; const clobbers_len: u31 = @truncate(extra.data.flags); var extra_i: usize = extra.end; - const outputs: []const Air.Inst.Ref = @ptrCast(self.air.extra[extra_i..][0..extra.data.outputs_len]); + const outputs: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[extra_i..][0..extra.data.outputs_len]); extra_i += outputs.len; - const inputs: []const Air.Inst.Ref = @ptrCast(self.air.extra[extra_i..][0..extra.data.inputs_len]); + const inputs: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[extra_i..][0..extra.data.inputs_len]); extra_i += inputs.len; const dead = !is_volatile and self.liveness.isUnused(inst); @@ -5214,8 +5213,8 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void { if (output != .none) { return self.fail("TODO implement codegen for non-expr asm", .{}); } - const extra_bytes = std.mem.sliceAsBytes(self.air.extra[extra_i..]); - const constraint = std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra[extra_i..]), 0); + const extra_bytes = std.mem.sliceAsBytes(self.air.extra.items[extra_i..]); + const constraint = std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra.items[extra_i..]), 0); const name = std.mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0); // This equation accounts for the fact that even if we have exactly 4 bytes // for the string, we still use the next u32 for the null terminator. @@ -5225,7 +5224,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void { } else null; for (inputs) |input| { - const input_bytes = std.mem.sliceAsBytes(self.air.extra[extra_i..]); + const input_bytes = std.mem.sliceAsBytes(self.air.extra.items[extra_i..]); const constraint = std.mem.sliceTo(input_bytes, 0); const name = std.mem.sliceTo(input_bytes[constraint.len + 1 ..], 0); // This equation accounts for the fact that even if we have exactly 4 bytes @@ -5247,7 +5246,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void { { var clobber_i: u32 = 0; while (clobber_i < clobbers_len) : (clobber_i += 1) { - const clobber = std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra[extra_i..]), 0); + const clobber = std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra.items[extra_i..]), 0); // This equation accounts for the fact that even if we have exactly 4 bytes // for the string, we still use the next u32 for the null terminator. extra_i += clobber.len / 4 + 1; @@ -5256,7 +5255,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void { } } - const asm_source = std.mem.sliceAsBytes(self.air.extra[extra_i..])[0..extra.data.source_len]; + const asm_source = std.mem.sliceAsBytes(self.air.extra.items[extra_i..])[0..extra.data.source_len]; if (mem.eql(u8, asm_source, "svc #0")) { _ = try self.addInst(.{ @@ -5282,7 +5281,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void { }; simple: { - var buf = [1]Air.Inst.Ref{.none} ** (Liveness.bpi - 1); + var buf = [1]Air.Inst.Ref{.none} ** (Air.Liveness.bpi - 1); var buf_index: usize = 0; for (outputs) |output| { if (output == .none) continue; @@ -6021,14 +6020,14 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void { const vector_ty = self.typeOfIndex(inst); const len = vector_ty.vectorLen(zcu); const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; - const elements: []const Air.Inst.Ref = @ptrCast(self.air.extra[ty_pl.payload..][0..len]); + const elements: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[ty_pl.payload..][0..len]); const result: MCValue = res: { if (self.liveness.isUnused(inst)) break :res MCValue.dead; return self.fail("TODO implement airAggregateInit for arm", .{}); }; - if (elements.len <= Liveness.bpi - 1) { - var buf = [1]Air.Inst.Ref{.none} ** (Liveness.bpi - 1); + if (elements.len <= Air.Liveness.bpi - 1) { + var buf = [1]Air.Inst.Ref{.none} ** (Air.Liveness.bpi - 1); @memcpy(buf[0..elements.len], elements); return self.finishAir(inst, result, buf); } @@ -6065,7 +6064,7 @@ fn airTry(self: *Self, inst: Air.Inst.Index) !void { const pt = self.pt; const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; const extra = self.air.extraData(Air.Try, pl_op.payload); - const body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra.end..][0..extra.data.body_len]); + const body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[extra.end..][0..extra.data.body_len]); const result: MCValue = result: { const error_union_bind: ReadArg.Bind = .{ .inst = pl_op.operand }; const error_union_ty = self.typeOf(pl_op.operand); @@ -6092,7 +6091,7 @@ fn airTry(self: *Self, inst: Air.Inst.Index) !void { fn airTryPtr(self: *Self, inst: Air.Inst.Index) !void { const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; const extra = self.air.extraData(Air.TryPtr, ty_pl.payload); - const body = self.air.extra[extra.end..][0..extra.data.body_len]; + const body = self.air.extra.items[extra.end..][0..extra.data.body_len]; _ = body; return self.fail("TODO implement airTryPtr for arm", .{}); // return self.finishAir(inst, result, .{ extra.data.ptr, .none, .none }); diff --git a/src/arch/powerpc/CodeGen.zig b/src/arch/powerpc/CodeGen.zig index c8553f5d67f26225e64d3e7727b64ac509520163..6334b65ff8271f0b5ba7c47e817bec23ed8091d0 100644 --- a/src/arch/powerpc/CodeGen.zig +++ b/src/arch/powerpc/CodeGen.zig @@ -5,7 +5,6 @@ const Air = @import("../../Air.zig"); const codegen = @import("../../codegen.zig"); const InternPool = @import("../../InternPool.zig"); const link = @import("../../link.zig"); -const Liveness = @import("../../Liveness.zig"); const Zcu = @import("../../Zcu.zig"); const assert = std.debug.assert; @@ -17,7 +16,7 @@ pub fn generate( src_loc: Zcu.LazySrcLoc, func_index: InternPool.Index, air: Air, - liveness: Liveness, + liveness: Air.Liveness, code: *std.ArrayListUnmanaged(u8), debug_output: link.File.DebugInfoOutput, ) codegen.CodeGenError!void { diff --git a/src/arch/riscv64/CodeGen.zig b/src/arch/riscv64/CodeGen.zig index 0d5a9103db4f978965381838f1f3e494a7ba4a4a..edba985beb9b1fa01586752d765d00efcc8f246a 100644 --- a/src/arch/riscv64/CodeGen.zig +++ b/src/arch/riscv64/CodeGen.zig @@ -10,7 +10,6 @@ const Allocator = mem.Allocator; const Air = @import("../../Air.zig"); const Mir = @import("Mir.zig"); const Emit = @import("Emit.zig"); -const Liveness = @import("../../Liveness.zig"); const Type = @import("../../Type.zig"); const Value = @import("../../Value.zig"); const link = @import("../../link.zig"); @@ -54,7 +53,7 @@ const InnerError = CodeGenError || error{OutOfRegisters}; pt: Zcu.PerThread, air: Air, -liveness: Liveness, +liveness: Air.Liveness, bin_file: *link.File, gpa: Allocator, @@ -82,7 +81,7 @@ scope_generation: u32, /// which is a relative jump, based on the address following the reloc. exitlude_jump_relocs: std.ArrayListUnmanaged(usize) = .empty, -reused_operands: std.StaticBitSet(Liveness.bpi - 1) = undefined, +reused_operands: std.StaticBitSet(Air.Liveness.bpi - 1) = undefined, /// Whenever there is a runtime branch, we push a Branch onto this stack, /// and pop it off when the runtime branch joins. This provides an "overlay" @@ -739,7 +738,7 @@ pub fn generate( src_loc: Zcu.LazySrcLoc, func_index: InternPool.Index, air: Air, - liveness: Liveness, + liveness: Air.Liveness, code: *std.ArrayListUnmanaged(u8), debug_output: link.File.DebugInfoOutput, ) CodeGenError!void { @@ -1426,7 +1425,7 @@ fn genBody(func: *Func, body: []const Air.Inst.Index) InnerError!void { verbose_tracking_log.debug("{}", .{func.fmtTracking()}); const old_air_bookkeeping = func.air_bookkeeping; - try func.ensureProcessDeathCapacity(Liveness.bpi); + try func.ensureProcessDeathCapacity(Air.Liveness.bpi); func.reused_operands = @TypeOf(func.reused_operands).initEmpty(); try func.inst_tracking.ensureUnusedCapacity(func.gpa, 1); @@ -1731,7 +1730,7 @@ fn freeValue(func: *Func, value: MCValue) !void { } } -fn feed(func: *Func, bt: *Liveness.BigTomb, operand: Air.Inst.Ref) !void { +fn feed(func: *Func, bt: *Air.Liveness.BigTomb, operand: Air.Inst.Ref) !void { if (bt.feed()) if (operand.toIndex()) |inst| { log.debug("feed inst: %{}", .{inst}); try func.processDeath(inst); @@ -1776,11 +1775,11 @@ fn finishAir( func: *Func, inst: Air.Inst.Index, result: MCValue, - operands: [Liveness.bpi - 1]Air.Inst.Ref, + operands: [Air.Liveness.bpi - 1]Air.Inst.Ref, ) !void { const tomb_bits = func.liveness.getTombBits(inst); for (0.., operands) |op_index, op| { - if (tomb_bits & @as(Liveness.Bpi, 1) << @intCast(op_index) == 0) continue; + if (tomb_bits & @as(Air.Liveness.Bpi, 1) << @intCast(op_index) == 0) continue; if (func.reused_operands.isSet(op_index)) continue; try func.processDeath(op.toIndexAllowNone() orelse continue); } @@ -3651,7 +3650,7 @@ fn airTlvDllimportPtr(func: *Func, inst: Air.Inst.Index) !void { fn airTry(func: *Func, inst: Air.Inst.Index) !void { const pl_op = func.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; const extra = func.air.extraData(Air.Try, pl_op.payload); - const body: []const Air.Inst.Index = @ptrCast(func.air.extra[extra.end..][0..extra.data.body_len]); + const body: []const Air.Inst.Index = @ptrCast(func.air.extra.items[extra.end..][0..extra.data.body_len]); const operand_ty = func.typeOf(pl_op.operand); const result = try func.genTry(inst, pl_op.operand, body, operand_ty, false); return func.finishAir(inst, result, .{ .none, .none, .none }); @@ -4419,7 +4418,7 @@ fn reuseOperand( func: *Func, inst: Air.Inst.Index, operand: Air.Inst.Ref, - op_index: Liveness.OperandInt, + op_index: Air.Liveness.OperandInt, mcv: MCValue, ) bool { return func.reuseOperandAdvanced(inst, operand, op_index, mcv, inst); @@ -4429,7 +4428,7 @@ fn reuseOperandAdvanced( func: *Func, inst: Air.Inst.Index, operand: Air.Inst.Ref, - op_index: Liveness.OperandInt, + op_index: Air.Liveness.OperandInt, mcv: MCValue, maybe_tracked_inst: ?Air.Inst.Index, ) bool { @@ -4816,7 +4815,7 @@ fn airCall(func: *Func, inst: Air.Inst.Index, modifier: std.builtin.CallModifier const pl_op = func.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; const callee = pl_op.operand; const extra = func.air.extraData(Air.Call, pl_op.payload); - const arg_refs: []const Air.Inst.Ref = @ptrCast(func.air.extra[extra.end..][0..extra.data.args_len]); + const arg_refs: []const Air.Inst.Ref = @ptrCast(func.air.extra.items[extra.end..][0..extra.data.args_len]); const expected_num_args = 8; const ExpectedContents = extern struct { @@ -5232,7 +5231,7 @@ fn airDbgStmt(func: *Func, inst: Air.Inst.Index) !void { fn airDbgInlineBlock(func: *Func, inst: Air.Inst.Index) !void { const ty_pl = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; const extra = func.air.extraData(Air.DbgInlineBlock, ty_pl.payload); - try func.lowerBlock(inst, @ptrCast(func.air.extra[extra.end..][0..extra.data.body_len])); + try func.lowerBlock(inst, @ptrCast(func.air.extra.items[extra.end..][0..extra.data.body_len])); } fn airDbgVar(func: *Func, inst: Air.Inst.Index) InnerError!void { @@ -5284,8 +5283,8 @@ fn airCondBr(func: *Func, inst: Air.Inst.Index) !void { const cond = try func.resolveInst(pl_op.operand); const cond_ty = func.typeOf(pl_op.operand); const extra = func.air.extraData(Air.CondBr, pl_op.payload); - const then_body: []const Air.Inst.Index = @ptrCast(func.air.extra[extra.end..][0..extra.data.then_body_len]); - const else_body: []const Air.Inst.Index = @ptrCast(func.air.extra[extra.end + then_body.len ..][0..extra.data.else_body_len]); + const then_body: []const Air.Inst.Index = @ptrCast(func.air.extra.items[extra.end..][0..extra.data.then_body_len]); + const else_body: []const Air.Inst.Index = @ptrCast(func.air.extra.items[extra.end + then_body.len ..][0..extra.data.else_body_len]); const liveness_cond_br = func.liveness.getCondBr(inst); // If the condition dies here in this condbr instruction, process @@ -5644,7 +5643,7 @@ fn airLoop(func: *Func, inst: Air.Inst.Index) !void { // A loop is a setup to be able to jump back to the beginning. const ty_pl = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; const loop = func.air.extraData(Air.Block, ty_pl.payload); - const body: []const Air.Inst.Index = @ptrCast(func.air.extra[loop.end..][0..loop.data.body_len]); + const body: []const Air.Inst.Index = @ptrCast(func.air.extra.items[loop.end..][0..loop.data.body_len]); func.scope_generation += 1; const state = try func.saveState(); @@ -5674,7 +5673,7 @@ fn jump(func: *Func, index: Mir.Inst.Index) !Mir.Inst.Index { fn airBlock(func: *Func, inst: Air.Inst.Index) !void { const ty_pl = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; const extra = func.air.extraData(Air.Block, ty_pl.payload); - try func.lowerBlock(inst, @ptrCast(func.air.extra[extra.end..][0..extra.data.body_len])); + try func.lowerBlock(inst, @ptrCast(func.air.extra.items[extra.end..][0..extra.data.body_len])); } fn lowerBlock(func: *Func, inst: Air.Inst.Index, body: []const Air.Inst.Index) !void { @@ -6063,9 +6062,9 @@ fn airAsm(func: *Func, inst: Air.Inst.Index) !void { const clobbers_len: u31 = @truncate(extra.data.flags); var extra_i: usize = extra.end; const outputs: []const Air.Inst.Ref = - @ptrCast(func.air.extra[extra_i..][0..extra.data.outputs_len]); + @ptrCast(func.air.extra.items[extra_i..][0..extra.data.outputs_len]); extra_i += outputs.len; - const inputs: []const Air.Inst.Ref = @ptrCast(func.air.extra[extra_i..][0..extra.data.inputs_len]); + const inputs: []const Air.Inst.Ref = @ptrCast(func.air.extra.items[extra_i..][0..extra.data.inputs_len]); extra_i += inputs.len; var result: MCValue = .none; @@ -6083,8 +6082,8 @@ fn airAsm(func: *Func, inst: Air.Inst.Index) !void { var outputs_extra_i = extra_i; for (outputs) |output| { - const extra_bytes = mem.sliceAsBytes(func.air.extra[extra_i..]); - const constraint = mem.sliceTo(mem.sliceAsBytes(func.air.extra[extra_i..]), 0); + const extra_bytes = mem.sliceAsBytes(func.air.extra.items[extra_i..]); + const constraint = mem.sliceTo(mem.sliceAsBytes(func.air.extra.items[extra_i..]), 0); const name = mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0); // This equation accounts for the fact that even if we have exactly 4 bytes // for the string, we still use the next u32 for the null terminator. @@ -6141,7 +6140,7 @@ fn airAsm(func: *Func, inst: Air.Inst.Index) !void { } for (inputs) |input| { - const input_bytes = mem.sliceAsBytes(func.air.extra[extra_i..]); + const input_bytes = mem.sliceAsBytes(func.air.extra.items[extra_i..]); const constraint = mem.sliceTo(input_bytes, 0); const name = mem.sliceTo(input_bytes[constraint.len + 1 ..], 0); // This equation accounts for the fact that even if we have exactly 4 bytes @@ -6177,7 +6176,7 @@ fn airAsm(func: *Func, inst: Air.Inst.Index) !void { { var clobber_i: u32 = 0; while (clobber_i < clobbers_len) : (clobber_i += 1) { - const clobber = std.mem.sliceTo(std.mem.sliceAsBytes(func.air.extra[extra_i..]), 0); + const clobber = std.mem.sliceTo(std.mem.sliceAsBytes(func.air.extra.items[extra_i..]), 0); // This equation accounts for the fact that even if we have exactly 4 bytes // for the string, we still use the next u32 for the null terminator. extra_i += clobber.len / 4 + 1; @@ -6224,7 +6223,7 @@ fn airAsm(func: *Func, inst: Air.Inst.Index) !void { labels.deinit(func.gpa); } - const asm_source = std.mem.sliceAsBytes(func.air.extra[extra_i..])[0..extra.data.source_len]; + const asm_source = std.mem.sliceAsBytes(func.air.extra.items[extra_i..])[0..extra.data.source_len]; var line_it = mem.tokenizeAny(u8, asm_source, "\n\r;"); next_line: while (line_it.next()) |line| { var mnem_it = mem.tokenizeAny(u8, line, " \t"); @@ -6493,9 +6492,9 @@ fn airAsm(func: *Func, inst: Air.Inst.Index) !void { return func.fail("undefined label: '{s}'", .{label.key_ptr.*}); for (outputs, args.items[0..outputs.len]) |output, arg_mcv| { - const extra_bytes = mem.sliceAsBytes(func.air.extra[outputs_extra_i..]); + const extra_bytes = mem.sliceAsBytes(func.air.extra.items[outputs_extra_i..]); const constraint = - mem.sliceTo(mem.sliceAsBytes(func.air.extra[outputs_extra_i..]), 0); + mem.sliceTo(mem.sliceAsBytes(func.air.extra.items[outputs_extra_i..]), 0); const name = mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0); // This equation accounts for the fact that even if we have exactly 4 bytes // for the string, we still use the next u32 for the null terminator. @@ -6508,7 +6507,7 @@ fn airAsm(func: *Func, inst: Air.Inst.Index) !void { } simple: { - var buf = [1]Air.Inst.Ref{.none} ** (Liveness.bpi - 1); + var buf = [1]Air.Inst.Ref{.none} ** (Air.Liveness.bpi - 1); var buf_index: usize = 0; for (outputs) |output| { if (output == .none) continue; @@ -8027,7 +8026,7 @@ fn airAggregateInit(func: *Func, inst: Air.Inst.Index) !void { const result_ty = func.typeOfIndex(inst); const len: usize = @intCast(result_ty.arrayLen(zcu)); const ty_pl = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; - const elements: []const Air.Inst.Ref = @ptrCast(func.air.extra[ty_pl.payload..][0..len]); + const elements: []const Air.Inst.Ref = @ptrCast(func.air.extra.items[ty_pl.payload..][0..len]); const result: MCValue = result: { switch (result_ty.zigTypeTag(zcu)) { @@ -8113,8 +8112,8 @@ fn airAggregateInit(func: *Func, inst: Air.Inst.Index) !void { } }; - if (elements.len <= Liveness.bpi - 1) { - var buf = [1]Air.Inst.Ref{.none} ** (Liveness.bpi - 1); + if (elements.len <= Air.Liveness.bpi - 1) { + var buf = [1]Air.Inst.Ref{.none} ** (Air.Liveness.bpi - 1); @memcpy(buf[0..elements.len], elements); return func.finishAir(inst, result, buf); } diff --git a/src/arch/sparc64/CodeGen.zig b/src/arch/sparc64/CodeGen.zig index 4668d2acd4eb6a5995d6000a6d33a8a95f022c4d..4235de94f5896b4308f4a081dbe8653c55591c24 100644 --- a/src/arch/sparc64/CodeGen.zig +++ b/src/arch/sparc64/CodeGen.zig @@ -18,7 +18,6 @@ const codegen = @import("../../codegen.zig"); const Air = @import("../../Air.zig"); const Mir = @import("Mir.zig"); const Emit = @import("Emit.zig"); -const Liveness = @import("../../Liveness.zig"); const Type = @import("../../Type.zig"); const CodeGenError = codegen.CodeGenError; const Endian = std.builtin.Endian; @@ -50,7 +49,7 @@ const RegisterView = enum(u1) { gpa: Allocator, pt: Zcu.PerThread, air: Air, -liveness: Liveness, +liveness: Air.Liveness, bin_file: *link.File, target: *const std.Target, func_index: InternPool.Index, @@ -78,7 +77,7 @@ end_di_column: u32, /// which is a relative jump, based on the address following the reloc. exitlude_jump_relocs: std.ArrayListUnmanaged(usize) = .empty, -reused_operands: std.StaticBitSet(Liveness.bpi - 1) = undefined, +reused_operands: std.StaticBitSet(Air.Liveness.bpi - 1) = undefined, /// Whenever there is a runtime branch, we push a Branch onto this stack, /// and pop it off when the runtime branch joins. This provides an "overlay" @@ -240,7 +239,7 @@ const CallMCValues = struct { const BigTomb = struct { function: *Self, inst: Air.Inst.Index, - lbt: Liveness.BigTomb, + lbt: Air.Liveness.BigTomb, fn feed(bt: *BigTomb, op_ref: Air.Inst.Ref) void { const dies = bt.lbt.feed(); @@ -266,7 +265,7 @@ pub fn generate( src_loc: Zcu.LazySrcLoc, func_index: InternPool.Index, air: Air, - liveness: Liveness, + liveness: Air.Liveness, code: *std.ArrayListUnmanaged(u8), debug_output: link.File.DebugInfoOutput, ) CodeGenError!void { @@ -493,7 +492,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void { continue; const old_air_bookkeeping = self.air_bookkeeping; - try self.ensureProcessDeathCapacity(Liveness.bpi); + try self.ensureProcessDeathCapacity(Air.Liveness.bpi); self.reused_operands = @TypeOf(self.reused_operands).initEmpty(); switch (air_tags[@intFromEnum(inst)]) { @@ -839,14 +838,14 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void { const vector_ty = self.typeOfIndex(inst); const len = vector_ty.vectorLen(zcu); const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; - const elements = @as([]const Air.Inst.Ref, @ptrCast(self.air.extra[ty_pl.payload..][0..len])); + const elements: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[ty_pl.payload..][0..len]); const result: MCValue = res: { if (self.liveness.isUnused(inst)) break :res MCValue.dead; return self.fail("TODO implement airAggregateInit for {}", .{self.target.cpu.arch}); }; - if (elements.len <= Liveness.bpi - 1) { - var buf = [1]Air.Inst.Ref{.none} ** (Liveness.bpi - 1); + if (elements.len <= Air.Liveness.bpi - 1) { + var buf = [1]Air.Inst.Ref{.none} ** (Air.Liveness.bpi - 1); @memcpy(buf[0..elements.len], elements); return self.finishAir(inst, result, buf); } @@ -876,7 +875,7 @@ fn airArrayToSlice(self: *Self, inst: Air.Inst.Index) !void { const ptr_ty = self.typeOf(ty_op.operand); const ptr = try self.resolveInst(ty_op.operand); const array_ty = ptr_ty.childType(zcu); - const array_len = @as(u32, @intCast(array_ty.arrayLen(zcu))); + const array_len: u32 = @intCast(array_ty.arrayLen(zcu)); const ptr_bytes = 8; const stack_offset = try self.allocMem(inst, ptr_bytes * 2, .@"8"); try self.genSetStack(ptr_ty, stack_offset, ptr); @@ -890,11 +889,11 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void { const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; const extra = self.air.extraData(Air.Asm, ty_pl.payload); const is_volatile = (extra.data.flags & 0x80000000) != 0; - const clobbers_len = @as(u31, @truncate(extra.data.flags)); + const clobbers_len: u31 = @truncate(extra.data.flags); var extra_i: usize = extra.end; - const outputs = @as([]const Air.Inst.Ref, @ptrCast(self.air.extra[extra_i .. extra_i + extra.data.outputs_len])); + const outputs: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[extra_i .. extra_i + extra.data.outputs_len]); extra_i += outputs.len; - const inputs = @as([]const Air.Inst.Ref, @ptrCast(self.air.extra[extra_i .. extra_i + extra.data.inputs_len])); + const inputs: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[extra_i .. extra_i + extra.data.inputs_len]); extra_i += inputs.len; const dead = !is_volatile and self.liveness.isUnused(inst); @@ -907,8 +906,8 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void { if (output != .none) { return self.fail("TODO implement codegen for non-expr asm", .{}); } - const extra_bytes = std.mem.sliceAsBytes(self.air.extra[extra_i..]); - const constraint = std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra[extra_i..]), 0); + const extra_bytes = std.mem.sliceAsBytes(self.air.extra.items[extra_i..]); + const constraint = std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra.items[extra_i..]), 0); const name = std.mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0); // This equation accounts for the fact that even if we have exactly 4 bytes // for the string, we still use the next u32 for the null terminator. @@ -918,7 +917,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void { } else null; for (inputs) |input| { - const input_bytes = std.mem.sliceAsBytes(self.air.extra[extra_i..]); + const input_bytes = std.mem.sliceAsBytes(self.air.extra.items[extra_i..]); const constraint = std.mem.sliceTo(input_bytes, 0); const name = std.mem.sliceTo(input_bytes[constraint.len + 1 ..], 0); // This equation accounts for the fact that even if we have exactly 4 bytes @@ -940,7 +939,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void { { var clobber_i: u32 = 0; while (clobber_i < clobbers_len) : (clobber_i += 1) { - const clobber = std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra[extra_i..]), 0); + const clobber = std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra.items[extra_i..]), 0); // This equation accounts for the fact that even if we have exactly 4 bytes // for the string, we still use the next u32 for the null terminator. extra_i += clobber.len / 4 + 1; @@ -949,7 +948,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void { } } - const asm_source = std.mem.sliceAsBytes(self.air.extra[extra_i..])[0..extra.data.source_len]; + const asm_source = std.mem.sliceAsBytes(self.air.extra.items[extra_i..])[0..extra.data.source_len]; if (mem.eql(u8, asm_source, "ta 0x6d")) { _ = try self.addInst(.{ @@ -980,7 +979,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void { }; simple: { - var buf = [1]Air.Inst.Ref{.none} ** (Liveness.bpi - 1); + var buf = [1]Air.Inst.Ref{.none} ** (Air.Liveness.bpi - 1); var buf_index: usize = 0; for (outputs) |output| { if (output == .none) continue; @@ -1124,7 +1123,7 @@ fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void { fn airBlock(self: *Self, inst: Air.Inst.Index) !void { const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; const extra = self.air.extraData(Air.Block, ty_pl.payload); - try self.lowerBlock(inst, @ptrCast(self.air.extra[extra.end..][0..extra.data.body_len])); + try self.lowerBlock(inst, @ptrCast(self.air.extra.items[extra.end..][0..extra.data.body_len])); } fn lowerBlock(self: *Self, inst: Air.Inst.Index, body: []const Air.Inst.Index) !void { @@ -1292,7 +1291,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; const callee = pl_op.operand; const extra = self.air.extraData(Air.Call, pl_op.payload); - const args = @as([]const Air.Inst.Ref, @ptrCast(self.air.extra[extra.end .. extra.end + extra.data.args_len])); + const args: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[extra.end .. extra.end + extra.data.args_len]); const ty = self.typeOf(callee); const pt = self.pt; const zcu = pt.zcu; @@ -1376,8 +1375,8 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier const result = info.return_value; - if (args.len + 1 <= Liveness.bpi - 1) { - var buf = [1]Air.Inst.Ref{.none} ** (Liveness.bpi - 1); + if (args.len + 1 <= Air.Liveness.bpi - 1) { + var buf = [1]Air.Inst.Ref{.none} ** (Air.Liveness.bpi - 1); buf[0] = callee; @memcpy(buf[1..][0..args.len], args); return self.finishAir(inst, result, buf); @@ -1477,8 +1476,8 @@ fn airCondBr(self: *Self, inst: Air.Inst.Index) !void { const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; const condition = try self.resolveInst(pl_op.operand); const extra = self.air.extraData(Air.CondBr, pl_op.payload); - const then_body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra.end..][0..extra.data.then_body_len]); - const else_body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra.end + then_body.len ..][0..extra.data.else_body_len]); + const then_body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[extra.end..][0..extra.data.then_body_len]); + const else_body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[extra.end + then_body.len ..][0..extra.data.else_body_len]); const liveness_condbr = self.liveness.getCondBr(inst); // Here we emit a branch to the false section. @@ -1629,7 +1628,7 @@ fn airDbgInlineBlock(self: *Self, inst: Air.Inst.Index) !void { const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; const extra = self.air.extraData(Air.DbgInlineBlock, ty_pl.payload); // TODO emit debug info for function change - try self.lowerBlock(inst, @ptrCast(self.air.extra[extra.end..][0..extra.data.body_len])); + try self.lowerBlock(inst, @ptrCast(self.air.extra.items[extra.end..][0..extra.data.body_len])); } fn airDbgStmt(self: *Self, inst: Air.Inst.Index) !void { @@ -1795,8 +1794,8 @@ fn airLoop(self: *Self, inst: Air.Inst.Index) !void { // A loop is a setup to be able to jump back to the beginning. const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; const loop = self.air.extraData(Air.Block, ty_pl.payload); - const body: []const Air.Inst.Index = @ptrCast(self.air.extra[loop.end .. loop.end + loop.data.body_len]); - const start = @as(u32, @intCast(self.mir_instructions.len)); + const body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[loop.end .. loop.end + loop.data.body_len]); + const start: u32 = @intCast(self.mir_instructions.len); try self.genBody(body); try self.jump(start); @@ -2514,7 +2513,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void { const zcu = self.pt.zcu; const mcv = try self.resolveInst(operand); const struct_ty = self.typeOf(operand); - const struct_field_offset = @as(u32, @intCast(struct_ty.structFieldOffset(index, zcu))); + const struct_field_offset: u32 = @intCast(struct_ty.structFieldOffset(index, zcu)); switch (mcv) { .dead, .unreach => unreachable, @@ -2612,7 +2611,7 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void { fn airTry(self: *Self, inst: Air.Inst.Index) !void { const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; const extra = self.air.extraData(Air.Try, pl_op.payload); - const body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra.end..][0..extra.data.body_len]); + const body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[extra.end..][0..extra.data.body_len]); const result: MCValue = result: { const error_union_ty = self.typeOf(pl_op.operand); const error_union = try self.resolveInst(pl_op.operand); @@ -3478,7 +3477,7 @@ fn errUnionPayload(self: *Self, error_union_mcv: MCValue, error_union_ty: Type) return MCValue.none; } - const payload_offset = @as(u32, @intCast(errUnionPayloadOffset(payload_ty, zcu))); + const payload_offset: u32 = @intCast(errUnionPayloadOffset(payload_ty, zcu)); switch (error_union_mcv) { .register => return self.fail("TODO errUnionPayload for registers", .{}), .stack_offset => |off| { @@ -3513,14 +3512,14 @@ fn finishAirBookkeeping(self: *Self) void { } } -fn finishAir(self: *Self, inst: Air.Inst.Index, result: MCValue, operands: [Liveness.bpi - 1]Air.Inst.Ref) void { +fn finishAir(self: *Self, inst: Air.Inst.Index, result: MCValue, operands: [Air.Liveness.bpi - 1]Air.Inst.Ref) void { const tomb_bits = self.liveness.getTombBits(inst); for (0.., operands) |op_index, op| { - if (tomb_bits & @as(Liveness.Bpi, 1) << @intCast(op_index) == 0) continue; + if (tomb_bits & @as(Air.Liveness.Bpi, 1) << @intCast(op_index) == 0) continue; if (self.reused_operands.isSet(op_index)) continue; self.processDeath(op.toIndexAllowNone() orelse continue); } - if (tomb_bits & 1 << (Liveness.bpi - 1) == 0) { + if (tomb_bits & 1 << (Air.Liveness.bpi - 1) == 0) { log.debug("%{d} => {}", .{ inst, result }); const branch = &self.branch_stack.items[self.branch_stack.items.len - 1]; branch.inst_table.putAssumeCapacityNoClobber(inst, result); @@ -3944,7 +3943,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerErro try self.genSetStack(wrapped_ty, stack_offset, .{ .register = rwo.reg }); const overflow_bit_ty = ty.fieldType(1, zcu); - const overflow_bit_offset = @as(u32, @intCast(ty.structFieldOffset(1, zcu))); + const overflow_bit_offset: u32 = @intCast(ty.structFieldOffset(1, zcu)); const cond_reg = try self.register_manager.allocReg(null, gp); // TODO handle floating point CCRs @@ -4449,7 +4448,7 @@ fn resolveCallingConventionValues(self: *Self, fn_ty: Type, role: RegisterView) }; for (fn_info.param_types.get(ip), result.args) |ty, *result_arg| { - const param_size = @as(u32, @intCast(Type.fromInterned(ty).abiSize(zcu))); + const param_size: u32 = @intCast(Type.fromInterned(ty).abiSize(zcu)); if (param_size <= 8) { if (next_register < argument_registers.len) { result_arg.* = .{ .register = argument_registers[next_register] }; @@ -4534,7 +4533,7 @@ fn ret(self: *Self, mcv: MCValue) !void { try self.exitlude_jump_relocs.append(self.gpa, index); } -fn reuseOperand(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, op_index: Liveness.OperandInt, mcv: MCValue) bool { +fn reuseOperand(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, op_index: Air.Liveness.OperandInt, mcv: MCValue) bool { if (!self.liveness.operandDies(inst, op_index)) return false; @@ -4664,7 +4663,7 @@ fn structFieldPtr(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, inde const mcv = try self.resolveInst(operand); const ptr_ty = self.typeOf(operand); const struct_ty = ptr_ty.childType(zcu); - const struct_field_offset = @as(u32, @intCast(struct_ty.structFieldOffset(index, zcu))); + const struct_field_offset: u32 = @intCast(struct_ty.structFieldOffset(index, zcu)); switch (mcv) { .ptr_stack_offset => |off| { break :result MCValue{ .ptr_stack_offset = off - struct_field_offset }; diff --git a/src/arch/sparc64/Emit.zig b/src/arch/sparc64/Emit.zig index 74537f023112b66497a0cd8ba5b8545fc3207342..baab0126c2da6011d7464133835fc71a1e8d7657 100644 --- a/src/arch/sparc64/Emit.zig +++ b/src/arch/sparc64/Emit.zig @@ -7,7 +7,6 @@ const assert = std.debug.assert; const link = @import("../../link.zig"); const Zcu = @import("../../Zcu.zig"); const ErrorMsg = Zcu.ErrorMsg; -const Liveness = @import("../../Liveness.zig"); const log = std.log.scoped(.sparcv9_emit); const Emit = @This(); diff --git a/src/arch/wasm/CodeGen.zig b/src/arch/wasm/CodeGen.zig index c3be73bbca07725e5ab78b12552bd600fa9badee..a48f7012f536af7fad4a296e4660c548d2917291 100644 --- a/src/arch/wasm/CodeGen.zig +++ b/src/arch/wasm/CodeGen.zig @@ -17,7 +17,6 @@ const Value = @import("../../Value.zig"); const Compilation = @import("../../Compilation.zig"); const link = @import("../../link.zig"); const Air = @import("../../Air.zig"); -const Liveness = @import("../../Liveness.zig"); const Mir = @import("Mir.zig"); const Emit = @import("Emit.zig"); const abi = @import("abi.zig"); @@ -39,7 +38,7 @@ owner_nav: InternPool.Nav.Index, /// and block block_depth: u32 = 0, air: Air, -liveness: Liveness, +liveness: Air.Liveness, gpa: mem.Allocator, func_index: InternPool.Index, /// Contains a list of current branches. @@ -771,7 +770,7 @@ fn resolveValue(cg: *CodeGen, val: Value) InnerError!WValue { /// NOTE: if result == .stack, it will be stored in .local fn finishAir(cg: *CodeGen, inst: Air.Inst.Index, result: WValue, operands: []const Air.Inst.Ref) InnerError!void { - assert(operands.len <= Liveness.bpi - 1); + assert(operands.len <= Air.Liveness.bpi - 1); var tomb_bits = cg.liveness.getTombBits(inst); for (operands) |operand| { const dies = @as(u1, @truncate(tomb_bits)) != 0; @@ -811,7 +810,7 @@ inline fn currentBranch(cg: *CodeGen) *Branch { const BigTomb = struct { gen: *CodeGen, inst: Air.Inst.Index, - lbt: Liveness.BigTomb, + lbt: Air.Liveness.BigTomb, fn feed(bt: *BigTomb, op_ref: Air.Inst.Ref) void { const dies = bt.lbt.feed(); @@ -1262,7 +1261,7 @@ pub fn function( pt: Zcu.PerThread, func_index: InternPool.Index, air: Air, - liveness: Liveness, + liveness: Air.Liveness, ) Error!Function { const zcu = pt.zcu; const gpa = zcu.gpa; @@ -2123,7 +2122,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { continue; } const old_bookkeeping_value = cg.air_bookkeeping; - try cg.currentBranch().values.ensureUnusedCapacity(cg.gpa, Liveness.bpi); + try cg.currentBranch().values.ensureUnusedCapacity(cg.gpa, Air.Liveness.bpi); try cg.genInst(inst); if (std.debug.runtime_safety and cg.air_bookkeeping < old_bookkeeping_value + 1) { @@ -2217,7 +2216,7 @@ fn airCall(cg: *CodeGen, inst: Air.Inst.Index, modifier: std.builtin.CallModifie if (modifier == .always_tail) return cg.fail("TODO implement tail calls for wasm", .{}); const pl_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; const extra = cg.air.extraData(Air.Call, pl_op.payload); - const args: []const Air.Inst.Ref = @ptrCast(cg.air.extra[extra.end..][0..extra.data.args_len]); + const args: []const Air.Inst.Ref = @ptrCast(cg.air.extra.items[extra.end..][0..extra.data.args_len]); const ty = cg.typeOf(pl_op.operand); const pt = cg.pt; @@ -3410,7 +3409,7 @@ fn emitUndefined(cg: *CodeGen, ty: Type) InnerError!WValue { fn airBlock(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { const ty_pl = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; const extra = cg.air.extraData(Air.Block, ty_pl.payload); - try cg.lowerBlock(inst, ty_pl.ty.toType(), @ptrCast(cg.air.extra[extra.end..][0..extra.data.body_len])); + try cg.lowerBlock(inst, ty_pl.ty.toType(), @ptrCast(cg.air.extra.items[extra.end..][0..extra.data.body_len])); } fn lowerBlock(cg: *CodeGen, inst: Air.Inst.Index, block_ty: Type, body: []const Air.Inst.Index) InnerError!void { @@ -3456,7 +3455,7 @@ fn endBlock(cg: *CodeGen) !void { fn airLoop(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { const ty_pl = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; const loop = cg.air.extraData(Air.Block, ty_pl.payload); - const body: []const Air.Inst.Index = @ptrCast(cg.air.extra[loop.end..][0..loop.data.body_len]); + const body: []const Air.Inst.Index = @ptrCast(cg.air.extra.items[loop.end..][0..loop.data.body_len]); // result type of loop is always 'noreturn', meaning we can always // emit the wasm type 'block_empty'. @@ -3475,8 +3474,8 @@ fn airCondBr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { const pl_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; const condition = try cg.resolveInst(pl_op.operand); const extra = cg.air.extraData(Air.CondBr, pl_op.payload); - const then_body: []const Air.Inst.Index = @ptrCast(cg.air.extra[extra.end..][0..extra.data.then_body_len]); - const else_body: []const Air.Inst.Index = @ptrCast(cg.air.extra[extra.end + then_body.len ..][0..extra.data.else_body_len]); + const then_body: []const Air.Inst.Index = @ptrCast(cg.air.extra.items[extra.end..][0..extra.data.then_body_len]); + const else_body: []const Air.Inst.Index = @ptrCast(cg.air.extra.items[extra.end + then_body.len ..][0..extra.data.else_body_len]); const liveness_condbr = cg.liveness.getCondBr(inst); // result type is always noreturn, so use `block_empty` as type. @@ -5238,7 +5237,7 @@ fn airAggregateInit(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { const ty_pl = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; const result_ty = cg.typeOfIndex(inst); const len = @as(usize, @intCast(result_ty.arrayLen(zcu))); - const elements = @as([]const Air.Inst.Ref, @ptrCast(cg.air.extra[ty_pl.payload..][0..len])); + const elements: []const Air.Inst.Ref = @ptrCast(cg.air.extra.items[ty_pl.payload..][0..len]); const result: WValue = result_value: { switch (result_ty.zigTypeTag(zcu)) { @@ -5352,8 +5351,8 @@ fn airAggregateInit(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { } }; - if (elements.len <= Liveness.bpi - 1) { - var buf = [1]Air.Inst.Ref{.none} ** (Liveness.bpi - 1); + if (elements.len <= Air.Liveness.bpi - 1) { + var buf = [1]Air.Inst.Ref{.none} ** (Air.Liveness.bpi - 1); @memcpy(buf[0..elements.len], elements); return cg.finishAir(inst, result, &buf); } @@ -6454,7 +6453,7 @@ fn airDbgInlineBlock(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { const ty_pl = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; const extra = cg.air.extraData(Air.DbgInlineBlock, ty_pl.payload); // TODO - try cg.lowerBlock(inst, ty_pl.ty.toType(), @ptrCast(cg.air.extra[extra.end..][0..extra.data.body_len])); + try cg.lowerBlock(inst, ty_pl.ty.toType(), @ptrCast(cg.air.extra.items[extra.end..][0..extra.data.body_len])); } fn airDbgVar( @@ -6472,7 +6471,7 @@ fn airTry(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { const pl_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; const err_union = try cg.resolveInst(pl_op.operand); const extra = cg.air.extraData(Air.Try, pl_op.payload); - const body: []const Air.Inst.Index = @ptrCast(cg.air.extra[extra.end..][0..extra.data.body_len]); + const body: []const Air.Inst.Index = @ptrCast(cg.air.extra.items[extra.end..][0..extra.data.body_len]); const err_union_ty = cg.typeOf(pl_op.operand); const result = try lowerTry(cg, inst, err_union, body, err_union_ty, false); return cg.finishAir(inst, result, &.{pl_op.operand}); @@ -6483,7 +6482,7 @@ fn airTryPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { const ty_pl = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; const extra = cg.air.extraData(Air.TryPtr, ty_pl.payload); const err_union_ptr = try cg.resolveInst(extra.data.ptr); - const body: []const Air.Inst.Index = @ptrCast(cg.air.extra[extra.end..][0..extra.data.body_len]); + const body: []const Air.Inst.Index = @ptrCast(cg.air.extra.items[extra.end..][0..extra.data.body_len]); const err_union_ty = cg.typeOf(extra.data.ptr).childType(zcu); const result = try lowerTry(cg, inst, err_union_ptr, body, err_union_ty, true); return cg.finishAir(inst, result, &.{extra.data.ptr}); diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig index 985a5446cc88f8ae71ab5bb13ee141c577acd3e9..0b2a17d19275343b46e1bf4065d80952de4b3452 100644 --- a/src/arch/x86_64/CodeGen.zig +++ b/src/arch/x86_64/CodeGen.zig @@ -10,7 +10,6 @@ const wip_mir_log = std.log.scoped(.wip_mir); const Air = @import("../../Air.zig"); const Allocator = std.mem.Allocator; const Emit = @import("Emit.zig"); -const Liveness = @import("../../Liveness.zig"); const Lower = @import("Lower.zig"); const Mir = @import("Mir.zig"); const Zcu = @import("../../Zcu.zig"); @@ -33,6 +32,11 @@ const FrameIndex = bits.FrameIndex; const InnerError = codegen.CodeGenError || error{OutOfRegisters}; +pub const legalize_features: Air.Legalize.Features = .{ + .remove_shift_vector_rhs_splat = false, + .reduce_one_elem_to_bitcast = true, +}; + /// Set this to `false` to uncover Sema OPV bugs. /// https://github.com/ziglang/zig/issues/22419 const hack_around_sema_opv_bugs = true; @@ -42,7 +46,7 @@ const err_ret_trace_index: Air.Inst.Index = @enumFromInt(std.math.maxInt(u32)); gpa: Allocator, pt: Zcu.PerThread, air: Air, -liveness: Liveness, +liveness: Air.Liveness, bin_file: *link.File, debug_output: link.File.DebugInfoOutput, target: *const std.Target, @@ -78,7 +82,7 @@ mir_table: std.ArrayListUnmanaged(Mir.Inst.Index) = .empty, /// which is a relative jump, based on the address following the reloc. epilogue_relocs: std.ArrayListUnmanaged(Mir.Inst.Index) = .empty, -reused_operands: std.StaticBitSet(Liveness.bpi - 1) = undefined, +reused_operands: std.StaticBitSet(Air.Liveness.bpi - 1) = undefined, inst_tracking: InstTrackingMap = .empty, // Key is the block instruction @@ -859,7 +863,7 @@ pub fn generate( src_loc: Zcu.LazySrcLoc, func_index: InternPool.Index, air: Air, - liveness: Liveness, + liveness: Air.Liveness, code: *std.ArrayListUnmanaged(u8), debug_output: link.File.DebugInfoOutput, ) codegen.CodeGenError!void { @@ -63335,7 +63339,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { const ty_pl = air_datas[@intFromEnum(inst)].ty_pl; const block = cg.air.extraData(Air.Block, ty_pl.payload); if (cg.debug_output != .none) try cg.asmPseudo(.pseudo_dbg_enter_block_none); - try cg.lowerBlock(inst, @ptrCast(cg.air.extra[block.end..][0..block.data.body_len])); + try cg.lowerBlock(inst, @ptrCast(cg.air.extra.items[block.end..][0..block.data.body_len])); if (cg.debug_output != .none) try cg.asmPseudo(.pseudo_dbg_leave_block_none); }, .loop => if (use_old) try cg.airLoop(inst) else { @@ -63346,7 +63350,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .target = @intCast(cg.mir_instructions.len), }); defer assert(cg.loops.remove(inst)); - try cg.genBodyBlock(@ptrCast(cg.air.extra[block.end..][0..block.data.body_len])); + try cg.genBodyBlock(@ptrCast(cg.air.extra.items[block.end..][0..block.data.body_len])); }, .repeat => if (use_old) try cg.airRepeat(inst) else { const repeat = air_datas[@intFromEnum(inst)].repeat; @@ -84360,7 +84364,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .ops = .pseudo_dbg_enter_inline_func, .data = .{ .func = dbg_inline_block.data.func }, }); - try cg.lowerBlock(inst, @ptrCast(cg.air.extra[dbg_inline_block.end..][0..dbg_inline_block.data.body_len])); + try cg.lowerBlock(inst, @ptrCast(cg.air.extra.items[dbg_inline_block.end..][0..dbg_inline_block.data.body_len])); if (cg.debug_output != .none) _ = try cg.addInst(.{ .tag = .pseudo, .ops = .pseudo_dbg_leave_inline_func, @@ -160620,7 +160624,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { var bt = cg.liveness.iterateBigTomb(inst); switch (ip.indexToKey(agg_ty.toIntern())) { inline .array_type, .vector_type => |sequence_type| { - const elems: []const Air.Inst.Ref = @ptrCast(cg.air.extra[ty_pl.payload..][0..@intCast(sequence_type.len)]); + const elems: []const Air.Inst.Ref = @ptrCast(cg.air.extra.items[ty_pl.payload..][0..@intCast(sequence_type.len)]); const elem_size = Type.fromInterned(sequence_type.child).abiSize(zcu); var elem_disp: u31 = 0; for (elems) |elem_ref| { @@ -160638,7 +160642,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { }, .struct_type => { const loaded_struct = ip.loadStructType(agg_ty.toIntern()); - const elems: []const Air.Inst.Ref = @ptrCast(cg.air.extra[ty_pl.payload..][0..loaded_struct.field_types.len]); + const elems: []const Air.Inst.Ref = @ptrCast(cg.air.extra.items[ty_pl.payload..][0..loaded_struct.field_types.len]); switch (loaded_struct.layout) { .auto, .@"extern" => { for (elems, 0..) |elem_ref, field_index| { @@ -160657,7 +160661,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { } }, .tuple_type => |tuple_type| { - const elems: []const Air.Inst.Ref = @ptrCast(cg.air.extra[ty_pl.payload..][0..tuple_type.types.len]); + const elems: []const Air.Inst.Ref = @ptrCast(cg.air.extra.items[ty_pl.payload..][0..tuple_type.types.len]); var elem_disp: u31 = 0; for (elems, 0..) |elem_ref, field_index| { const elem_dies = bt.feed(); @@ -162630,7 +162634,7 @@ fn freeValue(self: *CodeGen, value: MCValue) !void { } } -fn feed(self: *CodeGen, bt: *Liveness.BigTomb, operand: Air.Inst.Ref) !void { +fn feed(self: *CodeGen, bt: *Air.Liveness.BigTomb, operand: Air.Inst.Ref) !void { if (bt.feed()) if (operand.toIndex()) |inst| try self.processDeath(inst); } @@ -162657,11 +162661,11 @@ fn finishAir( self: *CodeGen, inst: Air.Inst.Index, result: MCValue, - operands: [Liveness.bpi - 1]Air.Inst.Ref, + operands: [Air.Liveness.bpi - 1]Air.Inst.Ref, ) !void { const tomb_bits = self.liveness.getTombBits(inst); for (0.., operands) |op_index, op| { - if (tomb_bits & @as(Liveness.Bpi, 1) << @intCast(op_index) == 0) continue; + if (tomb_bits & @as(Air.Liveness.Bpi, 1) << @intCast(op_index) == 0) continue; if (self.reused_operands.isSet(op_index)) continue; try self.processDeath(op.toIndexAllowNone() orelse continue); } @@ -167965,7 +167969,7 @@ fn reuseOperand( self: *CodeGen, inst: Air.Inst.Index, operand: Air.Inst.Ref, - op_index: Liveness.OperandInt, + op_index: Air.Liveness.OperandInt, mcv: MCValue, ) bool { return self.reuseOperandAdvanced(inst, operand, op_index, mcv, inst); @@ -167975,7 +167979,7 @@ fn reuseOperandAdvanced( self: *CodeGen, inst: Air.Inst.Index, operand: Air.Inst.Ref, - op_index: Liveness.OperandInt, + op_index: Air.Liveness.OperandInt, mcv: MCValue, maybe_tracked_inst: ?Air.Inst.Index, ) bool { @@ -172435,7 +172439,7 @@ fn airCall(self: *CodeGen, inst: Air.Inst.Index, modifier: std.builtin.CallModif const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; const extra = self.air.extraData(Air.Call, pl_op.payload); const arg_refs: []const Air.Inst.Ref = - @ptrCast(self.air.extra[extra.end..][0..extra.data.args_len]); + @ptrCast(self.air.extra.items[extra.end..][0..extra.data.args_len]); const ExpectedContents = extern struct { tys: [16][@sizeOf(Type)]u8 align(@alignOf(Type)), @@ -173349,7 +173353,7 @@ fn airCmpLtErrorsLen(self: *CodeGen, inst: Air.Inst.Index) !void { fn airTry(self: *CodeGen, inst: Air.Inst.Index) !void { const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; const extra = self.air.extraData(Air.Try, pl_op.payload); - const body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra.end..][0..extra.data.body_len]); + const body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[extra.end..][0..extra.data.body_len]); const operand_ty = self.typeOf(pl_op.operand); const result = try self.genTry(inst, pl_op.operand, body, operand_ty, false); return self.finishAir(inst, result, .{ .none, .none, .none }); @@ -173358,7 +173362,7 @@ fn airTry(self: *CodeGen, inst: Air.Inst.Index) !void { fn airTryPtr(self: *CodeGen, inst: Air.Inst.Index) !void { const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; const extra = self.air.extraData(Air.TryPtr, ty_pl.payload); - const body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra.end..][0..extra.data.body_len]); + const body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[extra.end..][0..extra.data.body_len]); const operand_ty = self.typeOf(extra.data.ptr); const result = try self.genTry(inst, extra.data.ptr, body, operand_ty, true); return self.finishAir(inst, result, .{ .none, .none, .none }); @@ -173449,9 +173453,9 @@ fn airCondBr(self: *CodeGen, inst: Air.Inst.Index) !void { const cond_ty = self.typeOf(pl_op.operand); const extra = self.air.extraData(Air.CondBr, pl_op.payload); const then_body: []const Air.Inst.Index = - @ptrCast(self.air.extra[extra.end..][0..extra.data.then_body_len]); + @ptrCast(self.air.extra.items[extra.end..][0..extra.data.then_body_len]); const else_body: []const Air.Inst.Index = - @ptrCast(self.air.extra[extra.end + then_body.len ..][0..extra.data.else_body_len]); + @ptrCast(self.air.extra.items[extra.end + then_body.len ..][0..extra.data.else_body_len]); const liveness_cond_br = self.liveness.getCondBr(inst); // If the condition dies here in this condbr instruction, process @@ -173838,7 +173842,7 @@ fn airLoop(self: *CodeGen, inst: Air.Inst.Index) !void { // A loop is a setup to be able to jump back to the beginning. const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; const loop = self.air.extraData(Air.Block, ty_pl.payload); - const body: []const Air.Inst.Index = @ptrCast(self.air.extra[loop.end..][0..loop.data.body_len]); + const body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[loop.end..][0..loop.data.body_len]); const state = try self.saveState(); @@ -174469,9 +174473,9 @@ fn airAsm(self: *CodeGen, inst: Air.Inst.Index) !void { const extra = self.air.extraData(Air.Asm, ty_pl.payload); const clobbers_len: u31 = @truncate(extra.data.flags); var extra_i: usize = extra.end; - const outputs: []const Air.Inst.Ref = @ptrCast(self.air.extra[extra_i..][0..extra.data.outputs_len]); + const outputs: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[extra_i..][0..extra.data.outputs_len]); extra_i += outputs.len; - const inputs: []const Air.Inst.Ref = @ptrCast(self.air.extra[extra_i..][0..extra.data.inputs_len]); + const inputs: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[extra_i..][0..extra.data.inputs_len]); extra_i += inputs.len; var result: MCValue = .none; @@ -174489,8 +174493,8 @@ fn airAsm(self: *CodeGen, inst: Air.Inst.Index) !void { var outputs_extra_i = extra_i; for (outputs) |output| { - const extra_bytes = std.mem.sliceAsBytes(self.air.extra[extra_i..]); - const constraint = std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra[extra_i..]), 0); + const extra_bytes = std.mem.sliceAsBytes(self.air.extra.items[extra_i..]); + const constraint = std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra.items[extra_i..]), 0); const name = std.mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0); // This equation accounts for the fact that even if we have exactly 4 bytes // for the string, we still use the next u32 for the null terminator. @@ -174575,7 +174579,7 @@ fn airAsm(self: *CodeGen, inst: Air.Inst.Index) !void { } for (inputs) |input| { - const input_bytes = std.mem.sliceAsBytes(self.air.extra[extra_i..]); + const input_bytes = std.mem.sliceAsBytes(self.air.extra.items[extra_i..]); const constraint = std.mem.sliceTo(input_bytes, 0); const name = std.mem.sliceTo(input_bytes[constraint.len + 1 ..], 0); // This equation accounts for the fact that even if we have exactly 4 bytes @@ -174663,7 +174667,7 @@ fn airAsm(self: *CodeGen, inst: Air.Inst.Index) !void { { var clobber_i: u32 = 0; while (clobber_i < clobbers_len) : (clobber_i += 1) { - const clobber = std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra[extra_i..]), 0); + const clobber = std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra.items[extra_i..]), 0); // This equation accounts for the fact that even if we have exactly 4 bytes // for the string, we still use the next u32 for the null terminator. extra_i += clobber.len / 4 + 1; @@ -174719,7 +174723,7 @@ fn airAsm(self: *CodeGen, inst: Air.Inst.Index) !void { labels.deinit(self.gpa); } - const asm_source = std.mem.sliceAsBytes(self.air.extra[extra_i..])[0..extra.data.source_len]; + const asm_source = std.mem.sliceAsBytes(self.air.extra.items[extra_i..])[0..extra.data.source_len]; var line_it = std.mem.tokenizeAny(u8, asm_source, "\n\r;"); next_line: while (line_it.next()) |line| { var mnem_it = std.mem.tokenizeAny(u8, line, " \t"); @@ -175131,9 +175135,9 @@ fn airAsm(self: *CodeGen, inst: Air.Inst.Index) !void { return self.fail("undefined label: '{s}'", .{label.key_ptr.*}); for (outputs, args.items[0..outputs.len]) |output, arg_mcv| { - const extra_bytes = std.mem.sliceAsBytes(self.air.extra[outputs_extra_i..]); + const extra_bytes = std.mem.sliceAsBytes(self.air.extra.items[outputs_extra_i..]); const constraint = - std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra[outputs_extra_i..]), 0); + std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra.items[outputs_extra_i..]), 0); const name = std.mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0); // This equation accounts for the fact that even if we have exactly 4 bytes // for the string, we still use the next u32 for the null terminator. @@ -175146,7 +175150,7 @@ fn airAsm(self: *CodeGen, inst: Air.Inst.Index) !void { } simple: { - var buf: [Liveness.bpi - 1]Air.Inst.Ref = @splat(.none); + var buf: [Air.Liveness.bpi - 1]Air.Inst.Ref = @splat(.none); var buf_index: usize = 0; for (outputs) |output| { if (output == .none) continue; @@ -179659,7 +179663,7 @@ fn airAggregateInit(self: *CodeGen, inst: Air.Inst.Index) !void { const result_ty = self.typeOfIndex(inst); const len: usize = @intCast(result_ty.arrayLen(zcu)); const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; - const elements: []const Air.Inst.Ref = @ptrCast(self.air.extra[ty_pl.payload..][0..len]); + const elements: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[ty_pl.payload..][0..len]); const result: MCValue = result: { switch (result_ty.zigTypeTag(zcu)) { .@"struct" => { @@ -179823,8 +179827,8 @@ fn airAggregateInit(self: *CodeGen, inst: Air.Inst.Index) !void { } }; - if (elements.len <= Liveness.bpi - 1) { - var buf: [Liveness.bpi - 1]Air.Inst.Ref = @splat(.none); + if (elements.len <= Air.Liveness.bpi - 1) { + var buf: [Air.Liveness.bpi - 1]Air.Inst.Ref = @splat(.none); @memcpy(buf[0..elements.len], elements); return self.finishAir(inst, result, buf); } @@ -186387,7 +186391,7 @@ const Temp = struct { for (0.., op_refs, op_temps) |op_index, op_ref, op_temp| { if (op_temp.index == temp.index) continue; if (op_temp.tracking(cg).short != .dead) try op_temp.die(cg); - if (tomb_bits & @as(Liveness.Bpi, 1) << @intCast(op_index) == 0) continue; + if (tomb_bits & @as(Air.Liveness.Bpi, 1) << @intCast(op_index) == 0) continue; if (cg.reused_operands.isSet(op_index)) continue; try cg.processDeath(op_ref.toIndexAllowNone() orelse continue); } @@ -186407,7 +186411,7 @@ const Temp = struct { } for (0.., op_refs, op_temps) |op_index, op_ref, op_temp| { if (op_temp.index != temp.index) continue; - if (tomb_bits & @as(Liveness.Bpi, 1) << @intCast(op_index) == 0) continue; + if (tomb_bits & @as(Air.Liveness.Bpi, 1) << @intCast(op_index) == 0) continue; if (cg.reused_operands.isSet(op_index)) continue; try cg.processDeath(op_ref.toIndexAllowNone() orelse continue); } diff --git a/src/codegen.zig b/src/codegen.zig index 6577667c7c38d7e99efb129bb51f97dbe6fe42b5..f5108162ca5cf7dd5438e8234591299999a05c8a 100644 --- a/src/codegen.zig +++ b/src/codegen.zig @@ -14,7 +14,6 @@ const Allocator = mem.Allocator; const Compilation = @import("Compilation.zig"); const ErrorMsg = Zcu.ErrorMsg; const InternPool = @import("InternPool.zig"); -const Liveness = @import("Liveness.zig"); const Zcu = @import("Zcu.zig"); const Type = @import("Type.zig"); @@ -33,15 +32,18 @@ fn devFeatureForBackend(comptime backend: std.builtin.CompilerBackend) dev.Featu return @field(dev.Feature, @tagName(backend)["stage2_".len..] ++ "_backend"); } -fn importBackend(comptime backend: std.builtin.CompilerBackend) type { +pub fn importBackend(comptime backend: std.builtin.CompilerBackend) ?type { return switch (backend) { .stage2_aarch64 => @import("arch/aarch64/CodeGen.zig"), .stage2_arm => @import("arch/arm/CodeGen.zig"), + .stage2_c => @import("codegen/c.zig"), + .stage2_llvm => @import("codegen/llvm.zig"), .stage2_powerpc => @import("arch/powerpc/CodeGen.zig"), .stage2_riscv64 => @import("arch/riscv64/CodeGen.zig"), .stage2_sparc64 => @import("arch/sparc64/CodeGen.zig"), + .stage2_spirv64 => @import("codegen/spirv.zig"), .stage2_x86_64 => @import("arch/x86_64/CodeGen.zig"), - else => unreachable, + else => null, }; } @@ -51,7 +53,7 @@ pub fn generateFunction( src_loc: Zcu.LazySrcLoc, func_index: InternPool.Index, air: Air, - liveness: Liveness, + liveness: Air.Liveness, code: *std.ArrayListUnmanaged(u8), debug_output: link.File.DebugInfoOutput, ) CodeGenError!void { @@ -68,7 +70,7 @@ pub fn generateFunction( .stage2_x86_64, => |backend| { dev.check(devFeatureForBackend(backend)); - return importBackend(backend).generate(lf, pt, src_loc, func_index, air, liveness, code, debug_output); + return importBackend(backend).?.generate(lf, pt, src_loc, func_index, air, liveness, code, debug_output); }, } } @@ -93,7 +95,7 @@ pub fn generateLazyFunction( .stage2_x86_64, => |backend| { dev.check(devFeatureForBackend(backend)); - return importBackend(backend).generateLazy(lf, pt, src_loc, lazy_sym, code, debug_output); + return importBackend(backend).?.generateLazy(lf, pt, src_loc, lazy_sym, code, debug_output); }, } } diff --git a/src/codegen/c.zig b/src/codegen/c.zig index 075d9ba303c569bdbcf18cc77449b5253a9d4034..8539efdbfe1969e2d37d75d78b7788341f7214e5 100644 --- a/src/codegen/c.zig +++ b/src/codegen/c.zig @@ -14,7 +14,6 @@ const C = link.File.C; const Decl = Zcu.Decl; const trace = @import("../tracy.zig").trace; const Air = @import("../Air.zig"); -const Liveness = @import("../Liveness.zig"); const InternPool = @import("../InternPool.zig"); const Alignment = InternPool.Alignment; @@ -356,7 +355,7 @@ pub fn isMangledIdent(ident: []const u8, solo: bool) bool { /// It is not available when generating .h file. pub const Function = struct { air: Air, - liveness: Liveness, + liveness: Air.Liveness, value_map: CValueMap, blocks: std.AutoHashMapUnmanaged(Air.Inst.Index, BlockData) = .empty, next_arg_index: u32 = 0, @@ -2323,9 +2322,9 @@ pub const DeclGen = struct { const pt = dg.pt; const zcu = pt.zcu; - const int_info = if (ty.isAbiInt(zcu)) ty.intInfo(zcu) else std.builtin.Type.Int{ + const int_info: std.builtin.Type.Int = if (ty.isAbiInt(zcu)) ty.intInfo(zcu) else .{ .signedness = .unsigned, - .bits = @as(u16, @intCast(ty.bitSize(zcu))), + .bits = @intCast(ty.bitSize(zcu)), }; if (is_big) try writer.print(", {}", .{int_info.signedness == .signed}); @@ -3179,7 +3178,7 @@ fn genBodyResolveState(f: *Function, inst: Air.Inst.Index, leading_deaths: []con // Remember how many locals there were before entering the body so that we can free any that // were newly introduced. Any new locals must necessarily be logically free after the then // branch is complete. - const pre_locals_len = @as(LocalIndex, @intCast(f.locals.items.len)); + const pre_locals_len: LocalIndex = @intCast(f.locals.items.len); for (leading_deaths) |death| { try die(f, inst, death.toRef()); @@ -4540,7 +4539,7 @@ fn airCall( const pl_op = f.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; const extra = f.air.extraData(Air.Call, pl_op.payload); - const args = @as([]const Air.Inst.Ref, @ptrCast(f.air.extra[extra.end..][0..extra.data.args_len])); + const args: []const Air.Inst.Ref = @ptrCast(f.air.extra.items[extra.end..][0..extra.data.args_len]); const resolved_args = try gpa.alloc(CValue, args.len); defer gpa.free(resolved_args); @@ -4708,7 +4707,7 @@ fn airDbgInlineBlock(f: *Function, inst: Air.Inst.Index) !CValue { const owner_nav = ip.getNav(zcu.funcInfo(extra.data.func).owner_nav); const writer = f.object.writer(); try writer.print("/* inline:{} */\n", .{owner_nav.fqn.fmt(&zcu.intern_pool)}); - return lowerBlock(f, inst, @ptrCast(f.air.extra[extra.end..][0..extra.data.body_len])); + return lowerBlock(f, inst, @ptrCast(f.air.extra.items[extra.end..][0..extra.data.body_len])); } fn airDbgVar(f: *Function, inst: Air.Inst.Index) !CValue { @@ -4729,7 +4728,7 @@ fn airDbgVar(f: *Function, inst: Air.Inst.Index) !CValue { fn airBlock(f: *Function, inst: Air.Inst.Index) !CValue { const ty_pl = f.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; const extra = f.air.extraData(Air.Block, ty_pl.payload); - return lowerBlock(f, inst, @ptrCast(f.air.extra[extra.end..][0..extra.data.body_len])); + return lowerBlock(f, inst, @ptrCast(f.air.extra.items[extra.end..][0..extra.data.body_len])); } fn lowerBlock(f: *Function, inst: Air.Inst.Index, body: []const Air.Inst.Index) !CValue { @@ -4781,7 +4780,7 @@ fn lowerBlock(f: *Function, inst: Air.Inst.Index, body: []const Air.Inst.Index) fn airTry(f: *Function, inst: Air.Inst.Index) !CValue { const pl_op = f.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; const extra = f.air.extraData(Air.Try, pl_op.payload); - const body: []const Air.Inst.Index = @ptrCast(f.air.extra[extra.end..][0..extra.data.body_len]); + const body: []const Air.Inst.Index = @ptrCast(f.air.extra.items[extra.end..][0..extra.data.body_len]); const err_union_ty = f.typeOf(pl_op.operand); return lowerTry(f, inst, pl_op.operand, body, err_union_ty, false); } @@ -4791,7 +4790,7 @@ fn airTryPtr(f: *Function, inst: Air.Inst.Index) !CValue { const zcu = pt.zcu; const ty_pl = f.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; const extra = f.air.extraData(Air.TryPtr, ty_pl.payload); - const body: []const Air.Inst.Index = @ptrCast(f.air.extra[extra.end..][0..extra.data.body_len]); + const body: []const Air.Inst.Index = @ptrCast(f.air.extra.items[extra.end..][0..extra.data.body_len]); const err_union_ty = f.typeOf(extra.data.ptr).childType(zcu); return lowerTry(f, inst, extra.data.ptr, body, err_union_ty, true); } @@ -5100,7 +5099,7 @@ fn airUnreach(f: *Function) !void { fn airLoop(f: *Function, inst: Air.Inst.Index) !void { const ty_pl = f.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; const loop = f.air.extraData(Air.Block, ty_pl.payload); - const body: []const Air.Inst.Index = @ptrCast(f.air.extra[loop.end..][0..loop.data.body_len]); + const body: []const Air.Inst.Index = @ptrCast(f.air.extra.items[loop.end..][0..loop.data.body_len]); const writer = f.object.writer(); // `repeat` instructions matching this loop will branch to @@ -5116,8 +5115,8 @@ fn airCondBr(f: *Function, inst: Air.Inst.Index) !void { const cond = try f.resolveInst(pl_op.operand); try reap(f, inst, &.{pl_op.operand}); const extra = f.air.extraData(Air.CondBr, pl_op.payload); - const then_body: []const Air.Inst.Index = @ptrCast(f.air.extra[extra.end..][0..extra.data.then_body_len]); - const else_body: []const Air.Inst.Index = @ptrCast(f.air.extra[extra.end + then_body.len ..][0..extra.data.else_body_len]); + const then_body: []const Air.Inst.Index = @ptrCast(f.air.extra.items[extra.end..][0..extra.data.then_body_len]); + const else_body: []const Air.Inst.Index = @ptrCast(f.air.extra.items[extra.end + then_body.len ..][0..extra.data.else_body_len]); const liveness_condbr = f.liveness.getCondBr(inst); const writer = f.object.writer(); @@ -5322,12 +5321,12 @@ fn airAsm(f: *Function, inst: Air.Inst.Index) !CValue { const ty_pl = f.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; const extra = f.air.extraData(Air.Asm, ty_pl.payload); const is_volatile = @as(u1, @truncate(extra.data.flags >> 31)) != 0; - const clobbers_len = @as(u31, @truncate(extra.data.flags)); + const clobbers_len: u31 = @truncate(extra.data.flags); const gpa = f.object.dg.gpa; var extra_i: usize = extra.end; - const outputs = @as([]const Air.Inst.Ref, @ptrCast(f.air.extra[extra_i..][0..extra.data.outputs_len])); + const outputs: []const Air.Inst.Ref = @ptrCast(f.air.extra.items[extra_i..][0..extra.data.outputs_len]); extra_i += outputs.len; - const inputs = @as([]const Air.Inst.Ref, @ptrCast(f.air.extra[extra_i..][0..extra.data.inputs_len])); + const inputs: []const Air.Inst.Ref = @ptrCast(f.air.extra.items[extra_i..][0..extra.data.inputs_len]); extra_i += inputs.len; const result = result: { @@ -5347,10 +5346,10 @@ fn airAsm(f: *Function, inst: Air.Inst.Index) !CValue { break :local inst_local; } else .none; - const locals_begin = @as(LocalIndex, @intCast(f.locals.items.len)); + const locals_begin: LocalIndex = @intCast(f.locals.items.len); const constraints_extra_begin = extra_i; for (outputs) |output| { - const extra_bytes = mem.sliceAsBytes(f.air.extra[extra_i..]); + const extra_bytes = mem.sliceAsBytes(f.air.extra.items[extra_i..]); const constraint = mem.sliceTo(extra_bytes, 0); const name = mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0); // This equation accounts for the fact that even if we have exactly 4 bytes @@ -5384,7 +5383,7 @@ fn airAsm(f: *Function, inst: Air.Inst.Index) !CValue { } } for (inputs) |input| { - const extra_bytes = mem.sliceAsBytes(f.air.extra[extra_i..]); + const extra_bytes = mem.sliceAsBytes(f.air.extra.items[extra_i..]); const constraint = mem.sliceTo(extra_bytes, 0); const name = mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0); // This equation accounts for the fact that even if we have exactly 4 bytes @@ -5419,14 +5418,14 @@ fn airAsm(f: *Function, inst: Air.Inst.Index) !CValue { } } for (0..clobbers_len) |_| { - const clobber = mem.sliceTo(mem.sliceAsBytes(f.air.extra[extra_i..]), 0); + const clobber = mem.sliceTo(mem.sliceAsBytes(f.air.extra.items[extra_i..]), 0); // This equation accounts for the fact that even if we have exactly 4 bytes // for the string, we still use the next u32 for the null terminator. extra_i += clobber.len / 4 + 1; } { - const asm_source = mem.sliceAsBytes(f.air.extra[extra_i..])[0..extra.data.source_len]; + const asm_source = mem.sliceAsBytes(f.air.extra.items[extra_i..])[0..extra.data.source_len]; var stack = std.heap.stackFallback(256, f.object.dg.gpa); const allocator = stack.get(); @@ -5484,7 +5483,7 @@ fn airAsm(f: *Function, inst: Air.Inst.Index) !CValue { var locals_index = locals_begin; try writer.writeByte(':'); for (outputs, 0..) |output, index| { - const extra_bytes = mem.sliceAsBytes(f.air.extra[extra_i..]); + const extra_bytes = mem.sliceAsBytes(f.air.extra.items[extra_i..]); const constraint = mem.sliceTo(extra_bytes, 0); const name = mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0); // This equation accounts for the fact that even if we have exactly 4 bytes @@ -5508,7 +5507,7 @@ fn airAsm(f: *Function, inst: Air.Inst.Index) !CValue { } try writer.writeByte(':'); for (inputs, 0..) |input, index| { - const extra_bytes = mem.sliceAsBytes(f.air.extra[extra_i..]); + const extra_bytes = mem.sliceAsBytes(f.air.extra.items[extra_i..]); const constraint = mem.sliceTo(extra_bytes, 0); const name = mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0); // This equation accounts for the fact that even if we have exactly 4 bytes @@ -5531,7 +5530,7 @@ fn airAsm(f: *Function, inst: Air.Inst.Index) !CValue { } try writer.writeByte(':'); for (0..clobbers_len) |clobber_i| { - const clobber = mem.sliceTo(mem.sliceAsBytes(f.air.extra[extra_i..]), 0); + const clobber = mem.sliceTo(mem.sliceAsBytes(f.air.extra.items[extra_i..]), 0); // This equation accounts for the fact that even if we have exactly 4 bytes // for the string, we still use the next u32 for the null terminator. extra_i += clobber.len / 4 + 1; @@ -5546,7 +5545,7 @@ fn airAsm(f: *Function, inst: Air.Inst.Index) !CValue { extra_i = constraints_extra_begin; locals_index = locals_begin; for (outputs) |output| { - const extra_bytes = mem.sliceAsBytes(f.air.extra[extra_i..]); + const extra_bytes = mem.sliceAsBytes(f.air.extra.items[extra_i..]); const constraint = mem.sliceTo(extra_bytes, 0); const name = mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0); // This equation accounts for the fact that even if we have exactly 4 bytes @@ -6725,7 +6724,7 @@ fn airAtomicRmw(f: *Function, inst: Air.Inst.Index) !CValue { const operand_mat = try Materialize.start(f, inst, ty, operand); try reap(f, inst, &.{ pl_op.operand, extra.operand }); - const repr_bits = @as(u16, @intCast(ty.abiSize(zcu) * 8)); + const repr_bits: u16 = @intCast(ty.abiSize(zcu) * 8); const is_float = ty.isRuntimeFloat(); const is_128 = repr_bits == 128; const repr_ty = if (is_float) pt.intType(.unsigned, repr_bits) catch unreachable else ty; @@ -7325,8 +7324,8 @@ fn airAggregateInit(f: *Function, inst: Air.Inst.Index) !CValue { const ip = &zcu.intern_pool; const ty_pl = f.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; const inst_ty = f.typeOfIndex(inst); - const len = @as(usize, @intCast(inst_ty.arrayLen(zcu))); - const elements = @as([]const Air.Inst.Ref, @ptrCast(f.air.extra[ty_pl.payload..][0..len])); + const len: usize = @intCast(inst_ty.arrayLen(zcu)); + const elements: []const Air.Inst.Ref = @ptrCast(f.air.extra.items[ty_pl.payload..][0..len]); const gpa = f.object.dg.gpa; const resolved_elements = try gpa.alloc(CValue, elements.len); defer gpa.free(resolved_elements); @@ -7830,7 +7829,7 @@ fn IndentWriter(comptime UnderlyingWriter: type) type { } pub fn write(self: *Self, bytes: []const u8) Error!usize { - if (bytes.len == 0) return @as(usize, 0); + if (bytes.len == 0) return 0; const current_indent = self.indent_count * Self.indent_delta; if (self.current_line_empty and current_indent > 0) { @@ -7860,7 +7859,7 @@ fn IndentWriter(comptime UnderlyingWriter: type) type { } fn writeNoIndent(self: *Self, bytes: []const u8) Error!usize { - if (bytes.len == 0) return @as(usize, 0); + if (bytes.len == 0) return 0; try self.underlying_writer.writeAll(bytes); if (bytes[bytes.len - 1] == '\n') { @@ -8048,7 +8047,7 @@ fn fmtStringLiteral(str: []const u8, sentinel: ?u8) std.fmt.Formatter(formatStri fn undefPattern(comptime IntType: type) IntType { const int_info = @typeInfo(IntType).int; const UnsignedType = std.meta.Int(.unsigned, int_info.bits); - return @as(IntType, @bitCast(@as(UnsignedType, (1 << (int_info.bits | 1)) / 3))); + return @bitCast(@as(UnsignedType, (1 << (int_info.bits | 1)) / 3)); } const FormatIntLiteralContext = struct { @@ -8188,9 +8187,9 @@ fn formatIntLiteral( wrap.len = wrap.limbs.len; const limbs_per_c_limb = @divExact(wrap.len, c_limb_info.count); - var c_limb_int_info = std.builtin.Type.Int{ + var c_limb_int_info: std.builtin.Type.Int = .{ .signedness = undefined, - .bits = @as(u16, @intCast(@divExact(c_bits, c_limb_info.count))), + .bits = @intCast(@divExact(c_bits, c_limb_info.count)), }; var c_limb_ctype: CType = undefined; @@ -8349,7 +8348,7 @@ fn lowersToArray(ty: Type, pt: Zcu.PerThread) bool { } fn reap(f: *Function, inst: Air.Inst.Index, operands: []const Air.Inst.Ref) !void { - assert(operands.len <= Liveness.bpi - 1); + assert(operands.len <= Air.Liveness.bpi - 1); var tomb_bits = f.liveness.getTombBits(inst); for (operands) |operand| { const dies = @as(u1, @truncate(tomb_bits)) != 0; @@ -8400,7 +8399,7 @@ fn freeLocal(f: *Function, inst: ?Air.Inst.Index, local_index: LocalIndex, ref_i const BigTomb = struct { f: *Function, inst: Air.Inst.Index, - lbt: Liveness.BigTomb, + lbt: Air.Liveness.BigTomb, fn feed(bt: *BigTomb, op_ref: Air.Inst.Ref) !void { const dies = bt.lbt.feed(); diff --git a/src/codegen/llvm.zig b/src/codegen/llvm.zig index dc1a8edf56cd69dabeea97ea8d28fb95b6de82d8..d2a72502ed83a9fbeac40744f7bd0b6d578758eb 100644 --- a/src/codegen/llvm.zig +++ b/src/codegen/llvm.zig @@ -18,7 +18,6 @@ const Zcu = @import("../Zcu.zig"); const InternPool = @import("../InternPool.zig"); const Package = @import("../Package.zig"); const Air = @import("../Air.zig"); -const Liveness = @import("../Liveness.zig"); const Value = @import("../Value.zig"); const Type = @import("../Type.zig"); const x86_64_abi = @import("../arch/x86_64/abi.zig"); @@ -1121,7 +1120,7 @@ pub const Object = struct { pt: Zcu.PerThread, func_index: InternPool.Index, air: Air, - liveness: Liveness, + liveness: Air.Liveness, ) !void { assert(std.meta.eql(pt, o.pt)); const zcu = pt.zcu; @@ -4616,7 +4615,7 @@ pub const FuncGen = struct { gpa: Allocator, ng: *NavGen, air: Air, - liveness: Liveness, + liveness: Air.Liveness, wip: Builder.WipFunction, is_naked: bool, fuzz: ?Fuzz, @@ -5183,7 +5182,7 @@ pub const FuncGen = struct { fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.builtin.CallModifier) !Builder.Value { const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; const extra = self.air.extraData(Air.Call, pl_op.payload); - const args: []const Air.Inst.Ref = @ptrCast(self.air.extra[extra.end..][0..extra.data.args_len]); + const args: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[extra.end..][0..extra.data.args_len]); const o = self.ng.object; const pt = o.pt; const zcu = pt.zcu; @@ -5856,7 +5855,7 @@ pub const FuncGen = struct { fn airBlock(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; const extra = self.air.extraData(Air.Block, ty_pl.payload); - return self.lowerBlock(inst, null, @ptrCast(self.air.extra[extra.end..][0..extra.data.body_len])); + return self.lowerBlock(inst, null, @ptrCast(self.air.extra.items[extra.end..][0..extra.data.body_len])); } fn lowerBlock( @@ -6140,8 +6139,8 @@ pub const FuncGen = struct { const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; const cond = try self.resolveInst(pl_op.operand); const extra = self.air.extraData(Air.CondBr, pl_op.payload); - const then_body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra.end..][0..extra.data.then_body_len]); - const else_body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra.end + then_body.len ..][0..extra.data.else_body_len]); + const then_body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[extra.end..][0..extra.data.then_body_len]); + const else_body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[extra.end + then_body.len ..][0..extra.data.else_body_len]); const Hint = enum { none, @@ -6205,7 +6204,7 @@ pub const FuncGen = struct { const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; const err_union = try self.resolveInst(pl_op.operand); const extra = self.air.extraData(Air.Try, pl_op.payload); - const body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra.end..][0..extra.data.body_len]); + const body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[extra.end..][0..extra.data.body_len]); const err_union_ty = self.typeOf(pl_op.operand); const payload_ty = self.typeOfIndex(inst); const can_elide_load = if (isByRef(payload_ty, zcu)) self.canElideLoad(body_tail) else false; @@ -6219,7 +6218,7 @@ pub const FuncGen = struct { const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; const extra = self.air.extraData(Air.TryPtr, ty_pl.payload); const err_union_ptr = try self.resolveInst(extra.data.ptr); - const body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra.end..][0..extra.data.body_len]); + const body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[extra.end..][0..extra.data.body_len]); const err_union_ty = self.typeOf(extra.data.ptr).childType(zcu); const is_unused = self.liveness.isUnused(inst); @@ -6550,7 +6549,7 @@ pub const FuncGen = struct { fn airLoop(self: *FuncGen, inst: Air.Inst.Index) !void { const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; const loop = self.air.extraData(Air.Block, ty_pl.payload); - const body: []const Air.Inst.Index = @ptrCast(self.air.extra[loop.end..][0..loop.data.body_len]); + const body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[loop.end..][0..loop.data.body_len]); const loop_block = try self.wip.block(1, "Loop"); // `airRepeat` will increment incoming each time _ = try self.wip.br(loop_block); @@ -7076,7 +7075,7 @@ pub const FuncGen = struct { const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; const extra = self.air.extraData(Air.DbgInlineBlock, ty_pl.payload); self.arg_inline_index = 0; - return self.lowerBlock(inst, extra.data.func, @ptrCast(self.air.extra[extra.end..][0..extra.data.body_len])); + return self.lowerBlock(inst, extra.data.func, @ptrCast(self.air.extra.items[extra.end..][0..extra.data.body_len])); } fn airDbgVarPtr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { @@ -7201,9 +7200,9 @@ pub const FuncGen = struct { const clobbers_len: u31 = @truncate(extra.data.flags); var extra_i: usize = extra.end; - const outputs: []const Air.Inst.Ref = @ptrCast(self.air.extra[extra_i..][0..extra.data.outputs_len]); + const outputs: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[extra_i..][0..extra.data.outputs_len]); extra_i += outputs.len; - const inputs: []const Air.Inst.Ref = @ptrCast(self.air.extra[extra_i..][0..extra.data.inputs_len]); + const inputs: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[extra_i..][0..extra.data.inputs_len]); extra_i += inputs.len; var llvm_constraints: std.ArrayListUnmanaged(u8) = .empty; @@ -7239,8 +7238,8 @@ pub const FuncGen = struct { var rw_extra_i = extra_i; for (outputs, llvm_ret_indirect, llvm_rw_vals) |output, *is_indirect, *llvm_rw_val| { - const extra_bytes = std.mem.sliceAsBytes(self.air.extra[extra_i..]); - const constraint = std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra[extra_i..]), 0); + const extra_bytes = std.mem.sliceAsBytes(self.air.extra.items[extra_i..]); + const constraint = std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra.items[extra_i..]), 0); const name = std.mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0); // This equation accounts for the fact that even if we have exactly 4 bytes // for the string, we still use the next u32 for the null terminator. @@ -7320,7 +7319,7 @@ pub const FuncGen = struct { } for (inputs) |input| { - const extra_bytes = std.mem.sliceAsBytes(self.air.extra[extra_i..]); + const extra_bytes = std.mem.sliceAsBytes(self.air.extra.items[extra_i..]); const constraint = std.mem.sliceTo(extra_bytes, 0); const name = std.mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0); // This equation accounts for the fact that even if we have exactly 4 bytes @@ -7385,8 +7384,8 @@ pub const FuncGen = struct { } for (outputs, llvm_ret_indirect, llvm_rw_vals, 0..) |output, is_indirect, llvm_rw_val, output_index| { - const extra_bytes = std.mem.sliceAsBytes(self.air.extra[rw_extra_i..]); - const constraint = std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra[rw_extra_i..]), 0); + const extra_bytes = std.mem.sliceAsBytes(self.air.extra.items[rw_extra_i..]); + const constraint = std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra.items[rw_extra_i..]), 0); const name = std.mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0); // This equation accounts for the fact that even if we have exactly 4 bytes // for the string, we still use the next u32 for the null terminator. @@ -7425,7 +7424,7 @@ pub const FuncGen = struct { { var clobber_i: u32 = 0; while (clobber_i < clobbers_len) : (clobber_i += 1) { - const clobber = std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra[extra_i..]), 0); + const clobber = std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra.items[extra_i..]), 0); // This equation accounts for the fact that even if we have exactly 4 bytes // for the string, we still use the next u32 for the null terminator. extra_i += clobber.len / 4 + 1; @@ -7465,7 +7464,7 @@ pub const FuncGen = struct { else => {}, } - const asm_source = std.mem.sliceAsBytes(self.air.extra[extra_i..])[0..extra.data.source_len]; + const asm_source = std.mem.sliceAsBytes(self.air.extra.items[extra_i..])[0..extra.data.source_len]; // hackety hacks until stage2 has proper inline asm in the frontend. var rendered_template = std.ArrayList(u8).init(self.gpa); @@ -10628,7 +10627,7 @@ pub const FuncGen = struct { const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; const result_ty = self.typeOfIndex(inst); const len: usize = @intCast(result_ty.arrayLen(zcu)); - const elements: []const Air.Inst.Ref = @ptrCast(self.air.extra[ty_pl.payload..][0..len]); + const elements: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[ty_pl.payload..][0..len]); const llvm_result_ty = try o.lowerType(result_ty); switch (result_ty.zigTypeTag(zcu)) { diff --git a/src/codegen/spirv.zig b/src/codegen/spirv.zig index b2ab76e2c734fbce89d45b01e924e88707b70ca6..5041634a75a3116d33b6f7fc9758f08dd115b5b2 100644 --- a/src/codegen/spirv.zig +++ b/src/codegen/spirv.zig @@ -10,7 +10,6 @@ const Decl = Zcu.Decl; const Type = @import("../Type.zig"); const Value = @import("../Value.zig"); const Air = @import("../Air.zig"); -const Liveness = @import("../Liveness.zig"); const InternPool = @import("../InternPool.zig"); const spec = @import("spirv/spec.zig"); @@ -195,7 +194,7 @@ pub const Object = struct { pt: Zcu.PerThread, nav_index: InternPool.Nav.Index, air: Air, - liveness: Liveness, + liveness: Air.Liveness, do_codegen: bool, ) !void { const zcu = pt.zcu; @@ -242,7 +241,7 @@ pub const Object = struct { pt: Zcu.PerThread, func_index: InternPool.Index, air: Air, - liveness: Liveness, + liveness: Air.Liveness, ) !void { const nav = pt.zcu.funcInfo(func_index).owner_nav; // TODO: Separate types for generating decls and functions? @@ -303,7 +302,7 @@ const NavGen = struct { /// The liveness analysis of the intermediate code for the declaration we are currently generating. /// Note: If the declaration is not a function, this value will be undefined! - liveness: Liveness, + liveness: Air.Liveness, /// An array of function argument result-ids. Each index corresponds with the /// function argument of the same index. @@ -4627,7 +4626,7 @@ const NavGen = struct { const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; const result_ty = self.typeOfIndex(inst); const len: usize = @intCast(result_ty.arrayLen(zcu)); - const elements: []const Air.Inst.Ref = @ptrCast(self.air.extra[ty_pl.payload..][0..len]); + const elements: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[ty_pl.payload..][0..len]); switch (result_ty.zigTypeTag(zcu)) { .@"struct" => { @@ -5474,7 +5473,7 @@ const NavGen = struct { fn airBlock(self: *NavGen, inst: Air.Inst.Index) !?IdRef { const inst_datas = self.air.instructions.items(.data); const extra = self.air.extraData(Air.Block, inst_datas[@intFromEnum(inst)].ty_pl.payload); - return self.lowerBlock(inst, @ptrCast(self.air.extra[extra.end..][0..extra.data.body_len])); + return self.lowerBlock(inst, @ptrCast(self.air.extra.items[extra.end..][0..extra.data.body_len])); } fn lowerBlock(self: *NavGen, inst: Air.Inst.Index, body: []const Air.Inst.Index) !?IdRef { @@ -5657,8 +5656,8 @@ const NavGen = struct { fn airCondBr(self: *NavGen, inst: Air.Inst.Index) !void { const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; const cond_br = self.air.extraData(Air.CondBr, pl_op.payload); - const then_body: []const Air.Inst.Index = @ptrCast(self.air.extra[cond_br.end..][0..cond_br.data.then_body_len]); - const else_body: []const Air.Inst.Index = @ptrCast(self.air.extra[cond_br.end + then_body.len ..][0..cond_br.data.else_body_len]); + const then_body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[cond_br.end..][0..cond_br.data.then_body_len]); + const else_body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[cond_br.end + then_body.len ..][0..cond_br.data.else_body_len]); const condition_id = try self.resolve(pl_op.operand); const then_label = self.spv.allocId(); @@ -5717,7 +5716,7 @@ const NavGen = struct { fn airLoop(self: *NavGen, inst: Air.Inst.Index) !void { const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; const loop = self.air.extraData(Air.Block, ty_pl.payload); - const body: []const Air.Inst.Index = @ptrCast(self.air.extra[loop.end..][0..loop.data.body_len]); + const body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[loop.end..][0..loop.data.body_len]); const body_label = self.spv.allocId(); @@ -5837,7 +5836,7 @@ const NavGen = struct { const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; const err_union_id = try self.resolve(pl_op.operand); const extra = self.air.extraData(Air.Try, pl_op.payload); - const body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra.end..][0..extra.data.body_len]); + const body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[extra.end..][0..extra.data.body_len]); const err_union_ty = self.typeOf(pl_op.operand); const payload_ty = self.typeOfIndex(inst); @@ -6344,7 +6343,7 @@ const NavGen = struct { const old_base_line = self.base_line; defer self.base_line = old_base_line; self.base_line = zcu.navSrcLine(zcu.funcInfo(extra.data.func).owner_nav); - return self.lowerBlock(inst, @ptrCast(self.air.extra[extra.end..][0..extra.data.body_len])); + return self.lowerBlock(inst, @ptrCast(self.air.extra.items[extra.end..][0..extra.data.body_len])); } fn airDbgVar(self: *NavGen, inst: Air.Inst.Index) !void { @@ -6365,9 +6364,9 @@ const NavGen = struct { if (!is_volatile and self.liveness.isUnused(inst)) return null; var extra_i: usize = extra.end; - const outputs: []const Air.Inst.Ref = @ptrCast(self.air.extra[extra_i..][0..extra.data.outputs_len]); + const outputs: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[extra_i..][0..extra.data.outputs_len]); extra_i += outputs.len; - const inputs: []const Air.Inst.Ref = @ptrCast(self.air.extra[extra_i..][0..extra.data.inputs_len]); + const inputs: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[extra_i..][0..extra.data.inputs_len]); extra_i += inputs.len; if (outputs.len > 1) { @@ -6386,15 +6385,15 @@ const NavGen = struct { if (output != .none) { return self.todo("implement inline asm with non-returned output", .{}); } - const extra_bytes = std.mem.sliceAsBytes(self.air.extra[extra_i..]); - const constraint = std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra[extra_i..]), 0); + const extra_bytes = std.mem.sliceAsBytes(self.air.extra.items[extra_i..]); + const constraint = std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra.items[extra_i..]), 0); const name = std.mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0); extra_i += (constraint.len + name.len + (2 + 3)) / 4; // TODO: Record output and use it somewhere. } for (inputs) |input| { - const extra_bytes = std.mem.sliceAsBytes(self.air.extra[extra_i..]); + const extra_bytes = std.mem.sliceAsBytes(self.air.extra.items[extra_i..]); const constraint = std.mem.sliceTo(extra_bytes, 0); const name = std.mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0); // This equation accounts for the fact that even if we have exactly 4 bytes @@ -6461,13 +6460,13 @@ const NavGen = struct { { var clobber_i: u32 = 0; while (clobber_i < clobbers_len) : (clobber_i += 1) { - const clobber = std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra[extra_i..]), 0); + const clobber = std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra.items[extra_i..]), 0); extra_i += clobber.len / 4 + 1; // TODO: Record clobber and use it somewhere. } } - const asm_source = std.mem.sliceAsBytes(self.air.extra[extra_i..])[0..extra.data.source_len]; + const asm_source = std.mem.sliceAsBytes(self.air.extra.items[extra_i..])[0..extra.data.source_len]; as.assemble(asm_source) catch |err| switch (err) { error.AssembleFail => { @@ -6501,8 +6500,8 @@ const NavGen = struct { for (outputs) |output| { _ = output; - const extra_bytes = std.mem.sliceAsBytes(self.air.extra[output_extra_i..]); - const constraint = std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra[output_extra_i..]), 0); + const extra_bytes = std.mem.sliceAsBytes(self.air.extra.items[output_extra_i..]); + const constraint = std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra.items[output_extra_i..]), 0); const name = std.mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0); output_extra_i += (constraint.len + name.len + (2 + 3)) / 4; @@ -6531,7 +6530,7 @@ const NavGen = struct { const zcu = pt.zcu; const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; const extra = self.air.extraData(Air.Call, pl_op.payload); - const args: []const Air.Inst.Ref = @ptrCast(self.air.extra[extra.end..][0..extra.data.args_len]); + const args: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[extra.end..][0..extra.data.args_len]); const callee_ty = self.typeOf(pl_op.operand); const zig_fn_ty = switch (callee_ty.zigTypeTag(zcu)) { .@"fn" => callee_ty, diff --git a/src/link.zig b/src/link.zig index cbc641a61ae9153ffa40f986b0a8a41c7a5c27df..8dfe3929a02e061802eeeadf44415d88c1824e8e 100644 --- a/src/link.zig +++ b/src/link.zig @@ -15,7 +15,6 @@ const Path = std.Build.Cache.Path; const Directory = std.Build.Cache.Directory; const Compilation = @import("Compilation.zig"); const LibCInstallation = std.zig.LibCInstallation; -const Liveness = @import("Liveness.zig"); const Zcu = @import("Zcu.zig"); const InternPool = @import("InternPool.zig"); const Type = @import("Type.zig"); @@ -738,7 +737,7 @@ pub const File = struct { pt: Zcu.PerThread, func_index: InternPool.Index, air: Air, - liveness: Liveness, + liveness: Air.Liveness, ) UpdateNavError!void { switch (base.tag) { inline else => |tag| { @@ -1601,8 +1600,9 @@ pub fn doTask(comp: *Compilation, tid: usize, task: Task) void { if (comp.remaining_prelink_tasks == 0) { const pt: Zcu.PerThread = .activate(comp.zcu.?, @enumFromInt(tid)); defer pt.deactivate(); - // This call takes ownership of `func.air`. - pt.linkerUpdateFunc(func.func, func.air) catch |err| switch (err) { + var air = func.air; + defer air.deinit(comp.gpa); + pt.linkerUpdateFunc(func.func, &air) catch |err| switch (err) { error.OutOfMemory => diags.setAllocFailure(), }; } else { diff --git a/src/link/C.zig b/src/link/C.zig index b5c03985c7a3a3531dcbe028d2aa5349954cb166..c32d8ba80ba07308512c188384636348c29128d2 100644 --- a/src/link/C.zig +++ b/src/link/C.zig @@ -18,7 +18,6 @@ const trace = @import("../tracy.zig").trace; const Type = @import("../Type.zig"); const Value = @import("../Value.zig"); const Air = @import("../Air.zig"); -const Liveness = @import("../Liveness.zig"); pub const zig_h = "#include \"zig.h\"\n"; @@ -180,7 +179,7 @@ pub fn updateFunc( pt: Zcu.PerThread, func_index: InternPool.Index, air: Air, - liveness: Liveness, + liveness: Air.Liveness, ) link.File.UpdateNavError!void { const zcu = pt.zcu; const gpa = zcu.gpa; diff --git a/src/link/Coff.zig b/src/link/Coff.zig index 2ae627a2a9bac90ec33757636afc3d8082cd2618..0d32e9c4c4d999396a257dc5fd2b3d3a4eb9fac2 100644 --- a/src/link/Coff.zig +++ b/src/link/Coff.zig @@ -1098,7 +1098,7 @@ pub fn updateFunc( pt: Zcu.PerThread, func_index: InternPool.Index, air: Air, - liveness: Liveness, + liveness: Air.Liveness, ) link.File.UpdateNavError!void { if (build_options.skip_non_native and builtin.object_format != .coff) { @panic("Attempted to compile for object format that was disabled by build configuration"); @@ -3802,7 +3802,6 @@ const trace = @import("../tracy.zig").trace; const Air = @import("../Air.zig"); const Compilation = @import("../Compilation.zig"); -const Liveness = @import("../Liveness.zig"); const LlvmObject = @import("../codegen/llvm.zig").Object; const Zcu = @import("../Zcu.zig"); const InternPool = @import("../InternPool.zig"); diff --git a/src/link/Elf.zig b/src/link/Elf.zig index 5f0023fb011e9a6dccba9c348f27ca9787e5ee44..41ecb4adc83c80379ea1ffa30f632b808910bc94 100644 --- a/src/link/Elf.zig +++ b/src/link/Elf.zig @@ -2385,7 +2385,7 @@ pub fn updateFunc( pt: Zcu.PerThread, func_index: InternPool.Index, air: Air, - liveness: Liveness, + liveness: Air.Liveness, ) link.File.UpdateNavError!void { if (build_options.skip_non_native and builtin.object_format != .elf) { @panic("Attempted to compile for object format that was disabled by build configuration"); @@ -5323,7 +5323,6 @@ const GotSection = synthetic_sections.GotSection; const GotPltSection = synthetic_sections.GotPltSection; const HashSection = synthetic_sections.HashSection; const LinkerDefined = @import("Elf/LinkerDefined.zig"); -const Liveness = @import("../Liveness.zig"); const LlvmObject = @import("../codegen/llvm.zig").Object; const Zcu = @import("../Zcu.zig"); const Object = @import("Elf/Object.zig"); diff --git a/src/link/Elf/ZigObject.zig b/src/link/Elf/ZigObject.zig index 5afa925f66a3bfec784ddcf49ff35958db16266e..af7167095a06c24e9a2a2ccfd535092b524819e7 100644 --- a/src/link/Elf/ZigObject.zig +++ b/src/link/Elf/ZigObject.zig @@ -1416,7 +1416,7 @@ pub fn updateFunc( pt: Zcu.PerThread, func_index: InternPool.Index, air: Air, - liveness: Liveness, + liveness: Air.Liveness, ) link.File.UpdateNavError!void { const tracy = trace(@src()); defer tracy.end(); @@ -2367,7 +2367,6 @@ const Dwarf = @import("../Dwarf.zig"); const Elf = @import("../Elf.zig"); const File = @import("file.zig").File; const InternPool = @import("../../InternPool.zig"); -const Liveness = @import("../../Liveness.zig"); const Zcu = @import("../../Zcu.zig"); const Object = @import("Object.zig"); const Symbol = @import("Symbol.zig"); diff --git a/src/link/Goff.zig b/src/link/Goff.zig index 55d2a15e56357b91a0009644cf4762f41d728262..6ed360be258bc79437ea002bc3e0836aa8f66f98 100644 --- a/src/link/Goff.zig +++ b/src/link/Goff.zig @@ -17,7 +17,6 @@ const link = @import("../link.zig"); const trace = @import("../tracy.zig").trace; const build_options = @import("build_options"); const Air = @import("../Air.zig"); -const Liveness = @import("../Liveness.zig"); const LlvmObject = @import("../codegen/llvm.zig").Object; base: link.File, @@ -79,7 +78,7 @@ pub fn updateFunc( pt: Zcu.PerThread, func_index: InternPool.Index, air: Air, - liveness: Liveness, + liveness: Air.Liveness, ) link.File.UpdateNavError!void { if (build_options.skip_non_native and builtin.object_format != .goff) @panic("Attempted to compile for object format that was disabled by build configuration"); diff --git a/src/link/MachO.zig b/src/link/MachO.zig index 1b2d9a18497db44b434a585470328c88db08a011..3ddc12a5b0aad6c14441cb5679ee0cfa93831a16 100644 --- a/src/link/MachO.zig +++ b/src/link/MachO.zig @@ -3074,7 +3074,7 @@ pub fn updateFunc( pt: Zcu.PerThread, func_index: InternPool.Index, air: Air, - liveness: Liveness, + liveness: Air.Liveness, ) link.File.UpdateNavError!void { if (build_options.skip_non_native and builtin.object_format != .macho) { @panic("Attempted to compile for object format that was disabled by build configuration"); @@ -5496,7 +5496,6 @@ const ObjcStubsSection = synthetic.ObjcStubsSection; const Object = @import("MachO/Object.zig"); const LazyBind = bind.LazyBind; const LaSymbolPtrSection = synthetic.LaSymbolPtrSection; -const Liveness = @import("../Liveness.zig"); const LlvmObject = @import("../codegen/llvm.zig").Object; const Md5 = std.crypto.hash.Md5; const Zcu = @import("../Zcu.zig"); diff --git a/src/link/MachO/ZigObject.zig b/src/link/MachO/ZigObject.zig index 63557db7e118673630fcddb40c0406a8535f8859..a0de86654421f473de50c75a613ba41fa754cce7 100644 --- a/src/link/MachO/ZigObject.zig +++ b/src/link/MachO/ZigObject.zig @@ -778,7 +778,7 @@ pub fn updateFunc( pt: Zcu.PerThread, func_index: InternPool.Index, air: Air, - liveness: Liveness, + liveness: Air.Liveness, ) link.File.UpdateNavError!void { const tracy = trace(@src()); defer tracy.end(); @@ -1820,7 +1820,6 @@ const Atom = @import("Atom.zig"); const Dwarf = @import("../Dwarf.zig"); const File = @import("file.zig").File; const InternPool = @import("../../InternPool.zig"); -const Liveness = @import("../../Liveness.zig"); const MachO = @import("../MachO.zig"); const Nlist = Object.Nlist; const Zcu = @import("../../Zcu.zig"); diff --git a/src/link/Plan9.zig b/src/link/Plan9.zig index e943bdabe1e449166b72c4645ffd63b21220d463..5cbb9287d7b171957390389a19e75917d8490a24 100644 --- a/src/link/Plan9.zig +++ b/src/link/Plan9.zig @@ -12,7 +12,6 @@ const trace = @import("../tracy.zig").trace; const File = link.File; const build_options = @import("build_options"); const Air = @import("../Air.zig"); -const Liveness = @import("../Liveness.zig"); const Type = @import("../Type.zig"); const Value = @import("../Value.zig"); const AnalUnit = InternPool.AnalUnit; @@ -389,7 +388,7 @@ pub fn updateFunc( pt: Zcu.PerThread, func_index: InternPool.Index, air: Air, - liveness: Liveness, + liveness: Air.Liveness, ) link.File.UpdateNavError!void { if (build_options.skip_non_native and builtin.object_format != .plan9) { @panic("Attempted to compile for object format that was disabled by build configuration"); diff --git a/src/link/SpirV.zig b/src/link/SpirV.zig index f3c2922725c8dcf1dccd62ae211624fcd2665bbb..a49771c3e2f24ae6a3843ffc706253f2f2543ea4 100644 --- a/src/link/SpirV.zig +++ b/src/link/SpirV.zig @@ -36,7 +36,6 @@ const codegen = @import("../codegen/spirv.zig"); const trace = @import("../tracy.zig").trace; const build_options = @import("build_options"); const Air = @import("../Air.zig"); -const Liveness = @import("../Liveness.zig"); const Type = @import("../Type.zig"); const Value = @import("../Value.zig"); @@ -118,7 +117,7 @@ pub fn updateFunc( pt: Zcu.PerThread, func_index: InternPool.Index, air: Air, - liveness: Liveness, + liveness: Air.Liveness, ) link.File.UpdateNavError!void { if (build_options.skip_non_native) { @panic("Attempted to compile for architecture that was disabled by build configuration"); diff --git a/src/link/Wasm.zig b/src/link/Wasm.zig index ff3682ffe4772e29705b7a30af92d75575bdd164..338eff4e7bbf32b0cf1ea12ca8609c24e50a106e 100644 --- a/src/link/Wasm.zig +++ b/src/link/Wasm.zig @@ -36,7 +36,6 @@ const abi = @import("../arch/wasm/abi.zig"); const Compilation = @import("../Compilation.zig"); const Dwarf = @import("Dwarf.zig"); const InternPool = @import("../InternPool.zig"); -const Liveness = @import("../Liveness.zig"); const LlvmObject = @import("../codegen/llvm.zig").Object; const Zcu = @import("../Zcu.zig"); const codegen = @import("../codegen.zig"); @@ -3193,7 +3192,7 @@ pub fn deinit(wasm: *Wasm) void { wasm.missing_exports.deinit(gpa); } -pub fn updateFunc(wasm: *Wasm, pt: Zcu.PerThread, func_index: InternPool.Index, air: Air, liveness: Liveness) !void { +pub fn updateFunc(wasm: *Wasm, pt: Zcu.PerThread, func_index: InternPool.Index, air: Air, liveness: Air.Liveness) !void { if (build_options.skip_non_native and builtin.object_format != .wasm) { @panic("Attempted to compile for object format that was disabled by build configuration"); } diff --git a/src/link/Xcoff.zig b/src/link/Xcoff.zig index e4d4caaac91306d748f85c971b0a2dca6a8daa03..525d99d39165ba6de963c40302b5d57ac896d4f5 100644 --- a/src/link/Xcoff.zig +++ b/src/link/Xcoff.zig @@ -17,7 +17,6 @@ const link = @import("../link.zig"); const trace = @import("../tracy.zig").trace; const build_options = @import("build_options"); const Air = @import("../Air.zig"); -const Liveness = @import("../Liveness.zig"); const LlvmObject = @import("../codegen/llvm.zig").Object; base: link.File, @@ -79,7 +78,7 @@ pub fn updateFunc( pt: Zcu.PerThread, func_index: InternPool.Index, air: Air, - liveness: Liveness, + liveness: Air.Liveness, ) link.File.UpdateNavError!void { if (build_options.skip_non_native and builtin.object_format != .xcoff) @panic("Attempted to compile for object format that was disabled by build configuration"); diff --git a/src/print_air.zig b/src/print_air.zig index 3436d915aa417be23749640c1ff0b4c57be55c72..0f658dcd9f84659d307386947e35ed98b15e920d 100644 --- a/src/print_air.zig +++ b/src/print_air.zig @@ -6,20 +6,19 @@ const Zcu = @import("Zcu.zig"); const Value = @import("Value.zig"); const Type = @import("Type.zig"); const Air = @import("Air.zig"); -const Liveness = @import("Liveness.zig"); const InternPool = @import("InternPool.zig"); -pub fn write(stream: anytype, pt: Zcu.PerThread, air: Air, liveness: ?Liveness) void { +pub fn write(stream: anytype, pt: Zcu.PerThread, air: Air, liveness: ?Air.Liveness) void { const instruction_bytes = air.instructions.len * // Here we don't use @sizeOf(Air.Inst.Data) because it would include // the debug safety tag but we want to measure release size. (@sizeOf(Air.Inst.Tag) + 8); - const extra_bytes = air.extra.len * @sizeOf(u32); + const extra_bytes = air.extra.items.len * @sizeOf(u32); const tomb_bytes = if (liveness) |l| l.tomb_bits.len * @sizeOf(usize) else 0; const liveness_extra_bytes = if (liveness) |l| l.extra.len * @sizeOf(u32) else 0; const liveness_special_bytes = if (liveness) |l| l.special.count() * 8 else 0; const total_bytes = @sizeOf(Air) + instruction_bytes + extra_bytes + - @sizeOf(Liveness) + liveness_extra_bytes + + @sizeOf(Air.Liveness) + liveness_extra_bytes + liveness_special_bytes + tomb_bytes; // zig fmt: off @@ -34,7 +33,7 @@ pub fn write(stream: anytype, pt: Zcu.PerThread, air: Air, liveness: ?Liveness) , .{ fmtIntSizeBin(total_bytes), air.instructions.len, fmtIntSizeBin(instruction_bytes), - air.extra.len, fmtIntSizeBin(extra_bytes), + air.extra.items.len, fmtIntSizeBin(extra_bytes), fmtIntSizeBin(tomb_bytes), if (liveness) |l| l.extra.len else 0, fmtIntSizeBin(liveness_extra_bytes), if (liveness) |l| l.special.count() else 0, fmtIntSizeBin(liveness_special_bytes), @@ -57,7 +56,7 @@ pub fn writeInst( inst: Air.Inst.Index, pt: Zcu.PerThread, air: Air, - liveness: ?Liveness, + liveness: ?Air.Liveness, ) void { var writer: Writer = .{ .pt = pt, @@ -70,11 +69,11 @@ pub fn writeInst( writer.writeInst(stream, inst) catch return; } -pub fn dump(pt: Zcu.PerThread, air: Air, liveness: ?Liveness) void { +pub fn dump(pt: Zcu.PerThread, air: Air, liveness: ?Air.Liveness) void { write(std.io.getStdErr().writer(), pt, air, liveness); } -pub fn dumpInst(inst: Air.Inst.Index, pt: Zcu.PerThread, air: Air, liveness: ?Liveness) void { +pub fn dumpInst(inst: Air.Inst.Index, pt: Zcu.PerThread, air: Air, liveness: ?Air.Liveness) void { writeInst(std.io.getStdErr().writer(), inst, pt, air, liveness); } @@ -82,7 +81,7 @@ const Writer = struct { pt: Zcu.PerThread, gpa: Allocator, air: Air, - liveness: ?Liveness, + liveness: ?Air.Liveness, indent: usize, skip_body: bool, @@ -391,15 +390,15 @@ const Writer = struct { }, else => unreachable, } - break :body w.air.extra[extra.end..][0..extra.data.body_len]; + break :body w.air.extra.items[extra.end..][0..extra.data.body_len]; }, else => unreachable, }); if (w.skip_body) return s.writeAll(", ..."); - const liveness_block = if (w.liveness) |liveness| + const liveness_block: Air.Liveness.BlockSlices = if (w.liveness) |liveness| liveness.getBlock(inst) else - Liveness.BlockSlices{ .deaths = &.{} }; + .{ .deaths = &.{} }; try s.writeAll(", {\n"); const old_indent = w.indent; @@ -417,7 +416,7 @@ const Writer = struct { fn writeLoop(w: *Writer, s: anytype, inst: Air.Inst.Index) @TypeOf(s).Error!void { const ty_pl = w.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; const extra = w.air.extraData(Air.Block, ty_pl.payload); - const body: []const Air.Inst.Index = @ptrCast(w.air.extra[extra.end..][0..extra.data.body_len]); + const body: []const Air.Inst.Index = @ptrCast(w.air.extra.items[extra.end..][0..extra.data.body_len]); try w.writeType(s, ty_pl.ty.toType()); if (w.skip_body) return s.writeAll(", ..."); @@ -435,7 +434,7 @@ const Writer = struct { const ty_pl = w.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; const vector_ty = ty_pl.ty.toType(); const len = @as(usize, @intCast(vector_ty.arrayLen(zcu))); - const elements = @as([]const Air.Inst.Ref, @ptrCast(w.air.extra[ty_pl.payload..][0..len])); + const elements = @as([]const Air.Inst.Ref, @ptrCast(w.air.extra.items[ty_pl.payload..][0..len])); try w.writeType(s, vector_ty); try s.writeAll(", ["); @@ -622,13 +621,13 @@ const Writer = struct { try s.writeAll(", volatile"); } - const outputs = @as([]const Air.Inst.Ref, @ptrCast(w.air.extra[extra_i..][0..extra.data.outputs_len])); + const outputs = @as([]const Air.Inst.Ref, @ptrCast(w.air.extra.items[extra_i..][0..extra.data.outputs_len])); extra_i += outputs.len; - const inputs = @as([]const Air.Inst.Ref, @ptrCast(w.air.extra[extra_i..][0..extra.data.inputs_len])); + const inputs = @as([]const Air.Inst.Ref, @ptrCast(w.air.extra.items[extra_i..][0..extra.data.inputs_len])); extra_i += inputs.len; for (outputs) |output| { - const extra_bytes = std.mem.sliceAsBytes(w.air.extra[extra_i..]); + const extra_bytes = std.mem.sliceAsBytes(w.air.extra.items[extra_i..]); const constraint = std.mem.sliceTo(extra_bytes, 0); const name = std.mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0); @@ -648,7 +647,7 @@ const Writer = struct { } for (inputs) |input| { - const extra_bytes = std.mem.sliceAsBytes(w.air.extra[extra_i..]); + const extra_bytes = std.mem.sliceAsBytes(w.air.extra.items[extra_i..]); const constraint = std.mem.sliceTo(extra_bytes, 0); const name = std.mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0); // This equation accounts for the fact that even if we have exactly 4 bytes @@ -665,7 +664,7 @@ const Writer = struct { { var clobber_i: u32 = 0; while (clobber_i < clobbers_len) : (clobber_i += 1) { - const extra_bytes = std.mem.sliceAsBytes(w.air.extra[extra_i..]); + const extra_bytes = std.mem.sliceAsBytes(w.air.extra.items[extra_i..]); const clobber = std.mem.sliceTo(extra_bytes, 0); // This equation accounts for the fact that even if we have exactly 4 bytes // for the string, we still use the next u32 for the null terminator. @@ -676,7 +675,7 @@ const Writer = struct { try s.writeAll("}"); } } - const asm_source = std.mem.sliceAsBytes(w.air.extra[extra_i..])[0..extra.data.source_len]; + const asm_source = std.mem.sliceAsBytes(w.air.extra.items[extra_i..])[0..extra.data.source_len]; try s.print(", \"{}\"", .{std.zig.fmtEscapes(asm_source)}); } @@ -695,7 +694,7 @@ const Writer = struct { fn writeCall(w: *Writer, s: anytype, inst: Air.Inst.Index) @TypeOf(s).Error!void { const pl_op = w.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; const extra = w.air.extraData(Air.Call, pl_op.payload); - const args = @as([]const Air.Inst.Ref, @ptrCast(w.air.extra[extra.end..][0..extra.data.args_len])); + const args = @as([]const Air.Inst.Ref, @ptrCast(w.air.extra.items[extra.end..][0..extra.data.args_len])); try w.writeOperand(s, inst, 0, pl_op.operand); try s.writeAll(", ["); for (args, 0..) |arg, i| { @@ -720,11 +719,11 @@ const Writer = struct { fn writeTry(w: *Writer, s: anytype, inst: Air.Inst.Index) @TypeOf(s).Error!void { const pl_op = w.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; const extra = w.air.extraData(Air.Try, pl_op.payload); - const body: []const Air.Inst.Index = @ptrCast(w.air.extra[extra.end..][0..extra.data.body_len]); - const liveness_condbr = if (w.liveness) |liveness| + const body: []const Air.Inst.Index = @ptrCast(w.air.extra.items[extra.end..][0..extra.data.body_len]); + const liveness_condbr: Air.Liveness.CondBrSlices = if (w.liveness) |liveness| liveness.getCondBr(inst) else - Liveness.CondBrSlices{ .then_deaths = &.{}, .else_deaths = &.{} }; + .{ .then_deaths = &.{}, .else_deaths = &.{} }; try w.writeOperand(s, inst, 0, pl_op.operand); if (w.skip_body) return s.writeAll(", ..."); @@ -754,11 +753,11 @@ const Writer = struct { fn writeTryPtr(w: *Writer, s: anytype, inst: Air.Inst.Index) @TypeOf(s).Error!void { const ty_pl = w.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; const extra = w.air.extraData(Air.TryPtr, ty_pl.payload); - const body: []const Air.Inst.Index = @ptrCast(w.air.extra[extra.end..][0..extra.data.body_len]); - const liveness_condbr = if (w.liveness) |liveness| + const body: []const Air.Inst.Index = @ptrCast(w.air.extra.items[extra.end..][0..extra.data.body_len]); + const liveness_condbr: Air.Liveness.CondBrSlices = if (w.liveness) |liveness| liveness.getCondBr(inst) else - Liveness.CondBrSlices{ .then_deaths = &.{}, .else_deaths = &.{} }; + .{ .then_deaths = &.{}, .else_deaths = &.{} }; try w.writeOperand(s, inst, 0, extra.data.ptr); @@ -791,12 +790,12 @@ const Writer = struct { fn writeCondBr(w: *Writer, s: anytype, inst: Air.Inst.Index) @TypeOf(s).Error!void { const pl_op = w.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; const extra = w.air.extraData(Air.CondBr, pl_op.payload); - const then_body: []const Air.Inst.Index = @ptrCast(w.air.extra[extra.end..][0..extra.data.then_body_len]); - const else_body: []const Air.Inst.Index = @ptrCast(w.air.extra[extra.end + then_body.len ..][0..extra.data.else_body_len]); - const liveness_condbr = if (w.liveness) |liveness| + const then_body: []const Air.Inst.Index = @ptrCast(w.air.extra.items[extra.end..][0..extra.data.then_body_len]); + const else_body: []const Air.Inst.Index = @ptrCast(w.air.extra.items[extra.end + then_body.len ..][0..extra.data.else_body_len]); + const liveness_condbr: Air.Liveness.CondBrSlices = if (w.liveness) |liveness| liveness.getCondBr(inst) else - Liveness.CondBrSlices{ .then_deaths = &.{}, .else_deaths = &.{} }; + .{ .then_deaths = &.{}, .else_deaths = &.{} }; try w.writeOperand(s, inst, 0, pl_op.operand); if (w.skip_body) return s.writeAll(", ..."); @@ -850,14 +849,14 @@ const Writer = struct { fn writeSwitchBr(w: *Writer, s: anytype, inst: Air.Inst.Index) @TypeOf(s).Error!void { const switch_br = w.air.unwrapSwitch(inst); - const liveness = if (w.liveness) |liveness| + const liveness: Air.Liveness.SwitchBrTable = if (w.liveness) |liveness| liveness.getSwitchBr(w.gpa, inst, switch_br.cases_len + 1) catch @panic("out of memory") else blk: { const slice = w.gpa.alloc([]const Air.Inst.Index, switch_br.cases_len + 1) catch @panic("out of memory"); @memset(slice, &.{}); - break :blk Liveness.SwitchBrTable{ .deaths = slice }; + break :blk .{ .deaths = slice }; }; defer w.gpa.free(liveness.deaths); @@ -956,10 +955,10 @@ const Writer = struct { op_index: usize, operand: Air.Inst.Ref, ) @TypeOf(s).Error!void { - const small_tomb_bits = Liveness.bpi - 1; + const small_tomb_bits = Air.Liveness.bpi - 1; const dies = if (w.liveness) |liveness| blk: { if (op_index < small_tomb_bits) - break :blk liveness.operandDies(inst, @as(Liveness.OperandInt, @intCast(op_index))); + break :blk liveness.operandDies(inst, @intCast(op_index)); var extra_index = liveness.special.get(inst).?; var tomb_op_index: usize = small_tomb_bits; while (true) {