diff --git a/src/codegen/wasm/CodeGen.zig b/src/codegen/wasm/CodeGen.zig index 5a266cebcbe3e2c6cb9eb7bb8d4fbf522a9ff6aa..cf68c7a738175e9ecdddd18f12bc1651d36e7892 100644 --- a/src/codegen/wasm/CodeGen.zig +++ b/src/codegen/wasm/CodeGen.zig @@ -136,13 +136,10 @@ air: Air, liveness: Air.Liveness, gpa: mem.Allocator, func_index: InternPool.Index, -/// Contains a list of current branches. -/// When we return from a branch, the branch will be popped from this list, -/// which means branches can only contain references from within its own branch, -/// or a branch higher (lower index) in the tree. -branches: std.ArrayList(Branch) = .empty, +/// Contains a list of current branches. Stores LocalPool before entering branch. +branches: std.ArrayList(LocalPool) = .empty, /// Table to save `WValue`'s generated by an `Air.Inst` -// values: ValueTable, +values: ValueTable = .empty, /// Mapping from Air.Inst.Index to block ids blocks: std.array_hash_map.Auto(Air.Inst.Index, struct { label: u32, @@ -150,10 +147,6 @@ blocks: std.array_hash_map.Auto(Air.Inst.Index, struct { }) = .{}, /// Maps `loop` instructions to their label. `br` to here repeats the loop. loops: std.AutoHashMapUnmanaged(Air.Inst.Index, u32) = .empty, -/// The index the next local generated will have -/// NOTE: arguments share the index with locals therefore the first variable -/// will have the index that comes after the last argument's index -local_index: u32, /// The index of the current argument. /// Used to track which argument is being referenced in `airArg`. arg_index: u32 = 0, @@ -207,23 +200,8 @@ stack_size: u32 = 0, /// However, local variables or the usage of `incoming_stack_alignment` in a `CallingConvention` can overwrite this default. stack_alignment: Alignment = .@"16", -// For each individual Wasm valtype we store a seperate free list which -// allows us to re-use locals that are no longer used. e.g. a temporary local. -/// A list of indexes which represents a local of valtype `i32`. -/// It is illegal to store a non-i32 valtype in this list. -free_locals_i32: std.ArrayList(u32) = .empty, -/// A list of indexes which represents a local of valtype `i64`. -/// It is illegal to store a non-i64 valtype in this list. -free_locals_i64: std.ArrayList(u32) = .empty, -/// A list of indexes which represents a local of valtype `f32`. -/// It is illegal to store a non-f32 valtype in this list. -free_locals_f32: std.ArrayList(u32) = .empty, -/// A list of indexes which represents a local of valtype `f64`. -/// It is illegal to store a non-f64 valtype in this list. -free_locals_f64: std.ArrayList(u32) = .empty, -/// A list of indexes which represents a local of valtype `v127`. -/// It is illegal to store a non-v128 valtype in this list. -free_locals_v128: std.ArrayList(u32) = .empty, +local_pool: LocalPool = .{}, +wasm_param_count: u32 = undefined, /// When in debug mode, this tracks if no `finishAir` was missed. /// Forgetting to call `finishAir` will cause the result to not be @@ -239,13 +217,7 @@ const WValue = union(enum) { none: void, /// The value lives on top of the stack stack: void, - /// Index of the local - local: struct { - /// Contains the index to the local - value: u32, - /// The amount of instructions referencing this `WValue` - references: u32, - }, + local: u32, // index /// An immediate 32bit value imm32: u32, /// An immediate 64bit value @@ -290,7 +262,7 @@ const WValue = union(enum) { switch (value) { .stack => { const new_local = try gen.allocLocal(ty); - try gen.addLocal(.local_set, new_local.local.value); + try gen.addLocal(.local_set, new_local.local); return new_local; }, else => return value, @@ -299,23 +271,9 @@ const WValue = union(enum) { /// Marks a local as no longer being referenced and essentially allows /// us to re-use it somewhere else within the function. - /// The valtype of the local is deducted by using the index of the given `WValue`. - fn free(value: *WValue, gen: *CodeGen) void { + fn free(value: *WValue, cg: *CodeGen) void { if (value.* != .local) return; - const local_value = value.local.value; - const reserved = gen.args.len + @intFromBool(gen.return_value != .none) + @intFromBool(gen.varargs != .none); - if (local_value < reserved + 2) return; // reserved locals may never be re-used. Also accounts for 2 stack locals. - - const index = local_value - reserved; - const valtype = gen.mir_locals.items[index]; - switch (valtype) { - .i32 => gen.free_locals_i32.append(gen.gpa, local_value) catch return, // It's ok to fail any of those, a new local can be allocated instead - .i64 => gen.free_locals_i64.append(gen.gpa, local_value) catch return, - .f32 => gen.free_locals_f32.append(gen.gpa, local_value) catch return, - .f64 => gen.free_locals_f64.append(gen.gpa, local_value) catch return, - .v128 => gen.free_locals_v128.append(gen.gpa, local_value) catch return, - } - log.debug("freed local ({d}) of type {}", .{ local_value, valtype }); + cg.releaseLocal(value.local); value.* = .dead; } }; @@ -323,6 +281,77 @@ const WValue = union(enum) { /// Hashmap to store generated `WValue` for each `Air.Inst.Ref` const ValueTable = std.array_hash_map.Auto(Air.Inst.Ref, WValue); +const LocalPool = struct { + const OptionalIndex = enum(u32) { + none = math.maxInt(u32), + _, + + fn init(index: u32) OptionalIndex { + const result: OptionalIndex = @fromBackingInt(index); + assert(result != .none); + return result; + } + + fn unwrap(index: OptionalIndex) ?u32 { + return if (index == .none) null else @backingInt(index); + } + }; + + const Local = union { + ref_count: u32, + next_free: OptionalIndex, + }; + + locals: std.ArrayList(Local) = .empty, + free_i32: OptionalIndex = .none, + free_i64: OptionalIndex = .none, + free_f32: OptionalIndex = .none, + free_f64: OptionalIndex = .none, + free_v128: OptionalIndex = .none, + + fn deinit(pool: *LocalPool, gpa: Allocator) void { + pool.locals.deinit(gpa); + pool.* = undefined; + } + + fn clone(pool: *LocalPool, gpa: Allocator) Allocator.Error!LocalPool { + return .{ + .locals = try pool.locals.clone(gpa), + .free_i32 = pool.free_i32, + .free_i64 = pool.free_i64, + .free_f32 = pool.free_f32, + .free_f64 = pool.free_f64, + .free_v128 = pool.free_v128, + }; + } + + fn freeList(pool: *LocalPool, valtype: std.wasm.Valtype) *OptionalIndex { + return switch (valtype) { + .i32 => &pool.free_i32, + .i64 => &pool.free_i64, + .f32 => &pool.free_f32, + .f64 => &pool.free_f64, + .v128 => &pool.free_v128, + }; + } + + fn restore(snapshot: *LocalPool, cg: *CodeGen) void { + const local_pool = &cg.local_pool; + const old_local_count = snapshot.locals.items.len; + assert(local_pool.locals.items.len >= old_local_count); + @memcpy(local_pool.locals.items[0..old_local_count], snapshot.locals.items); + local_pool.free_i32 = snapshot.free_i32; + local_pool.free_i64 = snapshot.free_i64; + local_pool.free_f32 = snapshot.free_f32; + local_pool.free_f64 = snapshot.free_f64; + local_pool.free_v128 = snapshot.free_v128; + + for (old_local_count..local_pool.locals.items.len) |index| { + cg.appendFreeLocal(cg.wasm_param_count + @as(u32, @intCast(index))); + } + } +}; + const bookkeeping_init = if (std.debug.runtime_safety) @as(usize, 0) else {}; const InnerError = Error || error{ @@ -336,14 +365,11 @@ pub fn deinit(cg: *CodeGen) void { const gpa = cg.gpa; for (cg.branches.items) |*branch| branch.deinit(gpa); cg.branches.deinit(gpa); + cg.values.deinit(gpa); cg.blocks.deinit(gpa); cg.loops.deinit(gpa); cg.simd_immediates.deinit(gpa); - cg.free_locals_i32.deinit(gpa); - cg.free_locals_i64.deinit(gpa); - cg.free_locals_f32.deinit(gpa); - cg.free_locals_f64.deinit(gpa); - cg.free_locals_v128.deinit(gpa); + cg.local_pool.deinit(gpa); cg.mir_instructions.deinit(gpa); cg.mir_extra.deinit(gpa); cg.mir_locals.deinit(gpa); @@ -362,19 +388,11 @@ pub fn fail(cg: *CodeGen, comptime fmt: []const u8, args: anytype) Error { /// Resolves the `WValue` for the given instruction `inst` /// When the given instruction has a `Value`, it returns a constant instead fn resolveInst(cg: *CodeGen, ref: Air.Inst.Ref) InnerError!WValue { - var branch_index = cg.branches.items.len; - while (branch_index > 0) : (branch_index -= 1) { - const branch = cg.branches.items[branch_index - 1]; - if (branch.values.get(ref)) |value| { - return value; - } - } + if (cg.values.get(ref)) |value| return value; // when we did not find an existing instruction, it // means we must generate it from a constant. - // We always store constants in the most outer branch as they must never - // be removed. The most outer branch is always at index 0. - const gop = try cg.branches.items[0].values.getOrPut(cg.gpa, ref); + const gop = try cg.values.getOrPut(cg.gpa, ref); assert(!gop.found_existing); const pt = cg.pt; @@ -413,20 +431,33 @@ fn resolveValue(cg: *CodeGen, val: Value) InnerError!WValue { /// NOTE: if result == .stack, it will be stored in .local fn finishAir(cg: *CodeGen, inst: Air.Inst.Index, result: WValue, operands: []const Air.Inst.Ref) InnerError!void { + return cg.finishAirWithBranchDeaths(inst, result, operands, &.{}); +} + +/// Takes `branch_deaths` from continuing branch and calls `processDeath` after result retaining logic. +fn finishAirWithBranchDeaths( + cg: *CodeGen, + inst: Air.Inst.Index, + result: WValue, + operands: []const Air.Inst.Ref, + branch_deaths: []const Air.Inst.Index, +) InnerError!void { assert(operands.len <= Air.Liveness.bpi - 1); - var final_result = result; - const aliased_operand: ?*WValue = if (result == .local) aliased: { + if (result == .local) aliased: { for (operands) |operand| { if (operand.toIndex() == null) continue; - const resolved_operand = cg.getResolvedInst(operand); - if (resolved_operand.* != .local or resolved_operand.local.value != result.local.value) continue; + const resolved_operand = cg.values.getPtr(operand) orelse unreachable; + if (resolved_operand.* != .local or resolved_operand.local != result.local) continue; - resolved_operand.local.references += 1; - break :aliased resolved_operand; + if (result.local >= cg.wasm_param_count) { + const local = &cg.local_pool.locals.items[result.local - cg.wasm_param_count]; + assert(local.ref_count > 0); + local.ref_count += 1; + } + break :aliased; } - break :aliased null; - } else null; + } var tomb_bits = cg.liveness.getTombBits(inst); for (operands) |operand| { @@ -435,8 +466,8 @@ fn finishAir(cg: *CodeGen, inst: Air.Inst.Index, result: WValue, operands: []con if (!dies) continue; processDeath(cg, operand); } - if (aliased_operand) |operand| final_result.local.references = operand.local.references; - try cg.finishAirResult(inst, final_result); + for (branch_deaths) |death| cg.processDeath(death.toRef()); + try cg.finishAirResult(inst, result); } fn finishAirResult(cg: *CodeGen, inst: Air.Inst.Index, result: WValue) InnerError!void { @@ -446,8 +477,7 @@ fn finishAirResult(cg: *CodeGen, inst: Air.Inst.Index, result: WValue) InnerErro result else try result.toLocal(cg, cg.typeOfIndex(inst)); - const branch = cg.currentBranch(); - branch.values.putAssumeCapacityNoClobber(inst.toRef(), trackable_result); + try cg.values.putNoClobber(cg.gpa, inst.toRef(), trackable_result); } if (std.debug.runtime_safety) { @@ -455,17 +485,16 @@ fn finishAirResult(cg: *CodeGen, inst: Air.Inst.Index, result: WValue) InnerErro } } -const Branch = struct { - values: ValueTable = .{}, +fn pushBranch(cg: *CodeGen) InnerError!void { + var snapshot = try cg.local_pool.clone(cg.gpa); + errdefer snapshot.deinit(cg.gpa); + try cg.branches.append(cg.gpa, snapshot); +} - fn deinit(branch: *Branch, gpa: Allocator) void { - branch.values.deinit(gpa); - branch.* = undefined; - } -}; - -inline fn currentBranch(cg: *CodeGen) *Branch { - return &cg.branches.items[cg.branches.items.len - 1]; +fn popBranch(cg: *CodeGen) void { + var snapshot = cg.branches.pop().?; + defer snapshot.deinit(cg.gpa); + snapshot.restore(cg); } fn feed(cg: *CodeGen, bt: *Air.Liveness.BigTomb, operand: Air.Inst.Ref) void { @@ -476,20 +505,9 @@ fn feed(cg: *CodeGen, bt: *Air.Liveness.BigTomb, operand: Air.Inst.Ref) void { fn processDeath(cg: *CodeGen, ref: Air.Inst.Ref) void { if (ref.toIndex() == null) return; - // Branches are currently only allowed to free locals allocated - // within their own branch. - // TODO: Upon branch consolidation free any locals if needed. - const value = cg.currentBranch().values.getPtr(ref) orelse return; + const value = cg.values.getPtr(ref) orelse unreachable; if (value.* != .local) return; - const reserved_indexes = cg.args.len + @intFromBool(cg.return_value != .none); - if (value.local.value < reserved_indexes) { - return; // function arguments can never be re-used - } - log.debug("Decreasing reference for ref: %{d}, using local '{d}'", .{ @backingInt(ref.toIndex().?), value.local.value }); - value.local.references -= 1; - if (value.local.references == 0) { - value.free(cg); - } + cg.releaseLocal(value.local); } pub fn addInst(cg: *CodeGen, inst: Mir.Inst) error{OutOfMemory}!void { @@ -708,7 +726,7 @@ fn emitWValue(cg: *CodeGen, value: WValue) InnerError!void { switch (value) { .dead => unreachable, // reference to free'd `WValue` .none, .stack => {}, // no-op - .local => |idx| try cg.addLocal(.local_get, idx.value), + .local => |idx| try cg.addLocal(.local_get, idx), .imm32 => |val| try cg.addImm32(val), .imm64 => |val| try cg.addImm64(val), .imm128 => |val| try cg.addImm128(val), @@ -780,22 +798,28 @@ fn emitWValue(cg: *CodeGen, value: WValue) InnerError!void { /// Emits the base address for a memory instruction. fn emitMemBase(cg: *CodeGen, ptr: WValue) InnerError!void { switch (ptr) { - .stack_offset => try cg.addLocal(.local_get, cg.bottom_stack_value.local.value), + .stack_offset => try cg.addLocal(.local_get, cg.bottom_stack_value.local), else => try cg.emitWValue(ptr), } } -/// From a reference, returns its resolved `WValue`. -/// It's illegal to provide a `Air.Inst.Ref` that hasn't been resolved yet. -fn getResolvedInst(cg: *CodeGen, ref: Air.Inst.Ref) *WValue { - var index = cg.branches.items.len; - while (index > 0) : (index -= 1) { - const branch = cg.branches.items[index - 1]; - if (branch.values.getPtr(ref)) |value| { - return value; - } - } - unreachable; // developer-error: This can only be called on resolved instructions. Use `resolveInst` instead. +fn appendFreeLocal(cg: *CodeGen, index: u32) void { + assert(index >= cg.wasm_param_count); + const mir_index = index - cg.wasm_param_count; + const valtype = cg.mir_locals.items[mir_index]; + const free_list = cg.local_pool.freeList(valtype); + cg.local_pool.locals.items[mir_index] = .{ .next_free = free_list.* }; + free_list.* = .init(index); + log.debug("freed local ({d}) of type {}", .{ index, valtype }); +} + +fn releaseLocal(cg: *CodeGen, index: u32) void { + if (index < cg.wasm_param_count) return; + const local = &cg.local_pool.locals.items[index - cg.wasm_param_count]; + assert(local.ref_count > 0); + local.ref_count -= 1; + if (local.ref_count > 0) return; + cg.appendFreeLocal(index); } /// Creates one locals for a given `Type`. @@ -803,16 +827,13 @@ fn getResolvedInst(cg: *CodeGen, ref: Air.Inst.Ref) *WValue { fn allocLocal(cg: *CodeGen, ty: Type) InnerError!WValue { const zcu = cg.pt.zcu; const valtype = typeToValtype(ty, zcu, cg.target); - const index_or_null = switch (valtype) { - .i32 => cg.free_locals_i32.pop(), - .i64 => cg.free_locals_i64.pop(), - .f32 => cg.free_locals_f32.pop(), - .f64 => cg.free_locals_f64.pop(), - .v128 => cg.free_locals_v128.pop(), - }; - if (index_or_null) |index| { + const free_list = cg.local_pool.freeList(valtype); + if (free_list.unwrap()) |index| { + const local = &cg.local_pool.locals.items[index - cg.wasm_param_count]; + free_list.* = local.next_free; + local.* = .{ .ref_count = 1 }; log.debug("reusing local ({d}) of type {}", .{ index, valtype }); - return .{ .local = .{ .value = index, .references = 1 } }; + return .{ .local = index }; } log.debug("new local of type {}", .{valtype}); return cg.ensureAllocLocal(ty); @@ -822,10 +843,13 @@ fn allocLocal(cg: *CodeGen, ty: Type) InnerError!WValue { /// to use a zero-initialized local. fn ensureAllocLocal(cg: *CodeGen, ty: Type) InnerError!WValue { const zcu = cg.pt.zcu; - try cg.mir_locals.append(cg.gpa, typeToValtype(ty, zcu, cg.target)); - const initial_index = cg.local_index; - cg.local_index += 1; - return .{ .local = .{ .value = initial_index, .references = 1 } }; + try cg.mir_locals.ensureUnusedCapacity(cg.gpa, 1); + try cg.local_pool.locals.ensureUnusedCapacity(cg.gpa, 1); + assert(cg.mir_locals.items.len == cg.local_pool.locals.items.len); + const initial_index = cg.wasm_param_count + @as(u32, @intCast(cg.local_pool.locals.items.len)); + cg.mir_locals.appendAssumeCapacity(typeToValtype(ty, zcu, cg.target)); + cg.local_pool.locals.appendAssumeCapacity(.{ .ref_count = 1 }); + return .{ .local = initial_index }; } pub const Error = codegen.Error; @@ -867,7 +891,6 @@ pub fn generate( .args = cc_result.args, .return_value = cc_result.return_value, .varargs = cc_result.varargs, - .local_index = cc_result.local_index, .mir_instructions = .empty, .mir_extra = .empty, .mir_locals = .empty, @@ -890,17 +913,10 @@ pub fn generate( fn generateInner(cg: *CodeGen, any_returns: bool) InnerError!Mir { const zcu = cg.pt.zcu; - // branch used for const values - try cg.branches.append(cg.gpa, .{}); - // func scope branch - try cg.branches.append(cg.gpa, .{}); - defer { - var func_branch = cg.branches.pop().?; - func_branch.deinit(cg.gpa); - var const_branch = cg.branches.pop().?; - const_branch.deinit(cg.gpa); - assert(cg.branches.items.len == 0); // missing branch merge - } + cg.wasm_param_count = @intCast(cg.args.len + @intFromBool(cg.return_value != .none) + @intFromBool(cg.varargs != .none)); + cg.initial_stack_value = try cg.ensureAllocLocal(Type.usize); + cg.bottom_stack_value = try cg.ensureAllocLocal(Type.usize); + defer assert(cg.branches.items.len == 0); // Generate MIR for function body try cg.genBody(cg.air.getMainBody()); @@ -925,11 +941,11 @@ fn generateInner(cg: *CodeGen, any_returns: bool) InnerError!Mir { .instructions = cg.mir_instructions.toOwnedSlice(), .extra = cg.mir_extra.toOwnedSliceAssert(), .locals = cg.mir_locals.toOwnedSliceAssert(), - .prologue = if (cg.initial_stack_value == .none) .none else .{ - .sp_local = cg.initial_stack_value.local.value, + .prologue = .{ + .sp_local = cg.initial_stack_value.local, .flags = .{ .stack_alignment = cg.stack_alignment }, .stack_size = cg.stack_size, - .bottom_stack_local = cg.bottom_stack_value.local.value, + .bottom_stack_local = cg.bottom_stack_value.local, }, .uavs = cg.mir_uavs.move(), .indirect_function_set = cg.mir_indirect_function_set.move(), @@ -976,7 +992,7 @@ fn resolveCallingConventionValues( if (firstParamSRet(fn_info.cc, Type.fromInterned(fn_info.return_type), zcu, target)) { // the sret arg will be passed as first argument, therefore we // set the `return_value` before allocating locals for regular args. - result.return_value = .{ .local = .{ .value = result.local_index, .references = 1 } }; + result.return_value = .{ .local = result.local_index }; result.local_index += 1; } @@ -987,7 +1003,7 @@ fn resolveCallingConventionValues( continue; } - try args.append(.{ .local = .{ .value = result.local_index, .references = 1 } }); + try args.append(.{ .local = result.local_index }); result.local_index += 1; } }, @@ -1000,17 +1016,17 @@ fn resolveCallingConventionValues( switch (abi.classifyType(param_ty, zcu, target)) { .direct, .indirect => { - try args.append(.{ .local = .{ .value = result.local_index, .references = 1 } }); + try args.append(.{ .local = result.local_index }); result.local_index += 1; }, .double_i64 => { - try args.append(.{ .local = .{ .value = result.local_index, .references = 1 } }); - try args.append(.{ .local = .{ .value = result.local_index + 1, .references = 1 } }); + try args.append(.{ .local = result.local_index }); + try args.append(.{ .local = result.local_index + 1 }); result.local_index += 2; }, .unrolled => |vector| { for (0..vector.len) |_| { - try args.append(.{ .local = .{ .value = result.local_index, .references = 1 } }); + try args.append(.{ .local = result.local_index }); result.local_index += 1; } }, @@ -1021,7 +1037,7 @@ fn resolveCallingConventionValues( } if (fn_info.is_var_args) { - result.varargs = .{ .local = .{ .value = result.local_index, .references = 1 } }; + result.varargs = .{ .local = result.local_index }; result.local_index += 1; } @@ -1088,23 +1104,9 @@ fn lowerArg(cg: *CodeGen, cc: std.lang.CallingConvention, ty: Type, value: WValu } } -/// Creates a local for the initial stack value -/// Asserts `initial_stack_value` is `.none` -fn initializeStack(cg: *CodeGen) !void { - assert(cg.initial_stack_value == .none); - // Reserve a local to store the current stack pointer - // We can later use this local to set the stack pointer back to the value - // we have stored here. - cg.initial_stack_value = try cg.ensureAllocLocal(Type.usize); - // Also reserve a local to store the bottom stack value - cg.bottom_stack_value = try cg.ensureAllocLocal(Type.usize); -} - /// Reads the stack pointer from `Context.initial_stack_value` and writes it /// to the global stack pointer variable fn restoreStackPointer(cg: *CodeGen) !void { - // only restore the pointer if it was initialized - if (cg.initial_stack_value == .none) return; // Get the original stack pointer's value try cg.emitWValue(cg.initial_stack_value); @@ -1144,10 +1146,6 @@ fn allocInt(cg: *CodeGen, int_ty: IntType) !WValue { fn allocStackBytes(cg: *CodeGen, size: u32, alignment: Alignment) !WValue { assert(size > 0); - if (cg.initial_stack_value == .none) { - try cg.initializeStack(); - } - cg.stack_alignment = cg.stack_alignment.max(alignment); const offset: u32 = @intCast(alignment.forward(cg.stack_size)); @@ -1166,10 +1164,6 @@ fn allocStackPtr(cg: *CodeGen, inst: Air.Inst.Index) !WValue { const ptr_ty = cg.typeOfIndex(inst); const pointee_ty = ptr_ty.childType(zcu); - if (cg.initial_stack_value == .none) { - try cg.initializeStack(); - } - const abi_alignment = ptr_ty.ptrAlignment(zcu); const abi_size = std.math.cast(u32, pointee_ty.abiSize(zcu)) orelse { return cg.fail("Type {f} with ABI size of {d} exceeds stack frame size", .{ @@ -1934,7 +1928,6 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { continue; } const old_bookkeeping_value = cg.air_bookkeeping; - try cg.currentBranch().values.ensureUnusedCapacity(cg.gpa, 1); try cg.genInst(inst); if (std.debug.runtime_safety and cg.air_bookkeeping < old_bookkeeping_value + 1) { @@ -2160,11 +2153,11 @@ fn airCall(cg: *CodeGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier) .direct => |scalar_type| { if (!isByRef(ret_ty, zcu, cg.target)) { const result_local = try cg.allocLocal(ret_ty); - try cg.addLocal(.local_set, result_local.local.value); + try cg.addLocal(.local_set, result_local.local); break :result_value result_local; } else { const result_local = try cg.allocLocal(scalar_type); - try cg.addLocal(.local_set, result_local.local.value); + try cg.addLocal(.local_set, result_local.local); const result = try cg.allocStack(ret_ty); try cg.store(result, result_local, scalar_type, .{}); break :result_value result; @@ -2175,7 +2168,7 @@ fn airCall(cg: *CodeGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier) assert(vector.len == 1); const result_local = try cg.allocLocal(vector.elem_type); // save call result from operand stack - try cg.addLocal(.local_set, result_local.local.value); + try cg.addLocal(.local_set, result_local.local); const result = try cg.allocStack(ret_ty); try cg.store(result, result_local, vector.elem_type, .{}); break :result_value result; @@ -2183,7 +2176,7 @@ fn airCall(cg: *CodeGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier) } } else { const result_local = try cg.allocLocal(ret_ty); - try cg.addLocal(.local_set, result_local.local.value); + try cg.addLocal(.local_set, result_local.local); break :result_value result_local; } }; @@ -2250,7 +2243,7 @@ fn airVaArg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { } } - try cg.addLocal(.local_set, arg_ptr.local.value); + try cg.addLocal(.local_set, arg_ptr.local); try cg.emitWValue(operand); try cg.emitWValue(arg_ptr); @@ -2272,7 +2265,7 @@ fn airVaArg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { try cg.addTag(.f32_demote_f64); const result_local = try cg.allocLocal(Type.f32); - try cg.addLocal(.local_set, result_local.local.value); + try cg.addLocal(.local_set, result_local.local); break :result result_local; } else try cg.load(arg_ptr, ty, .{}); @@ -3300,7 +3293,7 @@ fn intAbs(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue { var mask = try cg.allocLocal(Type.i32); defer mask.free(cg); - try cg.addLocal(.local_tee, mask.local.value); + try cg.addLocal(.local_tee, mask.local); try cg.emitWValue(operand); try cg.addTag(.i32_xor); @@ -3315,7 +3308,7 @@ fn intAbs(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue { var mask = try cg.allocLocal(Type.i64); defer mask.free(cg); - try cg.addLocal(.local_tee, mask.local.value); + try cg.addLocal(.local_tee, mask.local); try cg.emitWValue(operand); try cg.addTag(.i64_xor); @@ -3337,7 +3330,7 @@ fn intAbs(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue { var tmp = try cg.allocLocal(Type.u64); defer tmp.free(cg); - try cg.addLocal(.local_tee, tmp.local.value); + try cg.addLocal(.local_tee, tmp.local); try cg.store(.stack, .stack, .u64, .{ .offset = mask_offset }); try cg.emitWValue(tmp); try cg.store(.stack, .stack, .u64, .{ .offset = mask_offset + 8 }); @@ -4033,7 +4026,7 @@ fn toLocalInt(cg: *CodeGen, value: WValue, int_ty: IntType) InnerError!WValue { else => return cg.fail("TODO: Support toLocalInt for integer bitsize: {d}", .{int_ty.bits}), }; const new_local = try cg.allocLocal(ty); - try cg.addLocal(.local_set, new_local.local.value); + try cg.addLocal(.local_set, new_local.local); return new_local; }, .local, .stack_offset => return value, @@ -4112,7 +4105,7 @@ fn intSubOverflow(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerErro fn intMulOverflow(cg: *CodeGen, int_ty: IntType, lhs: WValue, rhs: WValue) InnerError!OverflowResult { const overflow_bit = try cg.allocLocal(Type.u32); try cg.addImm32(0); - try cg.addLocal(.local_set, overflow_bit.local.value); + try cg.addLocal(.local_set, overflow_bit.local); const result_val = if (int_ty.bits <= 32) blk: { const new_ty: IntType = .{ .is_signed = int_ty.is_signed, .bits = 64 }; @@ -4126,7 +4119,7 @@ fn intMulOverflow(cg: *CodeGen, int_ty: IntType, lhs: WValue, rhs: WValue) Inner const res_upcast = try cg.intCast(new_ty, int_ty, res_tmp); _ = try cg.intCmp(new_ty, .neq, res_upcast, bin_op); - try cg.addLocal(.local_set, overflow_bit.local.value); + try cg.addLocal(.local_set, overflow_bit.local); break :blk res_tmp; } else if (int_ty.bits <= 64) blk: { const new_ty: IntType = .{ .is_signed = int_ty.is_signed, .bits = 128 }; @@ -4140,7 +4133,7 @@ fn intMulOverflow(cg: *CodeGen, int_ty: IntType, lhs: WValue, rhs: WValue) Inner const res_upcast = try cg.intCast(new_ty, int_ty, res_tmp); _ = try cg.intCmp(new_ty, .neq, res_upcast, bin_op); - try cg.addLocal(.local_set, overflow_bit.local.value); + try cg.addLocal(.local_set, overflow_bit.local); break :blk res_tmp; } else if (int_ty.bits == 128 and int_ty.is_signed) blk: { const overflow_ret = try cg.allocStack(Type.i32); @@ -4151,7 +4144,7 @@ fn intMulOverflow(cg: *CodeGen, int_ty: IntType, lhs: WValue, rhs: WValue) Inner &.{ lhs, rhs, overflow_ret }, ); _ = try cg.load(overflow_ret, .i32, .{}); - try cg.addLocal(.local_set, overflow_bit.local.value); + try cg.addLocal(.local_set, overflow_bit.local); break :blk res; } else { const result = try cg.allocInt(int_ty); @@ -5232,17 +5225,17 @@ fn lowerBlock(cg: *CodeGen, inst: Air.Inst.Index, block_ty: Type, body: []const .label = cg.block_depth, .value = block_result, }); + defer assert(cg.blocks.swapRemove(inst)); { - try cg.branches.append(cg.gpa, .{}); - defer { - var branch = cg.branches.pop().?; - branch.deinit(cg.gpa); - } + try cg.pushBranch(); + defer cg.popBranch(); try cg.genBody(body); try cg.endBlock(); } + for (cg.liveness.getBlock(inst).deaths) |death| cg.processDeath(death.toRef()); + return cg.finishAir(inst, block_result, &.{}); } @@ -5282,6 +5275,7 @@ fn airCondBr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { const condition = try cg.resolveInst(cond_br.condition); const then_body = cond_br.then_body; const else_body = cond_br.else_body; + const branch_liveness = cg.liveness.getCondBr(inst); // result type is always noreturn, so use `block_empty` as type. try cg.startBlock(.block, .empty); @@ -5293,24 +5287,21 @@ fn airCondBr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { // and continue with the then codepath try cg.addLabel(.br_if, 0); - try cg.branches.ensureUnusedCapacity(cg.gpa, 2); + if (cg.liveness.operandDies(inst, 0)) cg.processDeath(cond_br.condition); + { - cg.branches.appendAssumeCapacity(.{}); - defer { - var else_stack = cg.branches.pop().?; - else_stack.deinit(cg.gpa); - } + try cg.pushBranch(); + defer cg.popBranch(); + for (branch_liveness.else_deaths) |death| cg.processDeath(death.toRef()); try cg.genBody(else_body); try cg.endBlock(); } // Outer block that matches the condition { - cg.branches.appendAssumeCapacity(.{}); - defer { - var then_stack = cg.branches.pop().?; - then_stack.deinit(cg.gpa); - } + try cg.pushBranch(); + defer cg.popBranch(); + for (branch_liveness.then_deaths) |death| cg.processDeath(death.toRef()); try cg.genBody(then_body); } @@ -5346,18 +5337,18 @@ fn airCmp(cg: *CodeGen, inst: Air.Inst.Index, op: std.math.CompareOperator) Inne try cg.startBlock(.block, .empty); try cg.addImm32(if (op == .eq) 0 else 1); - try cg.addLocal(.local_set, result.local.value); + try cg.addLocal(.local_set, result.local); _ = try cg.isNull(lhs, operand_ty, .i32_eq, .value); - try cg.addLocal(.local_tee, lhs_null.local.value); + try cg.addLocal(.local_tee, lhs_null.local); _ = try cg.isNull(rhs, operand_ty, .i32_eq, .value); try cg.addTag(.i32_ne); try cg.addLabel(.br_if, 0); try cg.addImm32(if (op == .eq) 1 else 0); - try cg.addLocal(.local_set, result.local.value); + try cg.addLocal(.local_set, result.local); - try cg.addLocal(.local_get, lhs_null.local.value); + try cg.addLocal(.local_get, lhs_null.local); try cg.addLabel(.br_if, 0); _ = try cg.load(lhs, payload_ty, .{}); @@ -5369,10 +5360,10 @@ fn airCmp(cg: *CodeGen, inst: Air.Inst.Index, op: std.math.CompareOperator) Inne _ = try cg.intCmp(.fromType(cg, payload_ty), op, .stack, .stack); } - try cg.addLocal(.local_set, result.local.value); + try cg.addLocal(.local_set, result.local); try cg.endBlock(); - try cg.addLocal(.local_get, result.local.value); + try cg.addLocal(.local_get, result.local); try cg.finishAir(inst, .stack, &.{ bin_op.lhs, bin_op.rhs }); } else { const result = try cg.intCmp(.fromType(cg, operand_ty), op, lhs, rhs); @@ -5570,7 +5561,7 @@ fn airBr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { if (block.value != .none) { const operand = try cg.resolveInst(br.operand); try cg.emitWValue(operand); - try cg.addLocal(.local_set, block.value.local.value); + try cg.addLocal(.local_set, block.value.local); } // We map every block to its block index. @@ -5854,24 +5845,29 @@ fn airSwitchBr(cg: *CodeGen, inst: Air.Inst.Index, is_dispatch_loop: bool) Inner const switch_br = cg.air.unwrapSwitch(inst); const target_ty = cg.typeOf(switch_br.operand); + const branch_liveness = try cg.liveness.getSwitchBr(cg.gpa, inst, switch_br.cases_len + 1); + defer cg.gpa.free(branch_liveness.deaths); assert(target_ty.hasRuntimeBits(zcu)); // swap target value with placeholder local, for dispatching + var owned_target: WValue = .none; + defer owned_target.free(cg); const target = if (is_dispatch_loop) target: { const initial_target = try cg.resolveInst(switch_br.operand); - const target: WValue = try cg.allocLocal(target_ty); + owned_target = try cg.allocLocal(target_ty); try cg.emitWValue(initial_target); - try cg.addLocal(.local_set, target.local.value); + try cg.addLocal(.local_set, owned_target.local); try cg.startBlock(.loop, .empty); // dispatch loop start try cg.blocks.putNoClobber(cg.gpa, inst, .{ .label = cg.block_depth, - .value = target, + .value = owned_target, }); - break :target target; + break :target owned_target; } else try cg.resolveInst(switch_br.operand); + defer if (is_dispatch_loop) assert(cg.blocks.swapRemove(inst)); const has_else_body = switch_br.else_body_len != 0; const branch_count = switch_br.cases_len + 1; // if else branch is missing, we trap when failing all conditions @@ -5883,12 +5879,14 @@ fn airSwitchBr(cg: *CodeGen, inst: Air.Inst.Index, is_dispatch_loop: bool) Inner var it = switch_br.iterateCases(); const else_body = it.elseBody(); - cg.branches.appendAssumeCapacity(.{}); - defer { - var else_branch = cg.branches.pop().?; - else_branch.deinit(cg.gpa); + if (cg.liveness.operandDies(inst, 0)) cg.processDeath(switch_br.operand); + + { + try cg.pushBranch(); + defer cg.popBranch(); + for (branch_liveness.deaths[0]) |death| cg.processDeath(death.toRef()); + try cg.genBody(else_body); } - try cg.genBody(else_body); if (is_dispatch_loop) { try cg.endBlock(); // dispatch loop end @@ -6012,16 +6010,18 @@ fn airSwitchBr(cg: *CodeGen, inst: Air.Inst.Index, is_dispatch_loop: bool) Inner try cg.addLabel(.br, branch_count - 1); } + if (cg.liveness.operandDies(inst, 0)) cg.processDeath(switch_br.operand); + var cases_it = switch_br.iterateCases(); while (cases_it.next()) |case| { try cg.endBlock(); - cg.branches.appendAssumeCapacity(.{}); - defer { - var case_branch = cg.branches.pop().?; - case_branch.deinit(cg.gpa); + { + try cg.pushBranch(); + defer cg.popBranch(); + for (branch_liveness.deaths[case.idx]) |death| cg.processDeath(death.toRef()); + try cg.genBody(case.body); } - try cg.genBody(case.body); try cg.addLabel(.br, branch_count - case.idx - 1); // matching case found and executed => exit switch } @@ -6030,12 +6030,12 @@ fn airSwitchBr(cg: *CodeGen, inst: Air.Inst.Index, is_dispatch_loop: bool) Inner if (has_else_body) { const else_body = cases_it.elseBody(); - cg.branches.appendAssumeCapacity(.{}); - defer { - var else_branch = cg.branches.pop().?; - else_branch.deinit(cg.gpa); + { + try cg.pushBranch(); + defer cg.popBranch(); + for (branch_liveness.deaths[branch_liveness.deaths.len - 1]) |death| cg.processDeath(death.toRef()); + try cg.genBody(else_body); } - try cg.genBody(else_body); } else { try cg.addTag(.@"unreachable"); } @@ -6055,7 +6055,7 @@ fn airSwitchDispatch(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { const operand = try cg.resolveInst(br.operand); try cg.emitWValue(operand); - try cg.addLocal(.local_set, switch_loop.value.local.value); + try cg.addLocal(.local_set, switch_loop.value.local); const idx: u32 = cg.block_depth - switch_loop.label; try cg.addLabel(.br, idx); @@ -6630,7 +6630,7 @@ fn memset(cg: *CodeGen, elem_ty: Type, ptr: WValue, len: WValue, value: WValue) try cg.addTag(.i64_mul); }, } - try cg.addLocal(.local_set, new_len.local.value); + try cg.addLocal(.local_set, new_len.local); break :blk new_len; } else len, }; @@ -6640,7 +6640,7 @@ fn memset(cg: *CodeGen, elem_ty: Type, ptr: WValue, len: WValue, value: WValue) var new_ptr = try cg.allocLocal(Type.usize); defer new_ptr.free(cg); try cg.emitWValue(ptr); - try cg.addLocal(.local_set, new_ptr.local.value); + try cg.addLocal(.local_set, new_ptr.local); // get the loop conditional: if current pointer address equals final pointer's address try cg.emitWValue(ptr); @@ -6649,7 +6649,7 @@ fn memset(cg: *CodeGen, elem_ty: Type, ptr: WValue, len: WValue, value: WValue) .wasm32 => try cg.addTag(.i32_add), .wasm64 => try cg.addTag(.i64_add), } - try cg.addLocal(.local_set, end_ptr.local.value); + try cg.addLocal(.local_set, end_ptr.local); // outer block to jump to when loop is done try cg.startBlock(.block, .empty); @@ -6679,7 +6679,7 @@ fn memset(cg: *CodeGen, elem_ty: Type, ptr: WValue, len: WValue, value: WValue) try cg.addTag(.i64_add); }, } - try cg.addLocal(.local_set, new_ptr.local.value); + try cg.addLocal(.local_set, new_ptr.local); // end of loop try cg.addLabel(.br, 0); // jump to start of loop @@ -7186,7 +7186,7 @@ fn airMemcpy(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { try cg.emitWValue(slice_len); try cg.emitWValue(.{ .imm32 = @as(u32, @intCast(ptr_elem_ty.abiSize(zcu))) }); try cg.addTag(.i32_mul); - try cg.addLocal(.local_set, slice_len.local.value); + try cg.addLocal(.local_set, slice_len.local); } break :blk slice_len; }, @@ -7217,7 +7217,7 @@ fn airMemmove(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { try cg.emitWValue(slice_len); try cg.emitWValue(.{ .imm32 = @as(u32, @intCast(ptr_elem_ty.abiSize(zcu))) }); try cg.addTag(.i32_mul); - try cg.addLocal(.local_set, slice_len.local.value); + try cg.addLocal(.local_set, slice_len.local); } break :blk slice_len; }, @@ -7303,8 +7303,13 @@ fn airTry(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { const body = unwrapped_try.else_body; const err_union = try cg.resolveInst(unwrapped_try.error_union); const err_union_ty = cg.typeOf(unwrapped_try.error_union); - const result = try lowerTry(cg, inst, err_union, body, err_union_ty, false); - return cg.finishAir(inst, result, &.{unwrapped_try.error_union}); + const result = try lowerTry(cg, inst, unwrapped_try.error_union, err_union, body, err_union_ty, false); + return cg.finishAirWithBranchDeaths( + inst, + result, + &.{unwrapped_try.error_union}, + cg.liveness.getCondBr(inst).then_deaths, + ); } fn airTryPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { @@ -7313,20 +7318,26 @@ fn airTryPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { const err_union_ptr = try cg.resolveInst(unwrapped_try.error_union_ptr); const body = unwrapped_try.else_body; const err_union_ty = cg.typeOf(unwrapped_try.error_union_ptr).childType(zcu); - const result = try lowerTry(cg, inst, err_union_ptr, body, err_union_ty, true); - return cg.finishAir(inst, result, &.{unwrapped_try.error_union_ptr}); + const result = try lowerTry(cg, inst, unwrapped_try.error_union_ptr, err_union_ptr, body, err_union_ty, true); + return cg.finishAirWithBranchDeaths( + inst, + result, + &.{unwrapped_try.error_union_ptr}, + cg.liveness.getCondBr(inst).then_deaths, + ); } fn lowerTry( cg: *CodeGen, inst: Air.Inst.Index, + operand_ref: Air.Inst.Ref, err_union: WValue, body: []const Air.Inst.Index, err_union_ty: Type, operand_is_ptr: bool, ) InnerError!WValue { - _ = inst; const zcu = cg.pt.zcu; + const branch_liveness = cg.liveness.getCondBr(inst); const pl_ty = err_union_ty.errorUnionPayload(zcu); const pl_has_bits = pl_ty.hasRuntimeBits(zcu); @@ -7349,12 +7360,13 @@ fn lowerTry( try cg.addTag(.i32_eqz); try cg.addLabel(.br_if, 0); // jump out of block when error is '0' - try cg.branches.append(cg.gpa, .{}); - defer { - var branch = cg.branches.pop().?; - branch.deinit(cg.gpa); + { + try cg.pushBranch(); + defer cg.popBranch(); + if (cg.liveness.operandDies(inst, 0)) cg.processDeath(operand_ref); + for (branch_liveness.else_deaths) |death| cg.processDeath(death.toRef()); + try cg.genBody(body); } - try cg.genBody(body); try cg.endBlock(); } @@ -7367,8 +7379,7 @@ fn lowerTry( if (operand_is_ptr or isByRef(pl_ty, zcu, cg.target)) { return cg.offsetPointer(err_union, pl_offset); } - const payload = try cg.load(err_union, pl_ty, .{ .offset = pl_offset }); - return payload.toLocal(cg, pl_ty); + return cg.load(err_union, pl_ty, .{ .offset = pl_offset }); } /// Calls a compiler-rt intrinsic by creating an undefined symbol, @@ -7512,13 +7523,13 @@ fn airErrorSetHasValue(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { // 'false' branch (i.e. error set does not have value // ensure we set local to 0 in case the local was re-used. try cg.addImm32(0); - try cg.addLocal(.local_set, result.local.value); + try cg.addLocal(.local_set, result.local); try cg.addLabel(.br, 1); try cg.endBlock(); // 'true' branch try cg.addImm32(1); - try cg.addLocal(.local_set, result.local.value); + try cg.addLocal(.local_set, result.local); try cg.addLabel(.br, 0); try cg.endBlock(); @@ -7561,9 +7572,9 @@ fn airCmpxchg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { .offset = ptr_operand.memOffset(), .alignment = ty.abiAlignment(zcu).toStdMem(), }); - try cg.addLocal(.local_tee, val_local.local.value); + try cg.addLocal(.local_tee, val_local.local); _ = try cg.intCmp(int_ty, .eq, .stack, expected_val); - try cg.addLocal(.local_set, cmp_result.local.value); + try cg.addLocal(.local_set, cmp_result.local); break :val val_local; } else val: { if (ty.abiSize(zcu) > 8) { @@ -7575,7 +7586,7 @@ fn airCmpxchg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { try cg.emitWValue(new_val); try cg.emitWValue(ptr_val); _ = try cg.intCmp(int_ty, .eq, ptr_val, expected_val); - try cg.addLocal(.local_tee, cmp_result.local.value); + try cg.addLocal(.local_tee, cmp_result.local); try cg.addTag(.select); try cg.store(.stack, .stack, ty, .{ .offset = ptr_operand.memOffset() }); @@ -7685,11 +7696,11 @@ fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { }, ); const select_res = try cg.allocLocal(ty); - try cg.addLocal(.local_tee, select_res.local.value); + try cg.addLocal(.local_tee, select_res.local); _ = try cg.intCmp(int_ty, .neq, .stack, value); // leave on stack so we can use it for br_if try cg.emitWValue(select_res); - try cg.addLocal(.local_set, value.local.value); + try cg.addLocal(.local_set, value.local); try cg.addLabel(.br_if, 0); try cg.endBlock(); @@ -7864,9 +7875,6 @@ fn airAtomicStore(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { } fn airFrameAddress(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { - if (cg.initial_stack_value == .none) { - try cg.initializeStack(); - } try cg.emitWValue(cg.bottom_stack_value); return cg.finishAir(inst, .stack, &.{}); } @@ -7912,7 +7920,7 @@ fn airAsm(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { return cg.fail("Self-hosted wasm backend requires output constraint to be equal \"=r\"", .{}); } - const gop = try local_map.getOrPutValue(cg.gpa, name, result.local.value); + const gop = try local_map.getOrPutValue(cg.gpa, name, result.local); assert(!gop.found_existing); // first value assert(it.next() == null); @@ -7933,7 +7941,7 @@ fn airAsm(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { try cg.emitWValue(operand); const op_local = try WValue.toLocal(.stack, cg, cg.typeOf(input.operand)); - const gop = try local_map.getOrPutValue(cg.gpa, name, op_local.local.value); + const gop = try local_map.getOrPutValue(cg.gpa, name, op_local.local); if (gop.found_existing) { return cg.fail("Duplicate asm variable name \"{s}\"", .{name}); }