From 5ec1668eb4cee3ce3069f6ff7b2e3af0d52f43a6 Mon Sep 17 00:00:00 2001 From: Pavel Verigo Date: Fri, 21 Aug 2026 00:44:06 +0200 Subject: [PATCH 1/8] wasm: remove reuseOperand --- src/codegen/wasm/CodeGen.zig | 100 ++++++++++++++--------------------- 1 file changed, 41 insertions(+), 59 deletions(-) diff --git a/src/codegen/wasm/CodeGen.zig b/src/codegen/wasm/CodeGen.zig index e3997d0b2e806cadb750dde7feac93fc05bab31c..0117f6da9f8f70c77fdd49cc98a3c38ef221b32a 100644 --- a/src/codegen/wasm/CodeGen.zig +++ b/src/codegen/wasm/CodeGen.zig @@ -421,6 +421,20 @@ fn resolveValue(cg: *CodeGen, val: Value) InnerError!WValue { /// NOTE: if result == .stack, it will be stored in .local fn finishAir(cg: *CodeGen, inst: Air.Inst.Index, result: WValue, operands: []const Air.Inst.Ref) InnerError!void { assert(operands.len <= Air.Liveness.bpi - 1); + + var final_result = result; + const aliased_operand: ?*WValue = if (result == .local) aliased: { + for (operands) |operand| { + if (operand.toIndex() == null) continue; + const resolved_operand = cg.getResolvedInst(operand); + if (resolved_operand.* != .local or resolved_operand.local.value != result.local.value) continue; + + resolved_operand.local.references += 1; + break :aliased resolved_operand; + } + break :aliased null; + } else null; + var tomb_bits = cg.liveness.getTombBits(inst); for (operands) |operand| { const dies = @as(u1, @truncate(tomb_bits)) != 0; @@ -428,7 +442,8 @@ fn finishAir(cg: *CodeGen, inst: Air.Inst.Index, result: WValue, operands: []con if (!dies) continue; processDeath(cg, operand); } - try cg.finishAirResult(inst, result); + if (aliased_operand) |operand| final_result.local.references = operand.local.references; + try cg.finishAirResult(inst, final_result); } fn finishAirResult(cg: *CodeGen, inst: Air.Inst.Index, result: WValue) InnerError!void { @@ -478,7 +493,7 @@ fn processDeath(cg: *CodeGen, ref: Air.Inst.Ref) void { return; // function arguments can never be re-used } log.debug("Decreasing reference for ref: %{d}, using local '{d}'", .{ @backingInt(ref.toIndex().?), value.local.value }); - value.local.references -= 1; // if this panics, a call to `reuseOperand` was forgotten by the developer + value.local.references -= 1; if (value.local.references == 0) { value.free(cg); } @@ -631,7 +646,7 @@ fn genBlockType(ty: Type, zcu: *const Zcu, target: *const std.Target) std.wasm.B /// Writes the bytecode depending on the given `WValue` in `val` fn emitWValue(cg: *CodeGen, value: WValue) InnerError!void { switch (value) { - .dead => unreachable, // reference to free'd `WValue` (missing reuseOperand?) + .dead => unreachable, // reference to free'd `WValue` .none, .stack => {}, // no-op .local => |idx| try cg.addLocal(.local_get, idx.value), .imm32 => |val| try cg.addImm32(val), @@ -699,23 +714,6 @@ fn emitWValue(cg: *CodeGen, value: WValue) InnerError!void { } } -/// If given a local or stack-offset, increases the reference count by 1. -/// The old `WValue` found at instruction `ref` is then replaced by the -/// modified `WValue` and returned. When given a non-local or non-stack-offset, -/// returns the given `operand` itfunc instead. -fn reuseOperand(cg: *CodeGen, ref: Air.Inst.Ref, operand: WValue) WValue { - if (operand != .local and operand != .stack_offset) return operand; - var new_value = operand; - switch (new_value) { - .local => |*local| local.references += 1, - .stack_offset => |*stack_offset| stack_offset.references += 1, - else => unreachable, - } - const old_value = cg.getResolvedInst(ref); - old_value.* = new_value; - return new_value; -} - /// From a reference, returns its resolved `WValue`. /// It's illegal to provide a `Air.Inst.Ref` that hasn't been resolved yet. fn getResolvedInst(cg: *CodeGen, ref: Air.Inst.Ref) *WValue { @@ -1598,17 +1596,7 @@ fn genInst(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { const src_int_ty: IntType = .fromType(cg, src_ty); const dest_int_ty: IntType = .fromType(cg, dest_ty); - const src_bits = src_int_ty.bits; - const dest_bits = dest_int_ty.bits; - - const same_class: bool = (src_bits <= 32 and dest_bits <= 32) or - (src_bits >= 33 and src_bits <= 64 and dest_bits >= 33 and dest_bits <= 64) or - (src_bits >= 65 and src_bits <= 128 and dest_bits >= 65 and dest_bits <= 128); - - const result = if (same_class) - cg.reuseOperand(ty_op.operand, operand) - else - try cg.intCast(dest_int_ty, src_int_ty, operand); + const result = try cg.intCast(dest_int_ty, src_int_ty, operand); try cg.finishAir(inst, result, &.{ty_op.operand}); }, @@ -1625,12 +1613,7 @@ fn genInst(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { const src_int_ty: IntType = .fromType(cg, src_ty); const dest_int_ty: IntType = .fromType(cg, dest_ty); - - const result = if (src_int_ty.bits == dest_int_ty.bits) - cg.reuseOperand(ty_op.operand, operand) - else blk: { - break :blk try cg.intTrunc(dest_int_ty, src_int_ty, operand); - }; + const result = try cg.intTrunc(dest_int_ty, src_int_ty, operand); try cg.finishAir(inst, result, &.{ty_op.operand}); }, @@ -4259,6 +4242,7 @@ fn intCast(cg: *CodeGen, dest_ty: IntType, src_ty: IntType, operand: WValue) Inn } fn intTrunc(cg: *CodeGen, dest_ty: IntType, src_ty: IntType, operand: WValue) InnerError!WValue { + assert(src_ty.bits > dest_ty.bits); var result = try cg.intCast(dest_ty, src_ty, operand); const dest_wasm_bits = cg.intBackingBits(dest_ty.bits); @@ -5584,8 +5568,7 @@ fn airNopCast(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { assert(operand_ty.abiAlignment(zcu) == dest_ty.abiAlignment(zcu)); const operand = try cg.resolveInst(ty_op.operand); - const result = cg.reuseOperand(ty_op.operand, operand); - return cg.finishAir(inst, result, &.{ty_op.operand}); + return cg.finishAir(inst, operand, &.{ty_op.operand}); } fn airIntFromPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { @@ -5601,7 +5584,7 @@ fn airIntFromPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { const operand = try cg.resolveInst(ty_op.operand); const result = switch (operand) { .stack_offset => try cg.buildPointerOffset(operand, 0, .new), - else => cg.reuseOperand(ty_op.operand, operand), + else => operand, }; return cg.finishAir(inst, result, &.{ty_op.operand}); } @@ -5627,7 +5610,7 @@ fn airBitcast(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { const dest_ty = cg.typeOfIndex(inst); const src_ty = cg.typeOf(ty_op.operand); - const result = (try cg.bitcast(dest_ty, src_ty, operand)) orelse cg.reuseOperand(ty_op.operand, operand); + const result = try cg.bitcast(dest_ty, src_ty, operand); return cg.finishAir(inst, result, &.{ty_op.operand}); } @@ -5654,8 +5637,8 @@ fn bitcastClass(cg: *CodeGen, ty: Type) BitcastClass { }; } -fn bitcast(cg: *CodeGen, dest_ty: Type, src_ty: Type, operand: WValue) InnerError!?WValue { - if (dest_ty.eql(src_ty)) return null; +fn bitcast(cg: *CodeGen, dest_ty: Type, src_ty: Type, operand: WValue) InnerError!WValue { + if (dest_ty.eql(src_ty)) return operand; const zcu = cg.pt.zcu; const src_class = cg.bitcastClass(src_ty); @@ -5674,7 +5657,7 @@ fn bitcast(cg: *CodeGen, dest_ty: Type, src_ty: Type, operand: WValue) InnerErro if (src_by_ref and dest_by_ref) { if (needs_wrapping) return try cg.intWrap(dest_class.int, operand); - return null; + return operand; } if (dest_by_ref) { @@ -5708,7 +5691,7 @@ fn bitcast(cg: *CodeGen, dest_ty: Type, src_ty: Type, operand: WValue) InnerErro switch (dest_class) { .float => |float_ty| switch (float_ty) { - .f16 => return null, + .f16 => return operand, .f32 => { try cg.emitWValue(operand); try cg.addTag(.f32_reinterpret_i32); @@ -5729,7 +5712,7 @@ fn bitcast(cg: *CodeGen, dest_ty: Type, src_ty: Type, operand: WValue) InnerErro return try cg.intWrap(dest_class.int, operand); } - return null; + return operand; } fn airStructFieldPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { @@ -5740,7 +5723,7 @@ fn airStructFieldPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { const struct_ptr = try cg.resolveInst(extra.data.struct_operand); const struct_ptr_ty = cg.typeOf(extra.data.struct_operand); const struct_ty = struct_ptr_ty.childType(zcu); - const result = try cg.structFieldPtr(inst, extra.data.struct_operand, struct_ptr, struct_ptr_ty, struct_ty, extra.data.field_index); + const result = try cg.structFieldPtr(inst, struct_ptr, struct_ptr_ty, struct_ty, extra.data.field_index); return cg.finishAir(inst, result, &.{extra.data.struct_operand}); } @@ -5751,14 +5734,13 @@ fn airStructFieldPtrIndex(cg: *CodeGen, inst: Air.Inst.Index, index: u32) InnerE const struct_ptr_ty = cg.typeOf(ty_op.operand); const struct_ty = struct_ptr_ty.childType(zcu); - const result = try cg.structFieldPtr(inst, ty_op.operand, struct_ptr, struct_ptr_ty, struct_ty, index); + const result = try cg.structFieldPtr(inst, struct_ptr, struct_ptr_ty, struct_ty, index); return cg.finishAir(inst, result, &.{ty_op.operand}); } fn structFieldPtr( cg: *CodeGen, inst: Air.Inst.Index, - ref: Air.Inst.Ref, struct_ptr: WValue, struct_ptr_ty: Type, struct_ty: Type, @@ -5785,7 +5767,7 @@ fn structFieldPtr( }; // save a load and store when we can simply reuse the operand if (offset == 0) { - return cg.reuseOperand(ref, struct_ptr); + return struct_ptr; } switch (struct_ptr) { .stack_offset => |stack_offset| { @@ -6092,7 +6074,7 @@ fn airUnwrapErrUnionPayload(cg: *CodeGen, inst: Air.Inst.Index, op_is_ptr: bool) const result: WValue = result: { if (!payload_ty.hasRuntimeBits(zcu)) { if (op_is_ptr) { - break :result cg.reuseOperand(ty_op.operand, operand); + break :result operand; } else { break :result .none; } @@ -6131,7 +6113,7 @@ fn airUnwrapErrUnionError(cg: *CodeGen, inst: Air.Inst.Index, op_is_ptr: bool) I break :result try cg.load(operand, Type.anyerror, err_offset); } else { assert(!payload_ty.hasRuntimeBits(zcu)); - break :result cg.reuseOperand(ty_op.operand, operand); + break :result operand; } }; return cg.finishAir(inst, result, &.{ty_op.operand}); @@ -6147,7 +6129,7 @@ fn airWrapErrUnionPayload(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { const pl_ty = cg.typeOf(ty_op.operand); const result = result: { if (!pl_ty.hasRuntimeBits(zcu)) { - break :result cg.reuseOperand(ty_op.operand, operand); + break :result operand; } const err_union = try cg.allocStack(err_ty); @@ -6177,7 +6159,7 @@ fn airWrapErrUnionErr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { const result = result: { if (!pl_ty.hasRuntimeBits(zcu)) { - break :result cg.reuseOperand(ty_op.operand, operand); + break :result operand; } const err_union = try cg.allocStack(err_ty); @@ -6257,7 +6239,7 @@ fn airOptionalPayload(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { const result = result: { const operand = try cg.resolveInst(ty_op.operand); - if (opt_ty.optionalReprIsPayload(zcu)) break :result cg.reuseOperand(ty_op.operand, operand); + if (opt_ty.optionalReprIsPayload(zcu)) break :result operand; if (isByRef(payload_ty, zcu, cg.target)) { break :result try cg.buildPointerOffset(operand, 0, .new); @@ -6277,7 +6259,7 @@ fn airOptionalPayloadPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { const result = result: { const payload_ty = opt_ty.optionalChild(zcu); if (!payload_ty.hasRuntimeBits(zcu) or opt_ty.optionalReprIsPayload(zcu)) { - break :result cg.reuseOperand(ty_op.operand, operand); + break :result operand; } break :result try cg.buildPointerOffset(operand, 0, .new); @@ -6327,7 +6309,7 @@ fn airWrapOptional(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { const operand = try cg.resolveInst(ty_op.operand); const op_ty = cg.typeOfIndex(inst); if (op_ty.optionalReprIsPayload(zcu)) { - break :result cg.reuseOperand(ty_op.operand, operand); + break :result operand; } const offset = std.math.cast(u32, payload_ty.abiSize(zcu)) orelse { return cg.fail("Optional type {f} too big to fit into stack frame", .{op_ty.fmt(pt)}); @@ -7146,7 +7128,7 @@ fn airErrUnionPayloadPtrSet(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void const result = result: { if (!payload_ty.hasRuntimeBits(zcu)) { - break :result cg.reuseOperand(ty_op.operand, operand); + break :result operand; } break :result try cg.buildPointerOffset(operand, @as(u32, @intCast(errUnionPayloadOffset(payload_ty, zcu))), .new); @@ -7182,7 +7164,7 @@ fn airFieldParentPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { try cg.addTag(.i32_sub); try cg.addLocal(.local_set, base.local.value); break :result base; - } else cg.reuseOperand(extra.field_ptr, field_ptr); + } else field_ptr; return cg.finishAir(inst, result, &.{extra.field_ptr}); } -- 2.54.0 From d12892e8904bbb27aac7baa69f483348e516a30e Mon Sep 17 00:00:00 2001 From: Pavel Verigo Date: Fri, 21 Aug 2026 12:43:10 +0200 Subject: [PATCH 2/8] wasm: store/load use memarg, support alignment --- src/codegen/wasm/CodeGen.zig | 511 ++++++++++++++++++---------------- src/codegen/wasm/Emit.zig | 4 +- src/codegen/wasm/Mir.zig | 11 +- src/codegen/wasm/assembly.zig | 8 +- 4 files changed, 288 insertions(+), 246 deletions(-) diff --git a/src/codegen/wasm/CodeGen.zig b/src/codegen/wasm/CodeGen.zig index 0117f6da9f8f70c77fdd49cc98a3c38ef221b32a..dac187ee04f99dd4642579678794459c810bcdd7 100644 --- a/src/codegen/wasm/CodeGen.zig +++ b/src/codegen/wasm/CodeGen.zig @@ -584,7 +584,7 @@ fn addCallIntrinsic(cg: *CodeGen, intrinsic: Mir.Intrinsic) error{OutOfMemory}!v /// Returns the index into `mir_extra` fn addExtra(cg: *CodeGen, extra: anytype) error{OutOfMemory}!u32 { const field_count = @typeInfo(@TypeOf(extra)).@"struct".field_names.len; - try cg.mir_extra.ensureUnusedCapacity(cg.gpa, field_count); + try cg.mir_extra.ensureUnusedCapacity(cg.gpa, field_count * 2); return cg.addExtraAssumeCapacity(extra); } @@ -594,14 +594,22 @@ fn addExtraAssumeCapacity(cg: *CodeGen, extra: anytype) error{OutOfMemory}!u32 { const info = @typeInfo(@TypeOf(extra)).@"struct"; const result: u32 = @intCast(cg.mir_extra.items.len); inline for (info.field_names, info.field_types) |field_name, field_type| { - cg.mir_extra.appendAssumeCapacity(switch (field_type) { - u32 => @field(extra, field_name), - i32 => @bitCast(@field(extra, field_name)), - InternPool.Index, - InternPool.Nav.Index, - => @backingInt(@field(extra, field_name)), - else => @compileError("Unsupported field type " ++ @typeName(field_type)), - }); + switch (field_type) { + u64 => { + const value = @field(extra, field_name); + cg.mir_extra.appendAssumeCapacity(@truncate(value)); + cg.mir_extra.appendAssumeCapacity(@truncate(value >> 32)); + }, + else => cg.mir_extra.appendAssumeCapacity(switch (field_type) { + u32 => @field(extra, field_name), + i32 => @bitCast(@field(extra, field_name)), + InternPool.Index, + InternPool.Nav.Index, + mem.Alignment, + => @backingInt(@field(extra, field_name)), + else => @compileError("Unsupported field type " ++ @typeName(field_type)), + }), + } } return result; } @@ -991,26 +999,26 @@ fn lowerArg(cg: *CodeGen, cc: std.lang.CallingConvention, ty: Type, value: WValu if (!isByRef(ty, zcu, cg.target)) { return cg.lowerToStack(value); } else { - _ = try cg.load(value, scalar_ty, 0); + _ = try cg.load(value, scalar_ty, .{}); } }, .double_i64 => { assert(ty.abiSize(zcu) == 16); // in this case we have an integer or float that must be lowered as 2 i64's. try cg.emitWValue(value); - try cg.addMemArg(.i64_load, .{ .offset = value.offset(), .alignment = 8 }); + try cg.addMemArg(.i64_load, .{ .offset = value.offset(), .alignment = .@"8" }); try cg.emitWValue(value); - try cg.addMemArg(.i64_load, .{ .offset = value.offset() + 8, .alignment = 8 }); + try cg.addMemArg(.i64_load, .{ .offset = value.offset() + 8, .alignment = .@"8" }); }, .indirect => { const stack_copy = try cg.allocStack(ty); - try cg.store(stack_copy, value, ty, 0); + try cg.store(stack_copy, value, ty, .{}); return cg.lowerToStack(stack_copy); }, .unrolled => |vector| { - const elem_size: u32 = @intCast(vector.elem_type.abiSize(zcu)); + const elem_size = vector.elem_type.abiSize(zcu); for (0..vector.len) |index| { - _ = try cg.load(value, vector.elem_type, @intCast(index * elem_size)); + _ = try cg.load(value, vector.elem_type, .{ .offset = index * elem_size }); } }, } @@ -1330,7 +1338,7 @@ fn genInst(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { try cg.addTag(.i32_add); const ptr = try WValue.toLocal(.stack, cg, Type.usize); - try cg.store(ptr, elem_val, elem_ty, 0); + try cg.store(ptr, elem_val, elem_ty, .{}); return cg.finishAir(inst, .none, &.{ pl_op.operand, bin_op.lhs, bin_op.rhs }); }, @@ -1507,8 +1515,8 @@ fn genInst(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { const result = try cg.allocStack(cg.typeOfIndex(inst)); const offset: u32 = @intCast(ty.abiSize(cg.pt.zcu)); - try cg.store(result, res_tmp, ty, 0); - try cg.store(result, ov_tmp, Type.u1, offset); + try cg.store(result, res_tmp, ty, .{}); + try cg.store(result, ov_tmp, .u1, .{ .offset = offset }); try cg.finishAir(inst, result, &.{ extra.lhs, extra.rhs }); }, @@ -1939,20 +1947,20 @@ fn airRet(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { // result must be stored in the stack and we return a pointer // to the stack instead if (cg.return_value != .none) { - try cg.store(cg.return_value, operand, ret_ty, 0); + try cg.store(cg.return_value, operand, ret_ty, .{}); } else if (fn_info.cc == .wasm_mvp and ret_ty.hasRuntimeBits(zcu)) { switch (abi.classifyType(ret_ty, zcu, cg.target)) { .direct => |scalar_type| { if (!isByRef(ret_ty, zcu, cg.target)) { try cg.emitWValue(operand); } else { - _ = try cg.load(operand, scalar_type, 0); + _ = try cg.load(operand, scalar_type, .{}); } }, .double_i64, .indirect => unreachable, .unrolled => |vector| { assert(vector.len == 1); - _ = try cg.load(operand, vector.elem_type, 0); + _ = try cg.load(operand, vector.elem_type, .{}); }, } } else { @@ -2002,15 +2010,15 @@ fn airRetLoad(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { } else if (!firstParamSRet(fn_info.cc, Type.fromInterned(fn_info.return_type), zcu, cg.target)) { if (fn_info.cc == .wasm_mvp) { switch (abi.classifyType(ret_ty, zcu, cg.target)) { - .direct => |scalar_type| _ = try cg.load(operand, scalar_type, 0), + .direct => |scalar_type| _ = try cg.load(operand, scalar_type, .{}), .double_i64, .indirect => unreachable, .unrolled => |vector| { assert(vector.len == 1); - _ = try cg.load(operand, vector.elem_type, 0); + _ = try cg.load(operand, vector.elem_type, .{}); }, } } else { - _ = try cg.load(operand, ret_ty, 0); + _ = try cg.load(operand, ret_ty, .{}); } } @@ -2092,7 +2100,7 @@ fn airCall(cg: *CodeGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier) const arg_align = arg_ty.abiAlignment(zcu); offset = @intCast(arg_align.forward(offset)); - try cg.store(buffer, arg_val, arg_ty, offset); + try cg.store(buffer, arg_val, arg_ty, .{ .offset = offset }); offset += arg_size; } @@ -2149,7 +2157,7 @@ fn airCall(cg: *CodeGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier) const result_local = try cg.allocLocal(scalar_type); try cg.addLocal(.local_set, result_local.local.value); const result = try cg.allocStack(ret_ty); - try cg.store(result, result_local, scalar_type, 0); + try cg.store(result, result_local, scalar_type, .{}); break :result_value result; } }, @@ -2160,7 +2168,7 @@ fn airCall(cg: *CodeGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier) // save call result from operand stack try cg.addLocal(.local_set, result_local.local.value); const result = try cg.allocStack(ret_ty); - try cg.store(result, result_local, vector.elem_type, 0); + try cg.store(result, result_local, vector.elem_type, .{}); break :result_value result; }, } @@ -2191,7 +2199,7 @@ fn airVaCopy(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op; const operand = try cg.resolveInst(ty_op.operand); - const result = try cg.load(operand, .usize, 0); + const result = try cg.load(operand, .usize, .{}); return cg.finishAir(inst, result, &.{ty_op.operand}); } @@ -2214,7 +2222,7 @@ fn airVaArg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { const abi_align: u32 = @intCast(load_ty.abiAlignment(zcu).toByteUnits().?); const arg_ptr = try cg.allocLocal(.usize); - _ = try cg.load(operand, .usize, 0); + _ = try cg.load(operand, .usize, .{}); if (abi_align > 1) { switch (cg.ptr_size) { @@ -2247,17 +2255,17 @@ fn airVaArg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { try cg.addTag(.i64_add); }, } - try cg.store(.stack, .stack, .usize, 0); + try cg.store(.stack, .stack, .usize, .{}); const result = if (is_f32_va_arg) result: { - const promoted = try cg.load(arg_ptr, Type.f64, 0); + const promoted = try cg.load(arg_ptr, .f64, .{}); try cg.emitWValue(promoted); try cg.addTag(.f32_demote_f64); const result_local = try cg.allocLocal(Type.f32); try cg.addLocal(.local_set, result_local.local.value); break :result result_local; - } else try cg.load(arg_ptr, ty, 0); + } else try cg.load(arg_ptr, ty, .{}); return cg.finishAir(inst, result, &.{ty_op.operand}); } @@ -2282,18 +2290,26 @@ fn airStore(cg: *CodeGen, inst: Air.Inst.Index, safety: bool) InnerError!void { return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs }); } - const offset: u32 = switch (ptr_info.flags.vector_index) { + const offset: u64 = switch (ptr_info.flags.vector_index) { .none => offset: { assert(ptr_info.packed_offset.host_size == 0); // legalize .expand_packed_store break :offset 0; }, - else => |index| @intCast(@backingInt(index) * elem_ty.abiSize(zcu)), + else => |index| @backingInt(index) * elem_ty.abiSize(zcu), }; - try cg.store(lhs, rhs, elem_ty, offset); + try cg.store(lhs, rhs, elem_ty, .{ + .offset = offset, + .alignment = ptr_ty.ptrAlignment(zcu), + }); return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs }); } -fn store(cg: *CodeGen, lhs: WValue, rhs: WValue, ty: Type, offset: u32) InnerError!void { +const MemArg = struct { + offset: u64 = 0, + alignment: Alignment = .none, +}; + +fn store(cg: *CodeGen, lhs: WValue, rhs: WValue, ty: Type, memarg: MemArg) InnerError!void { assert(!(lhs != .stack and rhs == .stack)); const pt = cg.pt; const zcu = pt.zcu; @@ -2301,12 +2317,19 @@ fn store(cg: *CodeGen, lhs: WValue, rhs: WValue, ty: Type, offset: u32) InnerErr if (!ty.hasRuntimeBits(zcu)) return; + const natural_alignment = ty.abiAlignment(zcu); + // wasm memory instruction alignment is <= access width + const alignment = if (memarg.alignment == .none) + natural_alignment + else + natural_alignment.minStrict(memarg.alignment); + if (isByRef(ty, zcu, cg.target)) { - const offset_ptr: WValue = switch (offset + lhs.offset()) { + const offset_ptr: WValue = switch (memarg.offset + lhs.offset()) { 0 => lhs, else => |total_offset| ptr: { try cg.emitWValue(lhs); - try cg.addImm32(total_offset); + try cg.addImm32(@intCast(total_offset)); try cg.addTag(.i32_add); break :ptr .stack; }, @@ -2320,10 +2343,11 @@ fn store(cg: *CodeGen, lhs: WValue, rhs: WValue, ty: Type, offset: u32) InnerErr // TODO: Add helper functions for simd opcodes const extra_index: u32 = @intCast(cg.mir_extra.items.len); // stores as := opcode, offset, alignment (opcode::memarg) - try cg.mir_extra.appendSlice(cg.gpa, &[_]u32{ - @backingInt(std.wasm.SimdOpcode.v128_store), - offset + lhs.offset(), - @intCast(ty.abiAlignment(zcu).toByteUnits().?), + try cg.mir_extra.ensureUnusedCapacity(cg.gpa, 4); + cg.mir_extra.appendAssumeCapacity(@backingInt(std.wasm.SimdOpcode.v128_store)); + _ = try cg.addExtraAssumeCapacity(Mir.MemArg{ + .offset = memarg.offset + lhs.offset(), + .alignment = alignment.toStdMem(), }); return cg.addInst(.{ .tag = .simd_prefix, .data = .{ .payload = extra_index } }); } @@ -2353,8 +2377,8 @@ fn store(cg: *CodeGen, lhs: WValue, rhs: WValue, ty: Type, offset: u32) InnerErr try cg.addMemArg( store_opcode, .{ - .offset = offset + lhs.offset(), - .alignment = @intCast(ty.abiAlignment(zcu).toByteUnits().?), + .offset = memarg.offset + lhs.offset(), + .alignment = alignment.toStdMem(), }, ); } @@ -2370,34 +2394,45 @@ fn airLoad(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { assert(elem_ty.hasRuntimeBits(zcu)); - const offset: u32 = switch (ptr_info.flags.vector_index) { + const offset: u64 = switch (ptr_info.flags.vector_index) { .none => offset: { assert(ptr_info.packed_offset.host_size == 0); // legalize .expand_packed_load break :offset 0; }, - else => |index| @intCast(@backingInt(index) * elem_ty.abiSize(zcu)), + else => |index| @backingInt(index) * elem_ty.abiSize(zcu), }; - const result = try cg.load(operand, elem_ty, offset); + const result = try cg.load(operand, elem_ty, .{ + .offset = offset, + .alignment = ptr_ty.ptrAlignment(zcu), + }); return cg.finishAir(inst, result, &.{ty_op.operand}); } /// Loads an operand from the linear memory section. /// NOTE: Leaves the value on the stack, if isByRef == false. -fn load(cg: *CodeGen, operand: WValue, ty: Type, offset: u32) InnerError!WValue { +fn load(cg: *CodeGen, operand: WValue, ty: Type, memarg: MemArg) InnerError!WValue { const zcu = cg.pt.zcu; + + const natural_alignment = ty.abiAlignment(zcu); + // wasm memory instruction alignment is <= access width + const alignment = if (memarg.alignment == .none) + natural_alignment + else + natural_alignment.minStrict(memarg.alignment); + if (isByRef(ty, zcu, cg.target)) { - const src_ptr_maybe_stack: WValue = switch (offset + operand.offset()) { + const src_ptr_maybe_stack: WValue = switch (memarg.offset + operand.offset()) { 0 => operand, else => |total_offset| ptr: { try cg.emitWValue(operand); - try cg.addImm32(total_offset); + try cg.addImm32(@intCast(total_offset)); try cg.addTag(.i32_add); break :ptr .stack; }, }; const src_ptr = try src_ptr_maybe_stack.toLocal(cg, .usize); const new_ptr = try cg.allocStack(ty); - try cg.store(new_ptr, src_ptr, ty, 0); + try cg.store(new_ptr, src_ptr, ty, .{}); return new_ptr; } @@ -2408,10 +2443,11 @@ fn load(cg: *CodeGen, operand: WValue, ty: Type, offset: u32) InnerError!WValue // TODO: Add helper functions for simd opcodes const extra_index: u32 = @intCast(cg.mir_extra.items.len); // stores as := opcode, offset, alignment (opcode::memarg) - try cg.mir_extra.appendSlice(cg.gpa, &[_]u32{ - @backingInt(std.wasm.SimdOpcode.v128_load), - offset + operand.offset(), - @intCast(ty.abiAlignment(zcu).toByteUnits().?), + try cg.mir_extra.ensureUnusedCapacity(cg.gpa, 4); + cg.mir_extra.appendAssumeCapacity(@backingInt(std.wasm.SimdOpcode.v128_load)); + _ = try cg.addExtraAssumeCapacity(Mir.MemArg{ + .offset = memarg.offset + operand.offset(), + .alignment = alignment.toStdMem(), }); try cg.addInst(.{ .tag = .simd_prefix, .data = .{ .payload = extra_index } }); return .stack; @@ -2441,8 +2477,8 @@ fn load(cg: *CodeGen, operand: WValue, ty: Type, offset: u32) InnerError!WValue try cg.addMemArg( load_opcode, .{ - .offset = offset + operand.offset(), - .alignment = @intCast(ty.abiAlignment(zcu).toByteUnits().?), + .offset = memarg.offset + operand.offset(), + .alignment = alignment.toStdMem(), }, ); @@ -2470,7 +2506,7 @@ fn airArg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { cg.arg_index += 1; if (isByRef(arg_ty, zcu, cg.target)) { const result = try cg.allocStack(arg_ty); - try cg.store(result, arg, scalar_type, 0); + try cg.store(result, arg, scalar_type, .{}); return cg.finishAir(inst, result, &.{}); } }, @@ -2478,15 +2514,15 @@ fn airArg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { .double_i64 => { cg.arg_index += 2; const result = try cg.allocStack(arg_ty); - try cg.store(result, arg, Type.u64, 0); - try cg.store(result, cg.args[arg_index + 1], Type.u64, 8); + try cg.store(result, arg, .u64, .{}); + try cg.store(result, cg.args[arg_index + 1], .u64, .{ .offset = 8 }); return cg.finishAir(inst, result, &.{}); }, .unrolled => |vector| { const result = try cg.allocStack(arg_ty); - const elem_size: u32 = @intCast(vector.elem_type.abiSize(zcu)); + const elem_size = vector.elem_type.abiSize(zcu); for (0..vector.len) |index| { - try cg.store(result, cg.args[cg.arg_index], vector.elem_type, @intCast(index * elem_size)); + try cg.store(result, cg.args[cg.arg_index], vector.elem_type, .{ .offset = index * elem_size }); cg.arg_index += 1; } return cg.finishAir(inst, result, &.{}); @@ -2588,15 +2624,15 @@ fn intAdd(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue 65...128 => { const result = try cg.allocStack(Type.u128); - var lhs_lsb = try (try cg.load(lhs, Type.u64, 0)).toLocal(cg, Type.u64); + var lhs_lsb = try (try cg.load(lhs, .u64, .{})).toLocal(cg, Type.u64); defer lhs_lsb.free(cg); - var rhs_lsb = try (try cg.load(rhs, Type.u64, 0)).toLocal(cg, Type.u64); + var rhs_lsb = try (try cg.load(rhs, .u64, .{})).toLocal(cg, Type.u64); defer rhs_lsb.free(cg); var op_lsb = try (try cg.intAdd(.u64, lhs_lsb, rhs_lsb)).toLocal(cg, Type.u64); defer op_lsb.free(cg); - const lhs_msb = try cg.load(lhs, Type.u64, 8); - const rhs_msb = try cg.load(rhs, Type.u64, 8); + const lhs_msb = try cg.load(lhs, .u64, .{ .offset = 8 }); + const rhs_msb = try cg.load(rhs, .u64, .{ .offset = 8 }); const op_msb = try cg.intAdd(.u64, lhs_msb, rhs_msb); const lt = try cg.intCmp(.u64, .lt, op_lsb, rhs_lsb); @@ -2604,8 +2640,8 @@ fn intAdd(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue var tmp_op = try (try cg.intAdd(.u64, op_msb, tmp)).toLocal(cg, Type.u64); defer tmp_op.free(cg); - try cg.store(result, op_lsb, Type.u64, 0); - try cg.store(result, tmp_op, Type.u64, 8); + try cg.store(result, op_lsb, .u64, .{}); + try cg.store(result, tmp_op, .u64, .{ .offset = 8 }); return result; }, else => { @@ -2641,15 +2677,15 @@ fn intSub(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue 65...128 => { const result = try cg.allocStack(Type.u128); - var lhs_lsb = try (try cg.load(lhs, Type.u64, 0)).toLocal(cg, Type.u64); + var lhs_lsb = try (try cg.load(lhs, .u64, .{})).toLocal(cg, Type.u64); defer lhs_lsb.free(cg); - var rhs_lsb = try (try cg.load(rhs, Type.u64, 0)).toLocal(cg, Type.u64); + var rhs_lsb = try (try cg.load(rhs, .u64, .{})).toLocal(cg, Type.u64); defer rhs_lsb.free(cg); var op_lsb = try (try cg.intSub(.u64, lhs_lsb, rhs_lsb)).toLocal(cg, Type.u64); defer op_lsb.free(cg); - const lhs_msb = try cg.load(lhs, Type.u64, 8); - const rhs_msb = try cg.load(rhs, Type.u64, 8); + const lhs_msb = try cg.load(lhs, .u64, .{ .offset = 8 }); + const rhs_msb = try cg.load(rhs, .u64, .{ .offset = 8 }); const op_msb = try cg.intSub(.u64, lhs_msb, rhs_msb); const lt = try cg.intCmp(.u64, .lt, lhs_lsb, rhs_lsb); @@ -2657,8 +2693,8 @@ fn intSub(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue var tmp_op = try (try cg.intSub(.u64, op_msb, tmp)).toLocal(cg, Type.u64); defer tmp_op.free(cg); - try cg.store(result, op_lsb, Type.u64, 0); - try cg.store(result, tmp_op, Type.u64, 8); + try cg.store(result, op_lsb, .u64, .{}); + try cg.store(result, tmp_op, .u64, .{ .offset = 8 }); return result; }, else => { @@ -2990,15 +3026,15 @@ fn intAnd(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue 65...128 => { const result = try cg.allocStack(Type.u128); - const lhs_lsb = try cg.load(lhs, Type.u64, 0); - const rhs_lsb = try cg.load(rhs, Type.u64, 0); + const lhs_lsb = try cg.load(lhs, .u64, .{}); + const rhs_lsb = try cg.load(rhs, .u64, .{}); const and_lsb = try (try cg.intAnd(.u64, lhs_lsb, rhs_lsb)).toLocal(cg, Type.u64); - try cg.store(result, and_lsb, Type.u64, 0); + try cg.store(result, and_lsb, .u64, .{}); - const lhs_msb = try cg.load(lhs, Type.u64, 8); - const rhs_msb = try cg.load(rhs, Type.u64, 8); + const lhs_msb = try cg.load(lhs, .u64, .{ .offset = 8 }); + const rhs_msb = try cg.load(rhs, .u64, .{ .offset = 8 }); const and_msb = try (try cg.intAnd(.u64, lhs_msb, rhs_msb)).toLocal(cg, Type.u64); - try cg.store(result, and_msb, Type.u64, 8); + try cg.store(result, and_msb, .u64, .{ .offset = 8 }); return result; }, @@ -3034,15 +3070,15 @@ fn intOr(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue 65...128 => { const result = try cg.allocStack(Type.u128); - const lhs_lsb = try cg.load(lhs, Type.u64, 0); - const rhs_lsb = try cg.load(rhs, Type.u64, 0); + const lhs_lsb = try cg.load(lhs, .u64, .{}); + const rhs_lsb = try cg.load(rhs, .u64, .{}); const or_lsb = try (try cg.intOr(.u64, lhs_lsb, rhs_lsb)).toLocal(cg, Type.u64); - try cg.store(result, or_lsb, Type.u64, 0); + try cg.store(result, or_lsb, .u64, .{}); - const lhs_msb = try cg.load(lhs, Type.u64, 8); - const rhs_msb = try cg.load(rhs, Type.u64, 8); + const lhs_msb = try cg.load(lhs, .u64, .{ .offset = 8 }); + const rhs_msb = try cg.load(rhs, .u64, .{ .offset = 8 }); const or_msb = try (try cg.intOr(.u64, lhs_msb, rhs_msb)).toLocal(cg, Type.u64); - try cg.store(result, or_msb, Type.u64, 8); + try cg.store(result, or_msb, .u64, .{ .offset = 8 }); return result; }, @@ -3078,15 +3114,15 @@ fn intXor(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue 65...128 => { const result = try cg.allocStack(Type.u128); - const lhs_lsb = try cg.load(lhs, Type.u64, 0); - const rhs_lsb = try cg.load(rhs, Type.u64, 0); + const lhs_lsb = try cg.load(lhs, .u64, .{}); + const rhs_lsb = try cg.load(rhs, .u64, .{}); const xor_lsb = try (try cg.intXor(.u64, lhs_lsb, rhs_lsb)).toLocal(cg, Type.u64); - try cg.store(result, xor_lsb, Type.u64, 0); + try cg.store(result, xor_lsb, .u64, .{}); - const lhs_msb = try cg.load(lhs, Type.u64, 8); - const rhs_msb = try cg.load(rhs, Type.u64, 8); + const lhs_msb = try cg.load(lhs, .u64, .{ .offset = 8 }); + const rhs_msb = try cg.load(rhs, .u64, .{ .offset = 8 }); const xor_msb = try (try cg.intXor(.u64, lhs_msb, rhs_msb)).toLocal(cg, Type.u64); - try cg.store(result, xor_msb, Type.u64, 8); + try cg.store(result, xor_msb, .u64, .{ .offset = 8 }); return result; }, @@ -3141,20 +3177,20 @@ fn intNot(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue { const result = try cg.allocStack(Type.u128); try cg.emitWValue(result); - _ = try cg.load(operand, Type.u64, 0); + _ = try cg.load(operand, .u64, .{}); try cg.addImm64(~@as(u64, 0)); try cg.addTag(.i64_xor); - try cg.store(.stack, .stack, Type.u64, result.offset()); + try cg.store(.stack, .stack, .u64, .{ .offset = result.offset() }); try cg.emitWValue(result); - _ = try cg.load(operand, Type.u64, 8); + _ = try cg.load(operand, .u64, .{ .offset = 8 }); const high_mask: u64 = if (ty.is_signed) ~@as(u64, 0) else ~@as(u64, 0) >> @intCast(128 - ty.bits); try cg.addImm64(high_mask); try cg.addTag(.i64_xor); - try cg.store(.stack, .stack, Type.u64, result.offset() + 8); + try cg.store(.stack, .stack, .u64, .{ .offset = result.offset() + 8 }); return result; }, @@ -3287,16 +3323,16 @@ fn intAbs(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue { try cg.emitWValue(mask); try cg.emitWValue(mask); - _ = try cg.load(operand, Type.u64, 8); + _ = try cg.load(operand, .u64, .{ .offset = 8 }); try cg.addImm64(63); try cg.addTag(.i64_shr_s); var tmp = try cg.allocLocal(Type.u64); defer tmp.free(cg); try cg.addLocal(.local_tee, tmp.local.value); - try cg.store(.stack, .stack, Type.u64, mask.offset() + 0); + try cg.store(.stack, .stack, .u64, .{ .offset = mask.offset() + 0 }); try cg.emitWValue(tmp); - try cg.store(.stack, .stack, Type.u64, mask.offset() + 8); + try cg.store(.stack, .stack, .u64, .{ .offset = mask.offset() + 8 }); const a = try cg.intXor(u128_ty, operand, mask); const b = try cg.intSub(u128_ty, a, mask); @@ -3364,7 +3400,7 @@ fn intClz(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue { return .stack; }, 65...128 => { - var msb = try (try cg.load(operand, Type.u64, 8)).toLocal(cg, Type.u64); + var msb = try (try cg.load(operand, .u64, .{ .offset = 8 })).toLocal(cg, Type.u64); defer msb.free(cg); if (ty.is_signed and ty.bits < 128) { @@ -3375,7 +3411,7 @@ fn intClz(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue { } try cg.addTag(.i64_clz); - _ = try cg.load(operand, Type.u64, 0); + _ = try cg.load(operand, .u64, .{}); try cg.addTag(.i64_clz); try cg.emitWValue(.{ .imm64 = 64 }); try cg.addTag(.i64_add); @@ -3423,13 +3459,13 @@ fn intCtz(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue { return .stack; }, 65...128 => { - var lsb = try (try cg.load(operand, Type.u64, 0)).toLocal(cg, Type.u64); + var lsb = try (try cg.load(operand, .u64, .{})).toLocal(cg, Type.u64); defer lsb.free(cg); try cg.emitWValue(lsb); try cg.addTag(.i64_ctz); - _ = try cg.load(operand, Type.u64, 8); + _ = try cg.load(operand, .u64, .{ .offset = 8 }); if (ty.bits < 128) { try cg.addImm64(@as(u64, 1) << @intCast(ty.bits - 64)); try cg.addTag(.i64_or); @@ -3475,9 +3511,9 @@ fn intPopCount(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue { return .stack; }, 65...128 => { - _ = try cg.load(operand, Type.u64, 0); + _ = try cg.load(operand, .u64, .{}); try cg.addTag(.i64_popcnt); - _ = try cg.load(operand, Type.u64, 8); + _ = try cg.load(operand, .u64, .{ .offset = 8 }); if (ty.is_signed and ty.bits < 128) { try cg.addImm64(128 - ty.bits); try cg.addTag(.i64_shl); @@ -3525,7 +3561,7 @@ fn intBitReverse(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue { const tmp = try cg.allocStack(Type.u128); try cg.emitWValue(tmp); - const hi = try cg.load(operand, Type.u64, 8); + const hi = try cg.load(operand, .u64, .{ .offset = 8 }); const hi_rev = try cg.callIntrinsic( .__bitreversedi2, &.{.u64_type}, @@ -3533,10 +3569,10 @@ fn intBitReverse(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue { &.{hi}, ); try cg.emitWValue(hi_rev); - try cg.store(.stack, .stack, Type.u64, tmp.offset()); + try cg.store(.stack, .stack, .u64, .{ .offset = tmp.offset() }); try cg.emitWValue(tmp); - const lo = try cg.load(operand, Type.u64, 0); + const lo = try cg.load(operand, .u64, .{}); const lo_rev = try cg.callIntrinsic( .__bitreversedi2, &.{.u64_type}, @@ -3544,7 +3580,7 @@ fn intBitReverse(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue { &.{lo}, ); try cg.emitWValue(lo_rev); - try cg.store(.stack, .stack, Type.u64, tmp.offset() + 8); + try cg.store(.stack, .stack, .u64, .{ .offset = tmp.offset() + 8 }); if (ty.bits < 128) { const shift_ty: IntType = .{ .is_signed = ty.is_signed, .bits = 128 }; @@ -3595,25 +3631,25 @@ fn intByteSwap(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue { try cg.emitWValue(result); - const low = try cg.load(operand, Type.u64, 0); + const low = try cg.load(operand, .u64, .{}); const swap_low = try cg.callIntrinsic( .__bswapdi2, &.{.u64_type}, Type.u64, &.{low}, ); - try cg.store(.stack, swap_low, Type.u64, result.offset() + 8); + try cg.store(.stack, swap_low, .u64, .{ .offset = result.offset() + 8 }); try cg.emitWValue(result); - const high = try cg.load(operand, Type.u64, 8); + const high = try cg.load(operand, .u64, .{ .offset = 8 }); const swap_high = try cg.callIntrinsic( .__bswapdi2, &.{.u64_type}, Type.u64, &.{high}, ); - try cg.store(.stack, swap_high, Type.u64, result.offset()); + try cg.store(.stack, swap_high, .u64, .{ .offset = result.offset() }); if (ty.bits < 128) { const shift_ty: IntType = .{ .is_signed = ty.is_signed, .bits = 128 }; @@ -3671,11 +3707,11 @@ fn intWrap(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue { const result = try cg.allocStack(Type.u128); try cg.emitWValue(result); - _ = try cg.load(operand, Type.u64, 0); - try cg.store(.stack, .stack, Type.u64, result.offset()); + _ = try cg.load(operand, .u64, .{}); + try cg.store(.stack, .stack, .u64, .{ .offset = result.offset() }); try cg.emitWValue(result); - _ = try cg.load(operand, Type.u64, 8); + _ = try cg.load(operand, .u64, .{ .offset = 8 }); if (ty.is_signed) { try cg.addImm64(128 - ty.bits); try cg.addTag(.i64_shl); @@ -3685,7 +3721,7 @@ fn intWrap(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue { try cg.addImm64(~@as(u64, 0) >> @intCast(128 - ty.bits)); try cg.addTag(.i64_and); } - try cg.store(.stack, .stack, Type.u64, result.offset() + 8); + try cg.store(.stack, .stack, .u64, .{ .offset = result.offset() + 8 }); return result; }, @@ -3703,7 +3739,7 @@ fn intWrap(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue { const pad = 64 - ty.bits % 64; try cg.emitWValue(result); - _ = try cg.load(operand, Type.u64, used_len - 8); + _ = try cg.load(operand, .u64, .{ .offset = used_len - 8 }); if (ty.is_signed) { try cg.addImm64(pad); try cg.addTag(.i64_shl); @@ -3713,7 +3749,7 @@ fn intWrap(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue { try cg.addImm64(~@as(u64, 0) >> @intCast(pad)); try cg.addTag(.i64_and); } - try cg.store(.stack, .stack, Type.u64, result.offset() + used_len - 8); + try cg.store(.stack, .stack, .u64, .{ .offset = result.offset() + used_len - 8 }); } else { try cg.memcpy(result, operand, .{ .imm32 = used_len }); } @@ -3722,13 +3758,13 @@ fn intWrap(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue { if (used_len + 8 == full_len) { // last limb needs sign extended try cg.emitWValue(result); if (ty.is_signed) { - _ = try cg.load(result, Type.u64, used_len - 8); + _ = try cg.load(result, .u64, .{ .offset = used_len - 8 }); try cg.addImm64(63); try cg.addTag(.i64_shr_s); } else { try cg.addImm64(0); } - try cg.store(.stack, .stack, Type.u64, result.offset() + used_len); + try cg.store(.stack, .stack, .u64, .{ .offset = result.offset() + used_len }); } return result; @@ -3751,12 +3787,12 @@ fn intMaxValue(cg: *CodeGen, int_ty: IntType) InnerError!WValue { } } else if (int_ty.bits <= 128) { const result = try cg.allocInt(int_ty); - try cg.store(result, .{ .imm64 = ~@as(u64, 0) }, Type.u64, 0); + try cg.store(result, .{ .imm64 = ~@as(u64, 0) }, .u64, .{}); if (int_ty.is_signed) { - try cg.store(result, .{ .imm64 = (~@as(u64, 0) >> @intCast(128 - int_ty.bits)) >> 1 }, Type.u64, 8); + try cg.store(result, .{ .imm64 = (~@as(u64, 0) >> @intCast(128 - int_ty.bits)) >> 1 }, .u64, .{ .offset = 8 }); } else { - try cg.store(result, .{ .imm64 = ~@as(u64, 0) >> @intCast(128 - int_ty.bits) }, Type.u64, 8); + try cg.store(result, .{ .imm64 = ~@as(u64, 0) >> @intCast(128 - int_ty.bits) }, .u64, .{ .offset = 8 }); } return result; } else { @@ -3767,13 +3803,13 @@ fn intMaxValue(cg: *CodeGen, int_ty: IntType) InnerError!WValue { try cg.memset(Type.u8, result, .{ .imm32 = used_len - 8 }, .{ .imm32 = 0xFF }); if (int_ty.is_signed) { - try cg.store(result, .{ .imm64 = (~@as(u64, 0) >> @intCast(used_len * 8 - int_ty.bits)) >> 1 }, Type.u64, used_len - 8); + try cg.store(result, .{ .imm64 = (~@as(u64, 0) >> @intCast(used_len * 8 - int_ty.bits)) >> 1 }, .u64, .{ .offset = used_len - 8 }); } else { - try cg.store(result, .{ .imm64 = ~@as(u64, 0) >> @intCast(used_len * 8 - int_ty.bits) }, Type.u64, used_len - 8); + try cg.store(result, .{ .imm64 = ~@as(u64, 0) >> @intCast(used_len * 8 - int_ty.bits) }, .u64, .{ .offset = used_len - 8 }); } if (used_len + 8 == full_len) { - try cg.store(result, .{ .imm64 = 0 }, Type.u64, full_len - 8); + try cg.store(result, .{ .imm64 = 0 }, .u64, .{ .offset = full_len - 8 }); } return result; @@ -3790,8 +3826,8 @@ fn intMinValue(cg: *CodeGen, int_ty: IntType) InnerError!WValue { return .{ .imm64 = ~@as(u64, 0) << @intCast(int_ty.bits - 1) }; } else if (int_ty.bits <= 128) { const result = try cg.allocInt(int_ty); - try cg.store(result, .{ .imm64 = 0 }, Type.u64, 0); - try cg.store(result, .{ .imm64 = ~@as(u64, 0) << @intCast(int_ty.bits - 65) }, Type.u64, 8); + try cg.store(result, .{ .imm64 = 0 }, .u64, .{}); + try cg.store(result, .{ .imm64 = ~@as(u64, 0) << @intCast(int_ty.bits - 65) }, .u64, .{ .offset = 8 }); return result; } else { const result = try cg.allocInt(int_ty); @@ -3799,10 +3835,10 @@ fn intMinValue(cg: *CodeGen, int_ty: IntType) InnerError!WValue { const used_len = @divCeil(int_ty.bits, 64) * 8; try cg.memset(Type.u8, result, .{ .imm32 = used_len - 8 }, .{ .imm32 = 0 }); - try cg.store(result, .{ .imm64 = ~@as(u64, 0) << @intCast(int_ty.bits - (used_len - 8) * 8 - 1) }, Type.u64, used_len - 8); + try cg.store(result, .{ .imm64 = ~@as(u64, 0) << @intCast(int_ty.bits - (used_len - 8) * 8 - 1) }, .u64, .{ .offset = used_len - 8 }); if (used_len + 8 == full_len) { - try cg.store(result, .{ .imm64 = ~@as(u64, 0) }, Type.u64, full_len - 8); + try cg.store(result, .{ .imm64 = ~@as(u64, 0) }, .u64, .{ .offset = full_len - 8 }); } return result; @@ -3962,8 +3998,8 @@ fn intZeroValue(cg: *CodeGen, int_ty: IntType) InnerError!WValue { 33...64 => return .{ .imm64 = 0 }, 65...128 => { const result = try cg.allocInt(int_ty); - try cg.store(result, .{ .imm64 = 0 }, Type.u64, 0); - try cg.store(result, .{ .imm64 = 0 }, Type.u64, 8); + try cg.store(result, .{ .imm64 = 0 }, .u64, .{}); + try cg.store(result, .{ .imm64 = 0 }, .u64, .{ .offset = 8 }); return result; }, else => { @@ -4103,7 +4139,7 @@ fn intMulOverflow(cg: *CodeGen, int_ty: IntType, lhs: WValue, rhs: WValue) Inner Type.i128, &.{ lhs, rhs, overflow_ret }, ); - _ = try cg.load(overflow_ret, Type.i32, 0); + _ = try cg.load(overflow_ret, .i32, .{}); try cg.addLocal(.local_set, overflow_bit.local.value); break :blk res; } else { @@ -4180,12 +4216,12 @@ fn intCast(cg: *CodeGen, dest_ty: IntType, src_ty: IntType, operand: WValue) Inn try cg.emitWValue(result); try cg.emitWValue(operand); try cg.addTag(if (src_ty.is_signed) .i64_extend_i32_s else .i64_extend_i32_u); - try cg.store(.stack, .stack, Type.u64, result.offset()); + try cg.store(.stack, .stack, .u64, .{ .offset = result.offset() }); src_len = 8; } else if (src_bits == 64) { try cg.emitWValue(result); try cg.emitWValue(operand); - try cg.store(.stack, .stack, Type.u64, result.offset()); + try cg.store(.stack, .stack, .u64, .{ .offset = result.offset() }); src_len = 8; } else { src_len = src_bits / 8; @@ -4202,9 +4238,9 @@ fn intCast(cg: *CodeGen, dest_ty: IntType, src_ty: IntType, operand: WValue) Inn try cg.emitWValue(operand); } else unreachable; const shr = try cg.intShr(IntType.i64, .stack, .{ .imm32 = 63 }); - try cg.store(.stack, shr, Type.u64, 8 + result.offset()); + try cg.store(.stack, shr, .u64, .{ .offset = 8 + result.offset() }); } else { - try cg.store(result, .{ .imm64 = 0 }, Type.u64, 8); + try cg.store(result, .{ .imm64 = 0 }, .u64, .{ .offset = 8 }); } } else { var pad = result; @@ -4219,7 +4255,7 @@ fn intCast(cg: *CodeGen, dest_ty: IntType, src_ty: IntType, operand: WValue) Inn _ = try cg.intShr(IntType.i64, .stack, .{ .imm32 = 63 }); try cg.addTag(.i32_wrap_i64); } else { - _ = try cg.load(operand, Type.u64, src_len - 8); + _ = try cg.load(operand, .u64, .{ .offset = src_len - 8 }); _ = try cg.intShr(IntType.i64, .stack, .{ .imm32 = 63 }); try cg.addTag(.i32_wrap_i64); } @@ -4237,7 +4273,7 @@ fn intCast(cg: *CodeGen, dest_ty: IntType, src_ty: IntType, operand: WValue) Inn assert(dest_bits <= 64); assert(src_bits >= 128); const load_ty = if (dest_bits == 32) Type.u32 else Type.u64; - return cg.load(operand, load_ty, 0); + return cg.load(operand, load_ty, .{}); } } @@ -4608,18 +4644,18 @@ fn floatNeg(cg: *CodeGen, ty: FloatType, arg: WValue) InnerError!WValue { const result = try cg.allocStack(Type.f128); try cg.emitWValue(result); try cg.emitWValue(arg); - try cg.addMemArg(.i64_load, .{ .offset = 0 + arg.offset(), .alignment = 2 }); - try cg.addMemArg(.i64_store, .{ .offset = 0 + result.offset(), .alignment = 2 }); + try cg.addMemArg(.i64_load, .{ .offset = 0 + arg.offset(), .alignment = .@"2" }); + try cg.addMemArg(.i64_store, .{ .offset = 0 + result.offset(), .alignment = .@"2" }); try cg.emitWValue(result); try cg.emitWValue(arg); - try cg.addMemArg(.i64_load, .{ .offset = 8 + arg.offset(), .alignment = 2 }); + try cg.addMemArg(.i64_load, .{ .offset = 8 + arg.offset(), .alignment = .@"2" }); if (ty == .f80) { try cg.addImm64(0x8000); } else { try cg.addImm64(0x8000000000000000); } try cg.addTag(.i64_xor); - try cg.addMemArg(.i64_store, .{ .offset = 8 + result.offset(), .alignment = 2 }); + try cg.addMemArg(.i64_store, .{ .offset = 8 + result.offset(), .alignment = .@"2" }); return result; }, } @@ -4647,18 +4683,18 @@ fn floatAbs(cg: *CodeGen, ty: FloatType, arg: WValue) InnerError!WValue { const result = try cg.allocStack(Type.f128); try cg.emitWValue(result); try cg.emitWValue(arg); - try cg.addMemArg(.i64_load, .{ .offset = 0 + arg.offset(), .alignment = 2 }); - try cg.addMemArg(.i64_store, .{ .offset = 0 + result.offset(), .alignment = 2 }); + try cg.addMemArg(.i64_load, .{ .offset = 0 + arg.offset(), .alignment = .@"2" }); + try cg.addMemArg(.i64_store, .{ .offset = 0 + result.offset(), .alignment = .@"2" }); try cg.emitWValue(result); try cg.emitWValue(arg); - try cg.addMemArg(.i64_load, .{ .offset = 8 + arg.offset(), .alignment = 2 }); + try cg.addMemArg(.i64_load, .{ .offset = 8 + arg.offset(), .alignment = .@"2" }); if (ty == .f80) { try cg.addImm64(0x7FFF); } else { try cg.addImm64(0x7FFFFFFFFFFFFFFF); } try cg.addTag(.i64_and); - try cg.addMemArg(.i64_store, .{ .offset = 8 + result.offset(), .alignment = 2 }); + try cg.addMemArg(.i64_store, .{ .offset = 8 + result.offset(), .alignment = .@"2" }); return result; }, } @@ -5309,8 +5345,8 @@ fn airCmp(cg: *CodeGen, inst: Air.Inst.Index, op: std.math.CompareOperator) Inne try cg.addLocal(.local_get, lhs_null.local.value); try cg.addLabel(.br_if, 0); - _ = try cg.load(lhs, payload_ty, 0); - _ = try cg.load(rhs, payload_ty, 0); + _ = try cg.load(lhs, payload_ty, .{}); + _ = try cg.load(rhs, payload_ty, .{}); if (payload_ty.isAnyFloat()) { _ = try cg.floatCmp(.fromType(cg, payload_ty), op, .stack, .stack); @@ -5370,16 +5406,16 @@ fn intCmp(cg: *CodeGen, ty: IntType, op: std.math.CompareOperator, lhs: WValue, return .stack; }, 65...128 => { - var lhs_msb = try (try cg.load(lhs, Type.u64, 8)).toLocal(cg, Type.u64); + var lhs_msb = try (try cg.load(lhs, .u64, .{ .offset = 8 })).toLocal(cg, Type.u64); defer lhs_msb.free(cg); - var rhs_msb = try (try cg.load(rhs, Type.u64, 8)).toLocal(cg, Type.u64); + var rhs_msb = try (try cg.load(rhs, .u64, .{ .offset = 8 })).toLocal(cg, Type.u64); defer rhs_msb.free(cg); switch (op) { .eq, .neq => { const xor_high = try cg.intXor(.u64, lhs_msb, rhs_msb); - const lhs_lsb = try cg.load(lhs, Type.u64, 0); - const rhs_lsb = try cg.load(rhs, Type.u64, 0); + const lhs_lsb = try cg.load(lhs, .u64, .{}); + const rhs_lsb = try cg.load(rhs, .u64, .{}); const xor_low = try cg.intXor(.u64, lhs_lsb, rhs_lsb); const or_result = try cg.intOr(.u64, xor_high, xor_low); @@ -5392,8 +5428,8 @@ fn intCmp(cg: *CodeGen, ty: IntType, op: std.math.CompareOperator, lhs: WValue, else => { const word_int_ty: IntType = if (ty.is_signed) .i64 else .u64; - const lhs_lsb = try cg.load(lhs, Type.u64, 0); - const rhs_lsb = try cg.load(rhs, Type.u64, 0); + const lhs_lsb = try cg.load(lhs, .u64, .{}); + const rhs_lsb = try cg.load(rhs, .u64, .{}); // leave values on stack for 'select' _ = try cg.intCmp(.u64, op, lhs_lsb, rhs_lsb); @@ -5599,7 +5635,7 @@ fn airUnionFromEnum(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { const enum_value = try cg.resolveInst(ty_op.operand); const result = try cg.allocStack(union_ty); - try cg.store(result, enum_value, enum_ty, @intCast(layout.tagOffset())); + try cg.store(result, enum_value, enum_ty, .{ .offset = layout.tagOffset() }); return cg.finishAir(inst, result, &.{ty_op.operand}); } @@ -5662,12 +5698,12 @@ fn bitcast(cg: *CodeGen, dest_ty: Type, src_ty: Type, operand: WValue) InnerErro if (dest_by_ref) { const result = try cg.allocStack(src_ty); - try cg.store(result, operand, src_ty, 0); + try cg.store(result, operand, src_ty, .{}); return result; } if (src_by_ref) { - return try cg.load(operand, dest_ty, 0); + return try cg.load(operand, dest_ty, .{}); } switch (src_class) { @@ -5803,7 +5839,7 @@ fn airAggFieldVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { else => break :result try cg.buildPointerOffset(operand, offset, .new), } } - break :result try cg.load(operand, field_ty, offset); + break :result try cg.load(operand, field_ty, .{ .offset = offset }); }, }; @@ -6048,7 +6084,7 @@ fn airIsErr(cg: *CodeGen, inst: Air.Inst.Index, opcode: std.wasm.Opcode, op_kind if (op_kind == .ptr or pl_ty.hasRuntimeBits(zcu)) { try cg.addMemArg(.i32_load16_u, .{ .offset = operand.offset() + @as(u32, @intCast(errUnionErrorOffset(pl_ty, zcu))), - .alignment = @intCast(Type.anyerror.abiAlignment(zcu).toByteUnits().?), + .alignment = Type.anyerror.abiAlignment(zcu).toStdMem(), }); } @@ -6085,7 +6121,7 @@ fn airUnwrapErrUnionPayload(cg: *CodeGen, inst: Air.Inst.Index, op_is_ptr: bool) break :result try cg.buildPointerOffset(operand, pl_offset, .new); } else { assert(isByRef(eu_ty, zcu, cg.target)); - break :result try cg.load(operand, payload_ty, pl_offset); + break :result try cg.load(operand, payload_ty, .{ .offset = pl_offset }); } }; return cg.finishAir(inst, result, &.{ty_op.operand}); @@ -6110,7 +6146,7 @@ fn airUnwrapErrUnionError(cg: *CodeGen, inst: Air.Inst.Index, op_is_ptr: bool) I const err_offset: u32 = @intCast(errUnionErrorOffset(payload_ty, zcu)); if (op_is_ptr or isByRef(eu_ty, zcu, cg.target)) { - break :result try cg.load(operand, Type.anyerror, err_offset); + break :result try cg.load(operand, .anyerror, .{ .offset = err_offset }); } else { assert(!payload_ty.hasRuntimeBits(zcu)); break :result operand; @@ -6134,7 +6170,7 @@ fn airWrapErrUnionPayload(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { const err_union = try cg.allocStack(err_ty); const payload_ptr = try cg.buildPointerOffset(err_union, @as(u32, @intCast(errUnionPayloadOffset(pl_ty, zcu))), .new); - try cg.store(payload_ptr, operand, pl_ty, 0); + try cg.store(payload_ptr, operand, pl_ty, .{}); // ensure we also write '0' to the error part, so any present stack value gets overwritten by it. try cg.emitWValue(err_union); @@ -6142,7 +6178,7 @@ fn airWrapErrUnionPayload(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { const err_val_offset: u32 = @intCast(errUnionErrorOffset(pl_ty, zcu)); try cg.addMemArg(.i32_store16, .{ .offset = err_union.offset() + err_val_offset, - .alignment = 2, + .alignment = .@"2", }); break :result err_union; }; @@ -6164,7 +6200,7 @@ fn airWrapErrUnionErr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { const err_union = try cg.allocStack(err_ty); // store error value - try cg.store(err_union, operand, Type.anyerror, @intCast(errUnionErrorOffset(pl_ty, zcu))); + try cg.store(err_union, operand, .anyerror, .{ .offset = errUnionErrorOffset(pl_ty, zcu) }); // write 'undefined' to the payload const payload_ptr = try cg.buildPointerOffset(err_union, @as(u32, @intCast(errUnionPayloadOffset(pl_ty, zcu))), .new); @@ -6205,18 +6241,18 @@ fn isNull(cg: *CodeGen, operand: WValue, op_ty: Type, opcode: std.wasm.Opcode, o const offset = std.math.cast(u32, payload_ty.abiSize(zcu)) orelse { return cg.fail("Optional type {f} too big to fit into stack frame", .{optional_ty.fmt(pt)}); }; - try cg.addMemArg(.i32_load8_u, .{ .offset = operand.offset() + offset, .alignment = 1 }); + try cg.addMemArg(.i32_load8_u, .{ .offset = operand.offset() + offset, .alignment = .@"1" }); } } else if (payload_ty.isSlice(zcu)) { switch (cg.ptr_size) { - .wasm32 => try cg.addMemArg(.i32_load, .{ .offset = operand.offset(), .alignment = 4 }), - .wasm64 => try cg.addMemArg(.i64_load, .{ .offset = operand.offset(), .alignment = 8 }), + .wasm32 => try cg.addMemArg(.i32_load, .{ .offset = operand.offset(), .alignment = .@"4" }), + .wasm64 => try cg.addMemArg(.i64_load, .{ .offset = operand.offset(), .alignment = .@"8" }), } } else { if (op_kind == .ptr) { try cg.addMemArg(.i32_load, .{ .offset = operand.offset(), - .alignment = 4, + .alignment = .@"4", }); } } @@ -6245,7 +6281,7 @@ fn airOptionalPayload(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { break :result try cg.buildPointerOffset(operand, 0, .new); } - break :result try cg.load(operand, payload_ty, 0); + break :result try cg.load(operand, payload_ty, .{}); }; return cg.finishAir(inst, result, &.{ty_op.operand}); } @@ -6285,7 +6321,7 @@ fn airOptionalPayloadPtrSet(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void try cg.emitWValue(operand); try cg.addImm32(1); - try cg.addMemArg(.i32_store8, .{ .offset = operand.offset() + offset, .alignment = 1 }); + try cg.addMemArg(.i32_store8, .{ .offset = operand.offset() + offset, .alignment = .@"1" }); const result = try cg.buildPointerOffset(operand, 0, .new); return cg.finishAir(inst, result, &.{ty_op.operand}); @@ -6302,7 +6338,7 @@ fn airWrapOptional(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { const non_null_bit = try cg.allocStack(Type.u1); try cg.emitWValue(non_null_bit); try cg.addImm32(1); - try cg.addMemArg(.i32_store8, .{ .offset = non_null_bit.offset(), .alignment = 1 }); + try cg.addMemArg(.i32_store8, .{ .offset = non_null_bit.offset(), .alignment = .@"1" }); break :result non_null_bit; } @@ -6319,10 +6355,10 @@ fn airWrapOptional(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { const result_ptr = try cg.allocStack(op_ty); try cg.emitWValue(result_ptr); try cg.addImm32(1); - try cg.addMemArg(.i32_store8, .{ .offset = result_ptr.offset() + offset, .alignment = 1 }); + try cg.addMemArg(.i32_store8, .{ .offset = result_ptr.offset() + offset, .alignment = .@"1" }); const payload_ptr = try cg.buildPointerOffset(result_ptr, 0, .new); - try cg.store(payload_ptr, operand, payload_ty, 0); + try cg.store(payload_ptr, operand, payload_ty, .{}); break :result result_ptr; }; @@ -6338,8 +6374,8 @@ fn airSlice(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { const slice_ty = cg.typeOfIndex(inst); const slice = try cg.allocStack(slice_ty); - try cg.store(slice, lhs, Type.usize, 0); - try cg.store(slice, rhs, Type.usize, cg.ptrSize()); + try cg.store(slice, lhs, .usize, .{}); + try cg.store(slice, rhs, .usize, .{ .offset = cg.ptrSize() }); return cg.finishAir(inst, slice, &.{ bin_op.lhs, bin_op.rhs }); } @@ -6362,7 +6398,7 @@ fn airSliceElemVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { const elem_size = elem_ty.abiSize(zcu); // load pointer onto stack - _ = try cg.load(slice, Type.usize, 0); + _ = try cg.load(slice, .usize, .{}); // calculate index into slice try cg.emitWValue(index); @@ -6370,7 +6406,7 @@ fn airSliceElemVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { try cg.addTag(.i32_mul); try cg.addTag(.i32_add); - const elem_result = try cg.load(.stack, elem_ty, 0); + const elem_result = try cg.load(.stack, elem_ty, .{}); return cg.finishAir(inst, elem_result, &.{ bin_op.lhs, bin_op.rhs }); } @@ -6386,7 +6422,7 @@ fn airSliceElemPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { const slice = try cg.resolveInst(bin_op.lhs); const index = try cg.resolveInst(bin_op.rhs); - _ = try cg.load(slice, Type.usize, 0); + _ = try cg.load(slice, .usize, .{}); // calculate index into slice try cg.emitWValue(index); @@ -6404,12 +6440,12 @@ fn airSlicePtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { } fn slicePtr(cg: *CodeGen, operand: WValue) InnerError!WValue { - const ptr = try cg.load(operand, Type.usize, 0); + const ptr = try cg.load(operand, .usize, .{}); return ptr.toLocal(cg, Type.usize); } fn sliceLen(cg: *CodeGen, operand: WValue) InnerError!WValue { - const len = try cg.load(operand, Type.usize, cg.ptrSize()); + const len = try cg.load(operand, .usize, .{ .offset = cg.ptrSize() }); return len.toLocal(cg, Type.usize); } @@ -6424,11 +6460,11 @@ fn airArrayToSlice(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { // create a slice on the stack const slice_local = try cg.allocStack(slice_ty); - try cg.store(slice_local, operand, Type.usize, 0); + try cg.store(slice_local, operand, .usize, .{}); // store the length of the array in the slice const array_len: u32 = @intCast(array_ty.arrayLen(zcu)); - try cg.store(slice_local, .{ .imm32 = array_len }, Type.usize, cg.ptrSize()); + try cg.store(slice_local, .{ .imm32 = array_len }, .usize, .{ .offset = cg.ptrSize() }); return cg.finishAir(inst, slice_local, &.{ty_op.operand}); } @@ -6445,7 +6481,7 @@ fn airPtrElemVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { // load pointer onto the stack if (ptr_ty.isSlice(zcu)) { - _ = try cg.load(ptr, Type.usize, 0); + _ = try cg.load(ptr, .usize, .{}); } else { try cg.lowerToStack(ptr); } @@ -6456,7 +6492,7 @@ fn airPtrElemVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { try cg.addTag(.i32_mul); try cg.addTag(.i32_add); - const elem_result = try cg.load(.stack, elem_ty, 0); + const elem_result = try cg.load(.stack, elem_ty, .{}); return cg.finishAir(inst, elem_result, &.{ bin_op.lhs, bin_op.rhs }); } @@ -6475,7 +6511,7 @@ fn airPtrElemPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { // load pointer onto the stack if (ptr_ty.isSlice(zcu)) { - _ = try cg.load(ptr, Type.usize, 0); + _ = try cg.load(ptr, .usize, .{}); } else { try cg.lowerToStack(ptr); } @@ -6645,7 +6681,7 @@ fn memset(cg: *CodeGen, elem_ty: Type, ptr: WValue, len: WValue, value: WValue) try cg.addLabel(.br_if, 1); // jump out of loop into outer block (finished) // store the value at the current position of the pointer - try cg.store(new_ptr, value, elem_ty, 0); + try cg.store(new_ptr, value, elem_ty, .{}); // move the pointer to the next element try cg.emitWValue(new_ptr); @@ -6708,7 +6744,7 @@ fn airArrayElemVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { }, else => { const stack_vec = try cg.allocStack(array_ty); - try cg.store(stack_vec, array, array_ty, 0); + try cg.store(stack_vec, array, array_ty, .{}); // Is a non-unrolled vector (v128) try cg.lowerToStack(stack_vec); @@ -6723,7 +6759,7 @@ fn airArrayElemVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { const result = if (isByRef(elem_ty, zcu, cg.target)) .stack else - try cg.load(.stack, elem_ty, 0); + try cg.load(.stack, elem_ty, .{}); return cg.finishAir(inst, result, &.{ bin_op.lhs, bin_op.rhs }); } @@ -6750,10 +6786,11 @@ fn airSplat(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { try cg.emitWValue(operand); const extra_index: u32 = @intCast(cg.mir_extra.items.len); // stores as := opcode, offset, alignment (opcode::memarg) - try cg.mir_extra.appendSlice(cg.gpa, &[_]u32{ - opcode, - operand.offset(), - @intCast(elem_ty.abiAlignment(zcu).toByteUnits().?), + try cg.mir_extra.ensureUnusedCapacity(cg.gpa, 4); + cg.mir_extra.appendAssumeCapacity(opcode); + _ = try cg.addExtraAssumeCapacity(Mir.MemArg{ + .offset = operand.offset(), + .alignment = elem_ty.abiAlignment(zcu).toStdMem(), }); try cg.addInst(.{ .tag = .simd_prefix, .data = .{ .payload = extra_index } }); return cg.finishAir(inst, .stack, &.{ty_op.operand}); @@ -6782,7 +6819,7 @@ fn airSplat(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { var index: usize = 0; var offset: u32 = 0; while (index < vector_len) : (index += 1) { - try cg.store(result, operand, elem_ty, offset); + try cg.store(result, operand, elem_ty, .{ .offset = offset }); offset += elem_byte_size; } @@ -6821,10 +6858,10 @@ fn airShuffleOne(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { for (mask, 0..) |mask_elem, out_idx| { try cg.emitWValue(dest_alloc); const elem_val = switch (mask_elem.unwrap()) { - .elem => |idx| try cg.load(operand, elem_ty, @intCast(elem_size * idx)), + .elem => |idx| try cg.load(operand, elem_ty, .{ .offset = elem_size * idx }), .value => |val| try cg.lowerConstant(.fromInterned(val)), }; - try cg.store(.stack, elem_val, elem_ty, @intCast(dest_alloc.offset() + elem_size * out_idx)); + try cg.store(.stack, elem_val, elem_ty, .{ .offset = dest_alloc.offset() + elem_size * out_idx }); } return cg.finishAir(inst, dest_alloc, &.{unwrapped.operand}); } @@ -6889,11 +6926,11 @@ fn airShuffleTwo(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { for (mask, 0..) |mask_elem, out_idx| { try cg.emitWValue(dest_alloc); const elem_val = switch (mask_elem.unwrap()) { - .a_elem => |idx| try cg.load(operand_a, elem_ty, @intCast(elem_size * idx)), - .b_elem => |idx| try cg.load(operand_b, elem_ty, @intCast(elem_size * idx)), + .a_elem => |idx| try cg.load(operand_a, elem_ty, .{ .offset = elem_size * idx }), + .b_elem => |idx| try cg.load(operand_b, elem_ty, .{ .offset = elem_size * idx }), .undef => try cg.emitUndefined(elem_ty), }; - try cg.store(.stack, elem_val, elem_ty, @intCast(dest_alloc.offset() + elem_size * out_idx)); + try cg.store(.stack, elem_val, elem_ty, .{ .offset = dest_alloc.offset() + elem_size * out_idx }); } return cg.finishAir(inst, dest_alloc, &.{ unwrapped.operand_a, unwrapped.operand_b }); } @@ -6931,7 +6968,7 @@ fn airAggregateInit(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { const offset = try cg.buildPointerOffset(result, 0, .new); for (elements, 0..) |elem, elem_index| { const elem_val = try cg.resolveInst(elem); - try cg.store(offset, elem_val, elem_ty, 0); + try cg.store(offset, elem_val, elem_ty, .{}); if (elem_index < elements.len - 1 or sentinel != null) { _ = try cg.buildPointerOffset(offset, elem_size, .modify); @@ -6939,18 +6976,18 @@ fn airAggregateInit(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { } if (sentinel) |s| { const val = try cg.resolveValue(s); - try cg.store(offset, val, elem_ty, 0); + try cg.store(offset, val, elem_ty, .{}); } } else { var offset: u32 = 0; for (elements) |elem| { const elem_val = try cg.resolveInst(elem); - try cg.store(result, elem_val, elem_ty, offset); + try cg.store(result, elem_val, elem_ty, .{ .offset = offset }); offset += elem_size; } if (sentinel) |s| { const val = try cg.resolveValue(s); - try cg.store(result, val, elem_ty, offset); + try cg.store(result, val, elem_ty, .{ .offset = offset }); } } break :result_value result; @@ -6967,7 +7004,7 @@ fn airAggregateInit(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { const offset = try cg.buildPointerOffset(result, field_offset, .new); const value = try cg.resolveInst(elem); - try cg.store(offset, value, elem_ty, 0); + try cg.store(offset, value, elem_ty, .{}); } break :result_value result; @@ -7016,22 +7053,22 @@ fn airUnionInit(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { if (layout.tag_align.compare(.gte, layout.payload_align)) { if (isByRef(field_ty, zcu, cg.target)) { const payload_ptr = try cg.buildPointerOffset(result_ptr, layout.tag_size, .new); - try cg.store(payload_ptr, payload, field_ty, 0); + try cg.store(payload_ptr, payload, field_ty, .{}); } else { - try cg.store(result_ptr, payload, field_ty, @intCast(layout.tag_size)); + try cg.store(result_ptr, payload, field_ty, .{ .offset = layout.tag_size }); } if (layout.tag_size > 0) { - try cg.store(result_ptr, tag_int, .fromInterned(union_obj.enum_tag_type), 0); + try cg.store(result_ptr, tag_int, .fromInterned(union_obj.enum_tag_type), .{}); } } else { - try cg.store(result_ptr, payload, field_ty, 0); + try cg.store(result_ptr, payload, field_ty, .{}); if (layout.tag_size > 0) { try cg.store( result_ptr, tag_int, .fromInterned(union_obj.enum_tag_type), - @intCast(layout.payload_size), + .{ .offset = layout.payload_size }, ); } } @@ -7077,7 +7114,7 @@ fn airSetUnionTag(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { const union_ptr = try cg.resolveInst(bin_op.lhs); const new_tag = try cg.resolveInst(bin_op.rhs); if (layout.payload_size == 0) { - try cg.store(union_ptr, new_tag, tag_ty, 0); + try cg.store(union_ptr, new_tag, tag_ty, .{}); return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs }); } @@ -7086,7 +7123,7 @@ fn airSetUnionTag(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { const offset: u32 = if (layout.tag_align.compare(.lt, layout.payload_align)) blk: { break :blk @intCast(layout.payload_size); } else 0; - try cg.store(union_ptr, new_tag, tag_ty, offset); + try cg.store(union_ptr, new_tag, tag_ty, .{ .offset = offset }); return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs }); } @@ -7106,7 +7143,7 @@ fn airGetUnionTag(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { @intCast(layout.payload_size) else 0; - const result = try cg.load(operand, tag_ty, offset); + const result = try cg.load(operand, tag_ty, .{ .offset = offset }); return cg.finishAir(inst, result, &.{ty_op.operand}); } @@ -7122,8 +7159,8 @@ fn airErrUnionPayloadPtrSet(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void try cg.store( operand, .{ .imm32 = 0 }, - Type.anyerror, - @intCast(errUnionErrorOffset(payload_ty, zcu)), + .anyerror, + .{ .offset = errUnionErrorOffset(payload_ty, zcu) }, ); const result = result: { @@ -7360,7 +7397,7 @@ fn lowerTry( const err_offset: u32 = @intCast(errUnionErrorOffset(pl_ty, zcu)); try cg.addMemArg(.i32_load16_u, .{ .offset = err_union.offset() + err_offset, - .alignment = @intCast(Type.anyerror.abiAlignment(zcu).toByteUnits().?), + .alignment = Type.anyerror.abiAlignment(zcu).toStdMem(), }); } try cg.addTag(.i32_eqz); @@ -7384,7 +7421,7 @@ fn lowerTry( if (operand_is_ptr or isByRef(pl_ty, zcu, cg.target)) { return buildPointerOffset(cg, err_union, pl_offset, .new); } - const payload = try cg.load(err_union, pl_ty, pl_offset); + const payload = try cg.load(err_union, pl_ty, .{ .offset = pl_offset }); return payload.toLocal(cg, pl_ty); } @@ -7588,7 +7625,7 @@ fn airCmpxchg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { else => |size| return cg.fail("TODO: implement `@cmpxchg` for types with abi size '{d}'", .{size}), }, .{ .offset = ptr_operand.offset(), - .alignment = @intCast(ty.abiAlignment(zcu).toByteUnits().?), + .alignment = ty.abiAlignment(zcu).toStdMem(), }); try cg.addLocal(.local_tee, val_local.local.value); _ = try cg.intCmp(int_ty, .eq, .stack, expected_val); @@ -7598,7 +7635,7 @@ fn airCmpxchg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { if (ty.abiSize(zcu) > 8) { return cg.fail("TODO: Implement `@cmpxchg` for types larger than abi size of 8 bytes", .{}); } - const ptr_val = try WValue.toLocal(try cg.load(ptr_operand, ty, 0), cg, ty); + const ptr_val = try WValue.toLocal(try cg.load(ptr_operand, ty, .{}), cg, ty); try cg.lowerToStack(ptr_operand); try cg.lowerToStack(new_val); @@ -7606,7 +7643,7 @@ fn airCmpxchg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { _ = try cg.intCmp(int_ty, .eq, ptr_val, expected_val); try cg.addLocal(.local_tee, cmp_result.local.value); try cg.addTag(.select); - try cg.store(.stack, .stack, ty, 0); + try cg.store(.stack, .stack, ty, .{}); break :val ptr_val; }; @@ -7619,8 +7656,8 @@ fn airCmpxchg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { try cg.addTag(.i32_and); const and_result = try WValue.toLocal(.stack, cg, Type.bool); const result_ptr = try cg.allocStack(result_ty); - try cg.store(result_ptr, and_result, Type.bool, @as(u32, @intCast(ty.abiSize(zcu)))); - try cg.store(result_ptr, ptr_val, ty, 0); + try cg.store(result_ptr, and_result, .bool, .{ .offset = ty.abiSize(zcu) }); + try cg.store(result_ptr, ptr_val, ty, .{}); break :val result_ptr; } else val: { try cg.addImm32(0); @@ -7650,10 +7687,10 @@ fn airAtomicLoad(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { try cg.emitWValue(ptr); try cg.addAtomicMemArg(tag, .{ .offset = ptr.offset(), - .alignment = @intCast(ty.abiAlignment(zcu).toByteUnits().?), + .alignment = ty.abiAlignment(zcu).toStdMem(), }); } else { - _ = try cg.load(ptr, ty, 0); + _ = try cg.load(ptr, ty, .{}); } return cg.finishAir(inst, .stack, &.{atomic_load.ptr}); @@ -7676,7 +7713,7 @@ fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { .Min, .Nand, => { - const tmp = try cg.load(ptr, ty, 0); + const tmp = try cg.load(ptr, ty, .{}); const value = try tmp.toLocal(cg, ty); // create a loop to cmpxchg the new value @@ -7710,7 +7747,7 @@ fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { }, .{ .offset = ptr.offset(), - .alignment = @intCast(ty.abiAlignment(zcu).toByteUnits().?), + .alignment = ty.abiAlignment(zcu).toStdMem(), }, ); const select_res = try cg.allocLocal(ty); @@ -7770,18 +7807,18 @@ fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { }; try cg.addAtomicMemArg(tag, .{ .offset = ptr.offset(), - .alignment = @intCast(ty.abiAlignment(zcu).toByteUnits().?), + .alignment = ty.abiAlignment(zcu).toStdMem(), }); return cg.finishAir(inst, .stack, &.{ pl_op.operand, extra.operand }); }, } } else { - const loaded = try cg.load(ptr, ty, 0); + const loaded = try cg.load(ptr, ty, .{}); const result = try loaded.toLocal(cg, ty); switch (op) { .Xchg => { - try cg.store(ptr, operand, ty, 0); + try cg.store(ptr, operand, ty, .{}); }, .Add, .Sub, @@ -7794,7 +7831,7 @@ fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { .Sub => try cg.floatSub(float_ty, result, operand), else => unreachable, }; - try cg.store(.stack, .stack, ty, ptr.offset()); + try cg.store(.stack, .stack, ty, .{ .offset = ptr.offset() }); } else { const int_ty: IntType = .fromType(cg, ty); try cg.emitWValue(ptr); @@ -7804,7 +7841,7 @@ fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { else => unreachable, }; _ = try cg.intWrap(int_ty, .stack); - try cg.store(.stack, .stack, ty, ptr.offset()); + try cg.store(.stack, .stack, ty, .{ .offset = ptr.offset() }); } }, .And, @@ -7819,7 +7856,7 @@ fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { .Xor => try cg.intXor(int_ty, result, operand), else => unreachable, }; - try cg.store(.stack, .stack, ty, ptr.offset()); + try cg.store(.stack, .stack, ty, .{ .offset = ptr.offset() }); }, .Max, .Min, @@ -7831,7 +7868,7 @@ fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { try cg.emitWValue(operand); _ = try cg.floatCmp(float_ty, if (op == .Max) .gt else .lt, result, operand); try cg.addTag(.select); - try cg.store(.stack, .stack, ty, ptr.offset()); + try cg.store(.stack, .stack, ty, .{ .offset = ptr.offset() }); } else { const int_ty: IntType = .fromType(cg, ty); try cg.emitWValue(ptr); @@ -7839,7 +7876,7 @@ fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { try cg.emitWValue(operand); _ = try cg.intCmp(int_ty, if (op == .Max) .gt else .lt, result, operand); try cg.addTag(.select); - try cg.store(.stack, .stack, ty, ptr.offset()); + try cg.store(.stack, .stack, ty, .{ .offset = ptr.offset() }); } }, .Nand => { @@ -7854,7 +7891,7 @@ fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { return cg.fail("TODO: `@atomicRmw` with operator `Nand` for types larger than 64 bits", .{}); } _ = try cg.intXor(int_ty, and_res, .stack); - try cg.store(.stack, .stack, ty, ptr.offset()); + try cg.store(.stack, .stack, ty, .{ .offset = ptr.offset() }); }, } @@ -7883,10 +7920,10 @@ fn airAtomicStore(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { try cg.lowerToStack(operand); try cg.addAtomicMemArg(tag, .{ .offset = ptr.offset(), - .alignment = @intCast(ty.abiAlignment(zcu).toByteUnits().?), + .alignment = ty.abiAlignment(zcu).toStdMem(), }); } else { - try cg.store(ptr, operand, ty, 0); + try cg.store(ptr, operand, ty, .{}); } return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs }); diff --git a/src/codegen/wasm/Emit.zig b/src/codegen/wasm/Emit.zig index a83597dfc4fb3b0749a66fdf0c7e62c8995457c5..077cb04266f4d21da0bbbb31ee90d1bae03515ca 100644 --- a/src/codegen/wasm/Emit.zig +++ b/src/codegen/wasm/Emit.zig @@ -988,9 +988,7 @@ pub fn lower(emit: *Emit) Error!void { /// Asserts 20 unused capacity. fn encodeMemArg(code: *ArrayList(u8), mem_arg: Mir.MemArg) void { assert(code.unusedCapacitySlice().len >= 20); - // Wasm encodes alignment as power of 2, rather than natural alignment. - const encoded_alignment = @ctz(mem_arg.alignment); - writeUleb128(code, encoded_alignment); + writeUleb128(code, @backingInt(mem_arg.alignment)); writeUleb128(code, mem_arg.offset); } diff --git a/src/codegen/wasm/Mir.zig b/src/codegen/wasm/Mir.zig index 5f602496e901acc56864a8e45143fd7e14d193f7..ea40a3f76ebfab55f17eb1ab20ce93527d441909 100644 --- a/src/codegen/wasm/Mir.zig +++ b/src/codegen/wasm/Mir.zig @@ -694,11 +694,18 @@ pub fn extraData(self: *const Mir, comptime T: type, index: usize) struct { data inline for (info.field_names, info.field_types) |field_name, field_type| { @field(result, field_name) = switch (field_type) { u32 => self.extra[i], + u64 => value: { + const lo = @as(u64, self.extra[i]); + const hi = @as(u64, self.extra[i + 1]) << 32; + i += 1; + break :value hi | lo; + }, i32 => @bitCast(self.extra[i]), Wasm.UavsObjIndex, Wasm.UavsExeIndex, InternPool.Nav.Index, InternPool.Index, + std.mem.Alignment, => @fromBackingInt(@intCast(self.extra[i])), else => @compileError("Unsupported field type " ++ @typeName(field_type)), }; @@ -747,8 +754,8 @@ pub const Float64 = struct { }; pub const MemArg = struct { - offset: u32, - alignment: u32, + offset: u64, + alignment: std.mem.Alignment, }; pub const UavRefOff = struct { diff --git a/src/codegen/wasm/assembly.zig b/src/codegen/wasm/assembly.zig index d52c83d461a07d252c164d35ea5162e9f973d293..7aa056aae95e22cc5da42cb9f03ebe56257d41f4 100644 --- a/src/codegen/wasm/assembly.zig +++ b/src/codegen/wasm/assembly.zig @@ -679,20 +679,20 @@ fn parseMemArg(cg: *CodeGen, inst: []const u8, word_it: *mem.TokenIterator(u8, . const offset_str = oa[0]; const alignment_str = oa[1]; - const offset = std.fmt.parseInt(i32, offset_str, 0) catch { + const offset = std.fmt.parseInt(i64, offset_str, 0) catch { return cg.fail("Malformed assembly, malformed offset \"{s}\"", .{offset_str}); }; const alignment_p2 = std.fmt.parseInt(u5, alignment_str, 0) catch { return cg.fail("Malformed assembly, malformed offset \"{s}\"", .{alignment_str}); }; - return .{ .offset = @bitCast(offset), .alignment = @as(u32, 1) << alignment_p2 }; + return .{ .offset = @bitCast(offset), .alignment = @fromBackingInt(@intCast(alignment_p2)) }; } else { const offset_str = mem_arg_str; - const offset = std.fmt.parseInt(i32, offset_str, 0) catch { + const offset = std.fmt.parseInt(i64, offset_str, 0) catch { return cg.fail("Malformed assembly, malformed offset \"{s}\"", .{offset_str}); }; - return .{ .offset = @bitCast(offset), .alignment = 1 }; + return .{ .offset = @bitCast(offset), .alignment = .@"1" }; } } -- 2.54.0 From b819da2d57316993c5bb77c93d50f626e32eea91 Mon Sep 17 00:00:00 2001 From: Pavel Verigo Date: Fri, 21 Aug 2026 14:36:33 +0200 Subject: [PATCH 3/8] wasm: remove lowerToStack and buildOffsetPointer rework --- src/codegen/wasm/CodeGen.zig | 745 ++++++++++++++++------------------- 1 file changed, 347 insertions(+), 398 deletions(-) diff --git a/src/codegen/wasm/CodeGen.zig b/src/codegen/wasm/CodeGen.zig index dac187ee04f99dd4642579678794459c810bcdd7..b51508d201b43ecf103c4ff0318335837ab4c1d3 100644 --- a/src/codegen/wasm/CodeGen.zig +++ b/src/codegen/wasm/CodeGen.zig @@ -271,23 +271,16 @@ const WValue = union(enum) { /// pointing to where the value lives. stack_offset: struct { /// Contains the actual value of the offset - value: u32, - /// The amount of instructions referencing this `WValue` - references: u32, + value: u64, }, - /// Returns the offset from the bottom of the stack. This is useful when - /// we use the load or store instruction to ensure we retrieve the value - /// from the correct position, rather than the value that lives at the - /// bottom of the stack. For instances where `WValue` is not `stack_value` - /// this will return 0, which allows us to simply call this function for all - /// loads and stores without requiring checks everywhere. - fn offset(value: WValue) u32 { - switch (value) { - .stack_offset => |stack_offset| return stack_offset.value, + /// Returns the offset from the bottom of the stack. + fn memOffset(value: WValue) u64 { + return switch (value) { + .stack_offset => |stack_offset| stack_offset.value, .dead => unreachable, - else => return 0, - } + else => 0, + }; } /// Promotes a `WValue` to a local when given value is on top of the stack. @@ -651,6 +644,30 @@ fn genBlockType(ty: Type, zcu: *const Zcu, target: *const std.Target) std.wasm.B }; } +/// Returns `ptr` with `offset` applied. +fn offsetPointer(cg: *CodeGen, ptr: WValue, offset: u64) InnerError!WValue { + if (offset == 0) return ptr; + return switch (ptr) { + .stack_offset => |stack_offset| .{ .stack_offset = .{ + .value = stack_offset.value + offset, + } }, + else => result: { + try cg.emitWValue(ptr); + switch (cg.ptr_size) { + .wasm32 => { + try cg.addImm32(@intCast(offset)); + try cg.addTag(.i32_add); + }, + .wasm64 => { + try cg.addImm64(offset); + try cg.addTag(.i64_add); + }, + } + break :result .stack; + }, + }; +} + /// Writes the bytecode depending on the given `WValue` in `val` fn emitWValue(cg: *CodeGen, value: WValue) InnerError!void { switch (value) { @@ -718,7 +735,18 @@ fn emitWValue(cg: *CodeGen, value: WValue) InnerError!void { }); } }, - .stack_offset => try cg.addLocal(.local_get, cg.bottom_stack_value.local.value), // caller must ensure to address the offset + .stack_offset => |stack_offset| { + try cg.emitWValue(cg.bottom_stack_value); + _ = try cg.offsetPointer(.stack, stack_offset.value); + }, + } +} + +/// Emits the base address for a memory instruction. +fn emitMemBase(cg: *CodeGen, ptr: WValue) InnerError!void { + switch (ptr) { + .stack_offset => try cg.addLocal(.local_get, cg.bottom_stack_value.local.value), + else => try cg.emitWValue(ptr), } } @@ -989,7 +1017,7 @@ pub fn firstParamSRet( /// it matches the ABI. fn lowerArg(cg: *CodeGen, cc: std.lang.CallingConvention, ty: Type, value: WValue) !void { if (cc != .wasm_mvp) { - return cg.lowerToStack(value); + return cg.emitWValue(value); } const zcu = cg.pt.zcu; @@ -997,7 +1025,7 @@ fn lowerArg(cg: *CodeGen, cc: std.lang.CallingConvention, ty: Type, value: WValu switch (abi.classifyType(ty, zcu, cg.target)) { .direct => |scalar_ty| { if (!isByRef(ty, zcu, cg.target)) { - return cg.lowerToStack(value); + return cg.emitWValue(value); } else { _ = try cg.load(value, scalar_ty, .{}); } @@ -1005,15 +1033,16 @@ fn lowerArg(cg: *CodeGen, cc: std.lang.CallingConvention, ty: Type, value: WValu .double_i64 => { assert(ty.abiSize(zcu) == 16); // in this case we have an integer or float that must be lowered as 2 i64's. - try cg.emitWValue(value); - try cg.addMemArg(.i64_load, .{ .offset = value.offset(), .alignment = .@"8" }); - try cg.emitWValue(value); - try cg.addMemArg(.i64_load, .{ .offset = value.offset() + 8, .alignment = .@"8" }); + const base_offset = value.memOffset(); + try cg.emitMemBase(value); + try cg.addMemArg(.i64_load, .{ .offset = base_offset, .alignment = .@"8" }); + try cg.emitMemBase(value); + try cg.addMemArg(.i64_load, .{ .offset = base_offset + 8, .alignment = .@"8" }); }, .indirect => { const stack_copy = try cg.allocStack(ty); try cg.store(stack_copy, value, ty, .{}); - return cg.lowerToStack(stack_copy); + return cg.emitWValue(stack_copy); }, .unrolled => |vector| { const elem_size = vector.elem_type.abiSize(zcu); @@ -1024,30 +1053,6 @@ fn lowerArg(cg: *CodeGen, cc: std.lang.CallingConvention, ty: Type, value: WValu } } -/// Lowers a `WValue` to the stack. This means when the `value` results in -/// `.stack_offset` we calculate the pointer of this offset and use that. -/// The value is left on the stack, and not stored in any temporary. -fn lowerToStack(cg: *CodeGen, value: WValue) !void { - switch (value) { - .stack_offset => |offset| { - try cg.emitWValue(value); - if (offset.value > 0) { - switch (cg.ptr_size) { - .wasm32 => { - try cg.addImm32(offset.value); - try cg.addTag(.i32_add); - }, - .wasm64 => { - try cg.addImm64(offset.value); - try cg.addTag(.i64_add); - }, - } - } - }, - else => try cg.emitWValue(value), - } -} - /// Creates a local for the initial stack value /// Asserts `initial_stack_value` is `.none` fn initializeStack(cg: *CodeGen) !void { @@ -1113,7 +1118,7 @@ fn allocStackBytes(cg: *CodeGen, size: u32, alignment: Alignment) !WValue { const offset: u32 = @intCast(alignment.forward(cg.stack_size)); defer cg.stack_size = offset + size; - return .{ .stack_offset = .{ .value = offset, .references = 1 } }; + return .{ .stack_offset = .{ .value = offset } }; } /// From a given AIR instruction generates a pointer to the stack where @@ -1141,7 +1146,7 @@ fn allocStackPtr(cg: *CodeGen, inst: Air.Inst.Index) !WValue { const offset: u32 = @intCast(abi_alignment.forward(cg.stack_size)); defer cg.stack_size = offset + abi_size; - return .{ .stack_offset = .{ .value = offset, .references = 1 } }; + return .{ .stack_offset = .{ .value = offset } }; } fn emitMemoryCopy(cg: *CodeGen, dst: WValue, src: WValue, len: WValue) !void { @@ -1171,8 +1176,8 @@ fn emitMemoryCopy(cg: *CodeGen, dst: WValue, src: WValue, len: WValue) !void { try cg.addLabel(.br_if, 0); } - try cg.lowerToStack(dst); - try cg.lowerToStack(src); + try cg.emitWValue(dst); + try cg.emitWValue(src); try cg.emitWValue(len); try cg.addExtended(.memory_copy); @@ -1187,8 +1192,8 @@ fn memcpy(cg: *CodeGen, dst: WValue, src: WValue, len: WValue) !void { return; } - try cg.lowerToStack(dst); - try cg.lowerToStack(src); + try cg.emitWValue(dst); + try cg.emitWValue(src); try cg.emitWValue(len); try cg.addCallIntrinsic(.memcpy); try cg.addTag(.drop); @@ -1200,8 +1205,8 @@ fn memmove(cg: *CodeGen, dst: WValue, src: WValue, len: WValue) !void { return; } - try cg.lowerToStack(dst); - try cg.lowerToStack(src); + try cg.emitWValue(dst); + try cg.emitWValue(src); try cg.emitWValue(len); try cg.addCallIntrinsic(.memmove); try cg.addTag(.drop); @@ -1283,34 +1288,6 @@ pub fn determineSimdStoreStrategy(ty: Type, zcu: *const Zcu, target: *const std. return .unrolled; } -/// Creates a new local for a pointer that points to memory with given offset. -/// This can be used to get a pointer to a struct field, error payload, etc. -/// By providing `modify` as action, it will modify the given `ptr_value` instead of making a new -/// local value to store the pointer. This allows for local re-use and improves binary size. -fn buildPointerOffset(cg: *CodeGen, ptr_value: WValue, offset: u64, action: enum { modify, new }) InnerError!WValue { - // do not perform arithmetic when offset is 0. - if (offset == 0 and ptr_value.offset() == 0 and action == .modify) return ptr_value; - const result_ptr: WValue = switch (action) { - .new => try cg.ensureAllocLocal(Type.usize), - .modify => ptr_value, - }; - try cg.emitWValue(ptr_value); - if (offset + ptr_value.offset() > 0) { - switch (cg.ptr_size) { - .wasm32 => { - try cg.addImm32(@intCast(offset + ptr_value.offset())); - try cg.addTag(.i32_add); - }, - .wasm64 => { - try cg.addImm64(offset + ptr_value.offset()); - try cg.addTag(.i64_add); - }, - } - } - try cg.addLocal(.local_set, result_ptr.local.value); - return result_ptr; -} - fn genInst(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { const zcu = cg.pt.zcu; const air_tags = cg.air.instructions.items(.tag); @@ -1331,7 +1308,7 @@ fn genInst(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { const elem_ty = cg.typeOf(bin_op.rhs); const elem_size = elem_ty.abiSize(zcu); - try cg.lowerToStack(vec_ptr); + try cg.emitWValue(vec_ptr); try cg.emitWValue(elem_idx); try cg.addImm32(@intCast(elem_size)); try cg.addTag(.i32_mul); @@ -2108,7 +2085,7 @@ fn airCall(cg: *CodeGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier) } else .none; if (first_param_sret) { - try cg.lowerToStack(sret); + try cg.emitWValue(sret); } for (args, 0..) |arg, arg_i| { @@ -2122,7 +2099,7 @@ fn airCall(cg: *CodeGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier) } if (fn_info.is_var_args) { - try cg.lowerToStack(varargs_buf); + try cg.emitWValue(varargs_buf); } if (callee) |nav_index| { @@ -2243,8 +2220,8 @@ fn airVaArg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { try cg.addLocal(.local_set, arg_ptr.local.value); - try cg.lowerToStack(operand); - try cg.lowerToStack(arg_ptr); + try cg.emitWValue(operand); + try cg.emitWValue(arg_ptr); switch (cg.ptr_size) { .wasm32 => { try cg.addImm32(abi_size); @@ -2325,28 +2302,22 @@ fn store(cg: *CodeGen, lhs: WValue, rhs: WValue, ty: Type, memarg: MemArg) Inner natural_alignment.minStrict(memarg.alignment); if (isByRef(ty, zcu, cg.target)) { - const offset_ptr: WValue = switch (memarg.offset + lhs.offset()) { - 0 => lhs, - else => |total_offset| ptr: { - try cg.emitWValue(lhs); - try cg.addImm32(@intCast(total_offset)); - try cg.addTag(.i32_add); - break :ptr .stack; - }, - }; + const offset_ptr = try cg.offsetPointer(lhs, memarg.offset); return cg.memcpy(offset_ptr, rhs, .{ .imm32 = @intCast(abi_size) }); } + const base_offset = lhs.memOffset(); + try cg.emitMemBase(lhs); + if (ty.zigTypeTag(zcu) == .vector) { - try cg.emitWValue(lhs); - try cg.lowerToStack(rhs); + try cg.emitWValue(rhs); // TODO: Add helper functions for simd opcodes const extra_index: u32 = @intCast(cg.mir_extra.items.len); // stores as := opcode, offset, alignment (opcode::memarg) try cg.mir_extra.ensureUnusedCapacity(cg.gpa, 4); cg.mir_extra.appendAssumeCapacity(@backingInt(std.wasm.SimdOpcode.v128_store)); _ = try cg.addExtraAssumeCapacity(Mir.MemArg{ - .offset = memarg.offset + lhs.offset(), + .offset = base_offset + memarg.offset, .alignment = alignment.toStdMem(), }); return cg.addInst(.{ .tag = .simd_prefix, .data = .{ .payload = extra_index } }); @@ -2371,13 +2342,12 @@ fn store(cg: *CodeGen, lhs: WValue, rhs: WValue, ty: Type, memarg: MemArg) Inner } }; - try cg.emitWValue(lhs); - try cg.lowerToStack(rhs); + try cg.emitWValue(rhs); try cg.addMemArg( store_opcode, .{ - .offset = memarg.offset + lhs.offset(), + .offset = base_offset + memarg.offset, .alignment = alignment.toStdMem(), }, ); @@ -2421,23 +2391,18 @@ fn load(cg: *CodeGen, operand: WValue, ty: Type, memarg: MemArg) InnerError!WVal natural_alignment.minStrict(memarg.alignment); if (isByRef(ty, zcu, cg.target)) { - const src_ptr_maybe_stack: WValue = switch (memarg.offset + operand.offset()) { - 0 => operand, - else => |total_offset| ptr: { - try cg.emitWValue(operand); - try cg.addImm32(@intCast(total_offset)); - try cg.addTag(.i32_add); - break :ptr .stack; - }, - }; - const src_ptr = try src_ptr_maybe_stack.toLocal(cg, .usize); + var src_ptr = try cg.offsetPointer(operand, memarg.offset); + const owns_src_ptr = src_ptr == .stack; + src_ptr = try src_ptr.toLocal(cg, .usize); + defer if (owns_src_ptr) src_ptr.free(cg); + const new_ptr = try cg.allocStack(ty); - try cg.store(new_ptr, src_ptr, ty, .{}); + try cg.memcpy(new_ptr, src_ptr, .{ .imm32 = @intCast(ty.abiSize(zcu)) }); return new_ptr; } - // load local's value from memory by its stack position - try cg.emitWValue(operand); + const base_offset = operand.memOffset(); + try cg.emitMemBase(operand); if (ty.zigTypeTag(zcu) == .vector) { // TODO: Add helper functions for simd opcodes @@ -2446,7 +2411,7 @@ fn load(cg: *CodeGen, operand: WValue, ty: Type, memarg: MemArg) InnerError!WVal try cg.mir_extra.ensureUnusedCapacity(cg.gpa, 4); cg.mir_extra.appendAssumeCapacity(@backingInt(std.wasm.SimdOpcode.v128_load)); _ = try cg.addExtraAssumeCapacity(Mir.MemArg{ - .offset = memarg.offset + operand.offset(), + .offset = base_offset + memarg.offset, .alignment = alignment.toStdMem(), }); try cg.addInst(.{ .tag = .simd_prefix, .data = .{ .payload = extra_index } }); @@ -2477,7 +2442,7 @@ fn load(cg: *CodeGen, operand: WValue, ty: Type, memarg: MemArg) InnerError!WVal try cg.addMemArg( load_opcode, .{ - .offset = memarg.offset + operand.offset(), + .offset = base_offset + memarg.offset, .alignment = alignment.toStdMem(), }, ); @@ -2647,9 +2612,9 @@ fn intAdd(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue else => { const result = try cg.allocInt(ty); - try cg.lowerToStack(result); - try cg.lowerToStack(lhs); - try cg.lowerToStack(rhs); + try cg.emitWValue(result); + try cg.emitWValue(lhs); + try cg.emitWValue(rhs); try cg.addImm32(@intFromBool(ty.is_signed)); try cg.addImm32(ty.bits); try cg.addCallIntrinsic(.__addo_limb64); @@ -2700,9 +2665,9 @@ fn intSub(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue else => { const result = try cg.allocInt(ty); - try cg.lowerToStack(result); - try cg.lowerToStack(lhs); - try cg.lowerToStack(rhs); + try cg.emitWValue(result); + try cg.emitWValue(lhs); + try cg.emitWValue(rhs); try cg.addImm32(@intFromBool(ty.is_signed)); try cg.addImm32(ty.bits); try cg.addCallIntrinsic(.__subo_limb64); @@ -2732,9 +2697,9 @@ fn intMul(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue else => { const result = try cg.allocInt(ty); - try cg.lowerToStack(result); - try cg.lowerToStack(lhs); - try cg.lowerToStack(rhs); + try cg.emitWValue(result); + try cg.emitWValue(lhs); + try cg.emitWValue(rhs); try cg.addImm32(@intFromBool(ty.is_signed)); try cg.addImm32(ty.bits); try cg.addCallIntrinsic(.__mulo_limb64); @@ -3041,9 +3006,9 @@ fn intAnd(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue else => { const result = try cg.allocInt(ty); - try cg.lowerToStack(result); - try cg.lowerToStack(lhs); - try cg.lowerToStack(rhs); + try cg.emitWValue(result); + try cg.emitWValue(lhs); + try cg.emitWValue(rhs); try cg.addImm32(ty.bits); try cg.addCallIntrinsic(.__and_limb64); @@ -3085,9 +3050,9 @@ fn intOr(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue else => { const result = try cg.allocInt(ty); - try cg.lowerToStack(result); - try cg.lowerToStack(lhs); - try cg.lowerToStack(rhs); + try cg.emitWValue(result); + try cg.emitWValue(lhs); + try cg.emitWValue(rhs); try cg.addImm32(ty.bits); try cg.addCallIntrinsic(.__or_limb64); @@ -3129,9 +3094,9 @@ fn intXor(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue else => { const result = try cg.allocInt(ty); - try cg.lowerToStack(result); - try cg.lowerToStack(lhs); - try cg.lowerToStack(rhs); + try cg.emitWValue(result); + try cg.emitWValue(lhs); + try cg.emitWValue(rhs); try cg.addImm32(ty.bits); try cg.addCallIntrinsic(.__xor_limb64); @@ -3176,13 +3141,14 @@ fn intNot(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue { 65...128 => { const result = try cg.allocStack(Type.u128); - try cg.emitWValue(result); + const result_offset = result.memOffset(); + try cg.emitMemBase(result); _ = try cg.load(operand, .u64, .{}); try cg.addImm64(~@as(u64, 0)); try cg.addTag(.i64_xor); - try cg.store(.stack, .stack, .u64, .{ .offset = result.offset() }); + try cg.store(.stack, .stack, .u64, .{ .offset = result_offset }); - try cg.emitWValue(result); + try cg.emitMemBase(result); _ = try cg.load(operand, .u64, .{ .offset = 8 }); const high_mask: u64 = if (ty.is_signed) ~@as(u64, 0) @@ -3190,15 +3156,15 @@ fn intNot(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue { ~@as(u64, 0) >> @intCast(128 - ty.bits); try cg.addImm64(high_mask); try cg.addTag(.i64_xor); - try cg.store(.stack, .stack, .u64, .{ .offset = result.offset() + 8 }); + try cg.store(.stack, .stack, .u64, .{ .offset = result_offset + 8 }); return result; }, else => { const result = try cg.allocInt(ty); - try cg.lowerToStack(result); - try cg.lowerToStack(operand); + try cg.emitWValue(result); + try cg.emitWValue(operand); try cg.addImm32(@intFromBool(ty.is_signed)); try cg.addImm32(ty.bits); try cg.addCallIntrinsic(.__not_limb64); @@ -3230,9 +3196,9 @@ fn intShl(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue else => { const result = try cg.allocInt(ty); - try cg.lowerToStack(result); - try cg.lowerToStack(lhs); - try cg.lowerToStack(rhs); + try cg.emitWValue(result); + try cg.emitWValue(lhs); + try cg.emitWValue(rhs); try cg.addImm32(@intFromBool(ty.is_signed)); try cg.addImm32(ty.bits); try cg.addCallIntrinsic(.__shlo_limb64); @@ -3270,9 +3236,9 @@ fn intShr(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue else => { const result = try cg.allocInt(ty); - try cg.lowerToStack(result); - try cg.lowerToStack(lhs); - try cg.lowerToStack(rhs); + try cg.emitWValue(result); + try cg.emitWValue(lhs); + try cg.emitWValue(rhs); try cg.addImm32(@intFromBool(ty.is_signed)); try cg.addImm32(ty.bits); try cg.addCallIntrinsic(.__shr_limb64); @@ -3320,8 +3286,9 @@ fn intAbs(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue { const u128_ty: IntType = .{ .is_signed = false, .bits = 128 }; const mask = try cg.allocStack(Type.u128); - try cg.emitWValue(mask); - try cg.emitWValue(mask); + const mask_offset = mask.memOffset(); + try cg.emitMemBase(mask); + try cg.emitMemBase(mask); _ = try cg.load(operand, .u64, .{ .offset = 8 }); try cg.addImm64(63); @@ -3330,9 +3297,9 @@ fn intAbs(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue { var tmp = try cg.allocLocal(Type.u64); defer tmp.free(cg); try cg.addLocal(.local_tee, tmp.local.value); - try cg.store(.stack, .stack, .u64, .{ .offset = mask.offset() + 0 }); + try cg.store(.stack, .stack, .u64, .{ .offset = mask_offset }); try cg.emitWValue(tmp); - try cg.store(.stack, .stack, .u64, .{ .offset = mask.offset() + 8 }); + try cg.store(.stack, .stack, .u64, .{ .offset = mask_offset + 8 }); const a = try cg.intXor(u128_ty, operand, mask); const b = try cg.intSub(u128_ty, a, mask); @@ -3341,8 +3308,8 @@ fn intAbs(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue { else => { const result = try cg.allocInt(ty); - try cg.lowerToStack(result); - try cg.lowerToStack(operand); + try cg.emitWValue(result); + try cg.emitWValue(operand); try cg.addImm32(ty.bits); try cg.addCallIntrinsic(.__abs_limb64); @@ -3352,16 +3319,16 @@ fn intAbs(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue { } fn intMax(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue { - try cg.lowerToStack(lhs); - try cg.lowerToStack(rhs); + try cg.emitWValue(lhs); + try cg.emitWValue(rhs); _ = try cg.intCmp(ty, .gt, lhs, rhs); try cg.addTag(.select); return .stack; } fn intMin(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue { - try cg.lowerToStack(lhs); - try cg.lowerToStack(rhs); + try cg.emitWValue(lhs); + try cg.emitWValue(rhs); _ = try cg.intCmp(ty, .lt, lhs, rhs); try cg.addTag(.select); return .stack; @@ -3427,7 +3394,7 @@ fn intClz(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue { return .stack; }, else => { - try cg.lowerToStack(operand); + try cg.emitWValue(operand); try cg.addImm32(ty.bits); try cg.addCallIntrinsic(.__clz_limb64); @@ -3479,7 +3446,7 @@ fn intCtz(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue { return .stack; }, else => { - try cg.lowerToStack(operand); + try cg.emitWValue(operand); try cg.addImm32(ty.bits); try cg.addCallIntrinsic(.__ctz_limb64); @@ -3525,7 +3492,7 @@ fn intPopCount(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue { return .stack; }, else => { - try cg.lowerToStack(operand); + try cg.emitWValue(operand); try cg.addImm32(ty.bits); try cg.addCallIntrinsic(.__popcount_limb64); @@ -3560,7 +3527,8 @@ fn intBitReverse(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue { 65...128 => { const tmp = try cg.allocStack(Type.u128); - try cg.emitWValue(tmp); + const tmp_offset = tmp.memOffset(); + try cg.emitMemBase(tmp); const hi = try cg.load(operand, .u64, .{ .offset = 8 }); const hi_rev = try cg.callIntrinsic( .__bitreversedi2, @@ -3569,9 +3537,9 @@ fn intBitReverse(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue { &.{hi}, ); try cg.emitWValue(hi_rev); - try cg.store(.stack, .stack, .u64, .{ .offset = tmp.offset() }); + try cg.store(.stack, .stack, .u64, .{ .offset = tmp_offset }); - try cg.emitWValue(tmp); + try cg.emitMemBase(tmp); const lo = try cg.load(operand, .u64, .{}); const lo_rev = try cg.callIntrinsic( .__bitreversedi2, @@ -3580,7 +3548,7 @@ fn intBitReverse(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue { &.{lo}, ); try cg.emitWValue(lo_rev); - try cg.store(.stack, .stack, .u64, .{ .offset = tmp.offset() + 8 }); + try cg.store(.stack, .stack, .u64, .{ .offset = tmp_offset + 8 }); if (ty.bits < 128) { const shift_ty: IntType = .{ .is_signed = ty.is_signed, .bits = 128 }; @@ -3592,8 +3560,8 @@ fn intBitReverse(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue { else => { const result = try cg.allocInt(ty); - try cg.lowerToStack(result); - try cg.lowerToStack(operand); + try cg.emitWValue(result); + try cg.emitWValue(operand); try cg.addImm32(@intFromBool(ty.is_signed)); try cg.addImm32(ty.bits); try cg.addCallIntrinsic(.__bitreverse_limb64); @@ -3629,7 +3597,8 @@ fn intByteSwap(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue { 65...128 => { const result = try cg.allocStack(Type.u128); - try cg.emitWValue(result); + const result_offset = result.memOffset(); + try cg.emitMemBase(result); const low = try cg.load(operand, .u64, .{}); const swap_low = try cg.callIntrinsic( @@ -3638,9 +3607,9 @@ fn intByteSwap(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue { Type.u64, &.{low}, ); - try cg.store(.stack, swap_low, .u64, .{ .offset = result.offset() + 8 }); + try cg.store(.stack, swap_low, .u64, .{ .offset = result_offset + 8 }); - try cg.emitWValue(result); + try cg.emitMemBase(result); const high = try cg.load(operand, .u64, .{ .offset = 8 }); const swap_high = try cg.callIntrinsic( @@ -3649,7 +3618,7 @@ fn intByteSwap(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue { Type.u64, &.{high}, ); - try cg.store(.stack, swap_high, .u64, .{ .offset = result.offset() }); + try cg.store(.stack, swap_high, .u64, .{ .offset = result_offset }); if (ty.bits < 128) { const shift_ty: IntType = .{ .is_signed = ty.is_signed, .bits = 128 }; @@ -3661,8 +3630,8 @@ fn intByteSwap(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue { else => { const result = try cg.allocInt(ty); - try cg.lowerToStack(result); - try cg.lowerToStack(operand); + try cg.emitWValue(result); + try cg.emitWValue(operand); try cg.addImm32(@intFromBool(ty.is_signed)); try cg.addImm32(ty.bits); try cg.addCallIntrinsic(.__byteswap_limb64); @@ -3706,11 +3675,12 @@ fn intWrap(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue { 65...127 => { const result = try cg.allocStack(Type.u128); - try cg.emitWValue(result); + const result_offset = result.memOffset(); + try cg.emitMemBase(result); _ = try cg.load(operand, .u64, .{}); - try cg.store(.stack, .stack, .u64, .{ .offset = result.offset() }); + try cg.store(.stack, .stack, .u64, .{ .offset = result_offset }); - try cg.emitWValue(result); + try cg.emitMemBase(result); _ = try cg.load(operand, .u64, .{ .offset = 8 }); if (ty.is_signed) { try cg.addImm64(128 - ty.bits); @@ -3721,7 +3691,7 @@ fn intWrap(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue { try cg.addImm64(~@as(u64, 0) >> @intCast(128 - ty.bits)); try cg.addTag(.i64_and); } - try cg.store(.stack, .stack, .u64, .{ .offset = result.offset() + 8 }); + try cg.store(.stack, .stack, .u64, .{ .offset = result_offset + 8 }); return result; }, @@ -3738,7 +3708,7 @@ fn intWrap(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue { try cg.memcpy(result, operand, .{ .imm32 = used_len - 8 }); const pad = 64 - ty.bits % 64; - try cg.emitWValue(result); + try cg.emitMemBase(result); _ = try cg.load(operand, .u64, .{ .offset = used_len - 8 }); if (ty.is_signed) { try cg.addImm64(pad); @@ -3749,14 +3719,14 @@ fn intWrap(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue { try cg.addImm64(~@as(u64, 0) >> @intCast(pad)); try cg.addTag(.i64_and); } - try cg.store(.stack, .stack, .u64, .{ .offset = result.offset() + used_len - 8 }); + try cg.store(.stack, .stack, .u64, .{ .offset = result.memOffset() + used_len - 8 }); } else { try cg.memcpy(result, operand, .{ .imm32 = used_len }); } const full_len = @divExact(bits, 8); if (used_len + 8 == full_len) { // last limb needs sign extended - try cg.emitWValue(result); + try cg.emitMemBase(result); if (ty.is_signed) { _ = try cg.load(result, .u64, .{ .offset = used_len - 8 }); try cg.addImm64(63); @@ -3764,7 +3734,7 @@ fn intWrap(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue { } else { try cg.addImm64(0); } - try cg.store(.stack, .stack, .u64, .{ .offset = result.offset() + used_len }); + try cg.store(.stack, .stack, .u64, .{ .offset = result.memOffset() + used_len }); } return result; @@ -3858,20 +3828,20 @@ fn intAddSat(cg: *CodeGen, int_ty: IntType, lhs: WValue, rhs: WValue) InnerError defer rhs_is_neg.free(cg); const min_val = try cg.intMinValue(int_ty); - try cg.lowerToStack(min_val); - try cg.lowerToStack(max_val); + try cg.emitWValue(min_val); + try cg.emitWValue(max_val); try cg.emitWValue(rhs_is_neg); try cg.addTag(.select); - try cg.lowerToStack(op_val); + try cg.emitWValue(op_val); const overflow_cmp = try cg.intCmp(int_ty, .lt, op_val, lhs); const is_overflow = try cg.intCmp(.u32, .neq, rhs_is_neg, overflow_cmp); try cg.emitWValue(is_overflow); try cg.addTag(.select); return .stack; } else { - try cg.lowerToStack(max_val); - try cg.lowerToStack(op_val); + try cg.emitWValue(max_val); + try cg.emitWValue(op_val); const is_overflow = try cg.intCmp(int_ty, .lt, op_val, lhs); try cg.emitWValue(is_overflow); @@ -3892,12 +3862,12 @@ fn intSubSat(cg: *CodeGen, int_ty: IntType, lhs: WValue, rhs: WValue) InnerError const max_val = try cg.intMaxValue(int_ty); const min_val = try cg.intMinValue(int_ty); - try cg.lowerToStack(max_val); - try cg.lowerToStack(min_val); + try cg.emitWValue(max_val); + try cg.emitWValue(min_val); try cg.emitWValue(rhs_is_neg); try cg.addTag(.select); - try cg.lowerToStack(op_val); + try cg.emitWValue(op_val); const overflow_cmp = try cg.intCmp(int_ty, .gt, op_val, lhs); const is_overflow = try cg.intCmp(.u32, .neq, rhs_is_neg, overflow_cmp); try cg.emitWValue(is_overflow); @@ -3906,8 +3876,8 @@ fn intSubSat(cg: *CodeGen, int_ty: IntType, lhs: WValue, rhs: WValue) InnerError } else { const zero = try cg.intZeroValue(int_ty); - try cg.lowerToStack(zero); - try cg.lowerToStack(op_val); + try cg.emitWValue(zero); + try cg.emitWValue(op_val); const is_overflow = try cg.intCmp(int_ty, .lt, lhs, rhs); try cg.emitWValue(is_overflow); try cg.addTag(.select); @@ -3931,10 +3901,10 @@ fn intMulSat(cg: *CodeGen, int_ty: IntType, lhs: WValue, rhs: WValue) InnerError if (int_ty.is_signed) { const min_val = try cg.intMinValue(int_ty); - try cg.lowerToStack(min_val); + try cg.emitWValue(min_val); - try cg.lowerToStack(max_val); - try cg.lowerToStack(op_val); + try cg.emitWValue(max_val); + try cg.emitWValue(op_val); const max_ext = try cg.intCast(ext_ty, int_ty, max_val); const ov_pos = try cg.intCmp(ext_ty, .lt, max_ext, mul_ext); try cg.emitWValue(ov_pos); @@ -3942,12 +3912,12 @@ fn intMulSat(cg: *CodeGen, int_ty: IntType, lhs: WValue, rhs: WValue) InnerError const min_ext = try cg.intCast(ext_ty, int_ty, min_val); const ov_neg = try cg.intCmp(ext_ty, .gt, min_ext, mul_ext); - try cg.lowerToStack(ov_neg); + try cg.emitWValue(ov_neg); try cg.addTag(.select); return .stack; } else { - try cg.lowerToStack(max_val); - try cg.lowerToStack(op_val); + try cg.emitWValue(max_val); + try cg.emitWValue(op_val); const max_ext = try cg.intCast(ext_ty, int_ty, max_val); const is_overflow = try cg.intCmp(ext_ty, .lt, max_ext, mul_ext); try cg.emitWValue(is_overflow); @@ -3970,20 +3940,20 @@ fn intShlSat(cg: *CodeGen, int_ty: IntType, lhs: WValue, rhs: WValue) InnerError const zero = try cg.intZeroValue(int_ty); const min_val = try cg.intMinValue(int_ty); - try cg.lowerToStack(min_val); - try cg.lowerToStack(max_val); + try cg.emitWValue(min_val); + try cg.emitWValue(max_val); const lhs_is_neg = try cg.intCmp(int_ty, .lt, lhs, zero); try cg.emitWValue(lhs_is_neg); try cg.addTag(.select); - try cg.lowerToStack(op_val); + try cg.emitWValue(op_val); const is_overflow = try cg.intCmp(int_ty, .neq, check_val, lhs); try cg.emitWValue(is_overflow); try cg.addTag(.select); return .stack; } else { - try cg.lowerToStack(max_val); - try cg.lowerToStack(op_val); + try cg.emitWValue(max_val); + try cg.emitWValue(op_val); const is_overflow = try cg.intCmp(int_ty, .neq, check_val, lhs); try cg.emitWValue(is_overflow); try cg.addTag(.select); @@ -4055,9 +4025,9 @@ fn intAddOverflow(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerErro else => { const result = try cg.allocInt(ty); - try cg.lowerToStack(result); - try cg.lowerToStack(lhs); - try cg.lowerToStack(rhs); + try cg.emitWValue(result); + try cg.emitWValue(lhs); + try cg.emitWValue(rhs); try cg.addImm32(@intFromBool(ty.is_signed)); try cg.addImm32(ty.bits); try cg.addCallIntrinsic(.__addo_limb64); @@ -4087,9 +4057,9 @@ fn intSubOverflow(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerErro else => { const result = try cg.allocInt(ty); - try cg.lowerToStack(result); - try cg.lowerToStack(lhs); - try cg.lowerToStack(rhs); + try cg.emitWValue(result); + try cg.emitWValue(lhs); + try cg.emitWValue(rhs); try cg.addImm32(@intFromBool(ty.is_signed)); try cg.addImm32(ty.bits); try cg.addCallIntrinsic(.__subo_limb64); @@ -4145,9 +4115,9 @@ fn intMulOverflow(cg: *CodeGen, int_ty: IntType, lhs: WValue, rhs: WValue) Inner } else { const result = try cg.allocInt(int_ty); - try cg.lowerToStack(result); - try cg.lowerToStack(lhs); - try cg.lowerToStack(rhs); + try cg.emitWValue(result); + try cg.emitWValue(lhs); + try cg.emitWValue(rhs); try cg.addImm32(@intFromBool(int_ty.is_signed)); try cg.addImm32(int_ty.bits); try cg.addCallIntrinsic(.__mulo_limb64); @@ -4174,9 +4144,9 @@ fn intShlOverflow(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerErro else => { const result = try cg.allocInt(ty); - try cg.lowerToStack(result); - try cg.lowerToStack(lhs); - try cg.lowerToStack(rhs); + try cg.emitWValue(result); + try cg.emitWValue(lhs); + try cg.emitWValue(rhs); try cg.addImm32(@intFromBool(ty.is_signed)); try cg.addImm32(ty.bits); try cg.addCallIntrinsic(.__shlo_limb64); @@ -4213,15 +4183,15 @@ fn intCast(cg: *CodeGen, dest_ty: IntType, src_ty: IntType, operand: WValue) Inn } else { var src_len: u32 = undefined; if (src_bits == 32) { - try cg.emitWValue(result); + try cg.emitMemBase(result); try cg.emitWValue(operand); try cg.addTag(if (src_ty.is_signed) .i64_extend_i32_s else .i64_extend_i32_u); - try cg.store(.stack, .stack, .u64, .{ .offset = result.offset() }); + try cg.store(.stack, .stack, .u64, .{ .offset = result.memOffset() }); src_len = 8; } else if (src_bits == 64) { - try cg.emitWValue(result); + try cg.emitMemBase(result); try cg.emitWValue(operand); - try cg.store(.stack, .stack, .u64, .{ .offset = result.offset() }); + try cg.store(.stack, .stack, .u64, .{ .offset = result.memOffset() }); src_len = 8; } else { src_len = src_bits / 8; @@ -4230,7 +4200,7 @@ fn intCast(cg: *CodeGen, dest_ty: IntType, src_ty: IntType, operand: WValue) Inn if (dest_bits == 128) { if (src_ty.is_signed) { - try cg.emitWValue(result); + try cg.emitMemBase(result); if (src_bits == 32) { try cg.emitWValue(operand); try cg.addTag(if (dest_ty.is_signed) .i64_extend_i32_s else .i64_extend_i32_u); @@ -4238,7 +4208,7 @@ fn intCast(cg: *CodeGen, dest_ty: IntType, src_ty: IntType, operand: WValue) Inn try cg.emitWValue(operand); } else unreachable; const shr = try cg.intShr(IntType.i64, .stack, .{ .imm32 = 63 }); - try cg.store(.stack, shr, .u64, .{ .offset = 8 + result.offset() }); + try cg.store(.stack, shr, .u64, .{ .offset = result.memOffset() + 8 }); } else { try cg.store(result, .{ .imm64 = 0 }, .u64, .{ .offset = 8 }); } @@ -4642,20 +4612,22 @@ fn floatNeg(cg: *CodeGen, ty: FloatType, arg: WValue) InnerError!WValue { }, .f80, .f128 => { const result = try cg.allocStack(Type.f128); - try cg.emitWValue(result); - try cg.emitWValue(arg); - try cg.addMemArg(.i64_load, .{ .offset = 0 + arg.offset(), .alignment = .@"2" }); - try cg.addMemArg(.i64_store, .{ .offset = 0 + result.offset(), .alignment = .@"2" }); - try cg.emitWValue(result); - try cg.emitWValue(arg); - try cg.addMemArg(.i64_load, .{ .offset = 8 + arg.offset(), .alignment = .@"2" }); + const result_offset = result.memOffset(); + const arg_offset = arg.memOffset(); + try cg.emitMemBase(result); + try cg.emitMemBase(arg); + try cg.addMemArg(.i64_load, .{ .offset = arg_offset, .alignment = .@"2" }); + try cg.addMemArg(.i64_store, .{ .offset = result_offset, .alignment = .@"2" }); + try cg.emitMemBase(result); + try cg.emitMemBase(arg); + try cg.addMemArg(.i64_load, .{ .offset = arg_offset + 8, .alignment = .@"2" }); if (ty == .f80) { try cg.addImm64(0x8000); } else { try cg.addImm64(0x8000000000000000); } try cg.addTag(.i64_xor); - try cg.addMemArg(.i64_store, .{ .offset = 8 + result.offset(), .alignment = .@"2" }); + try cg.addMemArg(.i64_store, .{ .offset = result_offset + 8, .alignment = .@"2" }); return result; }, } @@ -4681,20 +4653,22 @@ fn floatAbs(cg: *CodeGen, ty: FloatType, arg: WValue) InnerError!WValue { }, .f80, .f128 => { const result = try cg.allocStack(Type.f128); - try cg.emitWValue(result); - try cg.emitWValue(arg); - try cg.addMemArg(.i64_load, .{ .offset = 0 + arg.offset(), .alignment = .@"2" }); - try cg.addMemArg(.i64_store, .{ .offset = 0 + result.offset(), .alignment = .@"2" }); - try cg.emitWValue(result); - try cg.emitWValue(arg); - try cg.addMemArg(.i64_load, .{ .offset = 8 + arg.offset(), .alignment = .@"2" }); + const result_offset = result.memOffset(); + const arg_offset = arg.memOffset(); + try cg.emitMemBase(result); + try cg.emitMemBase(arg); + try cg.addMemArg(.i64_load, .{ .offset = arg_offset, .alignment = .@"2" }); + try cg.addMemArg(.i64_store, .{ .offset = result_offset, .alignment = .@"2" }); + try cg.emitMemBase(result); + try cg.emitMemBase(arg); + try cg.addMemArg(.i64_load, .{ .offset = arg_offset + 8, .alignment = .@"2" }); if (ty == .f80) { try cg.addImm64(0x7FFF); } else { try cg.addImm64(0x7FFFFFFFFFFFFFFF); } try cg.addTag(.i64_and); - try cg.addMemArg(.i64_store, .{ .offset = 8 + result.offset(), .alignment = .@"2" }); + try cg.addMemArg(.i64_store, .{ .offset = result_offset + 8, .alignment = .@"2" }); return result; }, } @@ -5377,8 +5351,8 @@ fn intCmp(cg: *CodeGen, ty: IntType, op: std.math.CompareOperator, lhs: WValue, 0 => unreachable, 1...32 => { // lhs or rhs could be stack pointers - try cg.lowerToStack(lhs); - try cg.lowerToStack(rhs); + try cg.emitWValue(lhs); + try cg.emitWValue(rhs); const opcode: Mir.Inst.Tag = switch (op) { .eq => .i32_eq, .neq => .i32_ne, @@ -5392,8 +5366,8 @@ fn intCmp(cg: *CodeGen, ty: IntType, op: std.math.CompareOperator, lhs: WValue, }, 33...64 => { // lhs or rhs could be stack pointers - try cg.lowerToStack(lhs); - try cg.lowerToStack(rhs); + try cg.emitWValue(lhs); + try cg.emitWValue(rhs); const opcode: Mir.Inst.Tag = switch (op) { .eq => .i64_eq, .neq => .i64_ne, @@ -5442,8 +5416,8 @@ fn intCmp(cg: *CodeGen, ty: IntType, op: std.math.CompareOperator, lhs: WValue, return .stack; }, else => { - try cg.lowerToStack(lhs); - try cg.lowerToStack(rhs); + try cg.emitWValue(lhs); + try cg.emitWValue(rhs); try cg.addImm32(@intFromBool(ty.is_signed)); try cg.addImm32(ty.bits); try cg.addCallIntrinsic(.__cmp_limb64); @@ -5554,7 +5528,7 @@ fn airBr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { // if operand has codegen bits we should break with a value if (block.value != .none) { const operand = try cg.resolveInst(br.operand); - try cg.lowerToStack(operand); + try cg.emitWValue(operand); try cg.addLocal(.local_set, block.value.local.value); } @@ -5618,11 +5592,12 @@ fn airIntFromPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { assert(operand_ty.abiAlignment(zcu) == dest_ty.abiAlignment(zcu)); const operand = try cg.resolveInst(ty_op.operand); - const result = switch (operand) { - .stack_offset => try cg.buildPointerOffset(operand, 0, .new), - else => operand, - }; - return cg.finishAir(inst, result, &.{ty_op.operand}); + if (operand == .stack_offset) { + // materialize operand, we assume precomputed value is more efficient + try cg.emitWValue(operand); + return cg.finishAir(inst, .stack, &.{ty_op.operand}); + } + return cg.finishAir(inst, operand, &.{ty_op.operand}); } fn airUnionFromEnum(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { @@ -5801,16 +5776,7 @@ fn structFieldPtr( }, else => struct_ty.structFieldOffset(index, zcu), }; - // save a load and store when we can simply reuse the operand - if (offset == 0) { - return struct_ptr; - } - switch (struct_ptr) { - .stack_offset => |stack_offset| { - return .{ .stack_offset = .{ .value = stack_offset.value + @as(u32, @intCast(offset)), .references = 1 } }; - }, - else => return cg.buildPointerOffset(struct_ptr, offset, .new), - } + return cg.offsetPointer(struct_ptr, offset); } fn airAggFieldVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { @@ -5832,12 +5798,7 @@ fn airAggFieldVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { return cg.fail("Field type '{f}' too big to fit into stack frame", .{field_ty.fmt(pt)}); }; if (isByRef(field_ty, zcu, cg.target)) { - switch (operand) { - .stack_offset => |stack_offset| { - break :result .{ .stack_offset = .{ .value = stack_offset.value + offset, .references = 1 } }; - }, - else => break :result try cg.buildPointerOffset(operand, offset, .new), - } + break :result try cg.offsetPointer(operand, offset); } break :result try cg.load(operand, field_ty, .{ .offset = offset }); }, @@ -5859,7 +5820,7 @@ fn airSwitchBr(cg: *CodeGen, inst: Air.Inst.Index, is_dispatch_loop: bool) Inner const target = if (is_dispatch_loop) target: { const initial_target = try cg.resolveInst(switch_br.operand); const target: WValue = try cg.allocLocal(target_ty); - try cg.lowerToStack(initial_target); + try cg.emitWValue(initial_target); try cg.addLocal(.local_set, target.local.value); try cg.startBlock(.loop, .empty); // dispatch loop start @@ -6052,7 +6013,7 @@ fn airSwitchDispatch(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { const switch_loop = cg.blocks.get(br.block_inst).?; const operand = try cg.resolveInst(br.operand); - try cg.lowerToStack(operand); + try cg.emitWValue(operand); try cg.addLocal(.local_set, switch_loop.value.local.value); const idx: u32 = cg.block_depth - switch_loop.label; @@ -6080,12 +6041,14 @@ fn airIsErr(cg: *CodeGen, inst: Air.Inst.Index, opcode: std.wasm.Opcode, op_kind } } - try cg.emitWValue(operand); if (op_kind == .ptr or pl_ty.hasRuntimeBits(zcu)) { + try cg.emitMemBase(operand); try cg.addMemArg(.i32_load16_u, .{ - .offset = operand.offset() + @as(u32, @intCast(errUnionErrorOffset(pl_ty, zcu))), + .offset = operand.memOffset() + errUnionErrorOffset(pl_ty, zcu), .alignment = Type.anyerror.abiAlignment(zcu).toStdMem(), }); + } else { + try cg.emitWValue(operand); } // Compare the error value with '0' @@ -6116,9 +6079,9 @@ fn airUnwrapErrUnionPayload(cg: *CodeGen, inst: Air.Inst.Index, op_is_ptr: bool) } } - const pl_offset: u32 = @intCast(errUnionPayloadOffset(payload_ty, zcu)); + const pl_offset = errUnionPayloadOffset(payload_ty, zcu); if (op_is_ptr or isByRef(payload_ty, zcu, cg.target)) { - break :result try cg.buildPointerOffset(operand, pl_offset, .new); + break :result try cg.offsetPointer(operand, pl_offset); } else { assert(isByRef(eu_ty, zcu, cg.target)); break :result try cg.load(operand, payload_ty, .{ .offset = pl_offset }); @@ -6169,15 +6132,14 @@ fn airWrapErrUnionPayload(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { } const err_union = try cg.allocStack(err_ty); - const payload_ptr = try cg.buildPointerOffset(err_union, @as(u32, @intCast(errUnionPayloadOffset(pl_ty, zcu))), .new); - try cg.store(payload_ptr, operand, pl_ty, .{}); + try cg.store(err_union, operand, pl_ty, .{ .offset = errUnionPayloadOffset(pl_ty, zcu) }); // ensure we also write '0' to the error part, so any present stack value gets overwritten by it. - try cg.emitWValue(err_union); + try cg.emitMemBase(err_union); try cg.addImm32(0); - const err_val_offset: u32 = @intCast(errUnionErrorOffset(pl_ty, zcu)); + const err_val_offset = errUnionErrorOffset(pl_ty, zcu); try cg.addMemArg(.i32_store16, .{ - .offset = err_union.offset() + err_val_offset, + .offset = err_union.memOffset() + err_val_offset, .alignment = .@"2", }); break :result err_union; @@ -6203,7 +6165,7 @@ fn airWrapErrUnionErr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { try cg.store(err_union, operand, .anyerror, .{ .offset = errUnionErrorOffset(pl_ty, zcu) }); // write 'undefined' to the payload - const payload_ptr = try cg.buildPointerOffset(err_union, @as(u32, @intCast(errUnionPayloadOffset(pl_ty, zcu))), .new); + const payload_ptr = try cg.offsetPointer(err_union, errUnionPayloadOffset(pl_ty, zcu)); const len = @as(u32, @intCast(err_ty.errorUnionPayload(zcu).abiSize(zcu))); try cg.memset(Type.u8, payload_ptr, .{ .imm32 = len }, .{ .imm32 = 0xaa }); @@ -6228,7 +6190,6 @@ fn airIsNull(cg: *CodeGen, inst: Air.Inst.Index, opcode: std.wasm.Opcode, op_kin fn isNull(cg: *CodeGen, operand: WValue, op_ty: Type, opcode: std.wasm.Opcode, op_kind: OpKind) InnerError!WValue { const pt = cg.pt; const zcu = pt.zcu; - try cg.emitWValue(operand); const optional_ty = switch (op_kind) { .value => op_ty, .ptr => op_ty.childType(zcu), @@ -6241,19 +6202,27 @@ fn isNull(cg: *CodeGen, operand: WValue, op_ty: Type, opcode: std.wasm.Opcode, o const offset = std.math.cast(u32, payload_ty.abiSize(zcu)) orelse { return cg.fail("Optional type {f} too big to fit into stack frame", .{optional_ty.fmt(pt)}); }; - try cg.addMemArg(.i32_load8_u, .{ .offset = operand.offset() + offset, .alignment = .@"1" }); + try cg.emitMemBase(operand); + try cg.addMemArg(.i32_load8_u, .{ .offset = operand.memOffset() + offset, .alignment = .@"1" }); + } else { + try cg.emitWValue(operand); } } else if (payload_ty.isSlice(zcu)) { + const base_offset = operand.memOffset(); + try cg.emitMemBase(operand); switch (cg.ptr_size) { - .wasm32 => try cg.addMemArg(.i32_load, .{ .offset = operand.offset(), .alignment = .@"4" }), - .wasm64 => try cg.addMemArg(.i64_load, .{ .offset = operand.offset(), .alignment = .@"8" }), + .wasm32 => try cg.addMemArg(.i32_load, .{ .offset = base_offset, .alignment = .@"4" }), + .wasm64 => try cg.addMemArg(.i64_load, .{ .offset = base_offset, .alignment = .@"8" }), } } else { if (op_kind == .ptr) { + try cg.emitMemBase(operand); try cg.addMemArg(.i32_load, .{ - .offset = operand.offset(), + .offset = operand.memOffset(), .alignment = .@"4", }); + } else { + try cg.emitWValue(operand); } } @@ -6278,7 +6247,7 @@ fn airOptionalPayload(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { if (opt_ty.optionalReprIsPayload(zcu)) break :result operand; if (isByRef(payload_ty, zcu, cg.target)) { - break :result try cg.buildPointerOffset(operand, 0, .new); + break :result operand; } break :result try cg.load(operand, payload_ty, .{}); @@ -6298,7 +6267,7 @@ fn airOptionalPayloadPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { break :result operand; } - break :result try cg.buildPointerOffset(operand, 0, .new); + break :result operand; }; return cg.finishAir(inst, result, &.{ty_op.operand}); } @@ -6319,12 +6288,11 @@ fn airOptionalPayloadPtrSet(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void return cg.fail("Optional type {f} too big to fit into stack frame", .{opt_ty.fmt(pt)}); }; - try cg.emitWValue(operand); + try cg.emitMemBase(operand); try cg.addImm32(1); - try cg.addMemArg(.i32_store8, .{ .offset = operand.offset() + offset, .alignment = .@"1" }); + try cg.addMemArg(.i32_store8, .{ .offset = operand.memOffset() + offset, .alignment = .@"1" }); - const result = try cg.buildPointerOffset(operand, 0, .new); - return cg.finishAir(inst, result, &.{ty_op.operand}); + return cg.finishAir(inst, operand, &.{ty_op.operand}); } fn airWrapOptional(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { @@ -6336,9 +6304,9 @@ fn airWrapOptional(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { const result = result: { if (!payload_ty.hasRuntimeBits(zcu)) { const non_null_bit = try cg.allocStack(Type.u1); - try cg.emitWValue(non_null_bit); + try cg.emitMemBase(non_null_bit); try cg.addImm32(1); - try cg.addMemArg(.i32_store8, .{ .offset = non_null_bit.offset(), .alignment = .@"1" }); + try cg.addMemArg(.i32_store8, .{ .offset = non_null_bit.memOffset(), .alignment = .@"1" }); break :result non_null_bit; } @@ -6353,12 +6321,11 @@ fn airWrapOptional(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { // Create optional type, set the non-null bit, and store the operand inside the optional type const result_ptr = try cg.allocStack(op_ty); - try cg.emitWValue(result_ptr); + try cg.emitMemBase(result_ptr); try cg.addImm32(1); - try cg.addMemArg(.i32_store8, .{ .offset = result_ptr.offset() + offset, .alignment = .@"1" }); + try cg.addMemArg(.i32_store8, .{ .offset = result_ptr.memOffset() + offset, .alignment = .@"1" }); - const payload_ptr = try cg.buildPointerOffset(result_ptr, 0, .new); - try cg.store(payload_ptr, operand, payload_ty, .{}); + try cg.store(result_ptr, operand, payload_ty, .{}); break :result result_ptr; }; @@ -6483,7 +6450,7 @@ fn airPtrElemVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { if (ptr_ty.isSlice(zcu)) { _ = try cg.load(ptr, .usize, .{}); } else { - try cg.lowerToStack(ptr); + try cg.emitWValue(ptr); } // calculate index into slice @@ -6513,7 +6480,7 @@ fn airPtrElemPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { if (ptr_ty.isSlice(zcu)) { _ = try cg.load(ptr, .usize, .{}); } else { - try cg.lowerToStack(ptr); + try cg.emitWValue(ptr); } // calculate index into ptr @@ -6538,7 +6505,7 @@ fn airPtrBinOp(cg: *CodeGen, inst: Air.Inst.Index, op: enum { add, sub }) InnerE else => ptr_ty.childType(zcu), }; - try cg.lowerToStack(ptr); + try cg.emitWValue(ptr); try cg.emitWValue(offset); switch (cg.ptr_size) { @@ -6620,7 +6587,7 @@ fn memset(cg: *CodeGen, elem_ty: Type, ptr: WValue, len: WValue, value: WValue) try cg.addLabel(.br_if, 0); } - try cg.lowerToStack(ptr); + try cg.emitWValue(ptr); try cg.emitWValue(value); try cg.emitWValue(len); try cg.addExtended(.memory_fill); @@ -6655,11 +6622,13 @@ fn memset(cg: *CodeGen, elem_ty: Type, ptr: WValue, len: WValue, value: WValue) var end_ptr = try cg.allocLocal(Type.usize); defer end_ptr.free(cg); - var new_ptr = try cg.buildPointerOffset(ptr, 0, .new); + var new_ptr = try cg.allocLocal(Type.usize); defer new_ptr.free(cg); + try cg.emitWValue(ptr); + try cg.addLocal(.local_set, new_ptr.local.value); // get the loop conditional: if current pointer address equals final pointer's address - try cg.lowerToStack(ptr); + try cg.emitWValue(ptr); try cg.emitWValue(final_len); switch (cg.ptr_size) { .wasm32 => try cg.addTag(.i32_add), @@ -6714,7 +6683,7 @@ fn airArrayElemVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { const elem_size = elem_ty.abiSize(zcu); if (isByRef(array_ty, zcu, cg.target)) { - try cg.lowerToStack(array); + try cg.emitWValue(array); try cg.emitWValue(index); try cg.addImm32(@intCast(elem_size)); try cg.addTag(.i32_mul); @@ -6747,7 +6716,7 @@ fn airArrayElemVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { try cg.store(stack_vec, array, array_ty, .{}); // Is a non-unrolled vector (v128) - try cg.lowerToStack(stack_vec); + try cg.emitWValue(stack_vec); try cg.emitWValue(index); try cg.addImm32(@intCast(elem_size)); try cg.addTag(.i32_mul); @@ -6783,13 +6752,13 @@ fn airSplat(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { 64 => @backingInt(std.wasm.SimdOpcode.v128_load64_splat), else => break :blk, // Cannot make use of simd-instructions }; - try cg.emitWValue(operand); + try cg.emitMemBase(operand); const extra_index: u32 = @intCast(cg.mir_extra.items.len); // stores as := opcode, offset, alignment (opcode::memarg) try cg.mir_extra.ensureUnusedCapacity(cg.gpa, 4); cg.mir_extra.appendAssumeCapacity(opcode); _ = try cg.addExtraAssumeCapacity(Mir.MemArg{ - .offset = operand.offset(), + .offset = operand.memOffset(), .alignment = elem_ty.abiAlignment(zcu).toStdMem(), }); try cg.addInst(.{ .tag = .simd_prefix, .data = .{ .payload = extra_index } }); @@ -6856,12 +6825,12 @@ fn airShuffleOne(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { const dest_alloc = try cg.allocStack(result_ty); for (mask, 0..) |mask_elem, out_idx| { - try cg.emitWValue(dest_alloc); + try cg.emitMemBase(dest_alloc); const elem_val = switch (mask_elem.unwrap()) { .elem => |idx| try cg.load(operand, elem_ty, .{ .offset = elem_size * idx }), .value => |val| try cg.lowerConstant(.fromInterned(val)), }; - try cg.store(.stack, elem_val, elem_ty, .{ .offset = dest_alloc.offset() + elem_size * out_idx }); + try cg.store(.stack, elem_val, elem_ty, .{ .offset = dest_alloc.memOffset() + elem_size * out_idx }); } return cg.finishAir(inst, dest_alloc, &.{unwrapped.operand}); } @@ -6924,13 +6893,13 @@ fn airShuffleTwo(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { const dest_alloc = try cg.allocStack(result_ty); for (mask, 0..) |mask_elem, out_idx| { - try cg.emitWValue(dest_alloc); + try cg.emitMemBase(dest_alloc); const elem_val = switch (mask_elem.unwrap()) { .a_elem => |idx| try cg.load(operand_a, elem_ty, .{ .offset = elem_size * idx }), .b_elem => |idx| try cg.load(operand_b, elem_ty, .{ .offset = elem_size * idx }), .undef => try cg.emitUndefined(elem_ty), }; - try cg.store(.stack, elem_val, elem_ty, .{ .offset = dest_alloc.offset() + elem_size * out_idx }); + try cg.store(.stack, elem_val, elem_ty, .{ .offset = dest_alloc.memOffset() + elem_size * out_idx }); } return cg.finishAir(inst, dest_alloc, &.{ unwrapped.operand_a, unwrapped.operand_b }); } @@ -6956,39 +6925,18 @@ fn airAggregateInit(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { .array, .vector => { const result = try cg.allocStack(result_ty); const elem_ty = result_ty.childType(zcu); - const elem_size = @as(u32, @intCast(elem_ty.abiSize(zcu))); + const elem_size = elem_ty.abiSize(zcu); const sentinel = result_ty.sentinel(zcu); - // When the element type is by reference, we must copy the entire - // value. It is therefore safer to move the offset pointer and store - // each value individually, instead of using store offsets. - if (isByRef(elem_ty, zcu, cg.target)) { - // copy stack pointer into a temporary local, which is - // moved for each element to store each value in the right position. - const offset = try cg.buildPointerOffset(result, 0, .new); - for (elements, 0..) |elem, elem_index| { - const elem_val = try cg.resolveInst(elem); - try cg.store(offset, elem_val, elem_ty, .{}); - - if (elem_index < elements.len - 1 or sentinel != null) { - _ = try cg.buildPointerOffset(offset, elem_size, .modify); - } - } - if (sentinel) |s| { - const val = try cg.resolveValue(s); - try cg.store(offset, val, elem_ty, .{}); - } - } else { - var offset: u32 = 0; - for (elements) |elem| { - const elem_val = try cg.resolveInst(elem); - try cg.store(result, elem_val, elem_ty, .{ .offset = offset }); - offset += elem_size; - } - if (sentinel) |s| { - const val = try cg.resolveValue(s); - try cg.store(result, val, elem_ty, .{ .offset = offset }); - } + var offset: u64 = 0; + for (elements) |elem| { + const elem_val = try cg.resolveInst(elem); + try cg.store(result, elem_val, elem_ty, .{ .offset = offset }); + offset += elem_size; + } + if (sentinel) |s| { + const val = try cg.resolveValue(s); + try cg.store(result, val, elem_ty, .{ .offset = offset }); } break :result_value result; }, @@ -7001,10 +6949,9 @@ fn airAggregateInit(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { const elem_ty = result_ty.fieldType(elem_index, zcu); const field_offset = result_ty.structFieldOffset(elem_index, zcu); - const offset = try cg.buildPointerOffset(result, field_offset, .new); const value = try cg.resolveInst(elem); - try cg.store(offset, value, elem_ty, .{}); + try cg.store(result, value, elem_ty, .{ .offset = field_offset }); } break :result_value result; @@ -7051,12 +6998,7 @@ fn airUnionInit(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { const result_ptr = try cg.allocStack(union_ty); const payload = try cg.resolveInst(extra.init); if (layout.tag_align.compare(.gte, layout.payload_align)) { - if (isByRef(field_ty, zcu, cg.target)) { - const payload_ptr = try cg.buildPointerOffset(result_ptr, layout.tag_size, .new); - try cg.store(payload_ptr, payload, field_ty, .{}); - } else { - try cg.store(result_ptr, payload, field_ty, .{ .offset = layout.tag_size }); - } + try cg.store(result_ptr, payload, field_ty, .{ .offset = layout.tag_size }); if (layout.tag_size > 0) { try cg.store(result_ptr, tag_int, .fromInterned(union_obj.enum_tag_type), .{}); @@ -7163,13 +7105,11 @@ fn airErrUnionPayloadPtrSet(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void .{ .offset = errUnionErrorOffset(payload_ty, zcu) }, ); - const result = result: { - if (!payload_ty.hasRuntimeBits(zcu)) { - break :result operand; - } + if (!payload_ty.hasRuntimeBits(zcu)) { + return cg.finishAir(inst, operand, &.{ty_op.operand}); + } - break :result try cg.buildPointerOffset(operand, @as(u32, @intCast(errUnionPayloadOffset(payload_ty, zcu))), .new); - }; + const result = try cg.offsetPointer(operand, errUnionPayloadOffset(payload_ty, zcu)); return cg.finishAir(inst, result, &.{ty_op.operand}); } @@ -7194,16 +7134,23 @@ fn airFieldParentPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { }, }; - const result = if (field_offset != 0) result: { - const base = try cg.buildPointerOffset(field_ptr, 0, .new); - try cg.addLocal(.local_get, base.local.value); - try cg.addImm32(@intCast(field_offset)); - try cg.addTag(.i32_sub); - try cg.addLocal(.local_set, base.local.value); - break :result base; - } else field_ptr; + if (field_offset == 0) { + return cg.finishAir(inst, field_ptr, &.{extra.field_ptr}); + } - return cg.finishAir(inst, result, &.{extra.field_ptr}); + // maybe use offsetPointer? sub looks better in disasm + try cg.emitWValue(field_ptr); + switch (cg.ptr_size) { + .wasm32 => { + try cg.addImm32(@intCast(field_offset)); + try cg.addTag(.i32_sub); + }, + .wasm64 => { + try cg.addImm64(field_offset); + try cg.addTag(.i64_sub); + }, + } + return cg.finishAir(inst, .stack, &.{extra.field_ptr}); } fn sliceOrArrayPtr(cg: *CodeGen, ptr: WValue, ptr_ty: Type) InnerError!WValue { @@ -7322,7 +7269,7 @@ fn airErrorName(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { fn airPtrSliceFieldPtr(cg: *CodeGen, inst: Air.Inst.Index, offset: u32) InnerError!void { const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op; const slice_ptr = try cg.resolveInst(ty_op.operand); - const result = try cg.buildPointerOffset(slice_ptr, offset, .new); + const result = try cg.offsetPointer(slice_ptr, offset); return cg.finishAir(inst, result, &.{ty_op.operand}); } @@ -7392,13 +7339,15 @@ fn lowerTry( try cg.startBlock(.block, .empty); // check if the error tag is set for the error union. - try cg.emitWValue(err_union); if (pl_has_bits or operand_is_ptr) { - const err_offset: u32 = @intCast(errUnionErrorOffset(pl_ty, zcu)); + try cg.emitMemBase(err_union); + const err_offset = errUnionErrorOffset(pl_ty, zcu); try cg.addMemArg(.i32_load16_u, .{ - .offset = err_union.offset() + err_offset, + .offset = err_union.memOffset() + err_offset, .alignment = Type.anyerror.abiAlignment(zcu).toStdMem(), }); + } else { + try cg.emitWValue(err_union); } try cg.addTag(.i32_eqz); try cg.addLabel(.br_if, 0); // jump out of block when error is '0' @@ -7417,9 +7366,9 @@ fn lowerTry( return .none; } - const pl_offset: u32 = @intCast(errUnionPayloadOffset(pl_ty, zcu)); + const pl_offset = errUnionPayloadOffset(pl_ty, zcu); if (operand_is_ptr or isByRef(pl_ty, zcu, cg.target)) { - return buildPointerOffset(cg, err_union, pl_offset, .new); + return cg.offsetPointer(err_union, pl_offset); } const payload = try cg.load(err_union, pl_ty, .{ .offset = pl_offset }); return payload.toLocal(cg, pl_ty); @@ -7448,7 +7397,7 @@ fn callIntrinsic( // and emit it as our first argument const sret = if (want_sret_param) blk: { const sret_local = try cg.allocStack(return_type); - try cg.lowerToStack(sret_local); + try cg.emitWValue(sret_local); break :blk sret_local; } else .none; @@ -7476,7 +7425,7 @@ fn airTagName(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { const enum_ty = cg.typeOf(un_op); try cg.addInst(.{ .tag = .enum_tag_name_table_ref, .data = .{ .ip_index = enum_ty.toIntern() } }); - try cg.lowerToStack(operand); + try cg.emitWValue(operand); try cg.addInst(.{ .tag = .call_tag_index, .data = .{ .ip_index = enum_ty.toIntern() } }); switch (cg.ptr_size) { @@ -7500,7 +7449,7 @@ fn airIsNamedEnumValue(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { const operand = try cg.resolveInst(un_op); const enum_ty = cg.typeOf(un_op); - try cg.lowerToStack(operand); + try cg.emitWValue(operand); try cg.addInst(.{ .tag = .call_tag_index, .data = .{ .ip_index = enum_ty.toIntern() } }); try cg.addImm32(~@as(u32, 0)); try cg.addTag(.i32_ne); @@ -7614,9 +7563,9 @@ fn airCmpxchg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { const ptr_val = if (cg.useAtomicFeature()) val: { const val_local = try cg.allocLocal(ty); - try cg.emitWValue(ptr_operand); - try cg.lowerToStack(expected_val); - try cg.lowerToStack(new_val); + try cg.emitMemBase(ptr_operand); + try cg.emitWValue(expected_val); + try cg.emitWValue(new_val); try cg.addAtomicMemArg(switch (ty.abiSize(zcu)) { 1 => .i32_atomic_rmw8_cmpxchg_u, 2 => .i32_atomic_rmw16_cmpxchg_u, @@ -7624,7 +7573,7 @@ fn airCmpxchg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { 8 => .i32_atomic_rmw_cmpxchg, else => |size| return cg.fail("TODO: implement `@cmpxchg` for types with abi size '{d}'", .{size}), }, .{ - .offset = ptr_operand.offset(), + .offset = ptr_operand.memOffset(), .alignment = ty.abiAlignment(zcu).toStdMem(), }); try cg.addLocal(.local_tee, val_local.local.value); @@ -7637,13 +7586,13 @@ fn airCmpxchg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { } const ptr_val = try WValue.toLocal(try cg.load(ptr_operand, ty, .{}), cg, ty); - try cg.lowerToStack(ptr_operand); - try cg.lowerToStack(new_val); + try cg.emitMemBase(ptr_operand); + try cg.emitWValue(new_val); try cg.emitWValue(ptr_val); _ = try cg.intCmp(int_ty, .eq, ptr_val, expected_val); try cg.addLocal(.local_tee, cmp_result.local.value); try cg.addTag(.select); - try cg.store(.stack, .stack, ty, .{}); + try cg.store(.stack, .stack, ty, .{ .offset = ptr_operand.memOffset() }); break :val ptr_val; }; @@ -7684,9 +7633,9 @@ fn airAtomicLoad(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { 8 => .i64_atomic_load, else => |size| return cg.fail("TODO: @atomicLoad for types with abi size {d}", .{size}), }; - try cg.emitWValue(ptr); + try cg.emitMemBase(ptr); try cg.addAtomicMemArg(tag, .{ - .offset = ptr.offset(), + .offset = ptr.memOffset(), .alignment = ty.abiAlignment(zcu).toStdMem(), }); } else { @@ -7719,7 +7668,7 @@ fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { // create a loop to cmpxchg the new value try cg.startBlock(.loop, .empty); - try cg.emitWValue(ptr); + try cg.emitMemBase(ptr); try cg.emitWValue(value); if (op == .Nand) { const and_res = try cg.intAnd(int_ty, value, operand); @@ -7746,7 +7695,7 @@ fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { else => return cg.fail("TODO: implement `@atomicRmw` with operation `{s}` for types larger than 64 bits", .{@tagName(op)}), }, .{ - .offset = ptr.offset(), + .offset = ptr.memOffset(), .alignment = ty.abiAlignment(zcu).toStdMem(), }, ); @@ -7764,7 +7713,7 @@ fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { // the other operations have their own instructions for Wasm. else => { - try cg.emitWValue(ptr); + try cg.emitMemBase(ptr); try cg.emitWValue(operand); const tag: std.wasm.AtomicsOpcode = switch (ty.abiSize(zcu)) { 1 => switch (op) { @@ -7806,7 +7755,7 @@ fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { else => |size| return cg.fail("TODO: Implement `@atomicRmw` for types with abi size {d}", .{size}), }; try cg.addAtomicMemArg(tag, .{ - .offset = ptr.offset(), + .offset = ptr.memOffset(), .alignment = ty.abiAlignment(zcu).toStdMem(), }); return cg.finishAir(inst, .stack, &.{ pl_op.operand, extra.operand }); @@ -7825,23 +7774,23 @@ fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { => { if (ty.isAnyFloat()) { const float_ty: FloatType = .fromType(cg, ty); - try cg.emitWValue(ptr); + try cg.emitMemBase(ptr); _ = switch (op) { .Add => try cg.floatAdd(float_ty, result, operand), .Sub => try cg.floatSub(float_ty, result, operand), else => unreachable, }; - try cg.store(.stack, .stack, ty, .{ .offset = ptr.offset() }); + try cg.store(.stack, .stack, ty, .{ .offset = ptr.memOffset() }); } else { const int_ty: IntType = .fromType(cg, ty); - try cg.emitWValue(ptr); + try cg.emitMemBase(ptr); _ = switch (op) { .Add => try cg.intAdd(int_ty, result, operand), .Sub => try cg.intSub(int_ty, result, operand), else => unreachable, }; _ = try cg.intWrap(int_ty, .stack); - try cg.store(.stack, .stack, ty, .{ .offset = ptr.offset() }); + try cg.store(.stack, .stack, ty, .{ .offset = ptr.memOffset() }); } }, .And, @@ -7849,39 +7798,39 @@ fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { .Xor, => { const int_ty: IntType = .fromType(cg, ty); - try cg.emitWValue(ptr); + try cg.emitMemBase(ptr); _ = switch (op) { .And => try cg.intAnd(int_ty, result, operand), .Or => try cg.intOr(int_ty, result, operand), .Xor => try cg.intXor(int_ty, result, operand), else => unreachable, }; - try cg.store(.stack, .stack, ty, .{ .offset = ptr.offset() }); + try cg.store(.stack, .stack, ty, .{ .offset = ptr.memOffset() }); }, .Max, .Min, => { if (ty.isAnyFloat()) { const float_ty: FloatType = .fromType(cg, ty); - try cg.emitWValue(ptr); + try cg.emitMemBase(ptr); try cg.emitWValue(result); try cg.emitWValue(operand); _ = try cg.floatCmp(float_ty, if (op == .Max) .gt else .lt, result, operand); try cg.addTag(.select); - try cg.store(.stack, .stack, ty, .{ .offset = ptr.offset() }); + try cg.store(.stack, .stack, ty, .{ .offset = ptr.memOffset() }); } else { const int_ty: IntType = .fromType(cg, ty); - try cg.emitWValue(ptr); + try cg.emitMemBase(ptr); try cg.emitWValue(result); try cg.emitWValue(operand); _ = try cg.intCmp(int_ty, if (op == .Max) .gt else .lt, result, operand); try cg.addTag(.select); - try cg.store(.stack, .stack, ty, .{ .offset = ptr.offset() }); + try cg.store(.stack, .stack, ty, .{ .offset = ptr.memOffset() }); } }, .Nand => { const int_ty: IntType = .fromType(cg, ty); - try cg.emitWValue(ptr); + try cg.emitMemBase(ptr); const and_res = try cg.intAnd(int_ty, result, operand); if (int_ty.bits <= 32) { try cg.addImm32(~@as(u32, 0)); @@ -7891,7 +7840,7 @@ fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { return cg.fail("TODO: `@atomicRmw` with operator `Nand` for types larger than 64 bits", .{}); } _ = try cg.intXor(int_ty, and_res, .stack); - try cg.store(.stack, .stack, ty, .{ .offset = ptr.offset() }); + try cg.store(.stack, .stack, ty, .{ .offset = ptr.memOffset() }); }, } @@ -7916,10 +7865,10 @@ fn airAtomicStore(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { 8 => .i64_atomic_store, else => |size| return cg.fail("TODO: @atomicLoad for types with abi size {d}", .{size}), }; - try cg.emitWValue(ptr); - try cg.lowerToStack(operand); + try cg.emitMemBase(ptr); + try cg.emitWValue(operand); try cg.addAtomicMemArg(tag, .{ - .offset = ptr.offset(), + .offset = ptr.memOffset(), .alignment = ty.abiAlignment(zcu).toStdMem(), }); } else { @@ -7996,7 +7945,7 @@ fn airAsm(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { return cg.fail("Self-hosted wasm backend requires input constraint to be equal \"r\"", .{}); } - try cg.lowerToStack(operand); + try cg.emitWValue(operand); const op_local = try WValue.toLocal(.stack, cg, cg.typeOf(input.operand)); const gop = try local_map.getOrPutValue(cg.gpa, name, op_local.local.value); -- 2.54.0 From 88c44442aed3c6b5d7b49e0978b80c5efdfab5f8 Mon Sep 17 00:00:00 2001 From: Pavel Verigo Date: Sat, 22 Aug 2026 00:41:59 +0200 Subject: [PATCH 4/8] wasm: better assembly for offset and memset --- src/codegen/wasm/CodeGen.zig | 155 ++++++++++++++++------------------- 1 file changed, 70 insertions(+), 85 deletions(-) diff --git a/src/codegen/wasm/CodeGen.zig b/src/codegen/wasm/CodeGen.zig index b51508d201b43ecf103c4ff0318335837ab4c1d3..5a266cebcbe3e2c6cb9eb7bb8d4fbf522a9ff6aa 100644 --- a/src/codegen/wasm/CodeGen.zig +++ b/src/codegen/wasm/CodeGen.zig @@ -668,6 +668,41 @@ fn offsetPointer(cg: *CodeGen, ptr: WValue, offset: u64) InnerError!WValue { }; } +const PointerOffsetOp = enum { add, sub }; + +/// The offset may only be known at runtime. Base pointer must be already on the stack. Result is left on it also. +fn emitScaledPointerOffset(cg: *CodeGen, offset: WValue, scale: u64, op: PointerOffsetOp) InnerError!void { + if (scale == 0 or switch (offset) { + .imm32 => |imm| imm == 0, + .imm64 => |imm| imm == 0, + else => false, + }) return; + + try cg.emitWValue(offset); + switch (cg.ptr_size) { + .wasm32 => { + if (scale != 1) { + try cg.addImm32(@intCast(scale)); + try cg.addTag(.i32_mul); + } + try cg.addTag(switch (op) { + .add => .i32_add, + .sub => .i32_sub, + }); + }, + .wasm64 => { + if (scale != 1) { + try cg.addImm64(scale); + try cg.addTag(.i64_mul); + } + try cg.addTag(switch (op) { + .add => .i64_add, + .sub => .i64_sub, + }); + }, + } +} + /// Writes the bytecode depending on the given `WValue` in `val` fn emitWValue(cg: *CodeGen, value: WValue) InnerError!void { switch (value) { @@ -1187,7 +1222,7 @@ fn emitMemoryCopy(cg: *CodeGen, dst: WValue, src: WValue, len: WValue) !void { } fn memcpy(cg: *CodeGen, dst: WValue, src: WValue, len: WValue) !void { - if (cg.target.cpu.has(.wasm, .bulk_memory)) { + if (cg.target.cpu.has(.wasm, .bulk_memory_opt)) { try cg.emitMemoryCopy(dst, src, len); return; } @@ -1200,7 +1235,7 @@ fn memcpy(cg: *CodeGen, dst: WValue, src: WValue, len: WValue) !void { } fn memmove(cg: *CodeGen, dst: WValue, src: WValue, len: WValue) !void { - if (cg.target.cpu.has(.wasm, .bulk_memory)) { + if (cg.target.cpu.has(.wasm, .bulk_memory_opt)) { try cg.emitMemoryCopy(dst, src, len); return; } @@ -1309,10 +1344,7 @@ fn genInst(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { const elem_size = elem_ty.abiSize(zcu); try cg.emitWValue(vec_ptr); - try cg.emitWValue(elem_idx); - try cg.addImm32(@intCast(elem_size)); - try cg.addTag(.i32_mul); - try cg.addTag(.i32_add); + try cg.emitScaledPointerOffset(elem_idx, elem_size, .add); const ptr = try WValue.toLocal(.stack, cg, Type.usize); try cg.store(ptr, elem_val, elem_ty, .{}); @@ -2258,15 +2290,17 @@ fn airStore(cg: *CodeGen, inst: Air.Inst.Index, safety: bool) InnerError!void { const bin_op = cg.air.instructions.items(.data)[@backingInt(inst)].bin_op; const lhs = try cg.resolveInst(bin_op.lhs); - const rhs = try cg.resolveInst(bin_op.rhs); const ptr_ty = cg.typeOf(bin_op.lhs); const ptr_info = ptr_ty.ptrInfo(zcu); const elem_ty = ptr_ty.childType(zcu); + const rhs_is_undef = if (bin_op.rhs.toInterned()) |i| Value.fromInterned(i).isUndef(zcu) else false; - if (!safety and bin_op.rhs == .undef) { + if (!safety and rhs_is_undef) { return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs }); } + const rhs = try cg.resolveInst(bin_op.rhs); + const offset: u64 = switch (ptr_info.flags.vector_index) { .none => offset: { assert(ptr_info.packed_offset.host_size == 0); // legalize .expand_packed_store @@ -2274,6 +2308,13 @@ fn airStore(cg: *CodeGen, inst: Air.Inst.Index, safety: bool) InnerError!void { }, else => |index| @backingInt(index) * elem_ty.abiSize(zcu), }; + + if (safety and rhs_is_undef and isByRef(elem_ty, zcu, cg.target)) { + const dst = try cg.offsetPointer(lhs, offset); + try cg.memset(.u8, dst, .{ .imm32 = @intCast(elem_ty.abiSize(zcu)) }, .{ .imm32 = 0xaa }); + return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs }); + } + try cg.store(lhs, rhs, elem_ty, .{ .offset = offset, .alignment = ptr_ty.ptrAlignment(zcu), @@ -6368,10 +6409,7 @@ fn airSliceElemVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { _ = try cg.load(slice, .usize, .{}); // calculate index into slice - try cg.emitWValue(index); - try cg.addImm32(@intCast(elem_size)); - try cg.addTag(.i32_mul); - try cg.addTag(.i32_add); + try cg.emitScaledPointerOffset(index, elem_size, .add); const elem_result = try cg.load(.stack, elem_ty, .{}); @@ -6392,10 +6430,7 @@ fn airSliceElemPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { _ = try cg.load(slice, .usize, .{}); // calculate index into slice - try cg.emitWValue(index); - try cg.addImm32(@intCast(elem_size)); - try cg.addTag(.i32_mul); - try cg.addTag(.i32_add); + try cg.emitScaledPointerOffset(index, elem_size, .add); return cg.finishAir(inst, .stack, &.{ bin_op.lhs, bin_op.rhs }); } @@ -6454,10 +6489,7 @@ fn airPtrElemVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { } // calculate index into slice - try cg.emitWValue(index); - try cg.addImm32(@intCast(elem_size)); - try cg.addTag(.i32_mul); - try cg.addTag(.i32_add); + try cg.emitScaledPointerOffset(index, elem_size, .add); const elem_result = try cg.load(.stack, elem_ty, .{}); @@ -6484,15 +6516,12 @@ fn airPtrElemPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { } // calculate index into ptr - try cg.emitWValue(index); - try cg.addImm32(@intCast(elem_size)); - try cg.addTag(.i32_mul); - try cg.addTag(.i32_add); + try cg.emitScaledPointerOffset(index, elem_size, .add); return cg.finishAir(inst, .stack, &.{ bin_op.lhs, bin_op.rhs }); } -fn airPtrBinOp(cg: *CodeGen, inst: Air.Inst.Index, op: enum { add, sub }) InnerError!void { +fn airPtrBinOp(cg: *CodeGen, inst: Air.Inst.Index, op: PointerOffsetOp) InnerError!void { const zcu = cg.pt.zcu; const ty_pl = cg.air.instructions.items(.data)[@backingInt(inst)].ty_pl; const bin_op = cg.air.extraData(Air.Bin, ty_pl.payload).data; @@ -6506,26 +6535,7 @@ fn airPtrBinOp(cg: *CodeGen, inst: Air.Inst.Index, op: enum { add, sub }) InnerE }; try cg.emitWValue(ptr); - try cg.emitWValue(offset); - - switch (cg.ptr_size) { - .wasm32 => { - try cg.addImm32(@intCast(pointee_ty.abiSize(zcu))); - try cg.addTag(.i32_mul); - try cg.addTag(switch (op) { - .add => .i32_add, - .sub => .i32_sub, - }); - }, - .wasm64 => { - try cg.addImm64(pointee_ty.abiSize(zcu)); - try cg.addTag(.i64_mul); - try cg.addTag(switch (op) { - .add => .i64_add, - .sub => .i64_sub, - }); - }, - } + try cg.emitScaledPointerOffset(offset, pointee_ty.abiSize(zcu), op); return cg.finishAir(inst, .stack, &.{ bin_op.lhs, bin_op.rhs }); } @@ -6559,19 +6569,24 @@ fn airMemset(cg: *CodeGen, inst: Air.Inst.Index, safety: bool) InnerError!void { } /// Sets a region of memory at `ptr` to the value of `value` -/// When the user has enabled the bulk_memory feature, we lower +/// When the user has enabled the bulk_memory_opt feature, we lower /// this to wasm's memset instruction. When the feature is not present, /// we implement it manually. fn memset(cg: *CodeGen, elem_ty: Type, ptr: WValue, len: WValue, value: WValue) InnerError!void { const zcu = cg.pt.zcu; const abi_size = @as(u32, @intCast(elem_ty.abiSize(zcu))); - - // When bulk_memory is enabled, we lower it to wasm's memset instruction. + const len_known_neq_0 = switch (len) { + .imm32 => |imm| if (imm != 0) true else return, + .imm64 => |imm| if (imm != 0) true else return, + else => false, + }; + // When bulk_memory_opt is enabled, we lower it to wasm's memset instruction. // If not, we lower it ourselves. - if (cg.target.cpu.has(.wasm, .bulk_memory) and abi_size == 1) { + if (cg.target.cpu.has(.wasm, .bulk_memory_opt) and abi_size == 1) { const len0_ok = cg.target.cpu.has(.wasm, .nontrapping_bulk_memory_len0); + const emit_check = !(len0_ok or len_known_neq_0); - if (!len0_ok) { + if (emit_check) { try cg.startBlock(.block, .empty); // Even if `len` is zero, the spec requires an implementation to trap if `ptr + len` is @@ -6592,7 +6607,7 @@ fn memset(cg: *CodeGen, elem_ty: Type, ptr: WValue, len: WValue, value: WValue) try cg.emitWValue(len); try cg.addExtended(.memory_fill); - if (!len0_ok) { + if (emit_check) { try cg.endBlock(); } @@ -6684,10 +6699,7 @@ fn airArrayElemVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { if (isByRef(array_ty, zcu, cg.target)) { try cg.emitWValue(array); - try cg.emitWValue(index); - try cg.addImm32(@intCast(elem_size)); - try cg.addTag(.i32_mul); - try cg.addTag(.i32_add); + try cg.emitScaledPointerOffset(index, elem_size, .add); } else { assert(array_ty.zigTypeTag(zcu) == .vector); @@ -6717,10 +6729,7 @@ fn airArrayElemVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { // Is a non-unrolled vector (v128) try cg.emitWValue(stack_vec); - try cg.emitWValue(index); - try cg.addImm32(@intCast(elem_size)); - try cg.addTag(.i32_mul); - try cg.addTag(.i32_add); + try cg.emitScaledPointerOffset(index, elem_size, .add); }, } } @@ -7249,19 +7258,7 @@ fn airErrorName(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { // Lowers to a i32.const or i64.const with the error table memory address. cg.error_name_table_ref_count += 1; try cg.addTag(.error_name_table_ref); - try cg.emitWValue(operand); - switch (cg.ptr_size) { - .wasm32 => { - try cg.addImm32(@intCast(abi_size)); - try cg.addTag(.i32_mul); - try cg.addTag(.i32_add); - }, - .wasm64 => { - try cg.addImm64(abi_size); - try cg.addTag(.i64_mul); - try cg.addTag(.i64_add); - }, - } + try cg.emitScaledPointerOffset(operand, abi_size, .add); return cg.finishAir(inst, .stack, &.{un_op}); } @@ -7427,19 +7424,7 @@ fn airTagName(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { try cg.addInst(.{ .tag = .enum_tag_name_table_ref, .data = .{ .ip_index = enum_ty.toIntern() } }); try cg.emitWValue(operand); try cg.addInst(.{ .tag = .call_tag_index, .data = .{ .ip_index = enum_ty.toIntern() } }); - - switch (cg.ptr_size) { - .wasm32 => { - try cg.addImm32(@intCast(8)); - try cg.addTag(.i32_mul); - try cg.addTag(.i32_add); - }, - .wasm64 => { - try cg.addImm64(8); - try cg.addTag(.i64_mul); - try cg.addTag(.i64_add); - }, - } + try cg.emitScaledPointerOffset(.stack, 8, .add); return cg.finishAir(inst, .stack, &.{un_op}); } -- 2.54.0 From 235e20d85f599a502183ebaee6e9e44202f44a0c Mon Sep 17 00:00:00 2001 From: Pavel Verigo Date: Sat, 22 Aug 2026 05:43:07 +0200 Subject: [PATCH 5/8] wasm: dont align stack if it default value --- src/codegen/wasm/Emit.zig | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/codegen/wasm/Emit.zig b/src/codegen/wasm/Emit.zig index 077cb04266f4d21da0bbbb31ee90d1bae03515ca..3c76c439ec948111832be4a0fa336abe79c38b21 100644 --- a/src/codegen/wasm/Emit.zig +++ b/src/codegen/wasm/Emit.zig @@ -56,13 +56,15 @@ pub fn lower(emit: *Emit) Error!void { writeSleb128(code, aligned_stack); // subtract it from the current stack pointer code.appendAssumeCapacity(@backingInt(std.wasm.Opcode.i32_sub)); - // Get negative stack alignment - const neg_stack_align = @as(i32, @intCast(align_bytes)) * -1; - code.appendAssumeCapacity(@backingInt(std.wasm.Opcode.i32_const)); - writeSleb128(code, neg_stack_align); - // Bitwise-and the value to get the new stack pointer to ensure the - // pointers are aligned with the abi alignment. - code.appendAssumeCapacity(@backingInt(std.wasm.Opcode.i32_and)); + if (align_bytes != 16) { + // Get negative stack alignment + const neg_stack_align = @as(i32, @intCast(align_bytes)) * -1; + code.appendAssumeCapacity(@backingInt(std.wasm.Opcode.i32_const)); + writeSleb128(code, neg_stack_align); + // Bitwise-and the value to get the new stack pointer to ensure the + // pointers are aligned with the abi alignment. + code.appendAssumeCapacity(@backingInt(std.wasm.Opcode.i32_and)); + } // The bottom will be used to calculate all stack pointer offsets. code.appendAssumeCapacity(@backingInt(std.wasm.Opcode.local_tee)); writeUleb128(code, mir.prologue.bottom_stack_local); -- 2.54.0 From 503c59263e358ae8ca16b271fc35799a508d03bb Mon Sep 17 00:00:00 2001 From: Pavel Verigo Date: Sat, 22 Aug 2026 23:24:37 +0200 Subject: [PATCH 6/8] wasm: better local tracking --- src/codegen/wasm/CodeGen.zig | 526 ++++++++++++++++++----------------- 1 file changed, 267 insertions(+), 259 deletions(-) diff --git a/src/codegen/wasm/CodeGen.zig b/src/codegen/wasm/CodeGen.zig index 5a266cebcbe3e2c6cb9eb7bb8d4fbf522a9ff6aa..cf68c7a738175e9ecdddd18f12bc1651d36e7892 100644 --- a/src/codegen/wasm/CodeGen.zig +++ b/src/codegen/wasm/CodeGen.zig @@ -136,13 +136,10 @@ air: Air, liveness: Air.Liveness, gpa: mem.Allocator, func_index: InternPool.Index, -/// Contains a list of current branches. -/// When we return from a branch, the branch will be popped from this list, -/// which means branches can only contain references from within its own branch, -/// or a branch higher (lower index) in the tree. -branches: std.ArrayList(Branch) = .empty, +/// Contains a list of current branches. Stores LocalPool before entering branch. +branches: std.ArrayList(LocalPool) = .empty, /// Table to save `WValue`'s generated by an `Air.Inst` -// values: ValueTable, +values: ValueTable = .empty, /// Mapping from Air.Inst.Index to block ids blocks: std.array_hash_map.Auto(Air.Inst.Index, struct { label: u32, @@ -150,10 +147,6 @@ blocks: std.array_hash_map.Auto(Air.Inst.Index, struct { }) = .{}, /// Maps `loop` instructions to their label. `br` to here repeats the loop. loops: std.AutoHashMapUnmanaged(Air.Inst.Index, u32) = .empty, -/// The index the next local generated will have -/// NOTE: arguments share the index with locals therefore the first variable -/// will have the index that comes after the last argument's index -local_index: u32, /// The index of the current argument. /// Used to track which argument is being referenced in `airArg`. arg_index: u32 = 0, @@ -207,23 +200,8 @@ stack_size: u32 = 0, /// However, local variables or the usage of `incoming_stack_alignment` in a `CallingConvention` can overwrite this default. stack_alignment: Alignment = .@"16", -// For each individual Wasm valtype we store a seperate free list which -// allows us to re-use locals that are no longer used. e.g. a temporary local. -/// A list of indexes which represents a local of valtype `i32`. -/// It is illegal to store a non-i32 valtype in this list. -free_locals_i32: std.ArrayList(u32) = .empty, -/// A list of indexes which represents a local of valtype `i64`. -/// It is illegal to store a non-i64 valtype in this list. -free_locals_i64: std.ArrayList(u32) = .empty, -/// A list of indexes which represents a local of valtype `f32`. -/// It is illegal to store a non-f32 valtype in this list. -free_locals_f32: std.ArrayList(u32) = .empty, -/// A list of indexes which represents a local of valtype `f64`. -/// It is illegal to store a non-f64 valtype in this list. -free_locals_f64: std.ArrayList(u32) = .empty, -/// A list of indexes which represents a local of valtype `v127`. -/// It is illegal to store a non-v128 valtype in this list. -free_locals_v128: std.ArrayList(u32) = .empty, +local_pool: LocalPool = .{}, +wasm_param_count: u32 = undefined, /// When in debug mode, this tracks if no `finishAir` was missed. /// Forgetting to call `finishAir` will cause the result to not be @@ -239,13 +217,7 @@ const WValue = union(enum) { none: void, /// The value lives on top of the stack stack: void, - /// Index of the local - local: struct { - /// Contains the index to the local - value: u32, - /// The amount of instructions referencing this `WValue` - references: u32, - }, + local: u32, // index /// An immediate 32bit value imm32: u32, /// An immediate 64bit value @@ -290,7 +262,7 @@ const WValue = union(enum) { switch (value) { .stack => { const new_local = try gen.allocLocal(ty); - try gen.addLocal(.local_set, new_local.local.value); + try gen.addLocal(.local_set, new_local.local); return new_local; }, else => return value, @@ -299,23 +271,9 @@ const WValue = union(enum) { /// Marks a local as no longer being referenced and essentially allows /// us to re-use it somewhere else within the function. - /// The valtype of the local is deducted by using the index of the given `WValue`. - fn free(value: *WValue, gen: *CodeGen) void { + fn free(value: *WValue, cg: *CodeGen) void { if (value.* != .local) return; - const local_value = value.local.value; - const reserved = gen.args.len + @intFromBool(gen.return_value != .none) + @intFromBool(gen.varargs != .none); - if (local_value < reserved + 2) return; // reserved locals may never be re-used. Also accounts for 2 stack locals. - - const index = local_value - reserved; - const valtype = gen.mir_locals.items[index]; - switch (valtype) { - .i32 => gen.free_locals_i32.append(gen.gpa, local_value) catch return, // It's ok to fail any of those, a new local can be allocated instead - .i64 => gen.free_locals_i64.append(gen.gpa, local_value) catch return, - .f32 => gen.free_locals_f32.append(gen.gpa, local_value) catch return, - .f64 => gen.free_locals_f64.append(gen.gpa, local_value) catch return, - .v128 => gen.free_locals_v128.append(gen.gpa, local_value) catch return, - } - log.debug("freed local ({d}) of type {}", .{ local_value, valtype }); + cg.releaseLocal(value.local); value.* = .dead; } }; @@ -323,6 +281,77 @@ const WValue = union(enum) { /// Hashmap to store generated `WValue` for each `Air.Inst.Ref` const ValueTable = std.array_hash_map.Auto(Air.Inst.Ref, WValue); +const LocalPool = struct { + const OptionalIndex = enum(u32) { + none = math.maxInt(u32), + _, + + fn init(index: u32) OptionalIndex { + const result: OptionalIndex = @fromBackingInt(index); + assert(result != .none); + return result; + } + + fn unwrap(index: OptionalIndex) ?u32 { + return if (index == .none) null else @backingInt(index); + } + }; + + const Local = union { + ref_count: u32, + next_free: OptionalIndex, + }; + + locals: std.ArrayList(Local) = .empty, + free_i32: OptionalIndex = .none, + free_i64: OptionalIndex = .none, + free_f32: OptionalIndex = .none, + free_f64: OptionalIndex = .none, + free_v128: OptionalIndex = .none, + + fn deinit(pool: *LocalPool, gpa: Allocator) void { + pool.locals.deinit(gpa); + pool.* = undefined; + } + + fn clone(pool: *LocalPool, gpa: Allocator) Allocator.Error!LocalPool { + return .{ + .locals = try pool.locals.clone(gpa), + .free_i32 = pool.free_i32, + .free_i64 = pool.free_i64, + .free_f32 = pool.free_f32, + .free_f64 = pool.free_f64, + .free_v128 = pool.free_v128, + }; + } + + fn freeList(pool: *LocalPool, valtype: std.wasm.Valtype) *OptionalIndex { + return switch (valtype) { + .i32 => &pool.free_i32, + .i64 => &pool.free_i64, + .f32 => &pool.free_f32, + .f64 => &pool.free_f64, + .v128 => &pool.free_v128, + }; + } + + fn restore(snapshot: *LocalPool, cg: *CodeGen) void { + const local_pool = &cg.local_pool; + const old_local_count = snapshot.locals.items.len; + assert(local_pool.locals.items.len >= old_local_count); + @memcpy(local_pool.locals.items[0..old_local_count], snapshot.locals.items); + local_pool.free_i32 = snapshot.free_i32; + local_pool.free_i64 = snapshot.free_i64; + local_pool.free_f32 = snapshot.free_f32; + local_pool.free_f64 = snapshot.free_f64; + local_pool.free_v128 = snapshot.free_v128; + + for (old_local_count..local_pool.locals.items.len) |index| { + cg.appendFreeLocal(cg.wasm_param_count + @as(u32, @intCast(index))); + } + } +}; + const bookkeeping_init = if (std.debug.runtime_safety) @as(usize, 0) else {}; const InnerError = Error || error{ @@ -336,14 +365,11 @@ pub fn deinit(cg: *CodeGen) void { const gpa = cg.gpa; for (cg.branches.items) |*branch| branch.deinit(gpa); cg.branches.deinit(gpa); + cg.values.deinit(gpa); cg.blocks.deinit(gpa); cg.loops.deinit(gpa); cg.simd_immediates.deinit(gpa); - cg.free_locals_i32.deinit(gpa); - cg.free_locals_i64.deinit(gpa); - cg.free_locals_f32.deinit(gpa); - cg.free_locals_f64.deinit(gpa); - cg.free_locals_v128.deinit(gpa); + cg.local_pool.deinit(gpa); cg.mir_instructions.deinit(gpa); cg.mir_extra.deinit(gpa); cg.mir_locals.deinit(gpa); @@ -362,19 +388,11 @@ pub fn fail(cg: *CodeGen, comptime fmt: []const u8, args: anytype) Error { /// Resolves the `WValue` for the given instruction `inst` /// When the given instruction has a `Value`, it returns a constant instead fn resolveInst(cg: *CodeGen, ref: Air.Inst.Ref) InnerError!WValue { - var branch_index = cg.branches.items.len; - while (branch_index > 0) : (branch_index -= 1) { - const branch = cg.branches.items[branch_index - 1]; - if (branch.values.get(ref)) |value| { - return value; - } - } + if (cg.values.get(ref)) |value| return value; // when we did not find an existing instruction, it // means we must generate it from a constant. - // We always store constants in the most outer branch as they must never - // be removed. The most outer branch is always at index 0. - const gop = try cg.branches.items[0].values.getOrPut(cg.gpa, ref); + const gop = try cg.values.getOrPut(cg.gpa, ref); assert(!gop.found_existing); const pt = cg.pt; @@ -413,20 +431,33 @@ fn resolveValue(cg: *CodeGen, val: Value) InnerError!WValue { /// NOTE: if result == .stack, it will be stored in .local fn finishAir(cg: *CodeGen, inst: Air.Inst.Index, result: WValue, operands: []const Air.Inst.Ref) InnerError!void { + return cg.finishAirWithBranchDeaths(inst, result, operands, &.{}); +} + +/// Takes `branch_deaths` from continuing branch and calls `processDeath` after result retaining logic. +fn finishAirWithBranchDeaths( + cg: *CodeGen, + inst: Air.Inst.Index, + result: WValue, + operands: []const Air.Inst.Ref, + branch_deaths: []const Air.Inst.Index, +) InnerError!void { assert(operands.len <= Air.Liveness.bpi - 1); - var final_result = result; - const aliased_operand: ?*WValue = if (result == .local) aliased: { + if (result == .local) aliased: { for (operands) |operand| { if (operand.toIndex() == null) continue; - const resolved_operand = cg.getResolvedInst(operand); - if (resolved_operand.* != .local or resolved_operand.local.value != result.local.value) continue; + const resolved_operand = cg.values.getPtr(operand) orelse unreachable; + if (resolved_operand.* != .local or resolved_operand.local != result.local) continue; - resolved_operand.local.references += 1; - break :aliased resolved_operand; + if (result.local >= cg.wasm_param_count) { + const local = &cg.local_pool.locals.items[result.local - cg.wasm_param_count]; + assert(local.ref_count > 0); + local.ref_count += 1; + } + break :aliased; } - break :aliased null; - } else null; + } var tomb_bits = cg.liveness.getTombBits(inst); for (operands) |operand| { @@ -435,8 +466,8 @@ fn finishAir(cg: *CodeGen, inst: Air.Inst.Index, result: WValue, operands: []con if (!dies) continue; processDeath(cg, operand); } - if (aliased_operand) |operand| final_result.local.references = operand.local.references; - try cg.finishAirResult(inst, final_result); + for (branch_deaths) |death| cg.processDeath(death.toRef()); + try cg.finishAirResult(inst, result); } fn finishAirResult(cg: *CodeGen, inst: Air.Inst.Index, result: WValue) InnerError!void { @@ -446,8 +477,7 @@ fn finishAirResult(cg: *CodeGen, inst: Air.Inst.Index, result: WValue) InnerErro result else try result.toLocal(cg, cg.typeOfIndex(inst)); - const branch = cg.currentBranch(); - branch.values.putAssumeCapacityNoClobber(inst.toRef(), trackable_result); + try cg.values.putNoClobber(cg.gpa, inst.toRef(), trackable_result); } if (std.debug.runtime_safety) { @@ -455,17 +485,16 @@ fn finishAirResult(cg: *CodeGen, inst: Air.Inst.Index, result: WValue) InnerErro } } -const Branch = struct { - values: ValueTable = .{}, +fn pushBranch(cg: *CodeGen) InnerError!void { + var snapshot = try cg.local_pool.clone(cg.gpa); + errdefer snapshot.deinit(cg.gpa); + try cg.branches.append(cg.gpa, snapshot); +} - fn deinit(branch: *Branch, gpa: Allocator) void { - branch.values.deinit(gpa); - branch.* = undefined; - } -}; - -inline fn currentBranch(cg: *CodeGen) *Branch { - return &cg.branches.items[cg.branches.items.len - 1]; +fn popBranch(cg: *CodeGen) void { + var snapshot = cg.branches.pop().?; + defer snapshot.deinit(cg.gpa); + snapshot.restore(cg); } fn feed(cg: *CodeGen, bt: *Air.Liveness.BigTomb, operand: Air.Inst.Ref) void { @@ -476,20 +505,9 @@ fn feed(cg: *CodeGen, bt: *Air.Liveness.BigTomb, operand: Air.Inst.Ref) void { fn processDeath(cg: *CodeGen, ref: Air.Inst.Ref) void { if (ref.toIndex() == null) return; - // Branches are currently only allowed to free locals allocated - // within their own branch. - // TODO: Upon branch consolidation free any locals if needed. - const value = cg.currentBranch().values.getPtr(ref) orelse return; + const value = cg.values.getPtr(ref) orelse unreachable; if (value.* != .local) return; - const reserved_indexes = cg.args.len + @intFromBool(cg.return_value != .none); - if (value.local.value < reserved_indexes) { - return; // function arguments can never be re-used - } - log.debug("Decreasing reference for ref: %{d}, using local '{d}'", .{ @backingInt(ref.toIndex().?), value.local.value }); - value.local.references -= 1; - if (value.local.references == 0) { - value.free(cg); - } + cg.releaseLocal(value.local); } pub fn addInst(cg: *CodeGen, inst: Mir.Inst) error{OutOfMemory}!void { @@ -708,7 +726,7 @@ fn emitWValue(cg: *CodeGen, value: WValue) InnerError!void { switch (value) { .dead => unreachable, // reference to free'd `WValue` .none, .stack => {}, // no-op - .local => |idx| try cg.addLocal(.local_get, idx.value), + .local => |idx| try cg.addLocal(.local_get, idx), .imm32 => |val| try cg.addImm32(val), .imm64 => |val| try cg.addImm64(val), .imm128 => |val| try cg.addImm128(val), @@ -780,22 +798,28 @@ fn emitWValue(cg: *CodeGen, value: WValue) InnerError!void { /// Emits the base address for a memory instruction. fn emitMemBase(cg: *CodeGen, ptr: WValue) InnerError!void { switch (ptr) { - .stack_offset => try cg.addLocal(.local_get, cg.bottom_stack_value.local.value), + .stack_offset => try cg.addLocal(.local_get, cg.bottom_stack_value.local), else => try cg.emitWValue(ptr), } } -/// From a reference, returns its resolved `WValue`. -/// It's illegal to provide a `Air.Inst.Ref` that hasn't been resolved yet. -fn getResolvedInst(cg: *CodeGen, ref: Air.Inst.Ref) *WValue { - var index = cg.branches.items.len; - while (index > 0) : (index -= 1) { - const branch = cg.branches.items[index - 1]; - if (branch.values.getPtr(ref)) |value| { - return value; - } - } - unreachable; // developer-error: This can only be called on resolved instructions. Use `resolveInst` instead. +fn appendFreeLocal(cg: *CodeGen, index: u32) void { + assert(index >= cg.wasm_param_count); + const mir_index = index - cg.wasm_param_count; + const valtype = cg.mir_locals.items[mir_index]; + const free_list = cg.local_pool.freeList(valtype); + cg.local_pool.locals.items[mir_index] = .{ .next_free = free_list.* }; + free_list.* = .init(index); + log.debug("freed local ({d}) of type {}", .{ index, valtype }); +} + +fn releaseLocal(cg: *CodeGen, index: u32) void { + if (index < cg.wasm_param_count) return; + const local = &cg.local_pool.locals.items[index - cg.wasm_param_count]; + assert(local.ref_count > 0); + local.ref_count -= 1; + if (local.ref_count > 0) return; + cg.appendFreeLocal(index); } /// Creates one locals for a given `Type`. @@ -803,16 +827,13 @@ fn getResolvedInst(cg: *CodeGen, ref: Air.Inst.Ref) *WValue { fn allocLocal(cg: *CodeGen, ty: Type) InnerError!WValue { const zcu = cg.pt.zcu; const valtype = typeToValtype(ty, zcu, cg.target); - const index_or_null = switch (valtype) { - .i32 => cg.free_locals_i32.pop(), - .i64 => cg.free_locals_i64.pop(), - .f32 => cg.free_locals_f32.pop(), - .f64 => cg.free_locals_f64.pop(), - .v128 => cg.free_locals_v128.pop(), - }; - if (index_or_null) |index| { + const free_list = cg.local_pool.freeList(valtype); + if (free_list.unwrap()) |index| { + const local = &cg.local_pool.locals.items[index - cg.wasm_param_count]; + free_list.* = local.next_free; + local.* = .{ .ref_count = 1 }; log.debug("reusing local ({d}) of type {}", .{ index, valtype }); - return .{ .local = .{ .value = index, .references = 1 } }; + return .{ .local = index }; } log.debug("new local of type {}", .{valtype}); return cg.ensureAllocLocal(ty); @@ -822,10 +843,13 @@ fn allocLocal(cg: *CodeGen, ty: Type) InnerError!WValue { /// to use a zero-initialized local. fn ensureAllocLocal(cg: *CodeGen, ty: Type) InnerError!WValue { const zcu = cg.pt.zcu; - try cg.mir_locals.append(cg.gpa, typeToValtype(ty, zcu, cg.target)); - const initial_index = cg.local_index; - cg.local_index += 1; - return .{ .local = .{ .value = initial_index, .references = 1 } }; + try cg.mir_locals.ensureUnusedCapacity(cg.gpa, 1); + try cg.local_pool.locals.ensureUnusedCapacity(cg.gpa, 1); + assert(cg.mir_locals.items.len == cg.local_pool.locals.items.len); + const initial_index = cg.wasm_param_count + @as(u32, @intCast(cg.local_pool.locals.items.len)); + cg.mir_locals.appendAssumeCapacity(typeToValtype(ty, zcu, cg.target)); + cg.local_pool.locals.appendAssumeCapacity(.{ .ref_count = 1 }); + return .{ .local = initial_index }; } pub const Error = codegen.Error; @@ -867,7 +891,6 @@ pub fn generate( .args = cc_result.args, .return_value = cc_result.return_value, .varargs = cc_result.varargs, - .local_index = cc_result.local_index, .mir_instructions = .empty, .mir_extra = .empty, .mir_locals = .empty, @@ -890,17 +913,10 @@ pub fn generate( fn generateInner(cg: *CodeGen, any_returns: bool) InnerError!Mir { const zcu = cg.pt.zcu; - // branch used for const values - try cg.branches.append(cg.gpa, .{}); - // func scope branch - try cg.branches.append(cg.gpa, .{}); - defer { - var func_branch = cg.branches.pop().?; - func_branch.deinit(cg.gpa); - var const_branch = cg.branches.pop().?; - const_branch.deinit(cg.gpa); - assert(cg.branches.items.len == 0); // missing branch merge - } + cg.wasm_param_count = @intCast(cg.args.len + @intFromBool(cg.return_value != .none) + @intFromBool(cg.varargs != .none)); + cg.initial_stack_value = try cg.ensureAllocLocal(Type.usize); + cg.bottom_stack_value = try cg.ensureAllocLocal(Type.usize); + defer assert(cg.branches.items.len == 0); // Generate MIR for function body try cg.genBody(cg.air.getMainBody()); @@ -925,11 +941,11 @@ fn generateInner(cg: *CodeGen, any_returns: bool) InnerError!Mir { .instructions = cg.mir_instructions.toOwnedSlice(), .extra = cg.mir_extra.toOwnedSliceAssert(), .locals = cg.mir_locals.toOwnedSliceAssert(), - .prologue = if (cg.initial_stack_value == .none) .none else .{ - .sp_local = cg.initial_stack_value.local.value, + .prologue = .{ + .sp_local = cg.initial_stack_value.local, .flags = .{ .stack_alignment = cg.stack_alignment }, .stack_size = cg.stack_size, - .bottom_stack_local = cg.bottom_stack_value.local.value, + .bottom_stack_local = cg.bottom_stack_value.local, }, .uavs = cg.mir_uavs.move(), .indirect_function_set = cg.mir_indirect_function_set.move(), @@ -976,7 +992,7 @@ fn resolveCallingConventionValues( if (firstParamSRet(fn_info.cc, Type.fromInterned(fn_info.return_type), zcu, target)) { // the sret arg will be passed as first argument, therefore we // set the `return_value` before allocating locals for regular args. - result.return_value = .{ .local = .{ .value = result.local_index, .references = 1 } }; + result.return_value = .{ .local = result.local_index }; result.local_index += 1; } @@ -987,7 +1003,7 @@ fn resolveCallingConventionValues( continue; } - try args.append(.{ .local = .{ .value = result.local_index, .references = 1 } }); + try args.append(.{ .local = result.local_index }); result.local_index += 1; } }, @@ -1000,17 +1016,17 @@ fn resolveCallingConventionValues( switch (abi.classifyType(param_ty, zcu, target)) { .direct, .indirect => { - try args.append(.{ .local = .{ .value = result.local_index, .references = 1 } }); + try args.append(.{ .local = result.local_index }); result.local_index += 1; }, .double_i64 => { - try args.append(.{ .local = .{ .value = result.local_index, .references = 1 } }); - try args.append(.{ .local = .{ .value = result.local_index + 1, .references = 1 } }); + try args.append(.{ .local = result.local_index }); + try args.append(.{ .local = result.local_index + 1 }); result.local_index += 2; }, .unrolled => |vector| { for (0..vector.len) |_| { - try args.append(.{ .local = .{ .value = result.local_index, .references = 1 } }); + try args.append(.{ .local = result.local_index }); result.local_index += 1; } }, @@ -1021,7 +1037,7 @@ fn resolveCallingConventionValues( } if (fn_info.is_var_args) { - result.varargs = .{ .local = .{ .value = result.local_index, .references = 1 } }; + result.varargs = .{ .local = result.local_index }; result.local_index += 1; } @@ -1088,23 +1104,9 @@ fn lowerArg(cg: *CodeGen, cc: std.lang.CallingConvention, ty: Type, value: WValu } } -/// Creates a local for the initial stack value -/// Asserts `initial_stack_value` is `.none` -fn initializeStack(cg: *CodeGen) !void { - assert(cg.initial_stack_value == .none); - // Reserve a local to store the current stack pointer - // We can later use this local to set the stack pointer back to the value - // we have stored here. - cg.initial_stack_value = try cg.ensureAllocLocal(Type.usize); - // Also reserve a local to store the bottom stack value - cg.bottom_stack_value = try cg.ensureAllocLocal(Type.usize); -} - /// Reads the stack pointer from `Context.initial_stack_value` and writes it /// to the global stack pointer variable fn restoreStackPointer(cg: *CodeGen) !void { - // only restore the pointer if it was initialized - if (cg.initial_stack_value == .none) return; // Get the original stack pointer's value try cg.emitWValue(cg.initial_stack_value); @@ -1144,10 +1146,6 @@ fn allocInt(cg: *CodeGen, int_ty: IntType) !WValue { fn allocStackBytes(cg: *CodeGen, size: u32, alignment: Alignment) !WValue { assert(size > 0); - if (cg.initial_stack_value == .none) { - try cg.initializeStack(); - } - cg.stack_alignment = cg.stack_alignment.max(alignment); const offset: u32 = @intCast(alignment.forward(cg.stack_size)); @@ -1166,10 +1164,6 @@ fn allocStackPtr(cg: *CodeGen, inst: Air.Inst.Index) !WValue { const ptr_ty = cg.typeOfIndex(inst); const pointee_ty = ptr_ty.childType(zcu); - if (cg.initial_stack_value == .none) { - try cg.initializeStack(); - } - const abi_alignment = ptr_ty.ptrAlignment(zcu); const abi_size = std.math.cast(u32, pointee_ty.abiSize(zcu)) orelse { return cg.fail("Type {f} with ABI size of {d} exceeds stack frame size", .{ @@ -1934,7 +1928,6 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { continue; } const old_bookkeeping_value = cg.air_bookkeeping; - try cg.currentBranch().values.ensureUnusedCapacity(cg.gpa, 1); try cg.genInst(inst); if (std.debug.runtime_safety and cg.air_bookkeeping < old_bookkeeping_value + 1) { @@ -2160,11 +2153,11 @@ fn airCall(cg: *CodeGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier) .direct => |scalar_type| { if (!isByRef(ret_ty, zcu, cg.target)) { const result_local = try cg.allocLocal(ret_ty); - try cg.addLocal(.local_set, result_local.local.value); + try cg.addLocal(.local_set, result_local.local); break :result_value result_local; } else { const result_local = try cg.allocLocal(scalar_type); - try cg.addLocal(.local_set, result_local.local.value); + try cg.addLocal(.local_set, result_local.local); const result = try cg.allocStack(ret_ty); try cg.store(result, result_local, scalar_type, .{}); break :result_value result; @@ -2175,7 +2168,7 @@ fn airCall(cg: *CodeGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier) assert(vector.len == 1); const result_local = try cg.allocLocal(vector.elem_type); // save call result from operand stack - try cg.addLocal(.local_set, result_local.local.value); + try cg.addLocal(.local_set, result_local.local); const result = try cg.allocStack(ret_ty); try cg.store(result, result_local, vector.elem_type, .{}); break :result_value result; @@ -2183,7 +2176,7 @@ fn airCall(cg: *CodeGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier) } } else { const result_local = try cg.allocLocal(ret_ty); - try cg.addLocal(.local_set, result_local.local.value); + try cg.addLocal(.local_set, result_local.local); break :result_value result_local; } }; @@ -2250,7 +2243,7 @@ fn airVaArg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { } } - try cg.addLocal(.local_set, arg_ptr.local.value); + try cg.addLocal(.local_set, arg_ptr.local); try cg.emitWValue(operand); try cg.emitWValue(arg_ptr); @@ -2272,7 +2265,7 @@ fn airVaArg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { try cg.addTag(.f32_demote_f64); const result_local = try cg.allocLocal(Type.f32); - try cg.addLocal(.local_set, result_local.local.value); + try cg.addLocal(.local_set, result_local.local); break :result result_local; } else try cg.load(arg_ptr, ty, .{}); @@ -3300,7 +3293,7 @@ fn intAbs(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue { var mask = try cg.allocLocal(Type.i32); defer mask.free(cg); - try cg.addLocal(.local_tee, mask.local.value); + try cg.addLocal(.local_tee, mask.local); try cg.emitWValue(operand); try cg.addTag(.i32_xor); @@ -3315,7 +3308,7 @@ fn intAbs(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue { var mask = try cg.allocLocal(Type.i64); defer mask.free(cg); - try cg.addLocal(.local_tee, mask.local.value); + try cg.addLocal(.local_tee, mask.local); try cg.emitWValue(operand); try cg.addTag(.i64_xor); @@ -3337,7 +3330,7 @@ fn intAbs(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue { var tmp = try cg.allocLocal(Type.u64); defer tmp.free(cg); - try cg.addLocal(.local_tee, tmp.local.value); + try cg.addLocal(.local_tee, tmp.local); try cg.store(.stack, .stack, .u64, .{ .offset = mask_offset }); try cg.emitWValue(tmp); try cg.store(.stack, .stack, .u64, .{ .offset = mask_offset + 8 }); @@ -4033,7 +4026,7 @@ fn toLocalInt(cg: *CodeGen, value: WValue, int_ty: IntType) InnerError!WValue { else => return cg.fail("TODO: Support toLocalInt for integer bitsize: {d}", .{int_ty.bits}), }; const new_local = try cg.allocLocal(ty); - try cg.addLocal(.local_set, new_local.local.value); + try cg.addLocal(.local_set, new_local.local); return new_local; }, .local, .stack_offset => return value, @@ -4112,7 +4105,7 @@ fn intSubOverflow(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerErro fn intMulOverflow(cg: *CodeGen, int_ty: IntType, lhs: WValue, rhs: WValue) InnerError!OverflowResult { const overflow_bit = try cg.allocLocal(Type.u32); try cg.addImm32(0); - try cg.addLocal(.local_set, overflow_bit.local.value); + try cg.addLocal(.local_set, overflow_bit.local); const result_val = if (int_ty.bits <= 32) blk: { const new_ty: IntType = .{ .is_signed = int_ty.is_signed, .bits = 64 }; @@ -4126,7 +4119,7 @@ fn intMulOverflow(cg: *CodeGen, int_ty: IntType, lhs: WValue, rhs: WValue) Inner const res_upcast = try cg.intCast(new_ty, int_ty, res_tmp); _ = try cg.intCmp(new_ty, .neq, res_upcast, bin_op); - try cg.addLocal(.local_set, overflow_bit.local.value); + try cg.addLocal(.local_set, overflow_bit.local); break :blk res_tmp; } else if (int_ty.bits <= 64) blk: { const new_ty: IntType = .{ .is_signed = int_ty.is_signed, .bits = 128 }; @@ -4140,7 +4133,7 @@ fn intMulOverflow(cg: *CodeGen, int_ty: IntType, lhs: WValue, rhs: WValue) Inner const res_upcast = try cg.intCast(new_ty, int_ty, res_tmp); _ = try cg.intCmp(new_ty, .neq, res_upcast, bin_op); - try cg.addLocal(.local_set, overflow_bit.local.value); + try cg.addLocal(.local_set, overflow_bit.local); break :blk res_tmp; } else if (int_ty.bits == 128 and int_ty.is_signed) blk: { const overflow_ret = try cg.allocStack(Type.i32); @@ -4151,7 +4144,7 @@ fn intMulOverflow(cg: *CodeGen, int_ty: IntType, lhs: WValue, rhs: WValue) Inner &.{ lhs, rhs, overflow_ret }, ); _ = try cg.load(overflow_ret, .i32, .{}); - try cg.addLocal(.local_set, overflow_bit.local.value); + try cg.addLocal(.local_set, overflow_bit.local); break :blk res; } else { const result = try cg.allocInt(int_ty); @@ -5232,17 +5225,17 @@ fn lowerBlock(cg: *CodeGen, inst: Air.Inst.Index, block_ty: Type, body: []const .label = cg.block_depth, .value = block_result, }); + defer assert(cg.blocks.swapRemove(inst)); { - try cg.branches.append(cg.gpa, .{}); - defer { - var branch = cg.branches.pop().?; - branch.deinit(cg.gpa); - } + try cg.pushBranch(); + defer cg.popBranch(); try cg.genBody(body); try cg.endBlock(); } + for (cg.liveness.getBlock(inst).deaths) |death| cg.processDeath(death.toRef()); + return cg.finishAir(inst, block_result, &.{}); } @@ -5282,6 +5275,7 @@ fn airCondBr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { const condition = try cg.resolveInst(cond_br.condition); const then_body = cond_br.then_body; const else_body = cond_br.else_body; + const branch_liveness = cg.liveness.getCondBr(inst); // result type is always noreturn, so use `block_empty` as type. try cg.startBlock(.block, .empty); @@ -5293,24 +5287,21 @@ fn airCondBr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { // and continue with the then codepath try cg.addLabel(.br_if, 0); - try cg.branches.ensureUnusedCapacity(cg.gpa, 2); + if (cg.liveness.operandDies(inst, 0)) cg.processDeath(cond_br.condition); + { - cg.branches.appendAssumeCapacity(.{}); - defer { - var else_stack = cg.branches.pop().?; - else_stack.deinit(cg.gpa); - } + try cg.pushBranch(); + defer cg.popBranch(); + for (branch_liveness.else_deaths) |death| cg.processDeath(death.toRef()); try cg.genBody(else_body); try cg.endBlock(); } // Outer block that matches the condition { - cg.branches.appendAssumeCapacity(.{}); - defer { - var then_stack = cg.branches.pop().?; - then_stack.deinit(cg.gpa); - } + try cg.pushBranch(); + defer cg.popBranch(); + for (branch_liveness.then_deaths) |death| cg.processDeath(death.toRef()); try cg.genBody(then_body); } @@ -5346,18 +5337,18 @@ fn airCmp(cg: *CodeGen, inst: Air.Inst.Index, op: std.math.CompareOperator) Inne try cg.startBlock(.block, .empty); try cg.addImm32(if (op == .eq) 0 else 1); - try cg.addLocal(.local_set, result.local.value); + try cg.addLocal(.local_set, result.local); _ = try cg.isNull(lhs, operand_ty, .i32_eq, .value); - try cg.addLocal(.local_tee, lhs_null.local.value); + try cg.addLocal(.local_tee, lhs_null.local); _ = try cg.isNull(rhs, operand_ty, .i32_eq, .value); try cg.addTag(.i32_ne); try cg.addLabel(.br_if, 0); try cg.addImm32(if (op == .eq) 1 else 0); - try cg.addLocal(.local_set, result.local.value); + try cg.addLocal(.local_set, result.local); - try cg.addLocal(.local_get, lhs_null.local.value); + try cg.addLocal(.local_get, lhs_null.local); try cg.addLabel(.br_if, 0); _ = try cg.load(lhs, payload_ty, .{}); @@ -5369,10 +5360,10 @@ fn airCmp(cg: *CodeGen, inst: Air.Inst.Index, op: std.math.CompareOperator) Inne _ = try cg.intCmp(.fromType(cg, payload_ty), op, .stack, .stack); } - try cg.addLocal(.local_set, result.local.value); + try cg.addLocal(.local_set, result.local); try cg.endBlock(); - try cg.addLocal(.local_get, result.local.value); + try cg.addLocal(.local_get, result.local); try cg.finishAir(inst, .stack, &.{ bin_op.lhs, bin_op.rhs }); } else { const result = try cg.intCmp(.fromType(cg, operand_ty), op, lhs, rhs); @@ -5570,7 +5561,7 @@ fn airBr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { if (block.value != .none) { const operand = try cg.resolveInst(br.operand); try cg.emitWValue(operand); - try cg.addLocal(.local_set, block.value.local.value); + try cg.addLocal(.local_set, block.value.local); } // We map every block to its block index. @@ -5854,24 +5845,29 @@ fn airSwitchBr(cg: *CodeGen, inst: Air.Inst.Index, is_dispatch_loop: bool) Inner const switch_br = cg.air.unwrapSwitch(inst); const target_ty = cg.typeOf(switch_br.operand); + const branch_liveness = try cg.liveness.getSwitchBr(cg.gpa, inst, switch_br.cases_len + 1); + defer cg.gpa.free(branch_liveness.deaths); assert(target_ty.hasRuntimeBits(zcu)); // swap target value with placeholder local, for dispatching + var owned_target: WValue = .none; + defer owned_target.free(cg); const target = if (is_dispatch_loop) target: { const initial_target = try cg.resolveInst(switch_br.operand); - const target: WValue = try cg.allocLocal(target_ty); + owned_target = try cg.allocLocal(target_ty); try cg.emitWValue(initial_target); - try cg.addLocal(.local_set, target.local.value); + try cg.addLocal(.local_set, owned_target.local); try cg.startBlock(.loop, .empty); // dispatch loop start try cg.blocks.putNoClobber(cg.gpa, inst, .{ .label = cg.block_depth, - .value = target, + .value = owned_target, }); - break :target target; + break :target owned_target; } else try cg.resolveInst(switch_br.operand); + defer if (is_dispatch_loop) assert(cg.blocks.swapRemove(inst)); const has_else_body = switch_br.else_body_len != 0; const branch_count = switch_br.cases_len + 1; // if else branch is missing, we trap when failing all conditions @@ -5883,12 +5879,14 @@ fn airSwitchBr(cg: *CodeGen, inst: Air.Inst.Index, is_dispatch_loop: bool) Inner var it = switch_br.iterateCases(); const else_body = it.elseBody(); - cg.branches.appendAssumeCapacity(.{}); - defer { - var else_branch = cg.branches.pop().?; - else_branch.deinit(cg.gpa); + if (cg.liveness.operandDies(inst, 0)) cg.processDeath(switch_br.operand); + + { + try cg.pushBranch(); + defer cg.popBranch(); + for (branch_liveness.deaths[0]) |death| cg.processDeath(death.toRef()); + try cg.genBody(else_body); } - try cg.genBody(else_body); if (is_dispatch_loop) { try cg.endBlock(); // dispatch loop end @@ -6012,16 +6010,18 @@ fn airSwitchBr(cg: *CodeGen, inst: Air.Inst.Index, is_dispatch_loop: bool) Inner try cg.addLabel(.br, branch_count - 1); } + if (cg.liveness.operandDies(inst, 0)) cg.processDeath(switch_br.operand); + var cases_it = switch_br.iterateCases(); while (cases_it.next()) |case| { try cg.endBlock(); - cg.branches.appendAssumeCapacity(.{}); - defer { - var case_branch = cg.branches.pop().?; - case_branch.deinit(cg.gpa); + { + try cg.pushBranch(); + defer cg.popBranch(); + for (branch_liveness.deaths[case.idx]) |death| cg.processDeath(death.toRef()); + try cg.genBody(case.body); } - try cg.genBody(case.body); try cg.addLabel(.br, branch_count - case.idx - 1); // matching case found and executed => exit switch } @@ -6030,12 +6030,12 @@ fn airSwitchBr(cg: *CodeGen, inst: Air.Inst.Index, is_dispatch_loop: bool) Inner if (has_else_body) { const else_body = cases_it.elseBody(); - cg.branches.appendAssumeCapacity(.{}); - defer { - var else_branch = cg.branches.pop().?; - else_branch.deinit(cg.gpa); + { + try cg.pushBranch(); + defer cg.popBranch(); + for (branch_liveness.deaths[branch_liveness.deaths.len - 1]) |death| cg.processDeath(death.toRef()); + try cg.genBody(else_body); } - try cg.genBody(else_body); } else { try cg.addTag(.@"unreachable"); } @@ -6055,7 +6055,7 @@ fn airSwitchDispatch(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { const operand = try cg.resolveInst(br.operand); try cg.emitWValue(operand); - try cg.addLocal(.local_set, switch_loop.value.local.value); + try cg.addLocal(.local_set, switch_loop.value.local); const idx: u32 = cg.block_depth - switch_loop.label; try cg.addLabel(.br, idx); @@ -6630,7 +6630,7 @@ fn memset(cg: *CodeGen, elem_ty: Type, ptr: WValue, len: WValue, value: WValue) try cg.addTag(.i64_mul); }, } - try cg.addLocal(.local_set, new_len.local.value); + try cg.addLocal(.local_set, new_len.local); break :blk new_len; } else len, }; @@ -6640,7 +6640,7 @@ fn memset(cg: *CodeGen, elem_ty: Type, ptr: WValue, len: WValue, value: WValue) var new_ptr = try cg.allocLocal(Type.usize); defer new_ptr.free(cg); try cg.emitWValue(ptr); - try cg.addLocal(.local_set, new_ptr.local.value); + try cg.addLocal(.local_set, new_ptr.local); // get the loop conditional: if current pointer address equals final pointer's address try cg.emitWValue(ptr); @@ -6649,7 +6649,7 @@ fn memset(cg: *CodeGen, elem_ty: Type, ptr: WValue, len: WValue, value: WValue) .wasm32 => try cg.addTag(.i32_add), .wasm64 => try cg.addTag(.i64_add), } - try cg.addLocal(.local_set, end_ptr.local.value); + try cg.addLocal(.local_set, end_ptr.local); // outer block to jump to when loop is done try cg.startBlock(.block, .empty); @@ -6679,7 +6679,7 @@ fn memset(cg: *CodeGen, elem_ty: Type, ptr: WValue, len: WValue, value: WValue) try cg.addTag(.i64_add); }, } - try cg.addLocal(.local_set, new_ptr.local.value); + try cg.addLocal(.local_set, new_ptr.local); // end of loop try cg.addLabel(.br, 0); // jump to start of loop @@ -7186,7 +7186,7 @@ fn airMemcpy(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { try cg.emitWValue(slice_len); try cg.emitWValue(.{ .imm32 = @as(u32, @intCast(ptr_elem_ty.abiSize(zcu))) }); try cg.addTag(.i32_mul); - try cg.addLocal(.local_set, slice_len.local.value); + try cg.addLocal(.local_set, slice_len.local); } break :blk slice_len; }, @@ -7217,7 +7217,7 @@ fn airMemmove(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { try cg.emitWValue(slice_len); try cg.emitWValue(.{ .imm32 = @as(u32, @intCast(ptr_elem_ty.abiSize(zcu))) }); try cg.addTag(.i32_mul); - try cg.addLocal(.local_set, slice_len.local.value); + try cg.addLocal(.local_set, slice_len.local); } break :blk slice_len; }, @@ -7303,8 +7303,13 @@ fn airTry(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { const body = unwrapped_try.else_body; const err_union = try cg.resolveInst(unwrapped_try.error_union); const err_union_ty = cg.typeOf(unwrapped_try.error_union); - const result = try lowerTry(cg, inst, err_union, body, err_union_ty, false); - return cg.finishAir(inst, result, &.{unwrapped_try.error_union}); + const result = try lowerTry(cg, inst, unwrapped_try.error_union, err_union, body, err_union_ty, false); + return cg.finishAirWithBranchDeaths( + inst, + result, + &.{unwrapped_try.error_union}, + cg.liveness.getCondBr(inst).then_deaths, + ); } fn airTryPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { @@ -7313,20 +7318,26 @@ fn airTryPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { const err_union_ptr = try cg.resolveInst(unwrapped_try.error_union_ptr); const body = unwrapped_try.else_body; const err_union_ty = cg.typeOf(unwrapped_try.error_union_ptr).childType(zcu); - const result = try lowerTry(cg, inst, err_union_ptr, body, err_union_ty, true); - return cg.finishAir(inst, result, &.{unwrapped_try.error_union_ptr}); + const result = try lowerTry(cg, inst, unwrapped_try.error_union_ptr, err_union_ptr, body, err_union_ty, true); + return cg.finishAirWithBranchDeaths( + inst, + result, + &.{unwrapped_try.error_union_ptr}, + cg.liveness.getCondBr(inst).then_deaths, + ); } fn lowerTry( cg: *CodeGen, inst: Air.Inst.Index, + operand_ref: Air.Inst.Ref, err_union: WValue, body: []const Air.Inst.Index, err_union_ty: Type, operand_is_ptr: bool, ) InnerError!WValue { - _ = inst; const zcu = cg.pt.zcu; + const branch_liveness = cg.liveness.getCondBr(inst); const pl_ty = err_union_ty.errorUnionPayload(zcu); const pl_has_bits = pl_ty.hasRuntimeBits(zcu); @@ -7349,12 +7360,13 @@ fn lowerTry( try cg.addTag(.i32_eqz); try cg.addLabel(.br_if, 0); // jump out of block when error is '0' - try cg.branches.append(cg.gpa, .{}); - defer { - var branch = cg.branches.pop().?; - branch.deinit(cg.gpa); + { + try cg.pushBranch(); + defer cg.popBranch(); + if (cg.liveness.operandDies(inst, 0)) cg.processDeath(operand_ref); + for (branch_liveness.else_deaths) |death| cg.processDeath(death.toRef()); + try cg.genBody(body); } - try cg.genBody(body); try cg.endBlock(); } @@ -7367,8 +7379,7 @@ fn lowerTry( if (operand_is_ptr or isByRef(pl_ty, zcu, cg.target)) { return cg.offsetPointer(err_union, pl_offset); } - const payload = try cg.load(err_union, pl_ty, .{ .offset = pl_offset }); - return payload.toLocal(cg, pl_ty); + return cg.load(err_union, pl_ty, .{ .offset = pl_offset }); } /// Calls a compiler-rt intrinsic by creating an undefined symbol, @@ -7512,13 +7523,13 @@ fn airErrorSetHasValue(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { // 'false' branch (i.e. error set does not have value // ensure we set local to 0 in case the local was re-used. try cg.addImm32(0); - try cg.addLocal(.local_set, result.local.value); + try cg.addLocal(.local_set, result.local); try cg.addLabel(.br, 1); try cg.endBlock(); // 'true' branch try cg.addImm32(1); - try cg.addLocal(.local_set, result.local.value); + try cg.addLocal(.local_set, result.local); try cg.addLabel(.br, 0); try cg.endBlock(); @@ -7561,9 +7572,9 @@ fn airCmpxchg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { .offset = ptr_operand.memOffset(), .alignment = ty.abiAlignment(zcu).toStdMem(), }); - try cg.addLocal(.local_tee, val_local.local.value); + try cg.addLocal(.local_tee, val_local.local); _ = try cg.intCmp(int_ty, .eq, .stack, expected_val); - try cg.addLocal(.local_set, cmp_result.local.value); + try cg.addLocal(.local_set, cmp_result.local); break :val val_local; } else val: { if (ty.abiSize(zcu) > 8) { @@ -7575,7 +7586,7 @@ fn airCmpxchg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { try cg.emitWValue(new_val); try cg.emitWValue(ptr_val); _ = try cg.intCmp(int_ty, .eq, ptr_val, expected_val); - try cg.addLocal(.local_tee, cmp_result.local.value); + try cg.addLocal(.local_tee, cmp_result.local); try cg.addTag(.select); try cg.store(.stack, .stack, ty, .{ .offset = ptr_operand.memOffset() }); @@ -7685,11 +7696,11 @@ fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { }, ); const select_res = try cg.allocLocal(ty); - try cg.addLocal(.local_tee, select_res.local.value); + try cg.addLocal(.local_tee, select_res.local); _ = try cg.intCmp(int_ty, .neq, .stack, value); // leave on stack so we can use it for br_if try cg.emitWValue(select_res); - try cg.addLocal(.local_set, value.local.value); + try cg.addLocal(.local_set, value.local); try cg.addLabel(.br_if, 0); try cg.endBlock(); @@ -7864,9 +7875,6 @@ fn airAtomicStore(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { } fn airFrameAddress(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { - if (cg.initial_stack_value == .none) { - try cg.initializeStack(); - } try cg.emitWValue(cg.bottom_stack_value); return cg.finishAir(inst, .stack, &.{}); } @@ -7912,7 +7920,7 @@ fn airAsm(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { return cg.fail("Self-hosted wasm backend requires output constraint to be equal \"=r\"", .{}); } - const gop = try local_map.getOrPutValue(cg.gpa, name, result.local.value); + const gop = try local_map.getOrPutValue(cg.gpa, name, result.local); assert(!gop.found_existing); // first value assert(it.next() == null); @@ -7933,7 +7941,7 @@ fn airAsm(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { try cg.emitWValue(operand); const op_local = try WValue.toLocal(.stack, cg, cg.typeOf(input.operand)); - const gop = try local_map.getOrPutValue(cg.gpa, name, op_local.local.value); + const gop = try local_map.getOrPutValue(cg.gpa, name, op_local.local); if (gop.found_existing) { return cg.fail("Duplicate asm variable name \"{s}\"", .{name}); } -- 2.54.0 From 65b732cd6118eaac95929c08f262514e58a4e3b0 Mon Sep 17 00:00:00 2001 From: Pavel Verigo Date: Tue, 15 Sep 2026 19:32:39 +0200 Subject: [PATCH 7/8] wasm: local reuse and free verification + better blocks codegen --- src/codegen/wasm/CodeGen.zig | 226 +++++++++++++++++++++++------------ 1 file changed, 151 insertions(+), 75 deletions(-) diff --git a/src/codegen/wasm/CodeGen.zig b/src/codegen/wasm/CodeGen.zig index cf68c7a738175e9ecdddd18f12bc1651d36e7892..aec24e1e8ce59b81d43f7bc31facaa6b9ab4195e 100644 --- a/src/codegen/wasm/CodeGen.zig +++ b/src/codegen/wasm/CodeGen.zig @@ -473,11 +473,20 @@ fn finishAirWithBranchDeaths( fn finishAirResult(cg: *CodeGen, inst: Air.Inst.Index, result: WValue) InnerError!void { // results of `none` can never be referenced. if (result != .none) { - const trackable_result = if (result != .stack) - result - else - try result.toLocal(cg, cg.typeOfIndex(inst)); - try cg.values.putNoClobber(cg.gpa, inst.toRef(), trackable_result); + if (cg.liveness.isUnused(inst)) { + if (result == .stack) { + try cg.addTag(.drop); + } else { + var unused_result = result; + unused_result.free(cg); + } + } else { + const trackable_result = if (result != .stack) + result + else + try result.toLocal(cg, cg.typeOfIndex(inst)); + try cg.values.putNoClobber(cg.gpa, inst.toRef(), trackable_result); + } } if (std.debug.runtime_safety) { @@ -510,6 +519,27 @@ fn processDeath(cg: *CodeGen, ref: Air.Inst.Ref) void { cg.releaseLocal(value.local); } +fn verifyLocalPool(cg: *CodeGen) void { + if (true) return; // TODO: test more + if (!std.debug.runtime_safety) return; + + var free_count: usize = 0; + for ([_]std.wasm.Valtype{ .i32, .i64, .f32, .f64, .v128 }) |valtype| { + var next = cg.local_pool.freeList(valtype).*; + while (next.unwrap()) |index| { + assert(index >= cg.wasm_param_count); + const pool_index = index - cg.wasm_param_count; + assert(pool_index < cg.local_pool.locals.items.len); + assert(index != cg.initial_stack_value.local); + assert(index != cg.bottom_stack_value.local); + assert(cg.mir_locals.items[pool_index] == valtype); + free_count += 1; + next = cg.local_pool.locals.items[pool_index].next_free; + } + } + assert(free_count + 2 == cg.local_pool.locals.items.len); +} + pub fn addInst(cg: *CodeGen, inst: Mir.Inst) error{OutOfMemory}!void { try cg.mir_instructions.append(cg.gpa, inst); } @@ -656,10 +686,8 @@ pub fn typeToValtype(ty: Type, zcu: *const Zcu, target: *const std.Target) std.w /// Differently from `typeToValtype` this also allows `void` to create a block /// with no return type fn genBlockType(ty: Type, zcu: *const Zcu, target: *const std.Target) std.wasm.BlockType { - return switch (ty.ip_index) { - .void_type, .noreturn_type => .empty, - else => .fromValtype(typeToValtype(ty, zcu, target)), - }; + if (!ty.hasRuntimeBits(zcu)) return .empty; + return .fromValtype(typeToValtype(ty, zcu, target)); } /// Returns `ptr` with `offset` applied. @@ -1339,7 +1367,8 @@ fn genInst(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { try cg.emitWValue(vec_ptr); try cg.emitScaledPointerOffset(elem_idx, elem_size, .add); - const ptr = try WValue.toLocal(.stack, cg, Type.usize); + var ptr = try WValue.toLocal(.stack, cg, Type.usize); + defer ptr.free(cg); try cg.store(ptr, elem_val, elem_ty, .{}); @@ -1975,7 +2004,8 @@ fn airRet(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { try cg.restoreStackPointer(); try cg.addTag(.@"return"); - return cg.finishAir(inst, .none, &.{un_op}); + try cg.finishAir(inst, .none, &.{un_op}); + cg.verifyLocalPool(); } fn airRetPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { @@ -2026,7 +2056,8 @@ fn airRetLoad(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { try cg.restoreStackPointer(); try cg.addTag(.@"return"); - return cg.finishAir(inst, .none, &.{un_op}); + try cg.finishAir(inst, .none, &.{un_op}); + cg.verifyLocalPool(); } fn airCall(cg: *CodeGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier) InnerError!void { @@ -2156,7 +2187,8 @@ fn airCall(cg: *CodeGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier) try cg.addLocal(.local_set, result_local.local); break :result_value result_local; } else { - const result_local = try cg.allocLocal(scalar_type); + var result_local = try cg.allocLocal(scalar_type); + defer result_local.free(cg); try cg.addLocal(.local_set, result_local.local); const result = try cg.allocStack(ret_ty); try cg.store(result, result_local, scalar_type, .{}); @@ -2166,7 +2198,8 @@ fn airCall(cg: *CodeGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier) .double_i64, .indirect => unreachable, .unrolled => |vector| { assert(vector.len == 1); - const result_local = try cg.allocLocal(vector.elem_type); + var result_local = try cg.allocLocal(vector.elem_type); + defer result_local.free(cg); // save call result from operand stack try cg.addLocal(.local_set, result_local.local); const result = try cg.allocStack(ret_ty); @@ -2223,7 +2256,8 @@ fn airVaArg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { const abi_size: u32 = @intCast(load_ty.abiSize(zcu)); const abi_align: u32 = @intCast(load_ty.abiAlignment(zcu).toByteUnits().?); - const arg_ptr = try cg.allocLocal(.usize); + var arg_ptr = try cg.allocLocal(.usize); + defer arg_ptr.free(cg); _ = try cg.load(operand, .usize, .{}); if (abi_align > 1) { @@ -3027,12 +3061,14 @@ fn intAnd(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue const lhs_lsb = try cg.load(lhs, .u64, .{}); const rhs_lsb = try cg.load(rhs, .u64, .{}); - const and_lsb = try (try cg.intAnd(.u64, lhs_lsb, rhs_lsb)).toLocal(cg, Type.u64); + var and_lsb = try (try cg.intAnd(.u64, lhs_lsb, rhs_lsb)).toLocal(cg, Type.u64); + defer and_lsb.free(cg); try cg.store(result, and_lsb, .u64, .{}); const lhs_msb = try cg.load(lhs, .u64, .{ .offset = 8 }); const rhs_msb = try cg.load(rhs, .u64, .{ .offset = 8 }); - const and_msb = try (try cg.intAnd(.u64, lhs_msb, rhs_msb)).toLocal(cg, Type.u64); + var and_msb = try (try cg.intAnd(.u64, lhs_msb, rhs_msb)).toLocal(cg, Type.u64); + defer and_msb.free(cg); try cg.store(result, and_msb, .u64, .{ .offset = 8 }); return result; @@ -3071,12 +3107,14 @@ fn intOr(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue const lhs_lsb = try cg.load(lhs, .u64, .{}); const rhs_lsb = try cg.load(rhs, .u64, .{}); - const or_lsb = try (try cg.intOr(.u64, lhs_lsb, rhs_lsb)).toLocal(cg, Type.u64); + var or_lsb = try (try cg.intOr(.u64, lhs_lsb, rhs_lsb)).toLocal(cg, Type.u64); + defer or_lsb.free(cg); try cg.store(result, or_lsb, .u64, .{}); const lhs_msb = try cg.load(lhs, .u64, .{ .offset = 8 }); const rhs_msb = try cg.load(rhs, .u64, .{ .offset = 8 }); - const or_msb = try (try cg.intOr(.u64, lhs_msb, rhs_msb)).toLocal(cg, Type.u64); + var or_msb = try (try cg.intOr(.u64, lhs_msb, rhs_msb)).toLocal(cg, Type.u64); + defer or_msb.free(cg); try cg.store(result, or_msb, .u64, .{ .offset = 8 }); return result; @@ -3115,12 +3153,14 @@ fn intXor(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue const lhs_lsb = try cg.load(lhs, .u64, .{}); const rhs_lsb = try cg.load(rhs, .u64, .{}); - const xor_lsb = try (try cg.intXor(.u64, lhs_lsb, rhs_lsb)).toLocal(cg, Type.u64); + var xor_lsb = try (try cg.intXor(.u64, lhs_lsb, rhs_lsb)).toLocal(cg, Type.u64); + defer xor_lsb.free(cg); try cg.store(result, xor_lsb, .u64, .{}); const lhs_msb = try cg.load(lhs, .u64, .{ .offset = 8 }); const rhs_msb = try cg.load(rhs, .u64, .{ .offset = 8 }); - const xor_msb = try (try cg.intXor(.u64, lhs_msb, rhs_msb)).toLocal(cg, Type.u64); + var xor_msb = try (try cg.intXor(.u64, lhs_msb, rhs_msb)).toLocal(cg, Type.u64); + defer xor_msb.free(cg); try cg.store(result, xor_msb, .u64, .{ .offset = 8 }); return result; @@ -4103,6 +4143,19 @@ fn intSubOverflow(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerErro } fn intMulOverflow(cg: *CodeGen, int_ty: IntType, lhs: WValue, rhs: WValue) InnerError!OverflowResult { + if (int_ty.bits > 64 and (int_ty.bits != 128 or !int_ty.is_signed)) { + const result = try cg.allocInt(int_ty); + + try cg.emitWValue(result); + try cg.emitWValue(lhs); + try cg.emitWValue(rhs); + try cg.addImm32(@intFromBool(int_ty.is_signed)); + try cg.addImm32(int_ty.bits); + try cg.addCallIntrinsic(.__mulo_limb64); + + return .{ .result = result, .ov = .stack }; + } + const overflow_bit = try cg.allocLocal(Type.u32); try cg.addImm32(0); try cg.addLocal(.local_set, overflow_bit.local); @@ -4112,7 +4165,8 @@ fn intMulOverflow(cg: *CodeGen, int_ty: IntType, lhs: WValue, rhs: WValue) Inner const lhs_upcast = try cg.intCast(new_ty, int_ty, lhs); const rhs_upcast = try cg.intCast(new_ty, int_ty, rhs); const mul_raw = try cg.intMul(new_ty, lhs_upcast, rhs_upcast); - const bin_op = try cg.toLocalInt(mul_raw, new_ty); + var bin_op = try cg.toLocalInt(mul_raw, new_ty); + defer bin_op.free(cg); const res = try cg.intTrunc(int_ty, new_ty, bin_op); const res_tmp = try cg.toLocalInt(res, int_ty); @@ -4126,7 +4180,8 @@ fn intMulOverflow(cg: *CodeGen, int_ty: IntType, lhs: WValue, rhs: WValue) Inner const lhs_upcast = try cg.intCast(new_ty, int_ty, lhs); const rhs_upcast = try cg.intCast(new_ty, int_ty, rhs); const mul_raw = try cg.intMul(new_ty, lhs_upcast, rhs_upcast); - const bin_op = try cg.toLocalInt(mul_raw, new_ty); + var bin_op = try cg.toLocalInt(mul_raw, new_ty); + defer bin_op.free(cg); const res = try cg.intTrunc(int_ty, new_ty, bin_op); const res_tmp = try cg.toLocalInt(res, int_ty); @@ -4135,7 +4190,8 @@ fn intMulOverflow(cg: *CodeGen, int_ty: IntType, lhs: WValue, rhs: WValue) Inner _ = try cg.intCmp(new_ty, .neq, res_upcast, bin_op); try cg.addLocal(.local_set, overflow_bit.local); break :blk res_tmp; - } else if (int_ty.bits == 128 and int_ty.is_signed) blk: { + } else blk: { + assert(int_ty.bits == 128 and int_ty.is_signed); const overflow_ret = try cg.allocStack(Type.i32); const res = try cg.callIntrinsic( .__muloti4, @@ -4146,17 +4202,6 @@ fn intMulOverflow(cg: *CodeGen, int_ty: IntType, lhs: WValue, rhs: WValue) Inner _ = try cg.load(overflow_ret, .i32, .{}); try cg.addLocal(.local_set, overflow_bit.local); break :blk res; - } else { - const result = try cg.allocInt(int_ty); - - try cg.emitWValue(result); - try cg.emitWValue(lhs); - try cg.emitWValue(rhs); - try cg.addImm32(@intFromBool(int_ty.is_signed)); - try cg.addImm32(int_ty.bits); - try cg.addCallIntrinsic(.__mulo_limb64); - - return .{ .result = result, .ov = .stack }; }; return .{ .result = result_val, .ov = .{ .local = overflow_bit.local } }; @@ -5212,13 +5257,19 @@ fn airBlock(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { fn lowerBlock(cg: *CodeGen, inst: Air.Inst.Index, block_ty: Type, body: []const Air.Inst.Index) InnerError!void { const zcu = cg.pt.zcu; - // if wasm_block_ty is non-empty, we create a register to store the temporary value + + if (block_ty.isNoReturn(zcu)) { + assert(cg.liveness.getBlock(inst).deaths.len == 0); + try cg.genBody(body); + return cg.finishAir(inst, .none, &.{}); + } + const block_result: WValue = if (block_ty.hasRuntimeBits(zcu)) - try cg.allocLocal(block_ty) + .stack else .none; - try cg.startBlock(.block, .empty); + try cg.startBlock(.block, genBlockType(block_ty, zcu, cg.target)); // Here we set the current block idx, so breaks know the depth to jump // to when breaking out. try cg.blocks.putNoClobber(cg.gpa, inst, .{ @@ -5266,6 +5317,7 @@ fn airLoop(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { try cg.genBody(block.body); try cg.endBlock(); + try cg.addTag(.@"unreachable"); return cg.finishAir(inst, .none, &.{}); } @@ -5557,11 +5609,13 @@ fn airBr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { const br = cg.air.instructions.items(.data)[@backingInt(inst)].br; const block = cg.blocks.get(br.block_inst).?; - // if operand has codegen bits we should break with a value - if (block.value != .none) { - const operand = try cg.resolveInst(br.operand); - try cg.emitWValue(operand); - try cg.addLocal(.local_set, block.value.local); + switch (block.value) { + .none => {}, + .stack => { + const operand = try cg.resolveInst(br.operand); + try cg.emitWValue(operand); + }, + else => unreachable, } // We map every block to its block index. @@ -5584,7 +5638,8 @@ fn airRepeat(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { fn airTrap(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { try cg.addTag(.@"unreachable"); - return cg.finishAir(inst, .none, &.{}); + try cg.finishAir(inst, .none, &.{}); + cg.verifyLocalPool(); } fn airBreakpoint(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { @@ -5596,7 +5651,8 @@ fn airBreakpoint(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { fn airUnreachable(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { try cg.addTag(.@"unreachable"); - return cg.finishAir(inst, .none, &.{}); + try cg.finishAir(inst, .none, &.{}); + cg.verifyLocalPool(); } fn airNopCast(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { @@ -5839,6 +5895,7 @@ fn airAggFieldVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { return cg.finishAir(inst, result, &.{struct_field.struct_operand}); } +// Both `.switch_br` and `.loop_switch_br` are noreturn. fn airSwitchBr(cg: *CodeGen, inst: Air.Inst.Index, is_dispatch_loop: bool) InnerError!void { const pt = cg.pt; const zcu = pt.zcu; @@ -5850,9 +5907,9 @@ fn airSwitchBr(cg: *CodeGen, inst: Air.Inst.Index, is_dispatch_loop: bool) Inner assert(target_ty.hasRuntimeBits(zcu)); - // swap target value with placeholder local, for dispatching + const operand_dies = cg.liveness.operandDies(inst, 0); var owned_target: WValue = .none; - defer owned_target.free(cg); + errdefer owned_target.free(cg); const target = if (is_dispatch_loop) target: { const initial_target = try cg.resolveInst(switch_br.operand); owned_target = try cg.allocLocal(target_ty); @@ -5879,7 +5936,8 @@ fn airSwitchBr(cg: *CodeGen, inst: Air.Inst.Index, is_dispatch_loop: bool) Inner var it = switch_br.iterateCases(); const else_body = it.elseBody(); - if (cg.liveness.operandDies(inst, 0)) cg.processDeath(switch_br.operand); + if (operand_dies) cg.processDeath(switch_br.operand); + owned_target.free(cg); { try cg.pushBranch(); @@ -5890,6 +5948,7 @@ fn airSwitchBr(cg: *CodeGen, inst: Air.Inst.Index, is_dispatch_loop: bool) Inner if (is_dispatch_loop) { try cg.endBlock(); // dispatch loop end + try cg.addTag(.@"unreachable"); } return cg.finishAir(inst, .none, &.{}); } @@ -5934,8 +5993,6 @@ fn airSwitchBr(cg: *CodeGen, inst: Air.Inst.Index, is_dispatch_loop: bool) Inner break :width width_bigint.toConst().toInt(u32) catch null; }; - try cg.startBlock(.block, .empty); // whole switch block start - for (0..branch_count) |_| { try cg.startBlock(.block, .empty); } @@ -6010,7 +6067,8 @@ fn airSwitchBr(cg: *CodeGen, inst: Air.Inst.Index, is_dispatch_loop: bool) Inner try cg.addLabel(.br, branch_count - 1); } - if (cg.liveness.operandDies(inst, 0)) cg.processDeath(switch_br.operand); + if (operand_dies) cg.processDeath(switch_br.operand); + owned_target.free(cg); var cases_it = switch_br.iterateCases(); while (cases_it.next()) |case| { @@ -6022,8 +6080,6 @@ fn airSwitchBr(cg: *CodeGen, inst: Air.Inst.Index, is_dispatch_loop: bool) Inner for (branch_liveness.deaths[case.idx]) |death| cg.processDeath(death.toRef()); try cg.genBody(case.body); } - - try cg.addLabel(.br, branch_count - case.idx - 1); // matching case found and executed => exit switch } try cg.endBlock(); @@ -6040,10 +6096,9 @@ fn airSwitchBr(cg: *CodeGen, inst: Air.Inst.Index, is_dispatch_loop: bool) Inner try cg.addTag(.@"unreachable"); } - try cg.endBlock(); // whole switch block end - if (is_dispatch_loop) { try cg.endBlock(); // dispatch loop end + try cg.addTag(.@"unreachable"); } return cg.finishAir(inst, .none, &.{}); @@ -6547,11 +6602,6 @@ fn airMemset(cg: *CodeGen, inst: Air.Inst.Index, safety: bool) InnerError!void { const ptr = try cg.resolveInst(bin_op.lhs); const ptr_ty = cg.typeOf(bin_op.lhs); const value = try cg.resolveInst(bin_op.rhs); - const len = switch (ptr_ty.ptrSize(zcu)) { - .slice => try cg.sliceLen(ptr), - .one => @as(WValue, .{ .imm32 = @as(u32, @intCast(ptr_ty.childType(zcu).arrayLen(zcu))) }), - .c, .many => unreachable, - }; const elem_ty = if (ptr_ty.ptrSize(zcu) == .one) ptr_ty.childType(zcu).childType(zcu) @@ -6562,7 +6612,14 @@ fn airMemset(cg: *CodeGen, inst: Air.Inst.Index, safety: bool) InnerError!void { return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs }); } - const dst_ptr = try cg.sliceOrArrayPtr(ptr, ptr_ty); + var dst_ptr = try cg.sliceOrArrayPtr(ptr, ptr_ty); + defer if (ptr_ty.isSlice(zcu)) dst_ptr.free(cg); + var len = switch (ptr_ty.ptrSize(zcu)) { + .slice => try cg.sliceLen(ptr), + .one => @as(WValue, .{ .imm32 = @as(u32, @intCast(ptr_ty.childType(zcu).arrayLen(zcu))) }), + .c, .many => unreachable, + }; + defer len.free(cg); try cg.memset(elem_ty, dst_ptr, len, value); return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs }); @@ -6614,11 +6671,13 @@ fn memset(cg: *CodeGen, elem_ty: Type, ptr: WValue, len: WValue, value: WValue) return; } + var owned_final_len: WValue = .none; + defer owned_final_len.free(cg); const final_len: WValue = switch (len) { .imm32 => |val| .{ .imm32 = val * abi_size }, .imm64 => |val| .{ .imm64 = val * abi_size }, else => if (abi_size != 1) blk: { - const new_len = try cg.ensureAllocLocal(Type.usize); + owned_final_len = try cg.allocLocal(Type.usize); try cg.emitWValue(len); switch (cg.ptr_size) { .wasm32 => { @@ -6630,8 +6689,8 @@ fn memset(cg: *CodeGen, elem_ty: Type, ptr: WValue, len: WValue, value: WValue) try cg.addTag(.i64_mul); }, } - try cg.addLocal(.local_set, new_len.local); - break :blk new_len; + try cg.addLocal(.local_set, owned_final_len.local); + break :blk owned_final_len; } else len, }; @@ -7179,7 +7238,7 @@ fn airMemcpy(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { const ptr_elem_ty = dst_ty.childType(zcu); const src = try cg.resolveInst(bin_op.rhs); const src_ty = cg.typeOf(bin_op.rhs); - const len = switch (dst_ty.ptrSize(zcu)) { + var len = switch (dst_ty.ptrSize(zcu)) { .slice => blk: { const slice_len = try cg.sliceLen(dst); if (ptr_elem_ty.abiSize(zcu) != 1) { @@ -7195,8 +7254,11 @@ fn airMemcpy(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { }), .c, .many => unreachable, }; - const dst_ptr = try cg.sliceOrArrayPtr(dst, dst_ty); - const src_ptr = try cg.sliceOrArrayPtr(src, src_ty); + defer len.free(cg); + var dst_ptr = try cg.sliceOrArrayPtr(dst, dst_ty); + defer if (dst_ty.isSlice(zcu)) dst_ptr.free(cg); + var src_ptr = try cg.sliceOrArrayPtr(src, src_ty); + defer if (src_ty.isSlice(zcu)) src_ptr.free(cg); try cg.memcpy(dst_ptr, src_ptr, len); return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs }); @@ -7210,7 +7272,7 @@ fn airMemmove(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { const ptr_elem_ty = dst_ty.childType(zcu); const src = try cg.resolveInst(bin_op.rhs); const src_ty = cg.typeOf(bin_op.rhs); - const len = switch (dst_ty.ptrSize(zcu)) { + var len = switch (dst_ty.ptrSize(zcu)) { .slice => blk: { const slice_len = try cg.sliceLen(dst); if (ptr_elem_ty.abiSize(zcu) != 1) { @@ -7226,8 +7288,11 @@ fn airMemmove(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { }), .c, .many => unreachable, }; - const dst_ptr = try cg.sliceOrArrayPtr(dst, dst_ty); - const src_ptr = try cg.sliceOrArrayPtr(src, src_ty); + defer len.free(cg); + var dst_ptr = try cg.sliceOrArrayPtr(dst, dst_ty); + defer if (dst_ty.isSlice(zcu)) dst_ptr.free(cg); + var src_ptr = try cg.sliceOrArrayPtr(src, src_ty); + defer if (src_ty.isSlice(zcu)) src_ptr.free(cg); try cg.memmove(dst_ptr, src_ptr, len); return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs }); @@ -7295,7 +7360,8 @@ fn airDbgVar( ) InnerError!void { _ = is_ptr; _ = local_tag; - return cg.finishAir(inst, .none, &.{}); + const pl_op = cg.air.instructions.items(.data)[@backingInt(inst)].pl_op; + return cg.finishAir(inst, .none, &.{pl_op.operand}); } fn airTry(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { @@ -7555,9 +7621,10 @@ fn airCmpxchg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { const expected_val = try cg.resolveInst(extra.expected_value); const new_val = try cg.resolveInst(extra.new_value); - const cmp_result = try cg.allocLocal(Type.bool); + var cmp_result = try cg.allocLocal(Type.bool); + defer cmp_result.free(cg); - const ptr_val = if (cg.useAtomicFeature()) val: { + var ptr_val = if (cg.useAtomicFeature()) val: { const val_local = try cg.allocLocal(ty); try cg.emitMemBase(ptr_operand); try cg.emitWValue(expected_val); @@ -7599,7 +7666,8 @@ fn airCmpxchg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { try cg.addTag(.i32_xor); try cg.addImm32(1); try cg.addTag(.i32_and); - const and_result = try WValue.toLocal(.stack, cg, Type.bool); + var and_result = try WValue.toLocal(.stack, cg, Type.bool); + defer and_result.free(cg); const result_ptr = try cg.allocStack(result_ty); try cg.store(result_ptr, and_result, .bool, .{ .offset = ty.abiSize(zcu) }); try cg.store(result_ptr, ptr_val, ty, .{}); @@ -7611,6 +7679,7 @@ fn airCmpxchg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { try cg.addTag(.select); break :val .stack; }; + defer ptr_val.free(cg); return cg.finishAir(inst, result, &.{ extra.ptr, extra.expected_value, extra.new_value }); } @@ -7695,7 +7764,8 @@ fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { .alignment = ty.abiAlignment(zcu).toStdMem(), }, ); - const select_res = try cg.allocLocal(ty); + var select_res = try cg.allocLocal(ty); + defer select_res.free(cg); try cg.addLocal(.local_tee, select_res.local); _ = try cg.intCmp(int_ty, .neq, .stack, value); // leave on stack so we can use it for br_if @@ -7908,6 +7978,11 @@ fn airAsm(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { if (unwrapped_asm.source.len != 0) { var local_map: assembly.LocalMap = .empty; defer local_map.deinit(cg.gpa); + var input_locals: std.ArrayList(WValue) = try .initCapacity(cg.gpa, inputs.len); + defer { + for (input_locals.items) |*input_local| input_local.free(cg); + input_locals.deinit(cg.gpa); + } { var it = unwrapped_asm.iterateOutputs(); @@ -7940,6 +8015,7 @@ fn airAsm(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { try cg.emitWValue(operand); const op_local = try WValue.toLocal(.stack, cg, cg.typeOf(input.operand)); + input_locals.appendAssumeCapacity(op_local); const gop = try local_map.getOrPutValue(cg.gpa, name, op_local.local); if (gop.found_existing) { -- 2.54.0 From 5119f0a70de2565f22e1edc5d43d49b655c6f7a7 Mon Sep 17 00:00:00 2001 From: Pavel Verigo Date: Wed, 16 Sep 2026 02:07:05 +0200 Subject: [PATCH 8/8] wasm: .toStdMem was removed --- src/codegen/wasm/CodeGen.zig | 26 +++++++++++++------------- src/codegen/wasm/Emit.zig | 2 +- src/codegen/wasm/Mir.zig | 4 ++-- src/codegen/wasm/assembly.zig | 2 +- 4 files changed, 17 insertions(+), 17 deletions(-) diff --git a/src/codegen/wasm/CodeGen.zig b/src/codegen/wasm/CodeGen.zig index aec24e1e8ce59b81d43f7bc31facaa6b9ab4195e..6db24a407587e22d736169a154788ad6a340e63c 100644 --- a/src/codegen/wasm/CodeGen.zig +++ b/src/codegen/wasm/CodeGen.zig @@ -646,7 +646,7 @@ fn addExtraAssumeCapacity(cg: *CodeGen, extra: anytype) error{OutOfMemory}!u32 { i32 => @bitCast(@field(extra, field_name)), InternPool.Index, InternPool.Nav.Index, - mem.Alignment, + Alignment, => @backingInt(@field(extra, field_name)), else => @compileError("Unsupported field type " ++ @typeName(field_type)), }), @@ -2386,7 +2386,7 @@ fn store(cg: *CodeGen, lhs: WValue, rhs: WValue, ty: Type, memarg: MemArg) Inner cg.mir_extra.appendAssumeCapacity(@backingInt(std.wasm.SimdOpcode.v128_store)); _ = try cg.addExtraAssumeCapacity(Mir.MemArg{ .offset = base_offset + memarg.offset, - .alignment = alignment.toStdMem(), + .alignment = alignment, }); return cg.addInst(.{ .tag = .simd_prefix, .data = .{ .payload = extra_index } }); } @@ -2416,7 +2416,7 @@ fn store(cg: *CodeGen, lhs: WValue, rhs: WValue, ty: Type, memarg: MemArg) Inner store_opcode, .{ .offset = base_offset + memarg.offset, - .alignment = alignment.toStdMem(), + .alignment = alignment, }, ); } @@ -2480,7 +2480,7 @@ fn load(cg: *CodeGen, operand: WValue, ty: Type, memarg: MemArg) InnerError!WVal cg.mir_extra.appendAssumeCapacity(@backingInt(std.wasm.SimdOpcode.v128_load)); _ = try cg.addExtraAssumeCapacity(Mir.MemArg{ .offset = base_offset + memarg.offset, - .alignment = alignment.toStdMem(), + .alignment = alignment, }); try cg.addInst(.{ .tag = .simd_prefix, .data = .{ .payload = extra_index } }); return .stack; @@ -2511,7 +2511,7 @@ fn load(cg: *CodeGen, operand: WValue, ty: Type, memarg: MemArg) InnerError!WVal load_opcode, .{ .offset = base_offset + memarg.offset, - .alignment = alignment.toStdMem(), + .alignment = alignment, }, ); @@ -6141,7 +6141,7 @@ fn airIsErr(cg: *CodeGen, inst: Air.Inst.Index, opcode: std.wasm.Opcode, op_kind try cg.emitMemBase(operand); try cg.addMemArg(.i32_load16_u, .{ .offset = operand.memOffset() + errUnionErrorOffset(pl_ty, zcu), - .alignment = Type.anyerror.abiAlignment(zcu).toStdMem(), + .alignment = Type.anyerror.abiAlignment(zcu), }); } else { try cg.emitWValue(operand); @@ -6827,7 +6827,7 @@ fn airSplat(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { cg.mir_extra.appendAssumeCapacity(opcode); _ = try cg.addExtraAssumeCapacity(Mir.MemArg{ .offset = operand.memOffset(), - .alignment = elem_ty.abiAlignment(zcu).toStdMem(), + .alignment = elem_ty.abiAlignment(zcu), }); try cg.addInst(.{ .tag = .simd_prefix, .data = .{ .payload = extra_index } }); return cg.finishAir(inst, .stack, &.{ty_op.operand}); @@ -7418,7 +7418,7 @@ fn lowerTry( const err_offset = errUnionErrorOffset(pl_ty, zcu); try cg.addMemArg(.i32_load16_u, .{ .offset = err_union.memOffset() + err_offset, - .alignment = Type.anyerror.abiAlignment(zcu).toStdMem(), + .alignment = Type.anyerror.abiAlignment(zcu), }); } else { try cg.emitWValue(err_union); @@ -7637,7 +7637,7 @@ fn airCmpxchg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { else => |size| return cg.fail("TODO: implement `@cmpxchg` for types with abi size '{d}'", .{size}), }, .{ .offset = ptr_operand.memOffset(), - .alignment = ty.abiAlignment(zcu).toStdMem(), + .alignment = ty.abiAlignment(zcu), }); try cg.addLocal(.local_tee, val_local.local); _ = try cg.intCmp(int_ty, .eq, .stack, expected_val); @@ -7701,7 +7701,7 @@ fn airAtomicLoad(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { try cg.emitMemBase(ptr); try cg.addAtomicMemArg(tag, .{ .offset = ptr.memOffset(), - .alignment = ty.abiAlignment(zcu).toStdMem(), + .alignment = ty.abiAlignment(zcu), }); } else { _ = try cg.load(ptr, ty, .{}); @@ -7761,7 +7761,7 @@ fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { }, .{ .offset = ptr.memOffset(), - .alignment = ty.abiAlignment(zcu).toStdMem(), + .alignment = ty.abiAlignment(zcu), }, ); var select_res = try cg.allocLocal(ty); @@ -7822,7 +7822,7 @@ fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { }; try cg.addAtomicMemArg(tag, .{ .offset = ptr.memOffset(), - .alignment = ty.abiAlignment(zcu).toStdMem(), + .alignment = ty.abiAlignment(zcu), }); return cg.finishAir(inst, .stack, &.{ pl_op.operand, extra.operand }); }, @@ -7935,7 +7935,7 @@ fn airAtomicStore(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { try cg.emitWValue(operand); try cg.addAtomicMemArg(tag, .{ .offset = ptr.memOffset(), - .alignment = ty.abiAlignment(zcu).toStdMem(), + .alignment = ty.abiAlignment(zcu), }); } else { try cg.store(ptr, operand, ty, .{}); diff --git a/src/codegen/wasm/Emit.zig b/src/codegen/wasm/Emit.zig index 3c76c439ec948111832be4a0fa336abe79c38b21..6193650464133168e21379e8cf810d8fb9e9ac12 100644 --- a/src/codegen/wasm/Emit.zig +++ b/src/codegen/wasm/Emit.zig @@ -990,7 +990,7 @@ pub fn lower(emit: *Emit) Error!void { /// Asserts 20 unused capacity. fn encodeMemArg(code: *ArrayList(u8), mem_arg: Mir.MemArg) void { assert(code.unusedCapacitySlice().len >= 20); - writeUleb128(code, @backingInt(mem_arg.alignment)); + writeUleb128(code, mem_arg.alignment.toLog2Units()); writeUleb128(code, mem_arg.offset); } diff --git a/src/codegen/wasm/Mir.zig b/src/codegen/wasm/Mir.zig index ea40a3f76ebfab55f17eb1ab20ce93527d441909..1f8502a2faaaa084770fad1e1665bb76c1185476 100644 --- a/src/codegen/wasm/Mir.zig +++ b/src/codegen/wasm/Mir.zig @@ -705,7 +705,7 @@ pub fn extraData(self: *const Mir, comptime T: type, index: usize) struct { data Wasm.UavsExeIndex, InternPool.Nav.Index, InternPool.Index, - std.mem.Alignment, + Alignment, => @fromBackingInt(@intCast(self.extra[i])), else => @compileError("Unsupported field type " ++ @typeName(field_type)), }; @@ -755,7 +755,7 @@ pub const Float64 = struct { pub const MemArg = struct { offset: u64, - alignment: std.mem.Alignment, + alignment: Alignment, }; pub const UavRefOff = struct { diff --git a/src/codegen/wasm/assembly.zig b/src/codegen/wasm/assembly.zig index 7aa056aae95e22cc5da42cb9f03ebe56257d41f4..445f2deeecb3172626435524ed52a2a82ee660d5 100644 --- a/src/codegen/wasm/assembly.zig +++ b/src/codegen/wasm/assembly.zig @@ -686,7 +686,7 @@ fn parseMemArg(cg: *CodeGen, inst: []const u8, word_it: *mem.TokenIterator(u8, . return cg.fail("Malformed assembly, malformed offset \"{s}\"", .{alignment_str}); }; - return .{ .offset = @bitCast(offset), .alignment = @fromBackingInt(@intCast(alignment_p2)) }; + return .{ .offset = @bitCast(offset), .alignment = .fromLog2Units(alignment_p2) }; } else { const offset_str = mem_arg_str; const offset = std.fmt.parseInt(i64, offset_str, 0) catch { -- 2.54.0