authorgravatar for paul.verigo@gmail.comPavel Verigo <paul.verigo@gmail.com> 2026-08-21 12:43:10+02:00
committergravatar for paul.verigo@gmail.comPavel Verigo <paul.verigo@gmail.com> 2026-09-17 00:04:51+02:00
logd12892e8904bbb27aac7baa69f483348e516a30e
treed129832a30921a505603594390425a2c45fdcdb1
parent5ec1668eb4cee3ce3069f6ff7b2e3af0d52f43a6

wasm: store/load use memarg, support alignment

4 files changed, 288 insertions(+), 246 deletions(-)

src/codegen/wasm/CodeGen.zig+274-237
...@@ -584,7 +584,7 @@ fn addCallIntrinsic(cg: *CodeGen, intrinsic: Mir.Intrinsic) error{OutOfMemory}!v...@@ -584,7 +584,7 @@ fn addCallIntrinsic(cg: *CodeGen, intrinsic: Mir.Intrinsic) error{OutOfMemory}!v
584/// Returns the index into `mir_extra`584/// Returns the index into `mir_extra`
585fn addExtra(cg: *CodeGen, extra: anytype) error{OutOfMemory}!u32 {585fn addExtra(cg: *CodeGen, extra: anytype) error{OutOfMemory}!u32 {
586 const field_count = @typeInfo(@TypeOf(extra)).@"struct".field_names.len;586 const field_count = @typeInfo(@TypeOf(extra)).@"struct".field_names.len;
587 try cg.mir_extra.ensureUnusedCapacity(cg.gpa, field_count);587 try cg.mir_extra.ensureUnusedCapacity(cg.gpa, field_count * 2);
588 return cg.addExtraAssumeCapacity(extra);588 return cg.addExtraAssumeCapacity(extra);
589}589}
590590
...@@ -594,14 +594,22 @@ fn addExtraAssumeCapacity(cg: *CodeGen, extra: anytype) error{OutOfMemory}!u32 {...@@ -594,14 +594,22 @@ fn addExtraAssumeCapacity(cg: *CodeGen, extra: anytype) error{OutOfMemory}!u32 {
594 const info = @typeInfo(@TypeOf(extra)).@"struct";594 const info = @typeInfo(@TypeOf(extra)).@"struct";
595 const result: u32 = @intCast(cg.mir_extra.items.len);595 const result: u32 = @intCast(cg.mir_extra.items.len);
596 inline for (info.field_names, info.field_types) |field_name, field_type| {596 inline for (info.field_names, info.field_types) |field_name, field_type| {
597 cg.mir_extra.appendAssumeCapacity(switch (field_type) {597 switch (field_type) {
598 u32 => @field(extra, field_name),598 u64 => {
599 i32 => @bitCast(@field(extra, field_name)),599 const value = @field(extra, field_name);
600 InternPool.Index,600 cg.mir_extra.appendAssumeCapacity(@truncate(value));
601 InternPool.Nav.Index,601 cg.mir_extra.appendAssumeCapacity(@truncate(value >> 32));
602 => @backingInt(@field(extra, field_name)),602 },
603 else => @compileError("Unsupported field type " ++ @typeName(field_type)),603 else => cg.mir_extra.appendAssumeCapacity(switch (field_type) {
604 });604 u32 => @field(extra, field_name),
605 i32 => @bitCast(@field(extra, field_name)),
606 InternPool.Index,
607 InternPool.Nav.Index,
608 mem.Alignment,
609 => @backingInt(@field(extra, field_name)),
610 else => @compileError("Unsupported field type " ++ @typeName(field_type)),
611 }),
612 }
605 }613 }
606 return result;614 return result;
607}615}
...@@ -991,26 +999,26 @@ fn lowerArg(cg: *CodeGen, cc: std.lang.CallingConvention, ty: Type, value: WValu...@@ -991,26 +999,26 @@ fn lowerArg(cg: *CodeGen, cc: std.lang.CallingConvention, ty: Type, value: WValu
991 if (!isByRef(ty, zcu, cg.target)) {999 if (!isByRef(ty, zcu, cg.target)) {
992 return cg.lowerToStack(value);1000 return cg.lowerToStack(value);
993 } else {1001 } else {
994 _ = try cg.load(value, scalar_ty, 0);1002 _ = try cg.load(value, scalar_ty, .{});
995 }1003 }
996 },1004 },
997 .double_i64 => {1005 .double_i64 => {
998 assert(ty.abiSize(zcu) == 16);1006 assert(ty.abiSize(zcu) == 16);
999 // in this case we have an integer or float that must be lowered as 2 i64's.1007 // in this case we have an integer or float that must be lowered as 2 i64's.
1000 try cg.emitWValue(value);1008 try cg.emitWValue(value);
1001 try cg.addMemArg(.i64_load, .{ .offset = value.offset(), .alignment = 8 });1009 try cg.addMemArg(.i64_load, .{ .offset = value.offset(), .alignment = .@"8" });
1002 try cg.emitWValue(value);1010 try cg.emitWValue(value);
1003 try cg.addMemArg(.i64_load, .{ .offset = value.offset() + 8, .alignment = 8 });1011 try cg.addMemArg(.i64_load, .{ .offset = value.offset() + 8, .alignment = .@"8" });
1004 },1012 },
1005 .indirect => {1013 .indirect => {
1006 const stack_copy = try cg.allocStack(ty);1014 const stack_copy = try cg.allocStack(ty);
1007 try cg.store(stack_copy, value, ty, 0);1015 try cg.store(stack_copy, value, ty, .{});
1008 return cg.lowerToStack(stack_copy);1016 return cg.lowerToStack(stack_copy);
1009 },1017 },
1010 .unrolled => |vector| {1018 .unrolled => |vector| {
1011 const elem_size: u32 = @intCast(vector.elem_type.abiSize(zcu));1019 const elem_size = vector.elem_type.abiSize(zcu);
1012 for (0..vector.len) |index| {1020 for (0..vector.len) |index| {
1013 _ = try cg.load(value, vector.elem_type, @intCast(index * elem_size));1021 _ = try cg.load(value, vector.elem_type, .{ .offset = index * elem_size });
1014 }1022 }
1015 },1023 },
1016 }1024 }
...@@ -1330,7 +1338,7 @@ fn genInst(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -1330,7 +1338,7 @@ fn genInst(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
1330 try cg.addTag(.i32_add);1338 try cg.addTag(.i32_add);
1331 const ptr = try WValue.toLocal(.stack, cg, Type.usize);1339 const ptr = try WValue.toLocal(.stack, cg, Type.usize);
13321340
1333 try cg.store(ptr, elem_val, elem_ty, 0);1341 try cg.store(ptr, elem_val, elem_ty, .{});
13341342
1335 return cg.finishAir(inst, .none, &.{ pl_op.operand, bin_op.lhs, bin_op.rhs });1343 return cg.finishAir(inst, .none, &.{ pl_op.operand, bin_op.lhs, bin_op.rhs });
1336 },1344 },
...@@ -1507,8 +1515,8 @@ fn genInst(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -1507,8 +1515,8 @@ fn genInst(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
1507 const result = try cg.allocStack(cg.typeOfIndex(inst));1515 const result = try cg.allocStack(cg.typeOfIndex(inst));
1508 const offset: u32 = @intCast(ty.abiSize(cg.pt.zcu));1516 const offset: u32 = @intCast(ty.abiSize(cg.pt.zcu));
15091517
1510 try cg.store(result, res_tmp, ty, 0);1518 try cg.store(result, res_tmp, ty, .{});
1511 try cg.store(result, ov_tmp, Type.u1, offset);1519 try cg.store(result, ov_tmp, .u1, .{ .offset = offset });
15121520
1513 try cg.finishAir(inst, result, &.{ extra.lhs, extra.rhs });1521 try cg.finishAir(inst, result, &.{ extra.lhs, extra.rhs });
1514 },1522 },
...@@ -1939,20 +1947,20 @@ fn airRet(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -1939,20 +1947,20 @@ fn airRet(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
1939 // result must be stored in the stack and we return a pointer1947 // result must be stored in the stack and we return a pointer
1940 // to the stack instead1948 // to the stack instead
1941 if (cg.return_value != .none) {1949 if (cg.return_value != .none) {
1942 try cg.store(cg.return_value, operand, ret_ty, 0);1950 try cg.store(cg.return_value, operand, ret_ty, .{});
1943 } else if (fn_info.cc == .wasm_mvp and ret_ty.hasRuntimeBits(zcu)) {1951 } else if (fn_info.cc == .wasm_mvp and ret_ty.hasRuntimeBits(zcu)) {
1944 switch (abi.classifyType(ret_ty, zcu, cg.target)) {1952 switch (abi.classifyType(ret_ty, zcu, cg.target)) {
1945 .direct => |scalar_type| {1953 .direct => |scalar_type| {
1946 if (!isByRef(ret_ty, zcu, cg.target)) {1954 if (!isByRef(ret_ty, zcu, cg.target)) {
1947 try cg.emitWValue(operand);1955 try cg.emitWValue(operand);
1948 } else {1956 } else {
1949 _ = try cg.load(operand, scalar_type, 0);1957 _ = try cg.load(operand, scalar_type, .{});
1950 }1958 }
1951 },1959 },
1952 .double_i64, .indirect => unreachable,1960 .double_i64, .indirect => unreachable,
1953 .unrolled => |vector| {1961 .unrolled => |vector| {
1954 assert(vector.len == 1);1962 assert(vector.len == 1);
1955 _ = try cg.load(operand, vector.elem_type, 0);1963 _ = try cg.load(operand, vector.elem_type, .{});
1956 },1964 },
1957 }1965 }
1958 } else {1966 } else {
...@@ -2002,15 +2010,15 @@ fn airRetLoad(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -2002,15 +2010,15 @@ fn airRetLoad(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
2002 } else if (!firstParamSRet(fn_info.cc, Type.fromInterned(fn_info.return_type), zcu, cg.target)) {2010 } else if (!firstParamSRet(fn_info.cc, Type.fromInterned(fn_info.return_type), zcu, cg.target)) {
2003 if (fn_info.cc == .wasm_mvp) {2011 if (fn_info.cc == .wasm_mvp) {
2004 switch (abi.classifyType(ret_ty, zcu, cg.target)) {2012 switch (abi.classifyType(ret_ty, zcu, cg.target)) {
2005 .direct => |scalar_type| _ = try cg.load(operand, scalar_type, 0),2013 .direct => |scalar_type| _ = try cg.load(operand, scalar_type, .{}),
2006 .double_i64, .indirect => unreachable,2014 .double_i64, .indirect => unreachable,
2007 .unrolled => |vector| {2015 .unrolled => |vector| {
2008 assert(vector.len == 1);2016 assert(vector.len == 1);
2009 _ = try cg.load(operand, vector.elem_type, 0);2017 _ = try cg.load(operand, vector.elem_type, .{});
2010 },2018 },
2011 }2019 }
2012 } else {2020 } else {
2013 _ = try cg.load(operand, ret_ty, 0);2021 _ = try cg.load(operand, ret_ty, .{});
2014 }2022 }
2015 }2023 }
20162024
...@@ -2092,7 +2100,7 @@ fn airCall(cg: *CodeGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier)...@@ -2092,7 +2100,7 @@ fn airCall(cg: *CodeGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier)
2092 const arg_align = arg_ty.abiAlignment(zcu);2100 const arg_align = arg_ty.abiAlignment(zcu);
20932101
2094 offset = @intCast(arg_align.forward(offset));2102 offset = @intCast(arg_align.forward(offset));
2095 try cg.store(buffer, arg_val, arg_ty, offset);2103 try cg.store(buffer, arg_val, arg_ty, .{ .offset = offset });
2096 offset += arg_size;2104 offset += arg_size;
2097 }2105 }
20982106
...@@ -2149,7 +2157,7 @@ fn airCall(cg: *CodeGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier)...@@ -2149,7 +2157,7 @@ fn airCall(cg: *CodeGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier)
2149 const result_local = try cg.allocLocal(scalar_type);2157 const result_local = try cg.allocLocal(scalar_type);
2150 try cg.addLocal(.local_set, result_local.local.value);2158 try cg.addLocal(.local_set, result_local.local.value);
2151 const result = try cg.allocStack(ret_ty);2159 const result = try cg.allocStack(ret_ty);
2152 try cg.store(result, result_local, scalar_type, 0);2160 try cg.store(result, result_local, scalar_type, .{});
2153 break :result_value result;2161 break :result_value result;
2154 }2162 }
2155 },2163 },
...@@ -2160,7 +2168,7 @@ fn airCall(cg: *CodeGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier)...@@ -2160,7 +2168,7 @@ fn airCall(cg: *CodeGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier)
2160 // save call result from operand stack2168 // save call result from operand stack
2161 try cg.addLocal(.local_set, result_local.local.value);2169 try cg.addLocal(.local_set, result_local.local.value);
2162 const result = try cg.allocStack(ret_ty);2170 const result = try cg.allocStack(ret_ty);
2163 try cg.store(result, result_local, vector.elem_type, 0);2171 try cg.store(result, result_local, vector.elem_type, .{});
2164 break :result_value result;2172 break :result_value result;
2165 },2173 },
2166 }2174 }
...@@ -2191,7 +2199,7 @@ fn airVaCopy(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -2191,7 +2199,7 @@ fn airVaCopy(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
2191 const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;2199 const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
2192 const operand = try cg.resolveInst(ty_op.operand);2200 const operand = try cg.resolveInst(ty_op.operand);
21932201
2194 const result = try cg.load(operand, .usize, 0);2202 const result = try cg.load(operand, .usize, .{});
21952203
2196 return cg.finishAir(inst, result, &.{ty_op.operand});2204 return cg.finishAir(inst, result, &.{ty_op.operand});
2197}2205}
...@@ -2214,7 +2222,7 @@ fn airVaArg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -2214,7 +2222,7 @@ fn airVaArg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
2214 const abi_align: u32 = @intCast(load_ty.abiAlignment(zcu).toByteUnits().?);2222 const abi_align: u32 = @intCast(load_ty.abiAlignment(zcu).toByteUnits().?);
22152223
2216 const arg_ptr = try cg.allocLocal(.usize);2224 const arg_ptr = try cg.allocLocal(.usize);
2217 _ = try cg.load(operand, .usize, 0);2225 _ = try cg.load(operand, .usize, .{});
22182226
2219 if (abi_align > 1) {2227 if (abi_align > 1) {
2220 switch (cg.ptr_size) {2228 switch (cg.ptr_size) {
...@@ -2247,17 +2255,17 @@ fn airVaArg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -2247,17 +2255,17 @@ fn airVaArg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
2247 try cg.addTag(.i64_add);2255 try cg.addTag(.i64_add);
2248 },2256 },
2249 }2257 }
2250 try cg.store(.stack, .stack, .usize, 0);2258 try cg.store(.stack, .stack, .usize, .{});
22512259
2252 const result = if (is_f32_va_arg) result: {2260 const result = if (is_f32_va_arg) result: {
2253 const promoted = try cg.load(arg_ptr, Type.f64, 0);2261 const promoted = try cg.load(arg_ptr, .f64, .{});
2254 try cg.emitWValue(promoted);2262 try cg.emitWValue(promoted);
2255 try cg.addTag(.f32_demote_f64);2263 try cg.addTag(.f32_demote_f64);
22562264
2257 const result_local = try cg.allocLocal(Type.f32);2265 const result_local = try cg.allocLocal(Type.f32);
2258 try cg.addLocal(.local_set, result_local.local.value);2266 try cg.addLocal(.local_set, result_local.local.value);
2259 break :result result_local;2267 break :result result_local;
2260 } else try cg.load(arg_ptr, ty, 0);2268 } else try cg.load(arg_ptr, ty, .{});
22612269
2262 return cg.finishAir(inst, result, &.{ty_op.operand});2270 return cg.finishAir(inst, result, &.{ty_op.operand});
2263}2271}
...@@ -2282,18 +2290,26 @@ fn airStore(cg: *CodeGen, inst: Air.Inst.Index, safety: bool) InnerError!void {...@@ -2282,18 +2290,26 @@ fn airStore(cg: *CodeGen, inst: Air.Inst.Index, safety: bool) InnerError!void {
2282 return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });2290 return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });
2283 }2291 }
22842292
2285 const offset: u32 = switch (ptr_info.flags.vector_index) {2293 const offset: u64 = switch (ptr_info.flags.vector_index) {
2286 .none => offset: {2294 .none => offset: {
2287 assert(ptr_info.packed_offset.host_size == 0); // legalize .expand_packed_store2295 assert(ptr_info.packed_offset.host_size == 0); // legalize .expand_packed_store
2288 break :offset 0;2296 break :offset 0;
2289 },2297 },
2290 else => |index| @intCast(@backingInt(index) * elem_ty.abiSize(zcu)),2298 else => |index| @backingInt(index) * elem_ty.abiSize(zcu),
2291 };2299 };
2292 try cg.store(lhs, rhs, elem_ty, offset);2300 try cg.store(lhs, rhs, elem_ty, .{
2301 .offset = offset,
2302 .alignment = ptr_ty.ptrAlignment(zcu),
2303 });
2293 return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });2304 return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });
2294}2305}
22952306
2296fn store(cg: *CodeGen, lhs: WValue, rhs: WValue, ty: Type, offset: u32) InnerError!void {2307const MemArg = struct {
2308 offset: u64 = 0,
2309 alignment: Alignment = .none,
2310};
2311
2312fn store(cg: *CodeGen, lhs: WValue, rhs: WValue, ty: Type, memarg: MemArg) InnerError!void {
2297 assert(!(lhs != .stack and rhs == .stack));2313 assert(!(lhs != .stack and rhs == .stack));
2298 const pt = cg.pt;2314 const pt = cg.pt;
2299 const zcu = pt.zcu;2315 const zcu = pt.zcu;
...@@ -2301,12 +2317,19 @@ fn store(cg: *CodeGen, lhs: WValue, rhs: WValue, ty: Type, offset: u32) InnerErr...@@ -2301,12 +2317,19 @@ fn store(cg: *CodeGen, lhs: WValue, rhs: WValue, ty: Type, offset: u32) InnerErr
23012317
2302 if (!ty.hasRuntimeBits(zcu)) return;2318 if (!ty.hasRuntimeBits(zcu)) return;
23032319
2320 const natural_alignment = ty.abiAlignment(zcu);
2321 // wasm memory instruction alignment is <= access width
2322 const alignment = if (memarg.alignment == .none)
2323 natural_alignment
2324 else
2325 natural_alignment.minStrict(memarg.alignment);
2326
2304 if (isByRef(ty, zcu, cg.target)) {2327 if (isByRef(ty, zcu, cg.target)) {
2305 const offset_ptr: WValue = switch (offset + lhs.offset()) {2328 const offset_ptr: WValue = switch (memarg.offset + lhs.offset()) {
2306 0 => lhs,2329 0 => lhs,
2307 else => |total_offset| ptr: {2330 else => |total_offset| ptr: {
2308 try cg.emitWValue(lhs);2331 try cg.emitWValue(lhs);
2309 try cg.addImm32(total_offset);2332 try cg.addImm32(@intCast(total_offset));
2310 try cg.addTag(.i32_add);2333 try cg.addTag(.i32_add);
2311 break :ptr .stack;2334 break :ptr .stack;
2312 },2335 },
...@@ -2320,10 +2343,11 @@ fn store(cg: *CodeGen, lhs: WValue, rhs: WValue, ty: Type, offset: u32) InnerErr...@@ -2320,10 +2343,11 @@ fn store(cg: *CodeGen, lhs: WValue, rhs: WValue, ty: Type, offset: u32) InnerErr
2320 // TODO: Add helper functions for simd opcodes2343 // TODO: Add helper functions for simd opcodes
2321 const extra_index: u32 = @intCast(cg.mir_extra.items.len);2344 const extra_index: u32 = @intCast(cg.mir_extra.items.len);
2322 // stores as := opcode, offset, alignment (opcode::memarg)2345 // stores as := opcode, offset, alignment (opcode::memarg)
2323 try cg.mir_extra.appendSlice(cg.gpa, &[_]u32{2346 try cg.mir_extra.ensureUnusedCapacity(cg.gpa, 4);
2324 @backingInt(std.wasm.SimdOpcode.v128_store),2347 cg.mir_extra.appendAssumeCapacity(@backingInt(std.wasm.SimdOpcode.v128_store));
2325 offset + lhs.offset(),2348 _ = try cg.addExtraAssumeCapacity(Mir.MemArg{
2326 @intCast(ty.abiAlignment(zcu).toByteUnits().?),2349 .offset = memarg.offset + lhs.offset(),
2350 .alignment = alignment.toStdMem(),
2327 });2351 });
2328 return cg.addInst(.{ .tag = .simd_prefix, .data = .{ .payload = extra_index } });2352 return cg.addInst(.{ .tag = .simd_prefix, .data = .{ .payload = extra_index } });
2329 }2353 }
...@@ -2353,8 +2377,8 @@ fn store(cg: *CodeGen, lhs: WValue, rhs: WValue, ty: Type, offset: u32) InnerErr...@@ -2353,8 +2377,8 @@ fn store(cg: *CodeGen, lhs: WValue, rhs: WValue, ty: Type, offset: u32) InnerErr
2353 try cg.addMemArg(2377 try cg.addMemArg(
2354 store_opcode,2378 store_opcode,
2355 .{2379 .{
2356 .offset = offset + lhs.offset(),2380 .offset = memarg.offset + lhs.offset(),
2357 .alignment = @intCast(ty.abiAlignment(zcu).toByteUnits().?),2381 .alignment = alignment.toStdMem(),
2358 },2382 },
2359 );2383 );
2360}2384}
...@@ -2370,34 +2394,45 @@ fn airLoad(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -2370,34 +2394,45 @@ fn airLoad(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
23702394
2371 assert(elem_ty.hasRuntimeBits(zcu));2395 assert(elem_ty.hasRuntimeBits(zcu));
23722396
2373 const offset: u32 = switch (ptr_info.flags.vector_index) {2397 const offset: u64 = switch (ptr_info.flags.vector_index) {
2374 .none => offset: {2398 .none => offset: {
2375 assert(ptr_info.packed_offset.host_size == 0); // legalize .expand_packed_load2399 assert(ptr_info.packed_offset.host_size == 0); // legalize .expand_packed_load
2376 break :offset 0;2400 break :offset 0;
2377 },2401 },
2378 else => |index| @intCast(@backingInt(index) * elem_ty.abiSize(zcu)),2402 else => |index| @backingInt(index) * elem_ty.abiSize(zcu),
2379 };2403 };
2380 const result = try cg.load(operand, elem_ty, offset);2404 const result = try cg.load(operand, elem_ty, .{
2405 .offset = offset,
2406 .alignment = ptr_ty.ptrAlignment(zcu),
2407 });
2381 return cg.finishAir(inst, result, &.{ty_op.operand});2408 return cg.finishAir(inst, result, &.{ty_op.operand});
2382}2409}
23832410
2384/// Loads an operand from the linear memory section.2411/// Loads an operand from the linear memory section.
2385/// NOTE: Leaves the value on the stack, if isByRef == false.2412/// NOTE: Leaves the value on the stack, if isByRef == false.
2386fn load(cg: *CodeGen, operand: WValue, ty: Type, offset: u32) InnerError!WValue {2413fn load(cg: *CodeGen, operand: WValue, ty: Type, memarg: MemArg) InnerError!WValue {
2387 const zcu = cg.pt.zcu;2414 const zcu = cg.pt.zcu;
2415
2416 const natural_alignment = ty.abiAlignment(zcu);
2417 // wasm memory instruction alignment is <= access width
2418 const alignment = if (memarg.alignment == .none)
2419 natural_alignment
2420 else
2421 natural_alignment.minStrict(memarg.alignment);
2422
2388 if (isByRef(ty, zcu, cg.target)) {2423 if (isByRef(ty, zcu, cg.target)) {
2389 const src_ptr_maybe_stack: WValue = switch (offset + operand.offset()) {2424 const src_ptr_maybe_stack: WValue = switch (memarg.offset + operand.offset()) {
2390 0 => operand,2425 0 => operand,
2391 else => |total_offset| ptr: {2426 else => |total_offset| ptr: {
2392 try cg.emitWValue(operand);2427 try cg.emitWValue(operand);
2393 try cg.addImm32(total_offset);2428 try cg.addImm32(@intCast(total_offset));
2394 try cg.addTag(.i32_add);2429 try cg.addTag(.i32_add);
2395 break :ptr .stack;2430 break :ptr .stack;
2396 },2431 },
2397 };2432 };
2398 const src_ptr = try src_ptr_maybe_stack.toLocal(cg, .usize);2433 const src_ptr = try src_ptr_maybe_stack.toLocal(cg, .usize);
2399 const new_ptr = try cg.allocStack(ty);2434 const new_ptr = try cg.allocStack(ty);
2400 try cg.store(new_ptr, src_ptr, ty, 0);2435 try cg.store(new_ptr, src_ptr, ty, .{});
2401 return new_ptr;2436 return new_ptr;
2402 }2437 }
24032438
...@@ -2408,10 +2443,11 @@ fn load(cg: *CodeGen, operand: WValue, ty: Type, offset: u32) InnerError!WValue...@@ -2408,10 +2443,11 @@ fn load(cg: *CodeGen, operand: WValue, ty: Type, offset: u32) InnerError!WValue
2408 // TODO: Add helper functions for simd opcodes2443 // TODO: Add helper functions for simd opcodes
2409 const extra_index: u32 = @intCast(cg.mir_extra.items.len);2444 const extra_index: u32 = @intCast(cg.mir_extra.items.len);
2410 // stores as := opcode, offset, alignment (opcode::memarg)2445 // stores as := opcode, offset, alignment (opcode::memarg)
2411 try cg.mir_extra.appendSlice(cg.gpa, &[_]u32{2446 try cg.mir_extra.ensureUnusedCapacity(cg.gpa, 4);
2412 @backingInt(std.wasm.SimdOpcode.v128_load),2447 cg.mir_extra.appendAssumeCapacity(@backingInt(std.wasm.SimdOpcode.v128_load));
2413 offset + operand.offset(),2448 _ = try cg.addExtraAssumeCapacity(Mir.MemArg{
2414 @intCast(ty.abiAlignment(zcu).toByteUnits().?),2449 .offset = memarg.offset + operand.offset(),
2450 .alignment = alignment.toStdMem(),
2415 });2451 });
2416 try cg.addInst(.{ .tag = .simd_prefix, .data = .{ .payload = extra_index } });2452 try cg.addInst(.{ .tag = .simd_prefix, .data = .{ .payload = extra_index } });
2417 return .stack;2453 return .stack;
...@@ -2441,8 +2477,8 @@ fn load(cg: *CodeGen, operand: WValue, ty: Type, offset: u32) InnerError!WValue...@@ -2441,8 +2477,8 @@ fn load(cg: *CodeGen, operand: WValue, ty: Type, offset: u32) InnerError!WValue
2441 try cg.addMemArg(2477 try cg.addMemArg(
2442 load_opcode,2478 load_opcode,
2443 .{2479 .{
2444 .offset = offset + operand.offset(),2480 .offset = memarg.offset + operand.offset(),
2445 .alignment = @intCast(ty.abiAlignment(zcu).toByteUnits().?),2481 .alignment = alignment.toStdMem(),
2446 },2482 },
2447 );2483 );
24482484
...@@ -2470,7 +2506,7 @@ fn airArg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -2470,7 +2506,7 @@ fn airArg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
2470 cg.arg_index += 1;2506 cg.arg_index += 1;
2471 if (isByRef(arg_ty, zcu, cg.target)) {2507 if (isByRef(arg_ty, zcu, cg.target)) {
2472 const result = try cg.allocStack(arg_ty);2508 const result = try cg.allocStack(arg_ty);
2473 try cg.store(result, arg, scalar_type, 0);2509 try cg.store(result, arg, scalar_type, .{});
2474 return cg.finishAir(inst, result, &.{});2510 return cg.finishAir(inst, result, &.{});
2475 }2511 }
2476 },2512 },
...@@ -2478,15 +2514,15 @@ fn airArg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -2478,15 +2514,15 @@ fn airArg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
2478 .double_i64 => {2514 .double_i64 => {
2479 cg.arg_index += 2;2515 cg.arg_index += 2;
2480 const result = try cg.allocStack(arg_ty);2516 const result = try cg.allocStack(arg_ty);
2481 try cg.store(result, arg, Type.u64, 0);2517 try cg.store(result, arg, .u64, .{});
2482 try cg.store(result, cg.args[arg_index + 1], Type.u64, 8);2518 try cg.store(result, cg.args[arg_index + 1], .u64, .{ .offset = 8 });
2483 return cg.finishAir(inst, result, &.{});2519 return cg.finishAir(inst, result, &.{});
2484 },2520 },
2485 .unrolled => |vector| {2521 .unrolled => |vector| {
2486 const result = try cg.allocStack(arg_ty);2522 const result = try cg.allocStack(arg_ty);
2487 const elem_size: u32 = @intCast(vector.elem_type.abiSize(zcu));2523 const elem_size = vector.elem_type.abiSize(zcu);
2488 for (0..vector.len) |index| {2524 for (0..vector.len) |index| {
2489 try cg.store(result, cg.args[cg.arg_index], vector.elem_type, @intCast(index * elem_size));2525 try cg.store(result, cg.args[cg.arg_index], vector.elem_type, .{ .offset = index * elem_size });
2490 cg.arg_index += 1;2526 cg.arg_index += 1;
2491 }2527 }
2492 return cg.finishAir(inst, result, &.{});2528 return cg.finishAir(inst, result, &.{});
...@@ -2588,15 +2624,15 @@ fn intAdd(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue...@@ -2588,15 +2624,15 @@ fn intAdd(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue
2588 65...128 => {2624 65...128 => {
2589 const result = try cg.allocStack(Type.u128);2625 const result = try cg.allocStack(Type.u128);
25902626
2591 var lhs_lsb = try (try cg.load(lhs, Type.u64, 0)).toLocal(cg, Type.u64);2627 var lhs_lsb = try (try cg.load(lhs, .u64, .{})).toLocal(cg, Type.u64);
2592 defer lhs_lsb.free(cg);2628 defer lhs_lsb.free(cg);
2593 var rhs_lsb = try (try cg.load(rhs, Type.u64, 0)).toLocal(cg, Type.u64);2629 var rhs_lsb = try (try cg.load(rhs, .u64, .{})).toLocal(cg, Type.u64);
2594 defer rhs_lsb.free(cg);2630 defer rhs_lsb.free(cg);
2595 var op_lsb = try (try cg.intAdd(.u64, lhs_lsb, rhs_lsb)).toLocal(cg, Type.u64);2631 var op_lsb = try (try cg.intAdd(.u64, lhs_lsb, rhs_lsb)).toLocal(cg, Type.u64);
2596 defer op_lsb.free(cg);2632 defer op_lsb.free(cg);
25972633
2598 const lhs_msb = try cg.load(lhs, Type.u64, 8);2634 const lhs_msb = try cg.load(lhs, .u64, .{ .offset = 8 });
2599 const rhs_msb = try cg.load(rhs, Type.u64, 8);2635 const rhs_msb = try cg.load(rhs, .u64, .{ .offset = 8 });
2600 const op_msb = try cg.intAdd(.u64, lhs_msb, rhs_msb);2636 const op_msb = try cg.intAdd(.u64, lhs_msb, rhs_msb);
26012637
2602 const lt = try cg.intCmp(.u64, .lt, op_lsb, rhs_lsb);2638 const lt = try cg.intCmp(.u64, .lt, op_lsb, rhs_lsb);
...@@ -2604,8 +2640,8 @@ fn intAdd(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue...@@ -2604,8 +2640,8 @@ fn intAdd(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue
2604 var tmp_op = try (try cg.intAdd(.u64, op_msb, tmp)).toLocal(cg, Type.u64);2640 var tmp_op = try (try cg.intAdd(.u64, op_msb, tmp)).toLocal(cg, Type.u64);
2605 defer tmp_op.free(cg);2641 defer tmp_op.free(cg);
26062642
2607 try cg.store(result, op_lsb, Type.u64, 0);2643 try cg.store(result, op_lsb, .u64, .{});
2608 try cg.store(result, tmp_op, Type.u64, 8);2644 try cg.store(result, tmp_op, .u64, .{ .offset = 8 });
2609 return result;2645 return result;
2610 },2646 },
2611 else => {2647 else => {
...@@ -2641,15 +2677,15 @@ fn intSub(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue...@@ -2641,15 +2677,15 @@ fn intSub(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue
2641 65...128 => {2677 65...128 => {
2642 const result = try cg.allocStack(Type.u128);2678 const result = try cg.allocStack(Type.u128);
26432679
2644 var lhs_lsb = try (try cg.load(lhs, Type.u64, 0)).toLocal(cg, Type.u64);2680 var lhs_lsb = try (try cg.load(lhs, .u64, .{})).toLocal(cg, Type.u64);
2645 defer lhs_lsb.free(cg);2681 defer lhs_lsb.free(cg);
2646 var rhs_lsb = try (try cg.load(rhs, Type.u64, 0)).toLocal(cg, Type.u64);2682 var rhs_lsb = try (try cg.load(rhs, .u64, .{})).toLocal(cg, Type.u64);
2647 defer rhs_lsb.free(cg);2683 defer rhs_lsb.free(cg);
2648 var op_lsb = try (try cg.intSub(.u64, lhs_lsb, rhs_lsb)).toLocal(cg, Type.u64);2684 var op_lsb = try (try cg.intSub(.u64, lhs_lsb, rhs_lsb)).toLocal(cg, Type.u64);
2649 defer op_lsb.free(cg);2685 defer op_lsb.free(cg);
26502686
2651 const lhs_msb = try cg.load(lhs, Type.u64, 8);2687 const lhs_msb = try cg.load(lhs, .u64, .{ .offset = 8 });
2652 const rhs_msb = try cg.load(rhs, Type.u64, 8);2688 const rhs_msb = try cg.load(rhs, .u64, .{ .offset = 8 });
2653 const op_msb = try cg.intSub(.u64, lhs_msb, rhs_msb);2689 const op_msb = try cg.intSub(.u64, lhs_msb, rhs_msb);
26542690
2655 const lt = try cg.intCmp(.u64, .lt, lhs_lsb, rhs_lsb);2691 const lt = try cg.intCmp(.u64, .lt, lhs_lsb, rhs_lsb);
...@@ -2657,8 +2693,8 @@ fn intSub(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue...@@ -2657,8 +2693,8 @@ fn intSub(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue
2657 var tmp_op = try (try cg.intSub(.u64, op_msb, tmp)).toLocal(cg, Type.u64);2693 var tmp_op = try (try cg.intSub(.u64, op_msb, tmp)).toLocal(cg, Type.u64);
2658 defer tmp_op.free(cg);2694 defer tmp_op.free(cg);
26592695
2660 try cg.store(result, op_lsb, Type.u64, 0);2696 try cg.store(result, op_lsb, .u64, .{});
2661 try cg.store(result, tmp_op, Type.u64, 8);2697 try cg.store(result, tmp_op, .u64, .{ .offset = 8 });
2662 return result;2698 return result;
2663 },2699 },
2664 else => {2700 else => {
...@@ -2990,15 +3026,15 @@ fn intAnd(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue...@@ -2990,15 +3026,15 @@ fn intAnd(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue
2990 65...128 => {3026 65...128 => {
2991 const result = try cg.allocStack(Type.u128);3027 const result = try cg.allocStack(Type.u128);
29923028
2993 const lhs_lsb = try cg.load(lhs, Type.u64, 0);3029 const lhs_lsb = try cg.load(lhs, .u64, .{});
2994 const rhs_lsb = try cg.load(rhs, Type.u64, 0);3030 const rhs_lsb = try cg.load(rhs, .u64, .{});
2995 const and_lsb = try (try cg.intAnd(.u64, lhs_lsb, rhs_lsb)).toLocal(cg, Type.u64);3031 const and_lsb = try (try cg.intAnd(.u64, lhs_lsb, rhs_lsb)).toLocal(cg, Type.u64);
2996 try cg.store(result, and_lsb, Type.u64, 0);3032 try cg.store(result, and_lsb, .u64, .{});
29973033
2998 const lhs_msb = try cg.load(lhs, Type.u64, 8);3034 const lhs_msb = try cg.load(lhs, .u64, .{ .offset = 8 });
2999 const rhs_msb = try cg.load(rhs, Type.u64, 8);3035 const rhs_msb = try cg.load(rhs, .u64, .{ .offset = 8 });
3000 const and_msb = try (try cg.intAnd(.u64, lhs_msb, rhs_msb)).toLocal(cg, Type.u64);3036 const and_msb = try (try cg.intAnd(.u64, lhs_msb, rhs_msb)).toLocal(cg, Type.u64);
3001 try cg.store(result, and_msb, Type.u64, 8);3037 try cg.store(result, and_msb, .u64, .{ .offset = 8 });
30023038
3003 return result;3039 return result;
3004 },3040 },
...@@ -3034,15 +3070,15 @@ fn intOr(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue...@@ -3034,15 +3070,15 @@ fn intOr(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue
3034 65...128 => {3070 65...128 => {
3035 const result = try cg.allocStack(Type.u128);3071 const result = try cg.allocStack(Type.u128);
30363072
3037 const lhs_lsb = try cg.load(lhs, Type.u64, 0);3073 const lhs_lsb = try cg.load(lhs, .u64, .{});
3038 const rhs_lsb = try cg.load(rhs, Type.u64, 0);3074 const rhs_lsb = try cg.load(rhs, .u64, .{});
3039 const or_lsb = try (try cg.intOr(.u64, lhs_lsb, rhs_lsb)).toLocal(cg, Type.u64);3075 const or_lsb = try (try cg.intOr(.u64, lhs_lsb, rhs_lsb)).toLocal(cg, Type.u64);
3040 try cg.store(result, or_lsb, Type.u64, 0);3076 try cg.store(result, or_lsb, .u64, .{});
30413077
3042 const lhs_msb = try cg.load(lhs, Type.u64, 8);3078 const lhs_msb = try cg.load(lhs, .u64, .{ .offset = 8 });
3043 const rhs_msb = try cg.load(rhs, Type.u64, 8);3079 const rhs_msb = try cg.load(rhs, .u64, .{ .offset = 8 });
3044 const or_msb = try (try cg.intOr(.u64, lhs_msb, rhs_msb)).toLocal(cg, Type.u64);3080 const or_msb = try (try cg.intOr(.u64, lhs_msb, rhs_msb)).toLocal(cg, Type.u64);
3045 try cg.store(result, or_msb, Type.u64, 8);3081 try cg.store(result, or_msb, .u64, .{ .offset = 8 });
30463082
3047 return result;3083 return result;
3048 },3084 },
...@@ -3078,15 +3114,15 @@ fn intXor(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue...@@ -3078,15 +3114,15 @@ fn intXor(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue
3078 65...128 => {3114 65...128 => {
3079 const result = try cg.allocStack(Type.u128);3115 const result = try cg.allocStack(Type.u128);
30803116
3081 const lhs_lsb = try cg.load(lhs, Type.u64, 0);3117 const lhs_lsb = try cg.load(lhs, .u64, .{});
3082 const rhs_lsb = try cg.load(rhs, Type.u64, 0);3118 const rhs_lsb = try cg.load(rhs, .u64, .{});
3083 const xor_lsb = try (try cg.intXor(.u64, lhs_lsb, rhs_lsb)).toLocal(cg, Type.u64);3119 const xor_lsb = try (try cg.intXor(.u64, lhs_lsb, rhs_lsb)).toLocal(cg, Type.u64);
3084 try cg.store(result, xor_lsb, Type.u64, 0);3120 try cg.store(result, xor_lsb, .u64, .{});
30853121
3086 const lhs_msb = try cg.load(lhs, Type.u64, 8);3122 const lhs_msb = try cg.load(lhs, .u64, .{ .offset = 8 });
3087 const rhs_msb = try cg.load(rhs, Type.u64, 8);3123 const rhs_msb = try cg.load(rhs, .u64, .{ .offset = 8 });
3088 const xor_msb = try (try cg.intXor(.u64, lhs_msb, rhs_msb)).toLocal(cg, Type.u64);3124 const xor_msb = try (try cg.intXor(.u64, lhs_msb, rhs_msb)).toLocal(cg, Type.u64);
3089 try cg.store(result, xor_msb, Type.u64, 8);3125 try cg.store(result, xor_msb, .u64, .{ .offset = 8 });
30903126
3091 return result;3127 return result;
3092 },3128 },
...@@ -3141,20 +3177,20 @@ fn intNot(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {...@@ -3141,20 +3177,20 @@ fn intNot(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3141 const result = try cg.allocStack(Type.u128);3177 const result = try cg.allocStack(Type.u128);
31423178
3143 try cg.emitWValue(result);3179 try cg.emitWValue(result);
3144 _ = try cg.load(operand, Type.u64, 0);3180 _ = try cg.load(operand, .u64, .{});
3145 try cg.addImm64(~@as(u64, 0));3181 try cg.addImm64(~@as(u64, 0));
3146 try cg.addTag(.i64_xor);3182 try cg.addTag(.i64_xor);
3147 try cg.store(.stack, .stack, Type.u64, result.offset());3183 try cg.store(.stack, .stack, .u64, .{ .offset = result.offset() });
31483184
3149 try cg.emitWValue(result);3185 try cg.emitWValue(result);
3150 _ = try cg.load(operand, Type.u64, 8);3186 _ = try cg.load(operand, .u64, .{ .offset = 8 });
3151 const high_mask: u64 = if (ty.is_signed)3187 const high_mask: u64 = if (ty.is_signed)
3152 ~@as(u64, 0)3188 ~@as(u64, 0)
3153 else3189 else
3154 ~@as(u64, 0) >> @intCast(128 - ty.bits);3190 ~@as(u64, 0) >> @intCast(128 - ty.bits);
3155 try cg.addImm64(high_mask);3191 try cg.addImm64(high_mask);
3156 try cg.addTag(.i64_xor);3192 try cg.addTag(.i64_xor);
3157 try cg.store(.stack, .stack, Type.u64, result.offset() + 8);3193 try cg.store(.stack, .stack, .u64, .{ .offset = result.offset() + 8 });
31583194
3159 return result;3195 return result;
3160 },3196 },
...@@ -3287,16 +3323,16 @@ fn intAbs(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {...@@ -3287,16 +3323,16 @@ fn intAbs(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3287 try cg.emitWValue(mask);3323 try cg.emitWValue(mask);
3288 try cg.emitWValue(mask);3324 try cg.emitWValue(mask);
32893325
3290 _ = try cg.load(operand, Type.u64, 8);3326 _ = try cg.load(operand, .u64, .{ .offset = 8 });
3291 try cg.addImm64(63);3327 try cg.addImm64(63);
3292 try cg.addTag(.i64_shr_s);3328 try cg.addTag(.i64_shr_s);
32933329
3294 var tmp = try cg.allocLocal(Type.u64);3330 var tmp = try cg.allocLocal(Type.u64);
3295 defer tmp.free(cg);3331 defer tmp.free(cg);
3296 try cg.addLocal(.local_tee, tmp.local.value);3332 try cg.addLocal(.local_tee, tmp.local.value);
3297 try cg.store(.stack, .stack, Type.u64, mask.offset() + 0);3333 try cg.store(.stack, .stack, .u64, .{ .offset = mask.offset() + 0 });
3298 try cg.emitWValue(tmp);3334 try cg.emitWValue(tmp);
3299 try cg.store(.stack, .stack, Type.u64, mask.offset() + 8);3335 try cg.store(.stack, .stack, .u64, .{ .offset = mask.offset() + 8 });
33003336
3301 const a = try cg.intXor(u128_ty, operand, mask);3337 const a = try cg.intXor(u128_ty, operand, mask);
3302 const b = try cg.intSub(u128_ty, a, mask);3338 const b = try cg.intSub(u128_ty, a, mask);
...@@ -3364,7 +3400,7 @@ fn intClz(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {...@@ -3364,7 +3400,7 @@ fn intClz(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3364 return .stack;3400 return .stack;
3365 },3401 },
3366 65...128 => {3402 65...128 => {
3367 var msb = try (try cg.load(operand, Type.u64, 8)).toLocal(cg, Type.u64);3403 var msb = try (try cg.load(operand, .u64, .{ .offset = 8 })).toLocal(cg, Type.u64);
3368 defer msb.free(cg);3404 defer msb.free(cg);
33693405
3370 if (ty.is_signed and ty.bits < 128) {3406 if (ty.is_signed and ty.bits < 128) {
...@@ -3375,7 +3411,7 @@ fn intClz(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {...@@ -3375,7 +3411,7 @@ fn intClz(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3375 }3411 }
33763412
3377 try cg.addTag(.i64_clz);3413 try cg.addTag(.i64_clz);
3378 _ = try cg.load(operand, Type.u64, 0);3414 _ = try cg.load(operand, .u64, .{});
3379 try cg.addTag(.i64_clz);3415 try cg.addTag(.i64_clz);
3380 try cg.emitWValue(.{ .imm64 = 64 });3416 try cg.emitWValue(.{ .imm64 = 64 });
3381 try cg.addTag(.i64_add);3417 try cg.addTag(.i64_add);
...@@ -3423,13 +3459,13 @@ fn intCtz(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {...@@ -3423,13 +3459,13 @@ fn intCtz(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3423 return .stack;3459 return .stack;
3424 },3460 },
3425 65...128 => {3461 65...128 => {
3426 var lsb = try (try cg.load(operand, Type.u64, 0)).toLocal(cg, Type.u64);3462 var lsb = try (try cg.load(operand, .u64, .{})).toLocal(cg, Type.u64);
3427 defer lsb.free(cg);3463 defer lsb.free(cg);
34283464
3429 try cg.emitWValue(lsb);3465 try cg.emitWValue(lsb);
3430 try cg.addTag(.i64_ctz);3466 try cg.addTag(.i64_ctz);
34313467
3432 _ = try cg.load(operand, Type.u64, 8);3468 _ = try cg.load(operand, .u64, .{ .offset = 8 });
3433 if (ty.bits < 128) {3469 if (ty.bits < 128) {
3434 try cg.addImm64(@as(u64, 1) << @intCast(ty.bits - 64));3470 try cg.addImm64(@as(u64, 1) << @intCast(ty.bits - 64));
3435 try cg.addTag(.i64_or);3471 try cg.addTag(.i64_or);
...@@ -3475,9 +3511,9 @@ fn intPopCount(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {...@@ -3475,9 +3511,9 @@ fn intPopCount(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3475 return .stack;3511 return .stack;
3476 },3512 },
3477 65...128 => {3513 65...128 => {
3478 _ = try cg.load(operand, Type.u64, 0);3514 _ = try cg.load(operand, .u64, .{});
3479 try cg.addTag(.i64_popcnt);3515 try cg.addTag(.i64_popcnt);
3480 _ = try cg.load(operand, Type.u64, 8);3516 _ = try cg.load(operand, .u64, .{ .offset = 8 });
3481 if (ty.is_signed and ty.bits < 128) {3517 if (ty.is_signed and ty.bits < 128) {
3482 try cg.addImm64(128 - ty.bits);3518 try cg.addImm64(128 - ty.bits);
3483 try cg.addTag(.i64_shl);3519 try cg.addTag(.i64_shl);
...@@ -3525,7 +3561,7 @@ fn intBitReverse(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {...@@ -3525,7 +3561,7 @@ fn intBitReverse(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3525 const tmp = try cg.allocStack(Type.u128);3561 const tmp = try cg.allocStack(Type.u128);
35263562
3527 try cg.emitWValue(tmp);3563 try cg.emitWValue(tmp);
3528 const hi = try cg.load(operand, Type.u64, 8);3564 const hi = try cg.load(operand, .u64, .{ .offset = 8 });
3529 const hi_rev = try cg.callIntrinsic(3565 const hi_rev = try cg.callIntrinsic(
3530 .__bitreversedi2,3566 .__bitreversedi2,
3531 &.{.u64_type},3567 &.{.u64_type},
...@@ -3533,10 +3569,10 @@ fn intBitReverse(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {...@@ -3533,10 +3569,10 @@ fn intBitReverse(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3533 &.{hi},3569 &.{hi},
3534 );3570 );
3535 try cg.emitWValue(hi_rev);3571 try cg.emitWValue(hi_rev);
3536 try cg.store(.stack, .stack, Type.u64, tmp.offset());3572 try cg.store(.stack, .stack, .u64, .{ .offset = tmp.offset() });
35373573
3538 try cg.emitWValue(tmp);3574 try cg.emitWValue(tmp);
3539 const lo = try cg.load(operand, Type.u64, 0);3575 const lo = try cg.load(operand, .u64, .{});
3540 const lo_rev = try cg.callIntrinsic(3576 const lo_rev = try cg.callIntrinsic(
3541 .__bitreversedi2,3577 .__bitreversedi2,
3542 &.{.u64_type},3578 &.{.u64_type},
...@@ -3544,7 +3580,7 @@ fn intBitReverse(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {...@@ -3544,7 +3580,7 @@ fn intBitReverse(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3544 &.{lo},3580 &.{lo},
3545 );3581 );
3546 try cg.emitWValue(lo_rev);3582 try cg.emitWValue(lo_rev);
3547 try cg.store(.stack, .stack, Type.u64, tmp.offset() + 8);3583 try cg.store(.stack, .stack, .u64, .{ .offset = tmp.offset() + 8 });
35483584
3549 if (ty.bits < 128) {3585 if (ty.bits < 128) {
3550 const shift_ty: IntType = .{ .is_signed = ty.is_signed, .bits = 128 };3586 const shift_ty: IntType = .{ .is_signed = ty.is_signed, .bits = 128 };
...@@ -3595,25 +3631,25 @@ fn intByteSwap(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {...@@ -3595,25 +3631,25 @@ fn intByteSwap(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
35953631
3596 try cg.emitWValue(result);3632 try cg.emitWValue(result);
35973633
3598 const low = try cg.load(operand, Type.u64, 0);3634 const low = try cg.load(operand, .u64, .{});
3599 const swap_low = try cg.callIntrinsic(3635 const swap_low = try cg.callIntrinsic(
3600 .__bswapdi2,3636 .__bswapdi2,
3601 &.{.u64_type},3637 &.{.u64_type},
3602 Type.u64,3638 Type.u64,
3603 &.{low},3639 &.{low},
3604 );3640 );
3605 try cg.store(.stack, swap_low, Type.u64, result.offset() + 8);3641 try cg.store(.stack, swap_low, .u64, .{ .offset = result.offset() + 8 });
36063642
3607 try cg.emitWValue(result);3643 try cg.emitWValue(result);
36083644
3609 const high = try cg.load(operand, Type.u64, 8);3645 const high = try cg.load(operand, .u64, .{ .offset = 8 });
3610 const swap_high = try cg.callIntrinsic(3646 const swap_high = try cg.callIntrinsic(
3611 .__bswapdi2,3647 .__bswapdi2,
3612 &.{.u64_type},3648 &.{.u64_type},
3613 Type.u64,3649 Type.u64,
3614 &.{high},3650 &.{high},
3615 );3651 );
3616 try cg.store(.stack, swap_high, Type.u64, result.offset());3652 try cg.store(.stack, swap_high, .u64, .{ .offset = result.offset() });
36173653
3618 if (ty.bits < 128) {3654 if (ty.bits < 128) {
3619 const shift_ty: IntType = .{ .is_signed = ty.is_signed, .bits = 128 };3655 const shift_ty: IntType = .{ .is_signed = ty.is_signed, .bits = 128 };
...@@ -3671,11 +3707,11 @@ fn intWrap(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {...@@ -3671,11 +3707,11 @@ fn intWrap(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3671 const result = try cg.allocStack(Type.u128);3707 const result = try cg.allocStack(Type.u128);
36723708
3673 try cg.emitWValue(result);3709 try cg.emitWValue(result);
3674 _ = try cg.load(operand, Type.u64, 0);3710 _ = try cg.load(operand, .u64, .{});
3675 try cg.store(.stack, .stack, Type.u64, result.offset());3711 try cg.store(.stack, .stack, .u64, .{ .offset = result.offset() });
36763712
3677 try cg.emitWValue(result);3713 try cg.emitWValue(result);
3678 _ = try cg.load(operand, Type.u64, 8);3714 _ = try cg.load(operand, .u64, .{ .offset = 8 });
3679 if (ty.is_signed) {3715 if (ty.is_signed) {
3680 try cg.addImm64(128 - ty.bits);3716 try cg.addImm64(128 - ty.bits);
3681 try cg.addTag(.i64_shl);3717 try cg.addTag(.i64_shl);
...@@ -3685,7 +3721,7 @@ fn intWrap(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {...@@ -3685,7 +3721,7 @@ fn intWrap(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3685 try cg.addImm64(~@as(u64, 0) >> @intCast(128 - ty.bits));3721 try cg.addImm64(~@as(u64, 0) >> @intCast(128 - ty.bits));
3686 try cg.addTag(.i64_and);3722 try cg.addTag(.i64_and);
3687 }3723 }
3688 try cg.store(.stack, .stack, Type.u64, result.offset() + 8);3724 try cg.store(.stack, .stack, .u64, .{ .offset = result.offset() + 8 });
36893725
3690 return result;3726 return result;
3691 },3727 },
...@@ -3703,7 +3739,7 @@ fn intWrap(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {...@@ -3703,7 +3739,7 @@ fn intWrap(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3703 const pad = 64 - ty.bits % 64;3739 const pad = 64 - ty.bits % 64;
37043740
3705 try cg.emitWValue(result);3741 try cg.emitWValue(result);
3706 _ = try cg.load(operand, Type.u64, used_len - 8);3742 _ = try cg.load(operand, .u64, .{ .offset = used_len - 8 });
3707 if (ty.is_signed) {3743 if (ty.is_signed) {
3708 try cg.addImm64(pad);3744 try cg.addImm64(pad);
3709 try cg.addTag(.i64_shl);3745 try cg.addTag(.i64_shl);
...@@ -3713,7 +3749,7 @@ fn intWrap(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {...@@ -3713,7 +3749,7 @@ fn intWrap(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3713 try cg.addImm64(~@as(u64, 0) >> @intCast(pad));3749 try cg.addImm64(~@as(u64, 0) >> @intCast(pad));
3714 try cg.addTag(.i64_and);3750 try cg.addTag(.i64_and);
3715 }3751 }
3716 try cg.store(.stack, .stack, Type.u64, result.offset() + used_len - 8);3752 try cg.store(.stack, .stack, .u64, .{ .offset = result.offset() + used_len - 8 });
3717 } else {3753 } else {
3718 try cg.memcpy(result, operand, .{ .imm32 = used_len });3754 try cg.memcpy(result, operand, .{ .imm32 = used_len });
3719 }3755 }
...@@ -3722,13 +3758,13 @@ fn intWrap(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {...@@ -3722,13 +3758,13 @@ fn intWrap(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3722 if (used_len + 8 == full_len) { // last limb needs sign extended3758 if (used_len + 8 == full_len) { // last limb needs sign extended
3723 try cg.emitWValue(result);3759 try cg.emitWValue(result);
3724 if (ty.is_signed) {3760 if (ty.is_signed) {
3725 _ = try cg.load(result, Type.u64, used_len - 8);3761 _ = try cg.load(result, .u64, .{ .offset = used_len - 8 });
3726 try cg.addImm64(63);3762 try cg.addImm64(63);
3727 try cg.addTag(.i64_shr_s);3763 try cg.addTag(.i64_shr_s);
3728 } else {3764 } else {
3729 try cg.addImm64(0);3765 try cg.addImm64(0);
3730 }3766 }
3731 try cg.store(.stack, .stack, Type.u64, result.offset() + used_len);3767 try cg.store(.stack, .stack, .u64, .{ .offset = result.offset() + used_len });
3732 }3768 }
37333769
3734 return result;3770 return result;
...@@ -3751,12 +3787,12 @@ fn intMaxValue(cg: *CodeGen, int_ty: IntType) InnerError!WValue {...@@ -3751,12 +3787,12 @@ fn intMaxValue(cg: *CodeGen, int_ty: IntType) InnerError!WValue {
3751 }3787 }
3752 } else if (int_ty.bits <= 128) {3788 } else if (int_ty.bits <= 128) {
3753 const result = try cg.allocInt(int_ty);3789 const result = try cg.allocInt(int_ty);
3754 try cg.store(result, .{ .imm64 = ~@as(u64, 0) }, Type.u64, 0);3790 try cg.store(result, .{ .imm64 = ~@as(u64, 0) }, .u64, .{});
37553791
3756 if (int_ty.is_signed) {3792 if (int_ty.is_signed) {
3757 try cg.store(result, .{ .imm64 = (~@as(u64, 0) >> @intCast(128 - int_ty.bits)) >> 1 }, Type.u64, 8);3793 try cg.store(result, .{ .imm64 = (~@as(u64, 0) >> @intCast(128 - int_ty.bits)) >> 1 }, .u64, .{ .offset = 8 });
3758 } else {3794 } else {
3759 try cg.store(result, .{ .imm64 = ~@as(u64, 0) >> @intCast(128 - int_ty.bits) }, Type.u64, 8);3795 try cg.store(result, .{ .imm64 = ~@as(u64, 0) >> @intCast(128 - int_ty.bits) }, .u64, .{ .offset = 8 });
3760 }3796 }
3761 return result;3797 return result;
3762 } else {3798 } else {
...@@ -3767,13 +3803,13 @@ fn intMaxValue(cg: *CodeGen, int_ty: IntType) InnerError!WValue {...@@ -3767,13 +3803,13 @@ fn intMaxValue(cg: *CodeGen, int_ty: IntType) InnerError!WValue {
3767 try cg.memset(Type.u8, result, .{ .imm32 = used_len - 8 }, .{ .imm32 = 0xFF });3803 try cg.memset(Type.u8, result, .{ .imm32 = used_len - 8 }, .{ .imm32 = 0xFF });
37683804
3769 if (int_ty.is_signed) {3805 if (int_ty.is_signed) {
3770 try cg.store(result, .{ .imm64 = (~@as(u64, 0) >> @intCast(used_len * 8 - int_ty.bits)) >> 1 }, Type.u64, used_len - 8);3806 try cg.store(result, .{ .imm64 = (~@as(u64, 0) >> @intCast(used_len * 8 - int_ty.bits)) >> 1 }, .u64, .{ .offset = used_len - 8 });
3771 } else {3807 } else {
3772 try cg.store(result, .{ .imm64 = ~@as(u64, 0) >> @intCast(used_len * 8 - int_ty.bits) }, Type.u64, used_len - 8);3808 try cg.store(result, .{ .imm64 = ~@as(u64, 0) >> @intCast(used_len * 8 - int_ty.bits) }, .u64, .{ .offset = used_len - 8 });
3773 }3809 }
37743810
3775 if (used_len + 8 == full_len) {3811 if (used_len + 8 == full_len) {
3776 try cg.store(result, .{ .imm64 = 0 }, Type.u64, full_len - 8);3812 try cg.store(result, .{ .imm64 = 0 }, .u64, .{ .offset = full_len - 8 });
3777 }3813 }
37783814
3779 return result;3815 return result;
...@@ -3790,8 +3826,8 @@ fn intMinValue(cg: *CodeGen, int_ty: IntType) InnerError!WValue {...@@ -3790,8 +3826,8 @@ fn intMinValue(cg: *CodeGen, int_ty: IntType) InnerError!WValue {
3790 return .{ .imm64 = ~@as(u64, 0) << @intCast(int_ty.bits - 1) };3826 return .{ .imm64 = ~@as(u64, 0) << @intCast(int_ty.bits - 1) };
3791 } else if (int_ty.bits <= 128) {3827 } else if (int_ty.bits <= 128) {
3792 const result = try cg.allocInt(int_ty);3828 const result = try cg.allocInt(int_ty);
3793 try cg.store(result, .{ .imm64 = 0 }, Type.u64, 0);3829 try cg.store(result, .{ .imm64 = 0 }, .u64, .{});
3794 try cg.store(result, .{ .imm64 = ~@as(u64, 0) << @intCast(int_ty.bits - 65) }, Type.u64, 8);3830 try cg.store(result, .{ .imm64 = ~@as(u64, 0) << @intCast(int_ty.bits - 65) }, .u64, .{ .offset = 8 });
3795 return result;3831 return result;
3796 } else {3832 } else {
3797 const result = try cg.allocInt(int_ty);3833 const result = try cg.allocInt(int_ty);
...@@ -3799,10 +3835,10 @@ fn intMinValue(cg: *CodeGen, int_ty: IntType) InnerError!WValue {...@@ -3799,10 +3835,10 @@ fn intMinValue(cg: *CodeGen, int_ty: IntType) InnerError!WValue {
3799 const used_len = @divCeil(int_ty.bits, 64) * 8;3835 const used_len = @divCeil(int_ty.bits, 64) * 8;
38003836
3801 try cg.memset(Type.u8, result, .{ .imm32 = used_len - 8 }, .{ .imm32 = 0 });3837 try cg.memset(Type.u8, result, .{ .imm32 = used_len - 8 }, .{ .imm32 = 0 });
3802 try cg.store(result, .{ .imm64 = ~@as(u64, 0) << @intCast(int_ty.bits - (used_len - 8) * 8 - 1) }, Type.u64, used_len - 8);3838 try cg.store(result, .{ .imm64 = ~@as(u64, 0) << @intCast(int_ty.bits - (used_len - 8) * 8 - 1) }, .u64, .{ .offset = used_len - 8 });
38033839
3804 if (used_len + 8 == full_len) {3840 if (used_len + 8 == full_len) {
3805 try cg.store(result, .{ .imm64 = ~@as(u64, 0) }, Type.u64, full_len - 8);3841 try cg.store(result, .{ .imm64 = ~@as(u64, 0) }, .u64, .{ .offset = full_len - 8 });
3806 }3842 }
38073843
3808 return result;3844 return result;
...@@ -3962,8 +3998,8 @@ fn intZeroValue(cg: *CodeGen, int_ty: IntType) InnerError!WValue {...@@ -3962,8 +3998,8 @@ fn intZeroValue(cg: *CodeGen, int_ty: IntType) InnerError!WValue {
3962 33...64 => return .{ .imm64 = 0 },3998 33...64 => return .{ .imm64 = 0 },
3963 65...128 => {3999 65...128 => {
3964 const result = try cg.allocInt(int_ty);4000 const result = try cg.allocInt(int_ty);
3965 try cg.store(result, .{ .imm64 = 0 }, Type.u64, 0);4001 try cg.store(result, .{ .imm64 = 0 }, .u64, .{});
3966 try cg.store(result, .{ .imm64 = 0 }, Type.u64, 8);4002 try cg.store(result, .{ .imm64 = 0 }, .u64, .{ .offset = 8 });
3967 return result;4003 return result;
3968 },4004 },
3969 else => {4005 else => {
...@@ -4103,7 +4139,7 @@ fn intMulOverflow(cg: *CodeGen, int_ty: IntType, lhs: WValue, rhs: WValue) Inner...@@ -4103,7 +4139,7 @@ fn intMulOverflow(cg: *CodeGen, int_ty: IntType, lhs: WValue, rhs: WValue) Inner
4103 Type.i128,4139 Type.i128,
4104 &.{ lhs, rhs, overflow_ret },4140 &.{ lhs, rhs, overflow_ret },
4105 );4141 );
4106 _ = try cg.load(overflow_ret, Type.i32, 0);4142 _ = try cg.load(overflow_ret, .i32, .{});
4107 try cg.addLocal(.local_set, overflow_bit.local.value);4143 try cg.addLocal(.local_set, overflow_bit.local.value);
4108 break :blk res;4144 break :blk res;
4109 } else {4145 } else {
...@@ -4180,12 +4216,12 @@ fn intCast(cg: *CodeGen, dest_ty: IntType, src_ty: IntType, operand: WValue) Inn...@@ -4180,12 +4216,12 @@ fn intCast(cg: *CodeGen, dest_ty: IntType, src_ty: IntType, operand: WValue) Inn
4180 try cg.emitWValue(result);4216 try cg.emitWValue(result);
4181 try cg.emitWValue(operand);4217 try cg.emitWValue(operand);
4182 try cg.addTag(if (src_ty.is_signed) .i64_extend_i32_s else .i64_extend_i32_u);4218 try cg.addTag(if (src_ty.is_signed) .i64_extend_i32_s else .i64_extend_i32_u);
4183 try cg.store(.stack, .stack, Type.u64, result.offset());4219 try cg.store(.stack, .stack, .u64, .{ .offset = result.offset() });
4184 src_len = 8;4220 src_len = 8;
4185 } else if (src_bits == 64) {4221 } else if (src_bits == 64) {
4186 try cg.emitWValue(result);4222 try cg.emitWValue(result);
4187 try cg.emitWValue(operand);4223 try cg.emitWValue(operand);
4188 try cg.store(.stack, .stack, Type.u64, result.offset());4224 try cg.store(.stack, .stack, .u64, .{ .offset = result.offset() });
4189 src_len = 8;4225 src_len = 8;
4190 } else {4226 } else {
4191 src_len = src_bits / 8;4227 src_len = src_bits / 8;
...@@ -4202,9 +4238,9 @@ fn intCast(cg: *CodeGen, dest_ty: IntType, src_ty: IntType, operand: WValue) Inn...@@ -4202,9 +4238,9 @@ fn intCast(cg: *CodeGen, dest_ty: IntType, src_ty: IntType, operand: WValue) Inn
4202 try cg.emitWValue(operand);4238 try cg.emitWValue(operand);
4203 } else unreachable;4239 } else unreachable;
4204 const shr = try cg.intShr(IntType.i64, .stack, .{ .imm32 = 63 });4240 const shr = try cg.intShr(IntType.i64, .stack, .{ .imm32 = 63 });
4205 try cg.store(.stack, shr, Type.u64, 8 + result.offset());4241 try cg.store(.stack, shr, .u64, .{ .offset = 8 + result.offset() });
4206 } else {4242 } else {
4207 try cg.store(result, .{ .imm64 = 0 }, Type.u64, 8);4243 try cg.store(result, .{ .imm64 = 0 }, .u64, .{ .offset = 8 });
4208 }4244 }
4209 } else {4245 } else {
4210 var pad = result;4246 var pad = result;
...@@ -4219,7 +4255,7 @@ fn intCast(cg: *CodeGen, dest_ty: IntType, src_ty: IntType, operand: WValue) Inn...@@ -4219,7 +4255,7 @@ fn intCast(cg: *CodeGen, dest_ty: IntType, src_ty: IntType, operand: WValue) Inn
4219 _ = try cg.intShr(IntType.i64, .stack, .{ .imm32 = 63 });4255 _ = try cg.intShr(IntType.i64, .stack, .{ .imm32 = 63 });
4220 try cg.addTag(.i32_wrap_i64);4256 try cg.addTag(.i32_wrap_i64);
4221 } else {4257 } else {
4222 _ = try cg.load(operand, Type.u64, src_len - 8);4258 _ = try cg.load(operand, .u64, .{ .offset = src_len - 8 });
4223 _ = try cg.intShr(IntType.i64, .stack, .{ .imm32 = 63 });4259 _ = try cg.intShr(IntType.i64, .stack, .{ .imm32 = 63 });
4224 try cg.addTag(.i32_wrap_i64);4260 try cg.addTag(.i32_wrap_i64);
4225 }4261 }
...@@ -4237,7 +4273,7 @@ fn intCast(cg: *CodeGen, dest_ty: IntType, src_ty: IntType, operand: WValue) Inn...@@ -4237,7 +4273,7 @@ fn intCast(cg: *CodeGen, dest_ty: IntType, src_ty: IntType, operand: WValue) Inn
4237 assert(dest_bits <= 64);4273 assert(dest_bits <= 64);
4238 assert(src_bits >= 128);4274 assert(src_bits >= 128);
4239 const load_ty = if (dest_bits == 32) Type.u32 else Type.u64;4275 const load_ty = if (dest_bits == 32) Type.u32 else Type.u64;
4240 return cg.load(operand, load_ty, 0);4276 return cg.load(operand, load_ty, .{});
4241 }4277 }
4242}4278}
42434279
...@@ -4608,18 +4644,18 @@ fn floatNeg(cg: *CodeGen, ty: FloatType, arg: WValue) InnerError!WValue {...@@ -4608,18 +4644,18 @@ fn floatNeg(cg: *CodeGen, ty: FloatType, arg: WValue) InnerError!WValue {
4608 const result = try cg.allocStack(Type.f128);4644 const result = try cg.allocStack(Type.f128);
4609 try cg.emitWValue(result);4645 try cg.emitWValue(result);
4610 try cg.emitWValue(arg);4646 try cg.emitWValue(arg);
4611 try cg.addMemArg(.i64_load, .{ .offset = 0 + arg.offset(), .alignment = 2 });4647 try cg.addMemArg(.i64_load, .{ .offset = 0 + arg.offset(), .alignment = .@"2" });
4612 try cg.addMemArg(.i64_store, .{ .offset = 0 + result.offset(), .alignment = 2 });4648 try cg.addMemArg(.i64_store, .{ .offset = 0 + result.offset(), .alignment = .@"2" });
4613 try cg.emitWValue(result);4649 try cg.emitWValue(result);
4614 try cg.emitWValue(arg);4650 try cg.emitWValue(arg);
4615 try cg.addMemArg(.i64_load, .{ .offset = 8 + arg.offset(), .alignment = 2 });4651 try cg.addMemArg(.i64_load, .{ .offset = 8 + arg.offset(), .alignment = .@"2" });
4616 if (ty == .f80) {4652 if (ty == .f80) {
4617 try cg.addImm64(0x8000);4653 try cg.addImm64(0x8000);
4618 } else {4654 } else {
4619 try cg.addImm64(0x8000000000000000);4655 try cg.addImm64(0x8000000000000000);
4620 }4656 }
4621 try cg.addTag(.i64_xor);4657 try cg.addTag(.i64_xor);
4622 try cg.addMemArg(.i64_store, .{ .offset = 8 + result.offset(), .alignment = 2 });4658 try cg.addMemArg(.i64_store, .{ .offset = 8 + result.offset(), .alignment = .@"2" });
4623 return result;4659 return result;
4624 },4660 },
4625 }4661 }
...@@ -4647,18 +4683,18 @@ fn floatAbs(cg: *CodeGen, ty: FloatType, arg: WValue) InnerError!WValue {...@@ -4647,18 +4683,18 @@ fn floatAbs(cg: *CodeGen, ty: FloatType, arg: WValue) InnerError!WValue {
4647 const result = try cg.allocStack(Type.f128);4683 const result = try cg.allocStack(Type.f128);
4648 try cg.emitWValue(result);4684 try cg.emitWValue(result);
4649 try cg.emitWValue(arg);4685 try cg.emitWValue(arg);
4650 try cg.addMemArg(.i64_load, .{ .offset = 0 + arg.offset(), .alignment = 2 });4686 try cg.addMemArg(.i64_load, .{ .offset = 0 + arg.offset(), .alignment = .@"2" });
4651 try cg.addMemArg(.i64_store, .{ .offset = 0 + result.offset(), .alignment = 2 });4687 try cg.addMemArg(.i64_store, .{ .offset = 0 + result.offset(), .alignment = .@"2" });
4652 try cg.emitWValue(result);4688 try cg.emitWValue(result);
4653 try cg.emitWValue(arg);4689 try cg.emitWValue(arg);
4654 try cg.addMemArg(.i64_load, .{ .offset = 8 + arg.offset(), .alignment = 2 });4690 try cg.addMemArg(.i64_load, .{ .offset = 8 + arg.offset(), .alignment = .@"2" });
4655 if (ty == .f80) {4691 if (ty == .f80) {
4656 try cg.addImm64(0x7FFF);4692 try cg.addImm64(0x7FFF);
4657 } else {4693 } else {
4658 try cg.addImm64(0x7FFFFFFFFFFFFFFF);4694 try cg.addImm64(0x7FFFFFFFFFFFFFFF);
4659 }4695 }
4660 try cg.addTag(.i64_and);4696 try cg.addTag(.i64_and);
4661 try cg.addMemArg(.i64_store, .{ .offset = 8 + result.offset(), .alignment = 2 });4697 try cg.addMemArg(.i64_store, .{ .offset = 8 + result.offset(), .alignment = .@"2" });
4662 return result;4698 return result;
4663 },4699 },
4664 }4700 }
...@@ -5309,8 +5345,8 @@ fn airCmp(cg: *CodeGen, inst: Air.Inst.Index, op: std.math.CompareOperator) Inne...@@ -5309,8 +5345,8 @@ fn airCmp(cg: *CodeGen, inst: Air.Inst.Index, op: std.math.CompareOperator) Inne
5309 try cg.addLocal(.local_get, lhs_null.local.value);5345 try cg.addLocal(.local_get, lhs_null.local.value);
5310 try cg.addLabel(.br_if, 0);5346 try cg.addLabel(.br_if, 0);
53115347
5312 _ = try cg.load(lhs, payload_ty, 0);5348 _ = try cg.load(lhs, payload_ty, .{});
5313 _ = try cg.load(rhs, payload_ty, 0);5349 _ = try cg.load(rhs, payload_ty, .{});
53145350
5315 if (payload_ty.isAnyFloat()) {5351 if (payload_ty.isAnyFloat()) {
5316 _ = try cg.floatCmp(.fromType(cg, payload_ty), op, .stack, .stack);5352 _ = try cg.floatCmp(.fromType(cg, payload_ty), op, .stack, .stack);
...@@ -5370,16 +5406,16 @@ fn intCmp(cg: *CodeGen, ty: IntType, op: std.math.CompareOperator, lhs: WValue,...@@ -5370,16 +5406,16 @@ fn intCmp(cg: *CodeGen, ty: IntType, op: std.math.CompareOperator, lhs: WValue,
5370 return .stack;5406 return .stack;
5371 },5407 },
5372 65...128 => {5408 65...128 => {
5373 var lhs_msb = try (try cg.load(lhs, Type.u64, 8)).toLocal(cg, Type.u64);5409 var lhs_msb = try (try cg.load(lhs, .u64, .{ .offset = 8 })).toLocal(cg, Type.u64);
5374 defer lhs_msb.free(cg);5410 defer lhs_msb.free(cg);
5375 var rhs_msb = try (try cg.load(rhs, Type.u64, 8)).toLocal(cg, Type.u64);5411 var rhs_msb = try (try cg.load(rhs, .u64, .{ .offset = 8 })).toLocal(cg, Type.u64);
5376 defer rhs_msb.free(cg);5412 defer rhs_msb.free(cg);
53775413
5378 switch (op) {5414 switch (op) {
5379 .eq, .neq => {5415 .eq, .neq => {
5380 const xor_high = try cg.intXor(.u64, lhs_msb, rhs_msb);5416 const xor_high = try cg.intXor(.u64, lhs_msb, rhs_msb);
5381 const lhs_lsb = try cg.load(lhs, Type.u64, 0);5417 const lhs_lsb = try cg.load(lhs, .u64, .{});
5382 const rhs_lsb = try cg.load(rhs, Type.u64, 0);5418 const rhs_lsb = try cg.load(rhs, .u64, .{});
5383 const xor_low = try cg.intXor(.u64, lhs_lsb, rhs_lsb);5419 const xor_low = try cg.intXor(.u64, lhs_lsb, rhs_lsb);
5384 const or_result = try cg.intOr(.u64, xor_high, xor_low);5420 const or_result = try cg.intOr(.u64, xor_high, xor_low);
53855421
...@@ -5392,8 +5428,8 @@ fn intCmp(cg: *CodeGen, ty: IntType, op: std.math.CompareOperator, lhs: WValue,...@@ -5392,8 +5428,8 @@ fn intCmp(cg: *CodeGen, ty: IntType, op: std.math.CompareOperator, lhs: WValue,
5392 else => {5428 else => {
5393 const word_int_ty: IntType = if (ty.is_signed) .i64 else .u64;5429 const word_int_ty: IntType = if (ty.is_signed) .i64 else .u64;
53945430
5395 const lhs_lsb = try cg.load(lhs, Type.u64, 0);5431 const lhs_lsb = try cg.load(lhs, .u64, .{});
5396 const rhs_lsb = try cg.load(rhs, Type.u64, 0);5432 const rhs_lsb = try cg.load(rhs, .u64, .{});
53975433
5398 // leave values on stack for 'select'5434 // leave values on stack for 'select'
5399 _ = try cg.intCmp(.u64, op, lhs_lsb, rhs_lsb);5435 _ = try cg.intCmp(.u64, op, lhs_lsb, rhs_lsb);
...@@ -5599,7 +5635,7 @@ fn airUnionFromEnum(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -5599,7 +5635,7 @@ fn airUnionFromEnum(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
55995635
5600 const enum_value = try cg.resolveInst(ty_op.operand);5636 const enum_value = try cg.resolveInst(ty_op.operand);
5601 const result = try cg.allocStack(union_ty);5637 const result = try cg.allocStack(union_ty);
5602 try cg.store(result, enum_value, enum_ty, @intCast(layout.tagOffset()));5638 try cg.store(result, enum_value, enum_ty, .{ .offset = layout.tagOffset() });
56035639
5604 return cg.finishAir(inst, result, &.{ty_op.operand});5640 return cg.finishAir(inst, result, &.{ty_op.operand});
5605}5641}
...@@ -5662,12 +5698,12 @@ fn bitcast(cg: *CodeGen, dest_ty: Type, src_ty: Type, operand: WValue) InnerErro...@@ -5662,12 +5698,12 @@ fn bitcast(cg: *CodeGen, dest_ty: Type, src_ty: Type, operand: WValue) InnerErro
56625698
5663 if (dest_by_ref) {5699 if (dest_by_ref) {
5664 const result = try cg.allocStack(src_ty);5700 const result = try cg.allocStack(src_ty);
5665 try cg.store(result, operand, src_ty, 0);5701 try cg.store(result, operand, src_ty, .{});
5666 return result;5702 return result;
5667 }5703 }
56685704
5669 if (src_by_ref) {5705 if (src_by_ref) {
5670 return try cg.load(operand, dest_ty, 0);5706 return try cg.load(operand, dest_ty, .{});
5671 }5707 }
56725708
5673 switch (src_class) {5709 switch (src_class) {
...@@ -5803,7 +5839,7 @@ fn airAggFieldVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -5803,7 +5839,7 @@ fn airAggFieldVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
5803 else => break :result try cg.buildPointerOffset(operand, offset, .new),5839 else => break :result try cg.buildPointerOffset(operand, offset, .new),
5804 }5840 }
5805 }5841 }
5806 break :result try cg.load(operand, field_ty, offset);5842 break :result try cg.load(operand, field_ty, .{ .offset = offset });
5807 },5843 },
5808 };5844 };
58095845
...@@ -6048,7 +6084,7 @@ fn airIsErr(cg: *CodeGen, inst: Air.Inst.Index, opcode: std.wasm.Opcode, op_kind...@@ -6048,7 +6084,7 @@ fn airIsErr(cg: *CodeGen, inst: Air.Inst.Index, opcode: std.wasm.Opcode, op_kind
6048 if (op_kind == .ptr or pl_ty.hasRuntimeBits(zcu)) {6084 if (op_kind == .ptr or pl_ty.hasRuntimeBits(zcu)) {
6049 try cg.addMemArg(.i32_load16_u, .{6085 try cg.addMemArg(.i32_load16_u, .{
6050 .offset = operand.offset() + @as(u32, @intCast(errUnionErrorOffset(pl_ty, zcu))),6086 .offset = operand.offset() + @as(u32, @intCast(errUnionErrorOffset(pl_ty, zcu))),
6051 .alignment = @intCast(Type.anyerror.abiAlignment(zcu).toByteUnits().?),6087 .alignment = Type.anyerror.abiAlignment(zcu).toStdMem(),
6052 });6088 });
6053 }6089 }
60546090
...@@ -6085,7 +6121,7 @@ fn airUnwrapErrUnionPayload(cg: *CodeGen, inst: Air.Inst.Index, op_is_ptr: bool)...@@ -6085,7 +6121,7 @@ fn airUnwrapErrUnionPayload(cg: *CodeGen, inst: Air.Inst.Index, op_is_ptr: bool)
6085 break :result try cg.buildPointerOffset(operand, pl_offset, .new);6121 break :result try cg.buildPointerOffset(operand, pl_offset, .new);
6086 } else {6122 } else {
6087 assert(isByRef(eu_ty, zcu, cg.target));6123 assert(isByRef(eu_ty, zcu, cg.target));
6088 break :result try cg.load(operand, payload_ty, pl_offset);6124 break :result try cg.load(operand, payload_ty, .{ .offset = pl_offset });
6089 }6125 }
6090 };6126 };
6091 return cg.finishAir(inst, result, &.{ty_op.operand});6127 return cg.finishAir(inst, result, &.{ty_op.operand});
...@@ -6110,7 +6146,7 @@ fn airUnwrapErrUnionError(cg: *CodeGen, inst: Air.Inst.Index, op_is_ptr: bool) I...@@ -6110,7 +6146,7 @@ fn airUnwrapErrUnionError(cg: *CodeGen, inst: Air.Inst.Index, op_is_ptr: bool) I
61106146
6111 const err_offset: u32 = @intCast(errUnionErrorOffset(payload_ty, zcu));6147 const err_offset: u32 = @intCast(errUnionErrorOffset(payload_ty, zcu));
6112 if (op_is_ptr or isByRef(eu_ty, zcu, cg.target)) {6148 if (op_is_ptr or isByRef(eu_ty, zcu, cg.target)) {
6113 break :result try cg.load(operand, Type.anyerror, err_offset);6149 break :result try cg.load(operand, .anyerror, .{ .offset = err_offset });
6114 } else {6150 } else {
6115 assert(!payload_ty.hasRuntimeBits(zcu));6151 assert(!payload_ty.hasRuntimeBits(zcu));
6116 break :result operand;6152 break :result operand;
...@@ -6134,7 +6170,7 @@ fn airWrapErrUnionPayload(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -6134,7 +6170,7 @@ fn airWrapErrUnionPayload(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
61346170
6135 const err_union = try cg.allocStack(err_ty);6171 const err_union = try cg.allocStack(err_ty);
6136 const payload_ptr = try cg.buildPointerOffset(err_union, @as(u32, @intCast(errUnionPayloadOffset(pl_ty, zcu))), .new);6172 const payload_ptr = try cg.buildPointerOffset(err_union, @as(u32, @intCast(errUnionPayloadOffset(pl_ty, zcu))), .new);
6137 try cg.store(payload_ptr, operand, pl_ty, 0);6173 try cg.store(payload_ptr, operand, pl_ty, .{});
61386174
6139 // ensure we also write '0' to the error part, so any present stack value gets overwritten by it.6175 // ensure we also write '0' to the error part, so any present stack value gets overwritten by it.
6140 try cg.emitWValue(err_union);6176 try cg.emitWValue(err_union);
...@@ -6142,7 +6178,7 @@ fn airWrapErrUnionPayload(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -6142,7 +6178,7 @@ fn airWrapErrUnionPayload(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6142 const err_val_offset: u32 = @intCast(errUnionErrorOffset(pl_ty, zcu));6178 const err_val_offset: u32 = @intCast(errUnionErrorOffset(pl_ty, zcu));
6143 try cg.addMemArg(.i32_store16, .{6179 try cg.addMemArg(.i32_store16, .{
6144 .offset = err_union.offset() + err_val_offset,6180 .offset = err_union.offset() + err_val_offset,
6145 .alignment = 2,6181 .alignment = .@"2",
6146 });6182 });
6147 break :result err_union;6183 break :result err_union;
6148 };6184 };
...@@ -6164,7 +6200,7 @@ fn airWrapErrUnionErr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -6164,7 +6200,7 @@ fn airWrapErrUnionErr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
61646200
6165 const err_union = try cg.allocStack(err_ty);6201 const err_union = try cg.allocStack(err_ty);
6166 // store error value6202 // store error value
6167 try cg.store(err_union, operand, Type.anyerror, @intCast(errUnionErrorOffset(pl_ty, zcu)));6203 try cg.store(err_union, operand, .anyerror, .{ .offset = errUnionErrorOffset(pl_ty, zcu) });
61686204
6169 // write 'undefined' to the payload6205 // write 'undefined' to the payload
6170 const payload_ptr = try cg.buildPointerOffset(err_union, @as(u32, @intCast(errUnionPayloadOffset(pl_ty, zcu))), .new);6206 const payload_ptr = try cg.buildPointerOffset(err_union, @as(u32, @intCast(errUnionPayloadOffset(pl_ty, zcu))), .new);
...@@ -6205,18 +6241,18 @@ fn isNull(cg: *CodeGen, operand: WValue, op_ty: Type, opcode: std.wasm.Opcode, o...@@ -6205,18 +6241,18 @@ fn isNull(cg: *CodeGen, operand: WValue, op_ty: Type, opcode: std.wasm.Opcode, o
6205 const offset = std.math.cast(u32, payload_ty.abiSize(zcu)) orelse {6241 const offset = std.math.cast(u32, payload_ty.abiSize(zcu)) orelse {
6206 return cg.fail("Optional type {f} too big to fit into stack frame", .{optional_ty.fmt(pt)});6242 return cg.fail("Optional type {f} too big to fit into stack frame", .{optional_ty.fmt(pt)});
6207 };6243 };
6208 try cg.addMemArg(.i32_load8_u, .{ .offset = operand.offset() + offset, .alignment = 1 });6244 try cg.addMemArg(.i32_load8_u, .{ .offset = operand.offset() + offset, .alignment = .@"1" });
6209 }6245 }
6210 } else if (payload_ty.isSlice(zcu)) {6246 } else if (payload_ty.isSlice(zcu)) {
6211 switch (cg.ptr_size) {6247 switch (cg.ptr_size) {
6212 .wasm32 => try cg.addMemArg(.i32_load, .{ .offset = operand.offset(), .alignment = 4 }),6248 .wasm32 => try cg.addMemArg(.i32_load, .{ .offset = operand.offset(), .alignment = .@"4" }),
6213 .wasm64 => try cg.addMemArg(.i64_load, .{ .offset = operand.offset(), .alignment = 8 }),6249 .wasm64 => try cg.addMemArg(.i64_load, .{ .offset = operand.offset(), .alignment = .@"8" }),
6214 }6250 }
6215 } else {6251 } else {
6216 if (op_kind == .ptr) {6252 if (op_kind == .ptr) {
6217 try cg.addMemArg(.i32_load, .{6253 try cg.addMemArg(.i32_load, .{
6218 .offset = operand.offset(),6254 .offset = operand.offset(),
6219 .alignment = 4,6255 .alignment = .@"4",
6220 });6256 });
6221 }6257 }
6222 }6258 }
...@@ -6245,7 +6281,7 @@ fn airOptionalPayload(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -6245,7 +6281,7 @@ fn airOptionalPayload(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6245 break :result try cg.buildPointerOffset(operand, 0, .new);6281 break :result try cg.buildPointerOffset(operand, 0, .new);
6246 }6282 }
62476283
6248 break :result try cg.load(operand, payload_ty, 0);6284 break :result try cg.load(operand, payload_ty, .{});
6249 };6285 };
6250 return cg.finishAir(inst, result, &.{ty_op.operand});6286 return cg.finishAir(inst, result, &.{ty_op.operand});
6251}6287}
...@@ -6285,7 +6321,7 @@ fn airOptionalPayloadPtrSet(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void...@@ -6285,7 +6321,7 @@ fn airOptionalPayloadPtrSet(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void
62856321
6286 try cg.emitWValue(operand);6322 try cg.emitWValue(operand);
6287 try cg.addImm32(1);6323 try cg.addImm32(1);
6288 try cg.addMemArg(.i32_store8, .{ .offset = operand.offset() + offset, .alignment = 1 });6324 try cg.addMemArg(.i32_store8, .{ .offset = operand.offset() + offset, .alignment = .@"1" });
62896325
6290 const result = try cg.buildPointerOffset(operand, 0, .new);6326 const result = try cg.buildPointerOffset(operand, 0, .new);
6291 return cg.finishAir(inst, result, &.{ty_op.operand});6327 return cg.finishAir(inst, result, &.{ty_op.operand});
...@@ -6302,7 +6338,7 @@ fn airWrapOptional(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -6302,7 +6338,7 @@ fn airWrapOptional(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6302 const non_null_bit = try cg.allocStack(Type.u1);6338 const non_null_bit = try cg.allocStack(Type.u1);
6303 try cg.emitWValue(non_null_bit);6339 try cg.emitWValue(non_null_bit);
6304 try cg.addImm32(1);6340 try cg.addImm32(1);
6305 try cg.addMemArg(.i32_store8, .{ .offset = non_null_bit.offset(), .alignment = 1 });6341 try cg.addMemArg(.i32_store8, .{ .offset = non_null_bit.offset(), .alignment = .@"1" });
6306 break :result non_null_bit;6342 break :result non_null_bit;
6307 }6343 }
63086344
...@@ -6319,10 +6355,10 @@ fn airWrapOptional(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -6319,10 +6355,10 @@ fn airWrapOptional(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6319 const result_ptr = try cg.allocStack(op_ty);6355 const result_ptr = try cg.allocStack(op_ty);
6320 try cg.emitWValue(result_ptr);6356 try cg.emitWValue(result_ptr);
6321 try cg.addImm32(1);6357 try cg.addImm32(1);
6322 try cg.addMemArg(.i32_store8, .{ .offset = result_ptr.offset() + offset, .alignment = 1 });6358 try cg.addMemArg(.i32_store8, .{ .offset = result_ptr.offset() + offset, .alignment = .@"1" });
63236359
6324 const payload_ptr = try cg.buildPointerOffset(result_ptr, 0, .new);6360 const payload_ptr = try cg.buildPointerOffset(result_ptr, 0, .new);
6325 try cg.store(payload_ptr, operand, payload_ty, 0);6361 try cg.store(payload_ptr, operand, payload_ty, .{});
6326 break :result result_ptr;6362 break :result result_ptr;
6327 };6363 };
63286364
...@@ -6338,8 +6374,8 @@ fn airSlice(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -6338,8 +6374,8 @@ fn airSlice(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6338 const slice_ty = cg.typeOfIndex(inst);6374 const slice_ty = cg.typeOfIndex(inst);
63396375
6340 const slice = try cg.allocStack(slice_ty);6376 const slice = try cg.allocStack(slice_ty);
6341 try cg.store(slice, lhs, Type.usize, 0);6377 try cg.store(slice, lhs, .usize, .{});
6342 try cg.store(slice, rhs, Type.usize, cg.ptrSize());6378 try cg.store(slice, rhs, .usize, .{ .offset = cg.ptrSize() });
63436379
6344 return cg.finishAir(inst, slice, &.{ bin_op.lhs, bin_op.rhs });6380 return cg.finishAir(inst, slice, &.{ bin_op.lhs, bin_op.rhs });
6345}6381}
...@@ -6362,7 +6398,7 @@ fn airSliceElemVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -6362,7 +6398,7 @@ fn airSliceElemVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6362 const elem_size = elem_ty.abiSize(zcu);6398 const elem_size = elem_ty.abiSize(zcu);
63636399
6364 // load pointer onto stack6400 // load pointer onto stack
6365 _ = try cg.load(slice, Type.usize, 0);6401 _ = try cg.load(slice, .usize, .{});
63666402
6367 // calculate index into slice6403 // calculate index into slice
6368 try cg.emitWValue(index);6404 try cg.emitWValue(index);
...@@ -6370,7 +6406,7 @@ fn airSliceElemVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -6370,7 +6406,7 @@ fn airSliceElemVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6370 try cg.addTag(.i32_mul);6406 try cg.addTag(.i32_mul);
6371 try cg.addTag(.i32_add);6407 try cg.addTag(.i32_add);
63726408
6373 const elem_result = try cg.load(.stack, elem_ty, 0);6409 const elem_result = try cg.load(.stack, elem_ty, .{});
63746410
6375 return cg.finishAir(inst, elem_result, &.{ bin_op.lhs, bin_op.rhs });6411 return cg.finishAir(inst, elem_result, &.{ bin_op.lhs, bin_op.rhs });
6376}6412}
...@@ -6386,7 +6422,7 @@ fn airSliceElemPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -6386,7 +6422,7 @@ fn airSliceElemPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6386 const slice = try cg.resolveInst(bin_op.lhs);6422 const slice = try cg.resolveInst(bin_op.lhs);
6387 const index = try cg.resolveInst(bin_op.rhs);6423 const index = try cg.resolveInst(bin_op.rhs);
63886424
6389 _ = try cg.load(slice, Type.usize, 0);6425 _ = try cg.load(slice, .usize, .{});
63906426
6391 // calculate index into slice6427 // calculate index into slice
6392 try cg.emitWValue(index);6428 try cg.emitWValue(index);
...@@ -6404,12 +6440,12 @@ fn airSlicePtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -6404,12 +6440,12 @@ fn airSlicePtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6404}6440}
64056441
6406fn slicePtr(cg: *CodeGen, operand: WValue) InnerError!WValue {6442fn slicePtr(cg: *CodeGen, operand: WValue) InnerError!WValue {
6407 const ptr = try cg.load(operand, Type.usize, 0);6443 const ptr = try cg.load(operand, .usize, .{});
6408 return ptr.toLocal(cg, Type.usize);6444 return ptr.toLocal(cg, Type.usize);
6409}6445}
64106446
6411fn sliceLen(cg: *CodeGen, operand: WValue) InnerError!WValue {6447fn sliceLen(cg: *CodeGen, operand: WValue) InnerError!WValue {
6412 const len = try cg.load(operand, Type.usize, cg.ptrSize());6448 const len = try cg.load(operand, .usize, .{ .offset = cg.ptrSize() });
6413 return len.toLocal(cg, Type.usize);6449 return len.toLocal(cg, Type.usize);
6414}6450}
64156451
...@@ -6424,11 +6460,11 @@ fn airArrayToSlice(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -6424,11 +6460,11 @@ fn airArrayToSlice(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6424 // create a slice on the stack6460 // create a slice on the stack
6425 const slice_local = try cg.allocStack(slice_ty);6461 const slice_local = try cg.allocStack(slice_ty);
64266462
6427 try cg.store(slice_local, operand, Type.usize, 0);6463 try cg.store(slice_local, operand, .usize, .{});
64286464
6429 // store the length of the array in the slice6465 // store the length of the array in the slice
6430 const array_len: u32 = @intCast(array_ty.arrayLen(zcu));6466 const array_len: u32 = @intCast(array_ty.arrayLen(zcu));
6431 try cg.store(slice_local, .{ .imm32 = array_len }, Type.usize, cg.ptrSize());6467 try cg.store(slice_local, .{ .imm32 = array_len }, .usize, .{ .offset = cg.ptrSize() });
64326468
6433 return cg.finishAir(inst, slice_local, &.{ty_op.operand});6469 return cg.finishAir(inst, slice_local, &.{ty_op.operand});
6434}6470}
...@@ -6445,7 +6481,7 @@ fn airPtrElemVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -6445,7 +6481,7 @@ fn airPtrElemVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
64456481
6446 // load pointer onto the stack6482 // load pointer onto the stack
6447 if (ptr_ty.isSlice(zcu)) {6483 if (ptr_ty.isSlice(zcu)) {
6448 _ = try cg.load(ptr, Type.usize, 0);6484 _ = try cg.load(ptr, .usize, .{});
6449 } else {6485 } else {
6450 try cg.lowerToStack(ptr);6486 try cg.lowerToStack(ptr);
6451 }6487 }
...@@ -6456,7 +6492,7 @@ fn airPtrElemVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -6456,7 +6492,7 @@ fn airPtrElemVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6456 try cg.addTag(.i32_mul);6492 try cg.addTag(.i32_mul);
6457 try cg.addTag(.i32_add);6493 try cg.addTag(.i32_add);
64586494
6459 const elem_result = try cg.load(.stack, elem_ty, 0);6495 const elem_result = try cg.load(.stack, elem_ty, .{});
64606496
6461 return cg.finishAir(inst, elem_result, &.{ bin_op.lhs, bin_op.rhs });6497 return cg.finishAir(inst, elem_result, &.{ bin_op.lhs, bin_op.rhs });
6462}6498}
...@@ -6475,7 +6511,7 @@ fn airPtrElemPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -6475,7 +6511,7 @@ fn airPtrElemPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
64756511
6476 // load pointer onto the stack6512 // load pointer onto the stack
6477 if (ptr_ty.isSlice(zcu)) {6513 if (ptr_ty.isSlice(zcu)) {
6478 _ = try cg.load(ptr, Type.usize, 0);6514 _ = try cg.load(ptr, .usize, .{});
6479 } else {6515 } else {
6480 try cg.lowerToStack(ptr);6516 try cg.lowerToStack(ptr);
6481 }6517 }
...@@ -6645,7 +6681,7 @@ fn memset(cg: *CodeGen, elem_ty: Type, ptr: WValue, len: WValue, value: WValue)...@@ -6645,7 +6681,7 @@ fn memset(cg: *CodeGen, elem_ty: Type, ptr: WValue, len: WValue, value: WValue)
6645 try cg.addLabel(.br_if, 1); // jump out of loop into outer block (finished)6681 try cg.addLabel(.br_if, 1); // jump out of loop into outer block (finished)
66466682
6647 // store the value at the current position of the pointer6683 // store the value at the current position of the pointer
6648 try cg.store(new_ptr, value, elem_ty, 0);6684 try cg.store(new_ptr, value, elem_ty, .{});
66496685
6650 // move the pointer to the next element6686 // move the pointer to the next element
6651 try cg.emitWValue(new_ptr);6687 try cg.emitWValue(new_ptr);
...@@ -6708,7 +6744,7 @@ fn airArrayElemVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -6708,7 +6744,7 @@ fn airArrayElemVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6708 },6744 },
6709 else => {6745 else => {
6710 const stack_vec = try cg.allocStack(array_ty);6746 const stack_vec = try cg.allocStack(array_ty);
6711 try cg.store(stack_vec, array, array_ty, 0);6747 try cg.store(stack_vec, array, array_ty, .{});
67126748
6713 // Is a non-unrolled vector (v128)6749 // Is a non-unrolled vector (v128)
6714 try cg.lowerToStack(stack_vec);6750 try cg.lowerToStack(stack_vec);
...@@ -6723,7 +6759,7 @@ fn airArrayElemVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -6723,7 +6759,7 @@ fn airArrayElemVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6723 const result = if (isByRef(elem_ty, zcu, cg.target))6759 const result = if (isByRef(elem_ty, zcu, cg.target))
6724 .stack6760 .stack
6725 else6761 else
6726 try cg.load(.stack, elem_ty, 0);6762 try cg.load(.stack, elem_ty, .{});
6727 return cg.finishAir(inst, result, &.{ bin_op.lhs, bin_op.rhs });6763 return cg.finishAir(inst, result, &.{ bin_op.lhs, bin_op.rhs });
6728}6764}
67296765
...@@ -6750,10 +6786,11 @@ fn airSplat(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -6750,10 +6786,11 @@ fn airSplat(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6750 try cg.emitWValue(operand);6786 try cg.emitWValue(operand);
6751 const extra_index: u32 = @intCast(cg.mir_extra.items.len);6787 const extra_index: u32 = @intCast(cg.mir_extra.items.len);
6752 // stores as := opcode, offset, alignment (opcode::memarg)6788 // stores as := opcode, offset, alignment (opcode::memarg)
6753 try cg.mir_extra.appendSlice(cg.gpa, &[_]u32{6789 try cg.mir_extra.ensureUnusedCapacity(cg.gpa, 4);
6754 opcode,6790 cg.mir_extra.appendAssumeCapacity(opcode);
6755 operand.offset(),6791 _ = try cg.addExtraAssumeCapacity(Mir.MemArg{
6756 @intCast(elem_ty.abiAlignment(zcu).toByteUnits().?),6792 .offset = operand.offset(),
6793 .alignment = elem_ty.abiAlignment(zcu).toStdMem(),
6757 });6794 });
6758 try cg.addInst(.{ .tag = .simd_prefix, .data = .{ .payload = extra_index } });6795 try cg.addInst(.{ .tag = .simd_prefix, .data = .{ .payload = extra_index } });
6759 return cg.finishAir(inst, .stack, &.{ty_op.operand});6796 return cg.finishAir(inst, .stack, &.{ty_op.operand});
...@@ -6782,7 +6819,7 @@ fn airSplat(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -6782,7 +6819,7 @@ fn airSplat(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6782 var index: usize = 0;6819 var index: usize = 0;
6783 var offset: u32 = 0;6820 var offset: u32 = 0;
6784 while (index < vector_len) : (index += 1) {6821 while (index < vector_len) : (index += 1) {
6785 try cg.store(result, operand, elem_ty, offset);6822 try cg.store(result, operand, elem_ty, .{ .offset = offset });
6786 offset += elem_byte_size;6823 offset += elem_byte_size;
6787 }6824 }
67886825
...@@ -6821,10 +6858,10 @@ fn airShuffleOne(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -6821,10 +6858,10 @@ fn airShuffleOne(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6821 for (mask, 0..) |mask_elem, out_idx| {6858 for (mask, 0..) |mask_elem, out_idx| {
6822 try cg.emitWValue(dest_alloc);6859 try cg.emitWValue(dest_alloc);
6823 const elem_val = switch (mask_elem.unwrap()) {6860 const elem_val = switch (mask_elem.unwrap()) {
6824 .elem => |idx| try cg.load(operand, elem_ty, @intCast(elem_size * idx)),6861 .elem => |idx| try cg.load(operand, elem_ty, .{ .offset = elem_size * idx }),
6825 .value => |val| try cg.lowerConstant(.fromInterned(val)),6862 .value => |val| try cg.lowerConstant(.fromInterned(val)),
6826 };6863 };
6827 try cg.store(.stack, elem_val, elem_ty, @intCast(dest_alloc.offset() + elem_size * out_idx));6864 try cg.store(.stack, elem_val, elem_ty, .{ .offset = dest_alloc.offset() + elem_size * out_idx });
6828 }6865 }
6829 return cg.finishAir(inst, dest_alloc, &.{unwrapped.operand});6866 return cg.finishAir(inst, dest_alloc, &.{unwrapped.operand});
6830}6867}
...@@ -6889,11 +6926,11 @@ fn airShuffleTwo(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -6889,11 +6926,11 @@ fn airShuffleTwo(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6889 for (mask, 0..) |mask_elem, out_idx| {6926 for (mask, 0..) |mask_elem, out_idx| {
6890 try cg.emitWValue(dest_alloc);6927 try cg.emitWValue(dest_alloc);
6891 const elem_val = switch (mask_elem.unwrap()) {6928 const elem_val = switch (mask_elem.unwrap()) {
6892 .a_elem => |idx| try cg.load(operand_a, elem_ty, @intCast(elem_size * idx)),6929 .a_elem => |idx| try cg.load(operand_a, elem_ty, .{ .offset = elem_size * idx }),
6893 .b_elem => |idx| try cg.load(operand_b, elem_ty, @intCast(elem_size * idx)),6930 .b_elem => |idx| try cg.load(operand_b, elem_ty, .{ .offset = elem_size * idx }),
6894 .undef => try cg.emitUndefined(elem_ty),6931 .undef => try cg.emitUndefined(elem_ty),
6895 };6932 };
6896 try cg.store(.stack, elem_val, elem_ty, @intCast(dest_alloc.offset() + elem_size * out_idx));6933 try cg.store(.stack, elem_val, elem_ty, .{ .offset = dest_alloc.offset() + elem_size * out_idx });
6897 }6934 }
6898 return cg.finishAir(inst, dest_alloc, &.{ unwrapped.operand_a, unwrapped.operand_b });6935 return cg.finishAir(inst, dest_alloc, &.{ unwrapped.operand_a, unwrapped.operand_b });
6899}6936}
...@@ -6931,7 +6968,7 @@ fn airAggregateInit(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -6931,7 +6968,7 @@ fn airAggregateInit(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6931 const offset = try cg.buildPointerOffset(result, 0, .new);6968 const offset = try cg.buildPointerOffset(result, 0, .new);
6932 for (elements, 0..) |elem, elem_index| {6969 for (elements, 0..) |elem, elem_index| {
6933 const elem_val = try cg.resolveInst(elem);6970 const elem_val = try cg.resolveInst(elem);
6934 try cg.store(offset, elem_val, elem_ty, 0);6971 try cg.store(offset, elem_val, elem_ty, .{});
69356972
6936 if (elem_index < elements.len - 1 or sentinel != null) {6973 if (elem_index < elements.len - 1 or sentinel != null) {
6937 _ = try cg.buildPointerOffset(offset, elem_size, .modify);6974 _ = try cg.buildPointerOffset(offset, elem_size, .modify);
...@@ -6939,18 +6976,18 @@ fn airAggregateInit(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -6939,18 +6976,18 @@ fn airAggregateInit(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6939 }6976 }
6940 if (sentinel) |s| {6977 if (sentinel) |s| {
6941 const val = try cg.resolveValue(s);6978 const val = try cg.resolveValue(s);
6942 try cg.store(offset, val, elem_ty, 0);6979 try cg.store(offset, val, elem_ty, .{});
6943 }6980 }
6944 } else {6981 } else {
6945 var offset: u32 = 0;6982 var offset: u32 = 0;
6946 for (elements) |elem| {6983 for (elements) |elem| {
6947 const elem_val = try cg.resolveInst(elem);6984 const elem_val = try cg.resolveInst(elem);
6948 try cg.store(result, elem_val, elem_ty, offset);6985 try cg.store(result, elem_val, elem_ty, .{ .offset = offset });
6949 offset += elem_size;6986 offset += elem_size;
6950 }6987 }
6951 if (sentinel) |s| {6988 if (sentinel) |s| {
6952 const val = try cg.resolveValue(s);6989 const val = try cg.resolveValue(s);
6953 try cg.store(result, val, elem_ty, offset);6990 try cg.store(result, val, elem_ty, .{ .offset = offset });
6954 }6991 }
6955 }6992 }
6956 break :result_value result;6993 break :result_value result;
...@@ -6967,7 +7004,7 @@ fn airAggregateInit(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -6967,7 +7004,7 @@ fn airAggregateInit(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6967 const offset = try cg.buildPointerOffset(result, field_offset, .new);7004 const offset = try cg.buildPointerOffset(result, field_offset, .new);
69687005
6969 const value = try cg.resolveInst(elem);7006 const value = try cg.resolveInst(elem);
6970 try cg.store(offset, value, elem_ty, 0);7007 try cg.store(offset, value, elem_ty, .{});
6971 }7008 }
69727009
6973 break :result_value result;7010 break :result_value result;
...@@ -7016,22 +7053,22 @@ fn airUnionInit(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -7016,22 +7053,22 @@ fn airUnionInit(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7016 if (layout.tag_align.compare(.gte, layout.payload_align)) {7053 if (layout.tag_align.compare(.gte, layout.payload_align)) {
7017 if (isByRef(field_ty, zcu, cg.target)) {7054 if (isByRef(field_ty, zcu, cg.target)) {
7018 const payload_ptr = try cg.buildPointerOffset(result_ptr, layout.tag_size, .new);7055 const payload_ptr = try cg.buildPointerOffset(result_ptr, layout.tag_size, .new);
7019 try cg.store(payload_ptr, payload, field_ty, 0);7056 try cg.store(payload_ptr, payload, field_ty, .{});
7020 } else {7057 } else {
7021 try cg.store(result_ptr, payload, field_ty, @intCast(layout.tag_size));7058 try cg.store(result_ptr, payload, field_ty, .{ .offset = layout.tag_size });
7022 }7059 }
70237060
7024 if (layout.tag_size > 0) {7061 if (layout.tag_size > 0) {
7025 try cg.store(result_ptr, tag_int, .fromInterned(union_obj.enum_tag_type), 0);7062 try cg.store(result_ptr, tag_int, .fromInterned(union_obj.enum_tag_type), .{});
7026 }7063 }
7027 } else {7064 } else {
7028 try cg.store(result_ptr, payload, field_ty, 0);7065 try cg.store(result_ptr, payload, field_ty, .{});
7029 if (layout.tag_size > 0) {7066 if (layout.tag_size > 0) {
7030 try cg.store(7067 try cg.store(
7031 result_ptr,7068 result_ptr,
7032 tag_int,7069 tag_int,
7033 .fromInterned(union_obj.enum_tag_type),7070 .fromInterned(union_obj.enum_tag_type),
7034 @intCast(layout.payload_size),7071 .{ .offset = layout.payload_size },
7035 );7072 );
7036 }7073 }
7037 }7074 }
...@@ -7077,7 +7114,7 @@ fn airSetUnionTag(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -7077,7 +7114,7 @@ fn airSetUnionTag(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7077 const union_ptr = try cg.resolveInst(bin_op.lhs);7114 const union_ptr = try cg.resolveInst(bin_op.lhs);
7078 const new_tag = try cg.resolveInst(bin_op.rhs);7115 const new_tag = try cg.resolveInst(bin_op.rhs);
7079 if (layout.payload_size == 0) {7116 if (layout.payload_size == 0) {
7080 try cg.store(union_ptr, new_tag, tag_ty, 0);7117 try cg.store(union_ptr, new_tag, tag_ty, .{});
7081 return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });7118 return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });
7082 }7119 }
70837120
...@@ -7086,7 +7123,7 @@ fn airSetUnionTag(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -7086,7 +7123,7 @@ fn airSetUnionTag(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7086 const offset: u32 = if (layout.tag_align.compare(.lt, layout.payload_align)) blk: {7123 const offset: u32 = if (layout.tag_align.compare(.lt, layout.payload_align)) blk: {
7087 break :blk @intCast(layout.payload_size);7124 break :blk @intCast(layout.payload_size);
7088 } else 0;7125 } else 0;
7089 try cg.store(union_ptr, new_tag, tag_ty, offset);7126 try cg.store(union_ptr, new_tag, tag_ty, .{ .offset = offset });
7090 return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });7127 return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });
7091}7128}
70927129
...@@ -7106,7 +7143,7 @@ fn airGetUnionTag(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -7106,7 +7143,7 @@ fn airGetUnionTag(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7106 @intCast(layout.payload_size)7143 @intCast(layout.payload_size)
7107 else7144 else
7108 0;7145 0;
7109 const result = try cg.load(operand, tag_ty, offset);7146 const result = try cg.load(operand, tag_ty, .{ .offset = offset });
7110 return cg.finishAir(inst, result, &.{ty_op.operand});7147 return cg.finishAir(inst, result, &.{ty_op.operand});
7111}7148}
71127149
...@@ -7122,8 +7159,8 @@ fn airErrUnionPayloadPtrSet(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void...@@ -7122,8 +7159,8 @@ fn airErrUnionPayloadPtrSet(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void
7122 try cg.store(7159 try cg.store(
7123 operand,7160 operand,
7124 .{ .imm32 = 0 },7161 .{ .imm32 = 0 },
7125 Type.anyerror,7162 .anyerror,
7126 @intCast(errUnionErrorOffset(payload_ty, zcu)),7163 .{ .offset = errUnionErrorOffset(payload_ty, zcu) },
7127 );7164 );
71287165
7129 const result = result: {7166 const result = result: {
...@@ -7360,7 +7397,7 @@ fn lowerTry(...@@ -7360,7 +7397,7 @@ fn lowerTry(
7360 const err_offset: u32 = @intCast(errUnionErrorOffset(pl_ty, zcu));7397 const err_offset: u32 = @intCast(errUnionErrorOffset(pl_ty, zcu));
7361 try cg.addMemArg(.i32_load16_u, .{7398 try cg.addMemArg(.i32_load16_u, .{
7362 .offset = err_union.offset() + err_offset,7399 .offset = err_union.offset() + err_offset,
7363 .alignment = @intCast(Type.anyerror.abiAlignment(zcu).toByteUnits().?),7400 .alignment = Type.anyerror.abiAlignment(zcu).toStdMem(),
7364 });7401 });
7365 }7402 }
7366 try cg.addTag(.i32_eqz);7403 try cg.addTag(.i32_eqz);
...@@ -7384,7 +7421,7 @@ fn lowerTry(...@@ -7384,7 +7421,7 @@ fn lowerTry(
7384 if (operand_is_ptr or isByRef(pl_ty, zcu, cg.target)) {7421 if (operand_is_ptr or isByRef(pl_ty, zcu, cg.target)) {
7385 return buildPointerOffset(cg, err_union, pl_offset, .new);7422 return buildPointerOffset(cg, err_union, pl_offset, .new);
7386 }7423 }
7387 const payload = try cg.load(err_union, pl_ty, pl_offset);7424 const payload = try cg.load(err_union, pl_ty, .{ .offset = pl_offset });
7388 return payload.toLocal(cg, pl_ty);7425 return payload.toLocal(cg, pl_ty);
7389}7426}
73907427
...@@ -7588,7 +7625,7 @@ fn airCmpxchg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -7588,7 +7625,7 @@ fn airCmpxchg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7588 else => |size| return cg.fail("TODO: implement `@cmpxchg` for types with abi size '{d}'", .{size}),7625 else => |size| return cg.fail("TODO: implement `@cmpxchg` for types with abi size '{d}'", .{size}),
7589 }, .{7626 }, .{
7590 .offset = ptr_operand.offset(),7627 .offset = ptr_operand.offset(),
7591 .alignment = @intCast(ty.abiAlignment(zcu).toByteUnits().?),7628 .alignment = ty.abiAlignment(zcu).toStdMem(),
7592 });7629 });
7593 try cg.addLocal(.local_tee, val_local.local.value);7630 try cg.addLocal(.local_tee, val_local.local.value);
7594 _ = try cg.intCmp(int_ty, .eq, .stack, expected_val);7631 _ = try cg.intCmp(int_ty, .eq, .stack, expected_val);
...@@ -7598,7 +7635,7 @@ fn airCmpxchg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -7598,7 +7635,7 @@ fn airCmpxchg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7598 if (ty.abiSize(zcu) > 8) {7635 if (ty.abiSize(zcu) > 8) {
7599 return cg.fail("TODO: Implement `@cmpxchg` for types larger than abi size of 8 bytes", .{});7636 return cg.fail("TODO: Implement `@cmpxchg` for types larger than abi size of 8 bytes", .{});
7600 }7637 }
7601 const ptr_val = try WValue.toLocal(try cg.load(ptr_operand, ty, 0), cg, ty);7638 const ptr_val = try WValue.toLocal(try cg.load(ptr_operand, ty, .{}), cg, ty);
76027639
7603 try cg.lowerToStack(ptr_operand);7640 try cg.lowerToStack(ptr_operand);
7604 try cg.lowerToStack(new_val);7641 try cg.lowerToStack(new_val);
...@@ -7606,7 +7643,7 @@ fn airCmpxchg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -7606,7 +7643,7 @@ fn airCmpxchg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7606 _ = try cg.intCmp(int_ty, .eq, ptr_val, expected_val);7643 _ = try cg.intCmp(int_ty, .eq, ptr_val, expected_val);
7607 try cg.addLocal(.local_tee, cmp_result.local.value);7644 try cg.addLocal(.local_tee, cmp_result.local.value);
7608 try cg.addTag(.select);7645 try cg.addTag(.select);
7609 try cg.store(.stack, .stack, ty, 0);7646 try cg.store(.stack, .stack, ty, .{});
76107647
7611 break :val ptr_val;7648 break :val ptr_val;
7612 };7649 };
...@@ -7619,8 +7656,8 @@ fn airCmpxchg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -7619,8 +7656,8 @@ fn airCmpxchg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7619 try cg.addTag(.i32_and);7656 try cg.addTag(.i32_and);
7620 const and_result = try WValue.toLocal(.stack, cg, Type.bool);7657 const and_result = try WValue.toLocal(.stack, cg, Type.bool);
7621 const result_ptr = try cg.allocStack(result_ty);7658 const result_ptr = try cg.allocStack(result_ty);
7622 try cg.store(result_ptr, and_result, Type.bool, @as(u32, @intCast(ty.abiSize(zcu))));7659 try cg.store(result_ptr, and_result, .bool, .{ .offset = ty.abiSize(zcu) });
7623 try cg.store(result_ptr, ptr_val, ty, 0);7660 try cg.store(result_ptr, ptr_val, ty, .{});
7624 break :val result_ptr;7661 break :val result_ptr;
7625 } else val: {7662 } else val: {
7626 try cg.addImm32(0);7663 try cg.addImm32(0);
...@@ -7650,10 +7687,10 @@ fn airAtomicLoad(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -7650,10 +7687,10 @@ fn airAtomicLoad(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7650 try cg.emitWValue(ptr);7687 try cg.emitWValue(ptr);
7651 try cg.addAtomicMemArg(tag, .{7688 try cg.addAtomicMemArg(tag, .{
7652 .offset = ptr.offset(),7689 .offset = ptr.offset(),
7653 .alignment = @intCast(ty.abiAlignment(zcu).toByteUnits().?),7690 .alignment = ty.abiAlignment(zcu).toStdMem(),
7654 });7691 });
7655 } else {7692 } else {
7656 _ = try cg.load(ptr, ty, 0);7693 _ = try cg.load(ptr, ty, .{});
7657 }7694 }
76587695
7659 return cg.finishAir(inst, .stack, &.{atomic_load.ptr});7696 return cg.finishAir(inst, .stack, &.{atomic_load.ptr});
...@@ -7676,7 +7713,7 @@ fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -7676,7 +7713,7 @@ fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7676 .Min,7713 .Min,
7677 .Nand,7714 .Nand,
7678 => {7715 => {
7679 const tmp = try cg.load(ptr, ty, 0);7716 const tmp = try cg.load(ptr, ty, .{});
7680 const value = try tmp.toLocal(cg, ty);7717 const value = try tmp.toLocal(cg, ty);
76817718
7682 // create a loop to cmpxchg the new value7719 // create a loop to cmpxchg the new value
...@@ -7710,7 +7747,7 @@ fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -7710,7 +7747,7 @@ fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7710 },7747 },
7711 .{7748 .{
7712 .offset = ptr.offset(),7749 .offset = ptr.offset(),
7713 .alignment = @intCast(ty.abiAlignment(zcu).toByteUnits().?),7750 .alignment = ty.abiAlignment(zcu).toStdMem(),
7714 },7751 },
7715 );7752 );
7716 const select_res = try cg.allocLocal(ty);7753 const select_res = try cg.allocLocal(ty);
...@@ -7770,18 +7807,18 @@ fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -7770,18 +7807,18 @@ fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7770 };7807 };
7771 try cg.addAtomicMemArg(tag, .{7808 try cg.addAtomicMemArg(tag, .{
7772 .offset = ptr.offset(),7809 .offset = ptr.offset(),
7773 .alignment = @intCast(ty.abiAlignment(zcu).toByteUnits().?),7810 .alignment = ty.abiAlignment(zcu).toStdMem(),
7774 });7811 });
7775 return cg.finishAir(inst, .stack, &.{ pl_op.operand, extra.operand });7812 return cg.finishAir(inst, .stack, &.{ pl_op.operand, extra.operand });
7776 },7813 },
7777 }7814 }
7778 } else {7815 } else {
7779 const loaded = try cg.load(ptr, ty, 0);7816 const loaded = try cg.load(ptr, ty, .{});
7780 const result = try loaded.toLocal(cg, ty);7817 const result = try loaded.toLocal(cg, ty);
77817818
7782 switch (op) {7819 switch (op) {
7783 .Xchg => {7820 .Xchg => {
7784 try cg.store(ptr, operand, ty, 0);7821 try cg.store(ptr, operand, ty, .{});
7785 },7822 },
7786 .Add,7823 .Add,
7787 .Sub,7824 .Sub,
...@@ -7794,7 +7831,7 @@ fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -7794,7 +7831,7 @@ fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7794 .Sub => try cg.floatSub(float_ty, result, operand),7831 .Sub => try cg.floatSub(float_ty, result, operand),
7795 else => unreachable,7832 else => unreachable,
7796 };7833 };
7797 try cg.store(.stack, .stack, ty, ptr.offset());7834 try cg.store(.stack, .stack, ty, .{ .offset = ptr.offset() });
7798 } else {7835 } else {
7799 const int_ty: IntType = .fromType(cg, ty);7836 const int_ty: IntType = .fromType(cg, ty);
7800 try cg.emitWValue(ptr);7837 try cg.emitWValue(ptr);
...@@ -7804,7 +7841,7 @@ fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -7804,7 +7841,7 @@ fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7804 else => unreachable,7841 else => unreachable,
7805 };7842 };
7806 _ = try cg.intWrap(int_ty, .stack);7843 _ = try cg.intWrap(int_ty, .stack);
7807 try cg.store(.stack, .stack, ty, ptr.offset());7844 try cg.store(.stack, .stack, ty, .{ .offset = ptr.offset() });
7808 }7845 }
7809 },7846 },
7810 .And,7847 .And,
...@@ -7819,7 +7856,7 @@ fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -7819,7 +7856,7 @@ fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7819 .Xor => try cg.intXor(int_ty, result, operand),7856 .Xor => try cg.intXor(int_ty, result, operand),
7820 else => unreachable,7857 else => unreachable,
7821 };7858 };
7822 try cg.store(.stack, .stack, ty, ptr.offset());7859 try cg.store(.stack, .stack, ty, .{ .offset = ptr.offset() });
7823 },7860 },
7824 .Max,7861 .Max,
7825 .Min,7862 .Min,
...@@ -7831,7 +7868,7 @@ fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -7831,7 +7868,7 @@ fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7831 try cg.emitWValue(operand);7868 try cg.emitWValue(operand);
7832 _ = try cg.floatCmp(float_ty, if (op == .Max) .gt else .lt, result, operand);7869 _ = try cg.floatCmp(float_ty, if (op == .Max) .gt else .lt, result, operand);
7833 try cg.addTag(.select);7870 try cg.addTag(.select);
7834 try cg.store(.stack, .stack, ty, ptr.offset());7871 try cg.store(.stack, .stack, ty, .{ .offset = ptr.offset() });
7835 } else {7872 } else {
7836 const int_ty: IntType = .fromType(cg, ty);7873 const int_ty: IntType = .fromType(cg, ty);
7837 try cg.emitWValue(ptr);7874 try cg.emitWValue(ptr);
...@@ -7839,7 +7876,7 @@ fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -7839,7 +7876,7 @@ fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7839 try cg.emitWValue(operand);7876 try cg.emitWValue(operand);
7840 _ = try cg.intCmp(int_ty, if (op == .Max) .gt else .lt, result, operand);7877 _ = try cg.intCmp(int_ty, if (op == .Max) .gt else .lt, result, operand);
7841 try cg.addTag(.select);7878 try cg.addTag(.select);
7842 try cg.store(.stack, .stack, ty, ptr.offset());7879 try cg.store(.stack, .stack, ty, .{ .offset = ptr.offset() });
7843 }7880 }
7844 },7881 },
7845 .Nand => {7882 .Nand => {
...@@ -7854,7 +7891,7 @@ fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -7854,7 +7891,7 @@ fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7854 return cg.fail("TODO: `@atomicRmw` with operator `Nand` for types larger than 64 bits", .{});7891 return cg.fail("TODO: `@atomicRmw` with operator `Nand` for types larger than 64 bits", .{});
7855 }7892 }
7856 _ = try cg.intXor(int_ty, and_res, .stack);7893 _ = try cg.intXor(int_ty, and_res, .stack);
7857 try cg.store(.stack, .stack, ty, ptr.offset());7894 try cg.store(.stack, .stack, ty, .{ .offset = ptr.offset() });
7858 },7895 },
7859 }7896 }
78607897
...@@ -7883,10 +7920,10 @@ fn airAtomicStore(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -7883,10 +7920,10 @@ fn airAtomicStore(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7883 try cg.lowerToStack(operand);7920 try cg.lowerToStack(operand);
7884 try cg.addAtomicMemArg(tag, .{7921 try cg.addAtomicMemArg(tag, .{
7885 .offset = ptr.offset(),7922 .offset = ptr.offset(),
7886 .alignment = @intCast(ty.abiAlignment(zcu).toByteUnits().?),7923 .alignment = ty.abiAlignment(zcu).toStdMem(),
7887 });7924 });
7888 } else {7925 } else {
7889 try cg.store(ptr, operand, ty, 0);7926 try cg.store(ptr, operand, ty, .{});
7890 }7927 }
78917928
7892 return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });7929 return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });
src/codegen/wasm/Emit.zig+1-3
...@@ -988,9 +988,7 @@ pub fn lower(emit: *Emit) Error!void {...@@ -988,9 +988,7 @@ pub fn lower(emit: *Emit) Error!void {
988/// Asserts 20 unused capacity.988/// Asserts 20 unused capacity.
989fn encodeMemArg(code: *ArrayList(u8), mem_arg: Mir.MemArg) void {989fn encodeMemArg(code: *ArrayList(u8), mem_arg: Mir.MemArg) void {
990 assert(code.unusedCapacitySlice().len >= 20);990 assert(code.unusedCapacitySlice().len >= 20);
991 // Wasm encodes alignment as power of 2, rather than natural alignment.991 writeUleb128(code, @backingInt(mem_arg.alignment));
992 const encoded_alignment = @ctz(mem_arg.alignment);
993 writeUleb128(code, encoded_alignment);
994 writeUleb128(code, mem_arg.offset);992 writeUleb128(code, mem_arg.offset);
995}993}
996994
src/codegen/wasm/Mir.zig+9-2
...@@ -694,11 +694,18 @@ pub fn extraData(self: *const Mir, comptime T: type, index: usize) struct { data...@@ -694,11 +694,18 @@ pub fn extraData(self: *const Mir, comptime T: type, index: usize) struct { data
694 inline for (info.field_names, info.field_types) |field_name, field_type| {694 inline for (info.field_names, info.field_types) |field_name, field_type| {
695 @field(result, field_name) = switch (field_type) {695 @field(result, field_name) = switch (field_type) {
696 u32 => self.extra[i],696 u32 => self.extra[i],
697 u64 => value: {
698 const lo = @as(u64, self.extra[i]);
699 const hi = @as(u64, self.extra[i + 1]) << 32;
700 i += 1;
701 break :value hi | lo;
702 },
697 i32 => @bitCast(self.extra[i]),703 i32 => @bitCast(self.extra[i]),
698 Wasm.UavsObjIndex,704 Wasm.UavsObjIndex,
699 Wasm.UavsExeIndex,705 Wasm.UavsExeIndex,
700 InternPool.Nav.Index,706 InternPool.Nav.Index,
701 InternPool.Index,707 InternPool.Index,
708 std.mem.Alignment,
702 => @fromBackingInt(@intCast(self.extra[i])),709 => @fromBackingInt(@intCast(self.extra[i])),
703 else => @compileError("Unsupported field type " ++ @typeName(field_type)),710 else => @compileError("Unsupported field type " ++ @typeName(field_type)),
704 };711 };
...@@ -747,8 +754,8 @@ pub const Float64 = struct {...@@ -747,8 +754,8 @@ pub const Float64 = struct {
747};754};
748755
749pub const MemArg = struct {756pub const MemArg = struct {
750 offset: u32,757 offset: u64,
751 alignment: u32,758 alignment: std.mem.Alignment,
752};759};
753760
754pub const UavRefOff = struct {761pub const UavRefOff = struct {
src/codegen/wasm/assembly.zig+4-4
...@@ -679,20 +679,20 @@ fn parseMemArg(cg: *CodeGen, inst: []const u8, word_it: *mem.TokenIterator(u8, ....@@ -679,20 +679,20 @@ fn parseMemArg(cg: *CodeGen, inst: []const u8, word_it: *mem.TokenIterator(u8, .
679 const offset_str = oa[0];679 const offset_str = oa[0];
680 const alignment_str = oa[1];680 const alignment_str = oa[1];
681681
682 const offset = std.fmt.parseInt(i32, offset_str, 0) catch {682 const offset = std.fmt.parseInt(i64, offset_str, 0) catch {
683 return cg.fail("Malformed assembly, malformed offset \"{s}\"", .{offset_str});683 return cg.fail("Malformed assembly, malformed offset \"{s}\"", .{offset_str});
684 };684 };
685 const alignment_p2 = std.fmt.parseInt(u5, alignment_str, 0) catch {685 const alignment_p2 = std.fmt.parseInt(u5, alignment_str, 0) catch {
686 return cg.fail("Malformed assembly, malformed offset \"{s}\"", .{alignment_str});686 return cg.fail("Malformed assembly, malformed offset \"{s}\"", .{alignment_str});
687 };687 };
688688
689 return .{ .offset = @bitCast(offset), .alignment = @as(u32, 1) << alignment_p2 };689 return .{ .offset = @bitCast(offset), .alignment = @fromBackingInt(@intCast(alignment_p2)) };
690 } else {690 } else {
691 const offset_str = mem_arg_str;691 const offset_str = mem_arg_str;
692 const offset = std.fmt.parseInt(i32, offset_str, 0) catch {692 const offset = std.fmt.parseInt(i64, offset_str, 0) catch {
693 return cg.fail("Malformed assembly, malformed offset \"{s}\"", .{offset_str});693 return cg.fail("Malformed assembly, malformed offset \"{s}\"", .{offset_str});
694 };694 };
695 return .{ .offset = @bitCast(offset), .alignment = 1 };695 return .{ .offset = @bitCast(offset), .alignment = .@"1" };
696 }696 }
697}697}
698698