authorgravatar for paul.verigo@gmail.comPavel Verigo <paul.verigo@gmail.com> 2026-08-21 14:36:33+02:00
committergravatar for paul.verigo@gmail.comPavel Verigo <paul.verigo@gmail.com> 2026-09-17 00:04:51+02:00
logb819da2d57316993c5bb77c93d50f626e32eea91
tree688b0efb2a4ce5d8bc5501de2dc4d4dd0e54f56c
parentd12892e8904bbb27aac7baa69f483348e516a30e

wasm: remove lowerToStack and buildOffsetPointer rework

1 files changed, 347 insertions(+), 398 deletions(-)

src/codegen/wasm/CodeGen.zig+347-398
......@@ -271,23 +271,16 @@ const WValue = union(enum) {
271271 /// pointing to where the value lives.
272272 stack_offset: struct {
273273 /// Contains the actual value of the offset
274 value: u32,
275 /// The amount of instructions referencing this `WValue`
276 references: u32,
274 value: u64,
277275 },
278276
279 /// Returns the offset from the bottom of the stack. This is useful when
280 /// we use the load or store instruction to ensure we retrieve the value
281 /// from the correct position, rather than the value that lives at the
282 /// bottom of the stack. For instances where `WValue` is not `stack_value`
283 /// this will return 0, which allows us to simply call this function for all
284 /// loads and stores without requiring checks everywhere.
285 fn offset(value: WValue) u32 {
286 switch (value) {
287 .stack_offset => |stack_offset| return stack_offset.value,
277 /// Returns the offset from the bottom of the stack.
278 fn memOffset(value: WValue) u64 {
279 return switch (value) {
280 .stack_offset => |stack_offset| stack_offset.value,
288281 .dead => unreachable,
289 else => return 0,
290 }
282 else => 0,
283 };
291284 }
292285
293286 /// Promotes a `WValue` to a local when given value is on top of the stack.
......@@ -651,6 +644,30 @@ fn genBlockType(ty: Type, zcu: *const Zcu, target: *const std.Target) std.wasm.B
651644 };
652645}
653646
647/// Returns `ptr` with `offset` applied.
648fn offsetPointer(cg: *CodeGen, ptr: WValue, offset: u64) InnerError!WValue {
649 if (offset == 0) return ptr;
650 return switch (ptr) {
651 .stack_offset => |stack_offset| .{ .stack_offset = .{
652 .value = stack_offset.value + offset,
653 } },
654 else => result: {
655 try cg.emitWValue(ptr);
656 switch (cg.ptr_size) {
657 .wasm32 => {
658 try cg.addImm32(@intCast(offset));
659 try cg.addTag(.i32_add);
660 },
661 .wasm64 => {
662 try cg.addImm64(offset);
663 try cg.addTag(.i64_add);
664 },
665 }
666 break :result .stack;
667 },
668 };
669}
670
654671/// Writes the bytecode depending on the given `WValue` in `val`
655672fn emitWValue(cg: *CodeGen, value: WValue) InnerError!void {
656673 switch (value) {
......@@ -718,7 +735,18 @@ fn emitWValue(cg: *CodeGen, value: WValue) InnerError!void {
718735 });
719736 }
720737 },
721 .stack_offset => try cg.addLocal(.local_get, cg.bottom_stack_value.local.value), // caller must ensure to address the offset
738 .stack_offset => |stack_offset| {
739 try cg.emitWValue(cg.bottom_stack_value);
740 _ = try cg.offsetPointer(.stack, stack_offset.value);
741 },
742 }
743}
744
745/// Emits the base address for a memory instruction.
746fn emitMemBase(cg: *CodeGen, ptr: WValue) InnerError!void {
747 switch (ptr) {
748 .stack_offset => try cg.addLocal(.local_get, cg.bottom_stack_value.local.value),
749 else => try cg.emitWValue(ptr),
722750 }
723751}
724752
......@@ -989,7 +1017,7 @@ pub fn firstParamSRet(
9891017/// it matches the ABI.
9901018fn lowerArg(cg: *CodeGen, cc: std.lang.CallingConvention, ty: Type, value: WValue) !void {
9911019 if (cc != .wasm_mvp) {
992 return cg.lowerToStack(value);
1020 return cg.emitWValue(value);
9931021 }
9941022
9951023 const zcu = cg.pt.zcu;
......@@ -997,7 +1025,7 @@ fn lowerArg(cg: *CodeGen, cc: std.lang.CallingConvention, ty: Type, value: WValu
9971025 switch (abi.classifyType(ty, zcu, cg.target)) {
9981026 .direct => |scalar_ty| {
9991027 if (!isByRef(ty, zcu, cg.target)) {
1000 return cg.lowerToStack(value);
1028 return cg.emitWValue(value);
10011029 } else {
10021030 _ = try cg.load(value, scalar_ty, .{});
10031031 }
......@@ -1005,15 +1033,16 @@ fn lowerArg(cg: *CodeGen, cc: std.lang.CallingConvention, ty: Type, value: WValu
10051033 .double_i64 => {
10061034 assert(ty.abiSize(zcu) == 16);
10071035 // in this case we have an integer or float that must be lowered as 2 i64's.
1008 try cg.emitWValue(value);
1009 try cg.addMemArg(.i64_load, .{ .offset = value.offset(), .alignment = .@"8" });
1010 try cg.emitWValue(value);
1011 try cg.addMemArg(.i64_load, .{ .offset = value.offset() + 8, .alignment = .@"8" });
1036 const base_offset = value.memOffset();
1037 try cg.emitMemBase(value);
1038 try cg.addMemArg(.i64_load, .{ .offset = base_offset, .alignment = .@"8" });
1039 try cg.emitMemBase(value);
1040 try cg.addMemArg(.i64_load, .{ .offset = base_offset + 8, .alignment = .@"8" });
10121041 },
10131042 .indirect => {
10141043 const stack_copy = try cg.allocStack(ty);
10151044 try cg.store(stack_copy, value, ty, .{});
1016 return cg.lowerToStack(stack_copy);
1045 return cg.emitWValue(stack_copy);
10171046 },
10181047 .unrolled => |vector| {
10191048 const elem_size = vector.elem_type.abiSize(zcu);
......@@ -1024,30 +1053,6 @@ fn lowerArg(cg: *CodeGen, cc: std.lang.CallingConvention, ty: Type, value: WValu
10241053 }
10251054}
10261055
1027/// Lowers a `WValue` to the stack. This means when the `value` results in
1028/// `.stack_offset` we calculate the pointer of this offset and use that.
1029/// The value is left on the stack, and not stored in any temporary.
1030fn lowerToStack(cg: *CodeGen, value: WValue) !void {
1031 switch (value) {
1032 .stack_offset => |offset| {
1033 try cg.emitWValue(value);
1034 if (offset.value > 0) {
1035 switch (cg.ptr_size) {
1036 .wasm32 => {
1037 try cg.addImm32(offset.value);
1038 try cg.addTag(.i32_add);
1039 },
1040 .wasm64 => {
1041 try cg.addImm64(offset.value);
1042 try cg.addTag(.i64_add);
1043 },
1044 }
1045 }
1046 },
1047 else => try cg.emitWValue(value),
1048 }
1049}
1050
10511056/// Creates a local for the initial stack value
10521057/// Asserts `initial_stack_value` is `.none`
10531058fn initializeStack(cg: *CodeGen) !void {
......@@ -1113,7 +1118,7 @@ fn allocStackBytes(cg: *CodeGen, size: u32, alignment: Alignment) !WValue {
11131118 const offset: u32 = @intCast(alignment.forward(cg.stack_size));
11141119 defer cg.stack_size = offset + size;
11151120
1116 return .{ .stack_offset = .{ .value = offset, .references = 1 } };
1121 return .{ .stack_offset = .{ .value = offset } };
11171122}
11181123
11191124/// From a given AIR instruction generates a pointer to the stack where
......@@ -1141,7 +1146,7 @@ fn allocStackPtr(cg: *CodeGen, inst: Air.Inst.Index) !WValue {
11411146 const offset: u32 = @intCast(abi_alignment.forward(cg.stack_size));
11421147 defer cg.stack_size = offset + abi_size;
11431148
1144 return .{ .stack_offset = .{ .value = offset, .references = 1 } };
1149 return .{ .stack_offset = .{ .value = offset } };
11451150}
11461151
11471152fn emitMemoryCopy(cg: *CodeGen, dst: WValue, src: WValue, len: WValue) !void {
......@@ -1171,8 +1176,8 @@ fn emitMemoryCopy(cg: *CodeGen, dst: WValue, src: WValue, len: WValue) !void {
11711176 try cg.addLabel(.br_if, 0);
11721177 }
11731178
1174 try cg.lowerToStack(dst);
1175 try cg.lowerToStack(src);
1179 try cg.emitWValue(dst);
1180 try cg.emitWValue(src);
11761181 try cg.emitWValue(len);
11771182 try cg.addExtended(.memory_copy);
11781183
......@@ -1187,8 +1192,8 @@ fn memcpy(cg: *CodeGen, dst: WValue, src: WValue, len: WValue) !void {
11871192 return;
11881193 }
11891194
1190 try cg.lowerToStack(dst);
1191 try cg.lowerToStack(src);
1195 try cg.emitWValue(dst);
1196 try cg.emitWValue(src);
11921197 try cg.emitWValue(len);
11931198 try cg.addCallIntrinsic(.memcpy);
11941199 try cg.addTag(.drop);
......@@ -1200,8 +1205,8 @@ fn memmove(cg: *CodeGen, dst: WValue, src: WValue, len: WValue) !void {
12001205 return;
12011206 }
12021207
1203 try cg.lowerToStack(dst);
1204 try cg.lowerToStack(src);
1208 try cg.emitWValue(dst);
1209 try cg.emitWValue(src);
12051210 try cg.emitWValue(len);
12061211 try cg.addCallIntrinsic(.memmove);
12071212 try cg.addTag(.drop);
......@@ -1283,34 +1288,6 @@ pub fn determineSimdStoreStrategy(ty: Type, zcu: *const Zcu, target: *const std.
12831288 return .unrolled;
12841289}
12851290
1286/// Creates a new local for a pointer that points to memory with given offset.
1287/// This can be used to get a pointer to a struct field, error payload, etc.
1288/// By providing `modify` as action, it will modify the given `ptr_value` instead of making a new
1289/// local value to store the pointer. This allows for local re-use and improves binary size.
1290fn buildPointerOffset(cg: *CodeGen, ptr_value: WValue, offset: u64, action: enum { modify, new }) InnerError!WValue {
1291 // do not perform arithmetic when offset is 0.
1292 if (offset == 0 and ptr_value.offset() == 0 and action == .modify) return ptr_value;
1293 const result_ptr: WValue = switch (action) {
1294 .new => try cg.ensureAllocLocal(Type.usize),
1295 .modify => ptr_value,
1296 };
1297 try cg.emitWValue(ptr_value);
1298 if (offset + ptr_value.offset() > 0) {
1299 switch (cg.ptr_size) {
1300 .wasm32 => {
1301 try cg.addImm32(@intCast(offset + ptr_value.offset()));
1302 try cg.addTag(.i32_add);
1303 },
1304 .wasm64 => {
1305 try cg.addImm64(offset + ptr_value.offset());
1306 try cg.addTag(.i64_add);
1307 },
1308 }
1309 }
1310 try cg.addLocal(.local_set, result_ptr.local.value);
1311 return result_ptr;
1312}
1313
13141291fn genInst(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
13151292 const zcu = cg.pt.zcu;
13161293 const air_tags = cg.air.instructions.items(.tag);
......@@ -1331,7 +1308,7 @@ fn genInst(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
13311308 const elem_ty = cg.typeOf(bin_op.rhs);
13321309 const elem_size = elem_ty.abiSize(zcu);
13331310
1334 try cg.lowerToStack(vec_ptr);
1311 try cg.emitWValue(vec_ptr);
13351312 try cg.emitWValue(elem_idx);
13361313 try cg.addImm32(@intCast(elem_size));
13371314 try cg.addTag(.i32_mul);
......@@ -2108,7 +2085,7 @@ fn airCall(cg: *CodeGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier)
21082085 } else .none;
21092086
21102087 if (first_param_sret) {
2111 try cg.lowerToStack(sret);
2088 try cg.emitWValue(sret);
21122089 }
21132090
21142091 for (args, 0..) |arg, arg_i| {
......@@ -2122,7 +2099,7 @@ fn airCall(cg: *CodeGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier)
21222099 }
21232100
21242101 if (fn_info.is_var_args) {
2125 try cg.lowerToStack(varargs_buf);
2102 try cg.emitWValue(varargs_buf);
21262103 }
21272104
21282105 if (callee) |nav_index| {
......@@ -2243,8 +2220,8 @@ fn airVaArg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
22432220
22442221 try cg.addLocal(.local_set, arg_ptr.local.value);
22452222
2246 try cg.lowerToStack(operand);
2247 try cg.lowerToStack(arg_ptr);
2223 try cg.emitWValue(operand);
2224 try cg.emitWValue(arg_ptr);
22482225 switch (cg.ptr_size) {
22492226 .wasm32 => {
22502227 try cg.addImm32(abi_size);
......@@ -2325,28 +2302,22 @@ fn store(cg: *CodeGen, lhs: WValue, rhs: WValue, ty: Type, memarg: MemArg) Inner
23252302 natural_alignment.minStrict(memarg.alignment);
23262303
23272304 if (isByRef(ty, zcu, cg.target)) {
2328 const offset_ptr: WValue = switch (memarg.offset + lhs.offset()) {
2329 0 => lhs,
2330 else => |total_offset| ptr: {
2331 try cg.emitWValue(lhs);
2332 try cg.addImm32(@intCast(total_offset));
2333 try cg.addTag(.i32_add);
2334 break :ptr .stack;
2335 },
2336 };
2305 const offset_ptr = try cg.offsetPointer(lhs, memarg.offset);
23372306 return cg.memcpy(offset_ptr, rhs, .{ .imm32 = @intCast(abi_size) });
23382307 }
23392308
2309 const base_offset = lhs.memOffset();
2310 try cg.emitMemBase(lhs);
2311
23402312 if (ty.zigTypeTag(zcu) == .vector) {
2341 try cg.emitWValue(lhs);
2342 try cg.lowerToStack(rhs);
2313 try cg.emitWValue(rhs);
23432314 // TODO: Add helper functions for simd opcodes
23442315 const extra_index: u32 = @intCast(cg.mir_extra.items.len);
23452316 // stores as := opcode, offset, alignment (opcode::memarg)
23462317 try cg.mir_extra.ensureUnusedCapacity(cg.gpa, 4);
23472318 cg.mir_extra.appendAssumeCapacity(@backingInt(std.wasm.SimdOpcode.v128_store));
23482319 _ = try cg.addExtraAssumeCapacity(Mir.MemArg{
2349 .offset = memarg.offset + lhs.offset(),
2320 .offset = base_offset + memarg.offset,
23502321 .alignment = alignment.toStdMem(),
23512322 });
23522323 return cg.addInst(.{ .tag = .simd_prefix, .data = .{ .payload = extra_index } });
......@@ -2371,13 +2342,12 @@ fn store(cg: *CodeGen, lhs: WValue, rhs: WValue, ty: Type, memarg: MemArg) Inner
23712342 }
23722343 };
23732344
2374 try cg.emitWValue(lhs);
2375 try cg.lowerToStack(rhs);
2345 try cg.emitWValue(rhs);
23762346
23772347 try cg.addMemArg(
23782348 store_opcode,
23792349 .{
2380 .offset = memarg.offset + lhs.offset(),
2350 .offset = base_offset + memarg.offset,
23812351 .alignment = alignment.toStdMem(),
23822352 },
23832353 );
......@@ -2421,23 +2391,18 @@ fn load(cg: *CodeGen, operand: WValue, ty: Type, memarg: MemArg) InnerError!WVal
24212391 natural_alignment.minStrict(memarg.alignment);
24222392
24232393 if (isByRef(ty, zcu, cg.target)) {
2424 const src_ptr_maybe_stack: WValue = switch (memarg.offset + operand.offset()) {
2425 0 => operand,
2426 else => |total_offset| ptr: {
2427 try cg.emitWValue(operand);
2428 try cg.addImm32(@intCast(total_offset));
2429 try cg.addTag(.i32_add);
2430 break :ptr .stack;
2431 },
2432 };
2433 const src_ptr = try src_ptr_maybe_stack.toLocal(cg, .usize);
2394 var src_ptr = try cg.offsetPointer(operand, memarg.offset);
2395 const owns_src_ptr = src_ptr == .stack;
2396 src_ptr = try src_ptr.toLocal(cg, .usize);
2397 defer if (owns_src_ptr) src_ptr.free(cg);
2398
24342399 const new_ptr = try cg.allocStack(ty);
2435 try cg.store(new_ptr, src_ptr, ty, .{});
2400 try cg.memcpy(new_ptr, src_ptr, .{ .imm32 = @intCast(ty.abiSize(zcu)) });
24362401 return new_ptr;
24372402 }
24382403
2439 // load local's value from memory by its stack position
2440 try cg.emitWValue(operand);
2404 const base_offset = operand.memOffset();
2405 try cg.emitMemBase(operand);
24412406
24422407 if (ty.zigTypeTag(zcu) == .vector) {
24432408 // TODO: Add helper functions for simd opcodes
......@@ -2446,7 +2411,7 @@ fn load(cg: *CodeGen, operand: WValue, ty: Type, memarg: MemArg) InnerError!WVal
24462411 try cg.mir_extra.ensureUnusedCapacity(cg.gpa, 4);
24472412 cg.mir_extra.appendAssumeCapacity(@backingInt(std.wasm.SimdOpcode.v128_load));
24482413 _ = try cg.addExtraAssumeCapacity(Mir.MemArg{
2449 .offset = memarg.offset + operand.offset(),
2414 .offset = base_offset + memarg.offset,
24502415 .alignment = alignment.toStdMem(),
24512416 });
24522417 try cg.addInst(.{ .tag = .simd_prefix, .data = .{ .payload = extra_index } });
......@@ -2477,7 +2442,7 @@ fn load(cg: *CodeGen, operand: WValue, ty: Type, memarg: MemArg) InnerError!WVal
24772442 try cg.addMemArg(
24782443 load_opcode,
24792444 .{
2480 .offset = memarg.offset + operand.offset(),
2445 .offset = base_offset + memarg.offset,
24812446 .alignment = alignment.toStdMem(),
24822447 },
24832448 );
......@@ -2647,9 +2612,9 @@ fn intAdd(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue
26472612 else => {
26482613 const result = try cg.allocInt(ty);
26492614
2650 try cg.lowerToStack(result);
2651 try cg.lowerToStack(lhs);
2652 try cg.lowerToStack(rhs);
2615 try cg.emitWValue(result);
2616 try cg.emitWValue(lhs);
2617 try cg.emitWValue(rhs);
26532618 try cg.addImm32(@intFromBool(ty.is_signed));
26542619 try cg.addImm32(ty.bits);
26552620 try cg.addCallIntrinsic(.__addo_limb64);
......@@ -2700,9 +2665,9 @@ fn intSub(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue
27002665 else => {
27012666 const result = try cg.allocInt(ty);
27022667
2703 try cg.lowerToStack(result);
2704 try cg.lowerToStack(lhs);
2705 try cg.lowerToStack(rhs);
2668 try cg.emitWValue(result);
2669 try cg.emitWValue(lhs);
2670 try cg.emitWValue(rhs);
27062671 try cg.addImm32(@intFromBool(ty.is_signed));
27072672 try cg.addImm32(ty.bits);
27082673 try cg.addCallIntrinsic(.__subo_limb64);
......@@ -2732,9 +2697,9 @@ fn intMul(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue
27322697 else => {
27332698 const result = try cg.allocInt(ty);
27342699
2735 try cg.lowerToStack(result);
2736 try cg.lowerToStack(lhs);
2737 try cg.lowerToStack(rhs);
2700 try cg.emitWValue(result);
2701 try cg.emitWValue(lhs);
2702 try cg.emitWValue(rhs);
27382703 try cg.addImm32(@intFromBool(ty.is_signed));
27392704 try cg.addImm32(ty.bits);
27402705 try cg.addCallIntrinsic(.__mulo_limb64);
......@@ -3041,9 +3006,9 @@ fn intAnd(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue
30413006 else => {
30423007 const result = try cg.allocInt(ty);
30433008
3044 try cg.lowerToStack(result);
3045 try cg.lowerToStack(lhs);
3046 try cg.lowerToStack(rhs);
3009 try cg.emitWValue(result);
3010 try cg.emitWValue(lhs);
3011 try cg.emitWValue(rhs);
30473012 try cg.addImm32(ty.bits);
30483013 try cg.addCallIntrinsic(.__and_limb64);
30493014
......@@ -3085,9 +3050,9 @@ fn intOr(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue
30853050 else => {
30863051 const result = try cg.allocInt(ty);
30873052
3088 try cg.lowerToStack(result);
3089 try cg.lowerToStack(lhs);
3090 try cg.lowerToStack(rhs);
3053 try cg.emitWValue(result);
3054 try cg.emitWValue(lhs);
3055 try cg.emitWValue(rhs);
30913056 try cg.addImm32(ty.bits);
30923057 try cg.addCallIntrinsic(.__or_limb64);
30933058
......@@ -3129,9 +3094,9 @@ fn intXor(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue
31293094 else => {
31303095 const result = try cg.allocInt(ty);
31313096
3132 try cg.lowerToStack(result);
3133 try cg.lowerToStack(lhs);
3134 try cg.lowerToStack(rhs);
3097 try cg.emitWValue(result);
3098 try cg.emitWValue(lhs);
3099 try cg.emitWValue(rhs);
31353100 try cg.addImm32(ty.bits);
31363101 try cg.addCallIntrinsic(.__xor_limb64);
31373102
......@@ -3176,13 +3141,14 @@ fn intNot(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
31763141 65...128 => {
31773142 const result = try cg.allocStack(Type.u128);
31783143
3179 try cg.emitWValue(result);
3144 const result_offset = result.memOffset();
3145 try cg.emitMemBase(result);
31803146 _ = try cg.load(operand, .u64, .{});
31813147 try cg.addImm64(~@as(u64, 0));
31823148 try cg.addTag(.i64_xor);
3183 try cg.store(.stack, .stack, .u64, .{ .offset = result.offset() });
3149 try cg.store(.stack, .stack, .u64, .{ .offset = result_offset });
31843150
3185 try cg.emitWValue(result);
3151 try cg.emitMemBase(result);
31863152 _ = try cg.load(operand, .u64, .{ .offset = 8 });
31873153 const high_mask: u64 = if (ty.is_signed)
31883154 ~@as(u64, 0)
......@@ -3190,15 +3156,15 @@ fn intNot(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
31903156 ~@as(u64, 0) >> @intCast(128 - ty.bits);
31913157 try cg.addImm64(high_mask);
31923158 try cg.addTag(.i64_xor);
3193 try cg.store(.stack, .stack, .u64, .{ .offset = result.offset() + 8 });
3159 try cg.store(.stack, .stack, .u64, .{ .offset = result_offset + 8 });
31943160
31953161 return result;
31963162 },
31973163 else => {
31983164 const result = try cg.allocInt(ty);
31993165
3200 try cg.lowerToStack(result);
3201 try cg.lowerToStack(operand);
3166 try cg.emitWValue(result);
3167 try cg.emitWValue(operand);
32023168 try cg.addImm32(@intFromBool(ty.is_signed));
32033169 try cg.addImm32(ty.bits);
32043170 try cg.addCallIntrinsic(.__not_limb64);
......@@ -3230,9 +3196,9 @@ fn intShl(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue
32303196 else => {
32313197 const result = try cg.allocInt(ty);
32323198
3233 try cg.lowerToStack(result);
3234 try cg.lowerToStack(lhs);
3235 try cg.lowerToStack(rhs);
3199 try cg.emitWValue(result);
3200 try cg.emitWValue(lhs);
3201 try cg.emitWValue(rhs);
32363202 try cg.addImm32(@intFromBool(ty.is_signed));
32373203 try cg.addImm32(ty.bits);
32383204 try cg.addCallIntrinsic(.__shlo_limb64);
......@@ -3270,9 +3236,9 @@ fn intShr(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue
32703236 else => {
32713237 const result = try cg.allocInt(ty);
32723238
3273 try cg.lowerToStack(result);
3274 try cg.lowerToStack(lhs);
3275 try cg.lowerToStack(rhs);
3239 try cg.emitWValue(result);
3240 try cg.emitWValue(lhs);
3241 try cg.emitWValue(rhs);
32763242 try cg.addImm32(@intFromBool(ty.is_signed));
32773243 try cg.addImm32(ty.bits);
32783244 try cg.addCallIntrinsic(.__shr_limb64);
......@@ -3320,8 +3286,9 @@ fn intAbs(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
33203286 const u128_ty: IntType = .{ .is_signed = false, .bits = 128 };
33213287
33223288 const mask = try cg.allocStack(Type.u128);
3323 try cg.emitWValue(mask);
3324 try cg.emitWValue(mask);
3289 const mask_offset = mask.memOffset();
3290 try cg.emitMemBase(mask);
3291 try cg.emitMemBase(mask);
33253292
33263293 _ = try cg.load(operand, .u64, .{ .offset = 8 });
33273294 try cg.addImm64(63);
......@@ -3330,9 +3297,9 @@ fn intAbs(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
33303297 var tmp = try cg.allocLocal(Type.u64);
33313298 defer tmp.free(cg);
33323299 try cg.addLocal(.local_tee, tmp.local.value);
3333 try cg.store(.stack, .stack, .u64, .{ .offset = mask.offset() + 0 });
3300 try cg.store(.stack, .stack, .u64, .{ .offset = mask_offset });
33343301 try cg.emitWValue(tmp);
3335 try cg.store(.stack, .stack, .u64, .{ .offset = mask.offset() + 8 });
3302 try cg.store(.stack, .stack, .u64, .{ .offset = mask_offset + 8 });
33363303
33373304 const a = try cg.intXor(u128_ty, operand, mask);
33383305 const b = try cg.intSub(u128_ty, a, mask);
......@@ -3341,8 +3308,8 @@ fn intAbs(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
33413308 else => {
33423309 const result = try cg.allocInt(ty);
33433310
3344 try cg.lowerToStack(result);
3345 try cg.lowerToStack(operand);
3311 try cg.emitWValue(result);
3312 try cg.emitWValue(operand);
33463313 try cg.addImm32(ty.bits);
33473314 try cg.addCallIntrinsic(.__abs_limb64);
33483315
......@@ -3352,16 +3319,16 @@ fn intAbs(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
33523319}
33533320
33543321fn intMax(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue {
3355 try cg.lowerToStack(lhs);
3356 try cg.lowerToStack(rhs);
3322 try cg.emitWValue(lhs);
3323 try cg.emitWValue(rhs);
33573324 _ = try cg.intCmp(ty, .gt, lhs, rhs);
33583325 try cg.addTag(.select);
33593326 return .stack;
33603327}
33613328
33623329fn intMin(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue {
3363 try cg.lowerToStack(lhs);
3364 try cg.lowerToStack(rhs);
3330 try cg.emitWValue(lhs);
3331 try cg.emitWValue(rhs);
33653332 _ = try cg.intCmp(ty, .lt, lhs, rhs);
33663333 try cg.addTag(.select);
33673334 return .stack;
......@@ -3427,7 +3394,7 @@ fn intClz(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
34273394 return .stack;
34283395 },
34293396 else => {
3430 try cg.lowerToStack(operand);
3397 try cg.emitWValue(operand);
34313398 try cg.addImm32(ty.bits);
34323399 try cg.addCallIntrinsic(.__clz_limb64);
34333400
......@@ -3479,7 +3446,7 @@ fn intCtz(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
34793446 return .stack;
34803447 },
34813448 else => {
3482 try cg.lowerToStack(operand);
3449 try cg.emitWValue(operand);
34833450 try cg.addImm32(ty.bits);
34843451 try cg.addCallIntrinsic(.__ctz_limb64);
34853452
......@@ -3525,7 +3492,7 @@ fn intPopCount(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
35253492 return .stack;
35263493 },
35273494 else => {
3528 try cg.lowerToStack(operand);
3495 try cg.emitWValue(operand);
35293496 try cg.addImm32(ty.bits);
35303497 try cg.addCallIntrinsic(.__popcount_limb64);
35313498
......@@ -3560,7 +3527,8 @@ fn intBitReverse(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
35603527 65...128 => {
35613528 const tmp = try cg.allocStack(Type.u128);
35623529
3563 try cg.emitWValue(tmp);
3530 const tmp_offset = tmp.memOffset();
3531 try cg.emitMemBase(tmp);
35643532 const hi = try cg.load(operand, .u64, .{ .offset = 8 });
35653533 const hi_rev = try cg.callIntrinsic(
35663534 .__bitreversedi2,
......@@ -3569,9 +3537,9 @@ fn intBitReverse(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
35693537 &.{hi},
35703538 );
35713539 try cg.emitWValue(hi_rev);
3572 try cg.store(.stack, .stack, .u64, .{ .offset = tmp.offset() });
3540 try cg.store(.stack, .stack, .u64, .{ .offset = tmp_offset });
35733541
3574 try cg.emitWValue(tmp);
3542 try cg.emitMemBase(tmp);
35753543 const lo = try cg.load(operand, .u64, .{});
35763544 const lo_rev = try cg.callIntrinsic(
35773545 .__bitreversedi2,
......@@ -3580,7 +3548,7 @@ fn intBitReverse(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
35803548 &.{lo},
35813549 );
35823550 try cg.emitWValue(lo_rev);
3583 try cg.store(.stack, .stack, .u64, .{ .offset = tmp.offset() + 8 });
3551 try cg.store(.stack, .stack, .u64, .{ .offset = tmp_offset + 8 });
35843552
35853553 if (ty.bits < 128) {
35863554 const shift_ty: IntType = .{ .is_signed = ty.is_signed, .bits = 128 };
......@@ -3592,8 +3560,8 @@ fn intBitReverse(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
35923560 else => {
35933561 const result = try cg.allocInt(ty);
35943562
3595 try cg.lowerToStack(result);
3596 try cg.lowerToStack(operand);
3563 try cg.emitWValue(result);
3564 try cg.emitWValue(operand);
35973565 try cg.addImm32(@intFromBool(ty.is_signed));
35983566 try cg.addImm32(ty.bits);
35993567 try cg.addCallIntrinsic(.__bitreverse_limb64);
......@@ -3629,7 +3597,8 @@ fn intByteSwap(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
36293597 65...128 => {
36303598 const result = try cg.allocStack(Type.u128);
36313599
3632 try cg.emitWValue(result);
3600 const result_offset = result.memOffset();
3601 try cg.emitMemBase(result);
36333602
36343603 const low = try cg.load(operand, .u64, .{});
36353604 const swap_low = try cg.callIntrinsic(
......@@ -3638,9 +3607,9 @@ fn intByteSwap(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
36383607 Type.u64,
36393608 &.{low},
36403609 );
3641 try cg.store(.stack, swap_low, .u64, .{ .offset = result.offset() + 8 });
3610 try cg.store(.stack, swap_low, .u64, .{ .offset = result_offset + 8 });
36423611
3643 try cg.emitWValue(result);
3612 try cg.emitMemBase(result);
36443613
36453614 const high = try cg.load(operand, .u64, .{ .offset = 8 });
36463615 const swap_high = try cg.callIntrinsic(
......@@ -3649,7 +3618,7 @@ fn intByteSwap(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
36493618 Type.u64,
36503619 &.{high},
36513620 );
3652 try cg.store(.stack, swap_high, .u64, .{ .offset = result.offset() });
3621 try cg.store(.stack, swap_high, .u64, .{ .offset = result_offset });
36533622
36543623 if (ty.bits < 128) {
36553624 const shift_ty: IntType = .{ .is_signed = ty.is_signed, .bits = 128 };
......@@ -3661,8 +3630,8 @@ fn intByteSwap(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
36613630 else => {
36623631 const result = try cg.allocInt(ty);
36633632
3664 try cg.lowerToStack(result);
3665 try cg.lowerToStack(operand);
3633 try cg.emitWValue(result);
3634 try cg.emitWValue(operand);
36663635 try cg.addImm32(@intFromBool(ty.is_signed));
36673636 try cg.addImm32(ty.bits);
36683637 try cg.addCallIntrinsic(.__byteswap_limb64);
......@@ -3706,11 +3675,12 @@ fn intWrap(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
37063675 65...127 => {
37073676 const result = try cg.allocStack(Type.u128);
37083677
3709 try cg.emitWValue(result);
3678 const result_offset = result.memOffset();
3679 try cg.emitMemBase(result);
37103680 _ = try cg.load(operand, .u64, .{});
3711 try cg.store(.stack, .stack, .u64, .{ .offset = result.offset() });
3681 try cg.store(.stack, .stack, .u64, .{ .offset = result_offset });
37123682
3713 try cg.emitWValue(result);
3683 try cg.emitMemBase(result);
37143684 _ = try cg.load(operand, .u64, .{ .offset = 8 });
37153685 if (ty.is_signed) {
37163686 try cg.addImm64(128 - ty.bits);
......@@ -3721,7 +3691,7 @@ fn intWrap(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
37213691 try cg.addImm64(~@as(u64, 0) >> @intCast(128 - ty.bits));
37223692 try cg.addTag(.i64_and);
37233693 }
3724 try cg.store(.stack, .stack, .u64, .{ .offset = result.offset() + 8 });
3694 try cg.store(.stack, .stack, .u64, .{ .offset = result_offset + 8 });
37253695
37263696 return result;
37273697 },
......@@ -3738,7 +3708,7 @@ fn intWrap(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
37383708 try cg.memcpy(result, operand, .{ .imm32 = used_len - 8 });
37393709 const pad = 64 - ty.bits % 64;
37403710
3741 try cg.emitWValue(result);
3711 try cg.emitMemBase(result);
37423712 _ = try cg.load(operand, .u64, .{ .offset = used_len - 8 });
37433713 if (ty.is_signed) {
37443714 try cg.addImm64(pad);
......@@ -3749,14 +3719,14 @@ fn intWrap(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
37493719 try cg.addImm64(~@as(u64, 0) >> @intCast(pad));
37503720 try cg.addTag(.i64_and);
37513721 }
3752 try cg.store(.stack, .stack, .u64, .{ .offset = result.offset() + used_len - 8 });
3722 try cg.store(.stack, .stack, .u64, .{ .offset = result.memOffset() + used_len - 8 });
37533723 } else {
37543724 try cg.memcpy(result, operand, .{ .imm32 = used_len });
37553725 }
37563726
37573727 const full_len = @divExact(bits, 8);
37583728 if (used_len + 8 == full_len) { // last limb needs sign extended
3759 try cg.emitWValue(result);
3729 try cg.emitMemBase(result);
37603730 if (ty.is_signed) {
37613731 _ = try cg.load(result, .u64, .{ .offset = used_len - 8 });
37623732 try cg.addImm64(63);
......@@ -3764,7 +3734,7 @@ fn intWrap(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
37643734 } else {
37653735 try cg.addImm64(0);
37663736 }
3767 try cg.store(.stack, .stack, .u64, .{ .offset = result.offset() + used_len });
3737 try cg.store(.stack, .stack, .u64, .{ .offset = result.memOffset() + used_len });
37683738 }
37693739
37703740 return result;
......@@ -3858,20 +3828,20 @@ fn intAddSat(cg: *CodeGen, int_ty: IntType, lhs: WValue, rhs: WValue) InnerError
38583828 defer rhs_is_neg.free(cg);
38593829 const min_val = try cg.intMinValue(int_ty);
38603830
3861 try cg.lowerToStack(min_val);
3862 try cg.lowerToStack(max_val);
3831 try cg.emitWValue(min_val);
3832 try cg.emitWValue(max_val);
38633833 try cg.emitWValue(rhs_is_neg);
38643834 try cg.addTag(.select);
38653835
3866 try cg.lowerToStack(op_val);
3836 try cg.emitWValue(op_val);
38673837 const overflow_cmp = try cg.intCmp(int_ty, .lt, op_val, lhs);
38683838 const is_overflow = try cg.intCmp(.u32, .neq, rhs_is_neg, overflow_cmp);
38693839 try cg.emitWValue(is_overflow);
38703840 try cg.addTag(.select);
38713841 return .stack;
38723842 } else {
3873 try cg.lowerToStack(max_val);
3874 try cg.lowerToStack(op_val);
3843 try cg.emitWValue(max_val);
3844 try cg.emitWValue(op_val);
38753845
38763846 const is_overflow = try cg.intCmp(int_ty, .lt, op_val, lhs);
38773847 try cg.emitWValue(is_overflow);
......@@ -3892,12 +3862,12 @@ fn intSubSat(cg: *CodeGen, int_ty: IntType, lhs: WValue, rhs: WValue) InnerError
38923862 const max_val = try cg.intMaxValue(int_ty);
38933863 const min_val = try cg.intMinValue(int_ty);
38943864
3895 try cg.lowerToStack(max_val);
3896 try cg.lowerToStack(min_val);
3865 try cg.emitWValue(max_val);
3866 try cg.emitWValue(min_val);
38973867 try cg.emitWValue(rhs_is_neg);
38983868 try cg.addTag(.select);
38993869
3900 try cg.lowerToStack(op_val);
3870 try cg.emitWValue(op_val);
39013871 const overflow_cmp = try cg.intCmp(int_ty, .gt, op_val, lhs);
39023872 const is_overflow = try cg.intCmp(.u32, .neq, rhs_is_neg, overflow_cmp);
39033873 try cg.emitWValue(is_overflow);
......@@ -3906,8 +3876,8 @@ fn intSubSat(cg: *CodeGen, int_ty: IntType, lhs: WValue, rhs: WValue) InnerError
39063876 } else {
39073877 const zero = try cg.intZeroValue(int_ty);
39083878
3909 try cg.lowerToStack(zero);
3910 try cg.lowerToStack(op_val);
3879 try cg.emitWValue(zero);
3880 try cg.emitWValue(op_val);
39113881 const is_overflow = try cg.intCmp(int_ty, .lt, lhs, rhs);
39123882 try cg.emitWValue(is_overflow);
39133883 try cg.addTag(.select);
......@@ -3931,10 +3901,10 @@ fn intMulSat(cg: *CodeGen, int_ty: IntType, lhs: WValue, rhs: WValue) InnerError
39313901 if (int_ty.is_signed) {
39323902 const min_val = try cg.intMinValue(int_ty);
39333903
3934 try cg.lowerToStack(min_val);
3904 try cg.emitWValue(min_val);
39353905
3936 try cg.lowerToStack(max_val);
3937 try cg.lowerToStack(op_val);
3906 try cg.emitWValue(max_val);
3907 try cg.emitWValue(op_val);
39383908 const max_ext = try cg.intCast(ext_ty, int_ty, max_val);
39393909 const ov_pos = try cg.intCmp(ext_ty, .lt, max_ext, mul_ext);
39403910 try cg.emitWValue(ov_pos);
......@@ -3942,12 +3912,12 @@ fn intMulSat(cg: *CodeGen, int_ty: IntType, lhs: WValue, rhs: WValue) InnerError
39423912
39433913 const min_ext = try cg.intCast(ext_ty, int_ty, min_val);
39443914 const ov_neg = try cg.intCmp(ext_ty, .gt, min_ext, mul_ext);
3945 try cg.lowerToStack(ov_neg);
3915 try cg.emitWValue(ov_neg);
39463916 try cg.addTag(.select);
39473917 return .stack;
39483918 } else {
3949 try cg.lowerToStack(max_val);
3950 try cg.lowerToStack(op_val);
3919 try cg.emitWValue(max_val);
3920 try cg.emitWValue(op_val);
39513921 const max_ext = try cg.intCast(ext_ty, int_ty, max_val);
39523922 const is_overflow = try cg.intCmp(ext_ty, .lt, max_ext, mul_ext);
39533923 try cg.emitWValue(is_overflow);
......@@ -3970,20 +3940,20 @@ fn intShlSat(cg: *CodeGen, int_ty: IntType, lhs: WValue, rhs: WValue) InnerError
39703940 const zero = try cg.intZeroValue(int_ty);
39713941 const min_val = try cg.intMinValue(int_ty);
39723942
3973 try cg.lowerToStack(min_val);
3974 try cg.lowerToStack(max_val);
3943 try cg.emitWValue(min_val);
3944 try cg.emitWValue(max_val);
39753945 const lhs_is_neg = try cg.intCmp(int_ty, .lt, lhs, zero);
39763946 try cg.emitWValue(lhs_is_neg);
39773947 try cg.addTag(.select);
39783948
3979 try cg.lowerToStack(op_val);
3949 try cg.emitWValue(op_val);
39803950 const is_overflow = try cg.intCmp(int_ty, .neq, check_val, lhs);
39813951 try cg.emitWValue(is_overflow);
39823952 try cg.addTag(.select);
39833953 return .stack;
39843954 } else {
3985 try cg.lowerToStack(max_val);
3986 try cg.lowerToStack(op_val);
3955 try cg.emitWValue(max_val);
3956 try cg.emitWValue(op_val);
39873957 const is_overflow = try cg.intCmp(int_ty, .neq, check_val, lhs);
39883958 try cg.emitWValue(is_overflow);
39893959 try cg.addTag(.select);
......@@ -4055,9 +4025,9 @@ fn intAddOverflow(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerErro
40554025 else => {
40564026 const result = try cg.allocInt(ty);
40574027
4058 try cg.lowerToStack(result);
4059 try cg.lowerToStack(lhs);
4060 try cg.lowerToStack(rhs);
4028 try cg.emitWValue(result);
4029 try cg.emitWValue(lhs);
4030 try cg.emitWValue(rhs);
40614031 try cg.addImm32(@intFromBool(ty.is_signed));
40624032 try cg.addImm32(ty.bits);
40634033 try cg.addCallIntrinsic(.__addo_limb64);
......@@ -4087,9 +4057,9 @@ fn intSubOverflow(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerErro
40874057 else => {
40884058 const result = try cg.allocInt(ty);
40894059
4090 try cg.lowerToStack(result);
4091 try cg.lowerToStack(lhs);
4092 try cg.lowerToStack(rhs);
4060 try cg.emitWValue(result);
4061 try cg.emitWValue(lhs);
4062 try cg.emitWValue(rhs);
40934063 try cg.addImm32(@intFromBool(ty.is_signed));
40944064 try cg.addImm32(ty.bits);
40954065 try cg.addCallIntrinsic(.__subo_limb64);
......@@ -4145,9 +4115,9 @@ fn intMulOverflow(cg: *CodeGen, int_ty: IntType, lhs: WValue, rhs: WValue) Inner
41454115 } else {
41464116 const result = try cg.allocInt(int_ty);
41474117
4148 try cg.lowerToStack(result);
4149 try cg.lowerToStack(lhs);
4150 try cg.lowerToStack(rhs);
4118 try cg.emitWValue(result);
4119 try cg.emitWValue(lhs);
4120 try cg.emitWValue(rhs);
41514121 try cg.addImm32(@intFromBool(int_ty.is_signed));
41524122 try cg.addImm32(int_ty.bits);
41534123 try cg.addCallIntrinsic(.__mulo_limb64);
......@@ -4174,9 +4144,9 @@ fn intShlOverflow(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerErro
41744144 else => {
41754145 const result = try cg.allocInt(ty);
41764146
4177 try cg.lowerToStack(result);
4178 try cg.lowerToStack(lhs);
4179 try cg.lowerToStack(rhs);
4147 try cg.emitWValue(result);
4148 try cg.emitWValue(lhs);
4149 try cg.emitWValue(rhs);
41804150 try cg.addImm32(@intFromBool(ty.is_signed));
41814151 try cg.addImm32(ty.bits);
41824152 try cg.addCallIntrinsic(.__shlo_limb64);
......@@ -4213,15 +4183,15 @@ fn intCast(cg: *CodeGen, dest_ty: IntType, src_ty: IntType, operand: WValue) Inn
42134183 } else {
42144184 var src_len: u32 = undefined;
42154185 if (src_bits == 32) {
4216 try cg.emitWValue(result);
4186 try cg.emitMemBase(result);
42174187 try cg.emitWValue(operand);
42184188 try cg.addTag(if (src_ty.is_signed) .i64_extend_i32_s else .i64_extend_i32_u);
4219 try cg.store(.stack, .stack, .u64, .{ .offset = result.offset() });
4189 try cg.store(.stack, .stack, .u64, .{ .offset = result.memOffset() });
42204190 src_len = 8;
42214191 } else if (src_bits == 64) {
4222 try cg.emitWValue(result);
4192 try cg.emitMemBase(result);
42234193 try cg.emitWValue(operand);
4224 try cg.store(.stack, .stack, .u64, .{ .offset = result.offset() });
4194 try cg.store(.stack, .stack, .u64, .{ .offset = result.memOffset() });
42254195 src_len = 8;
42264196 } else {
42274197 src_len = src_bits / 8;
......@@ -4230,7 +4200,7 @@ fn intCast(cg: *CodeGen, dest_ty: IntType, src_ty: IntType, operand: WValue) Inn
42304200
42314201 if (dest_bits == 128) {
42324202 if (src_ty.is_signed) {
4233 try cg.emitWValue(result);
4203 try cg.emitMemBase(result);
42344204 if (src_bits == 32) {
42354205 try cg.emitWValue(operand);
42364206 try cg.addTag(if (dest_ty.is_signed) .i64_extend_i32_s else .i64_extend_i32_u);
......@@ -4238,7 +4208,7 @@ fn intCast(cg: *CodeGen, dest_ty: IntType, src_ty: IntType, operand: WValue) Inn
42384208 try cg.emitWValue(operand);
42394209 } else unreachable;
42404210 const shr = try cg.intShr(IntType.i64, .stack, .{ .imm32 = 63 });
4241 try cg.store(.stack, shr, .u64, .{ .offset = 8 + result.offset() });
4211 try cg.store(.stack, shr, .u64, .{ .offset = result.memOffset() + 8 });
42424212 } else {
42434213 try cg.store(result, .{ .imm64 = 0 }, .u64, .{ .offset = 8 });
42444214 }
......@@ -4642,20 +4612,22 @@ fn floatNeg(cg: *CodeGen, ty: FloatType, arg: WValue) InnerError!WValue {
46424612 },
46434613 .f80, .f128 => {
46444614 const result = try cg.allocStack(Type.f128);
4645 try cg.emitWValue(result);
4646 try cg.emitWValue(arg);
4647 try cg.addMemArg(.i64_load, .{ .offset = 0 + arg.offset(), .alignment = .@"2" });
4648 try cg.addMemArg(.i64_store, .{ .offset = 0 + result.offset(), .alignment = .@"2" });
4649 try cg.emitWValue(result);
4650 try cg.emitWValue(arg);
4651 try cg.addMemArg(.i64_load, .{ .offset = 8 + arg.offset(), .alignment = .@"2" });
4615 const result_offset = result.memOffset();
4616 const arg_offset = arg.memOffset();
4617 try cg.emitMemBase(result);
4618 try cg.emitMemBase(arg);
4619 try cg.addMemArg(.i64_load, .{ .offset = arg_offset, .alignment = .@"2" });
4620 try cg.addMemArg(.i64_store, .{ .offset = result_offset, .alignment = .@"2" });
4621 try cg.emitMemBase(result);
4622 try cg.emitMemBase(arg);
4623 try cg.addMemArg(.i64_load, .{ .offset = arg_offset + 8, .alignment = .@"2" });
46524624 if (ty == .f80) {
46534625 try cg.addImm64(0x8000);
46544626 } else {
46554627 try cg.addImm64(0x8000000000000000);
46564628 }
46574629 try cg.addTag(.i64_xor);
4658 try cg.addMemArg(.i64_store, .{ .offset = 8 + result.offset(), .alignment = .@"2" });
4630 try cg.addMemArg(.i64_store, .{ .offset = result_offset + 8, .alignment = .@"2" });
46594631 return result;
46604632 },
46614633 }
......@@ -4681,20 +4653,22 @@ fn floatAbs(cg: *CodeGen, ty: FloatType, arg: WValue) InnerError!WValue {
46814653 },
46824654 .f80, .f128 => {
46834655 const result = try cg.allocStack(Type.f128);
4684 try cg.emitWValue(result);
4685 try cg.emitWValue(arg);
4686 try cg.addMemArg(.i64_load, .{ .offset = 0 + arg.offset(), .alignment = .@"2" });
4687 try cg.addMemArg(.i64_store, .{ .offset = 0 + result.offset(), .alignment = .@"2" });
4688 try cg.emitWValue(result);
4689 try cg.emitWValue(arg);
4690 try cg.addMemArg(.i64_load, .{ .offset = 8 + arg.offset(), .alignment = .@"2" });
4656 const result_offset = result.memOffset();
4657 const arg_offset = arg.memOffset();
4658 try cg.emitMemBase(result);
4659 try cg.emitMemBase(arg);
4660 try cg.addMemArg(.i64_load, .{ .offset = arg_offset, .alignment = .@"2" });
4661 try cg.addMemArg(.i64_store, .{ .offset = result_offset, .alignment = .@"2" });
4662 try cg.emitMemBase(result);
4663 try cg.emitMemBase(arg);
4664 try cg.addMemArg(.i64_load, .{ .offset = arg_offset + 8, .alignment = .@"2" });
46914665 if (ty == .f80) {
46924666 try cg.addImm64(0x7FFF);
46934667 } else {
46944668 try cg.addImm64(0x7FFFFFFFFFFFFFFF);
46954669 }
46964670 try cg.addTag(.i64_and);
4697 try cg.addMemArg(.i64_store, .{ .offset = 8 + result.offset(), .alignment = .@"2" });
4671 try cg.addMemArg(.i64_store, .{ .offset = result_offset + 8, .alignment = .@"2" });
46984672 return result;
46994673 },
47004674 }
......@@ -5377,8 +5351,8 @@ fn intCmp(cg: *CodeGen, ty: IntType, op: std.math.CompareOperator, lhs: WValue,
53775351 0 => unreachable,
53785352 1...32 => {
53795353 // lhs or rhs could be stack pointers
5380 try cg.lowerToStack(lhs);
5381 try cg.lowerToStack(rhs);
5354 try cg.emitWValue(lhs);
5355 try cg.emitWValue(rhs);
53825356 const opcode: Mir.Inst.Tag = switch (op) {
53835357 .eq => .i32_eq,
53845358 .neq => .i32_ne,
......@@ -5392,8 +5366,8 @@ fn intCmp(cg: *CodeGen, ty: IntType, op: std.math.CompareOperator, lhs: WValue,
53925366 },
53935367 33...64 => {
53945368 // lhs or rhs could be stack pointers
5395 try cg.lowerToStack(lhs);
5396 try cg.lowerToStack(rhs);
5369 try cg.emitWValue(lhs);
5370 try cg.emitWValue(rhs);
53975371 const opcode: Mir.Inst.Tag = switch (op) {
53985372 .eq => .i64_eq,
53995373 .neq => .i64_ne,
......@@ -5442,8 +5416,8 @@ fn intCmp(cg: *CodeGen, ty: IntType, op: std.math.CompareOperator, lhs: WValue,
54425416 return .stack;
54435417 },
54445418 else => {
5445 try cg.lowerToStack(lhs);
5446 try cg.lowerToStack(rhs);
5419 try cg.emitWValue(lhs);
5420 try cg.emitWValue(rhs);
54475421 try cg.addImm32(@intFromBool(ty.is_signed));
54485422 try cg.addImm32(ty.bits);
54495423 try cg.addCallIntrinsic(.__cmp_limb64);
......@@ -5554,7 +5528,7 @@ fn airBr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
55545528 // if operand has codegen bits we should break with a value
55555529 if (block.value != .none) {
55565530 const operand = try cg.resolveInst(br.operand);
5557 try cg.lowerToStack(operand);
5531 try cg.emitWValue(operand);
55585532 try cg.addLocal(.local_set, block.value.local.value);
55595533 }
55605534
......@@ -5618,11 +5592,12 @@ fn airIntFromPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
56185592 assert(operand_ty.abiAlignment(zcu) == dest_ty.abiAlignment(zcu));
56195593
56205594 const operand = try cg.resolveInst(ty_op.operand);
5621 const result = switch (operand) {
5622 .stack_offset => try cg.buildPointerOffset(operand, 0, .new),
5623 else => operand,
5624 };
5625 return cg.finishAir(inst, result, &.{ty_op.operand});
5595 if (operand == .stack_offset) {
5596 // materialize operand, we assume precomputed value is more efficient
5597 try cg.emitWValue(operand);
5598 return cg.finishAir(inst, .stack, &.{ty_op.operand});
5599 }
5600 return cg.finishAir(inst, operand, &.{ty_op.operand});
56265601}
56275602
56285603fn airUnionFromEnum(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
......@@ -5801,16 +5776,7 @@ fn structFieldPtr(
58015776 },
58025777 else => struct_ty.structFieldOffset(index, zcu),
58035778 };
5804 // save a load and store when we can simply reuse the operand
5805 if (offset == 0) {
5806 return struct_ptr;
5807 }
5808 switch (struct_ptr) {
5809 .stack_offset => |stack_offset| {
5810 return .{ .stack_offset = .{ .value = stack_offset.value + @as(u32, @intCast(offset)), .references = 1 } };
5811 },
5812 else => return cg.buildPointerOffset(struct_ptr, offset, .new),
5813 }
5779 return cg.offsetPointer(struct_ptr, offset);
58145780}
58155781
58165782fn airAggFieldVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
......@@ -5832,12 +5798,7 @@ fn airAggFieldVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
58325798 return cg.fail("Field type '{f}' too big to fit into stack frame", .{field_ty.fmt(pt)});
58335799 };
58345800 if (isByRef(field_ty, zcu, cg.target)) {
5835 switch (operand) {
5836 .stack_offset => |stack_offset| {
5837 break :result .{ .stack_offset = .{ .value = stack_offset.value + offset, .references = 1 } };
5838 },
5839 else => break :result try cg.buildPointerOffset(operand, offset, .new),
5840 }
5801 break :result try cg.offsetPointer(operand, offset);
58415802 }
58425803 break :result try cg.load(operand, field_ty, .{ .offset = offset });
58435804 },
......@@ -5859,7 +5820,7 @@ fn airSwitchBr(cg: *CodeGen, inst: Air.Inst.Index, is_dispatch_loop: bool) Inner
58595820 const target = if (is_dispatch_loop) target: {
58605821 const initial_target = try cg.resolveInst(switch_br.operand);
58615822 const target: WValue = try cg.allocLocal(target_ty);
5862 try cg.lowerToStack(initial_target);
5823 try cg.emitWValue(initial_target);
58635824 try cg.addLocal(.local_set, target.local.value);
58645825
58655826 try cg.startBlock(.loop, .empty); // dispatch loop start
......@@ -6052,7 +6013,7 @@ fn airSwitchDispatch(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
60526013 const switch_loop = cg.blocks.get(br.block_inst).?;
60536014
60546015 const operand = try cg.resolveInst(br.operand);
6055 try cg.lowerToStack(operand);
6016 try cg.emitWValue(operand);
60566017 try cg.addLocal(.local_set, switch_loop.value.local.value);
60576018
60586019 const idx: u32 = cg.block_depth - switch_loop.label;
......@@ -6080,12 +6041,14 @@ fn airIsErr(cg: *CodeGen, inst: Air.Inst.Index, opcode: std.wasm.Opcode, op_kind
60806041 }
60816042 }
60826043
6083 try cg.emitWValue(operand);
60846044 if (op_kind == .ptr or pl_ty.hasRuntimeBits(zcu)) {
6045 try cg.emitMemBase(operand);
60856046 try cg.addMemArg(.i32_load16_u, .{
6086 .offset = operand.offset() + @as(u32, @intCast(errUnionErrorOffset(pl_ty, zcu))),
6047 .offset = operand.memOffset() + errUnionErrorOffset(pl_ty, zcu),
60876048 .alignment = Type.anyerror.abiAlignment(zcu).toStdMem(),
60886049 });
6050 } else {
6051 try cg.emitWValue(operand);
60896052 }
60906053
60916054 // Compare the error value with '0'
......@@ -6116,9 +6079,9 @@ fn airUnwrapErrUnionPayload(cg: *CodeGen, inst: Air.Inst.Index, op_is_ptr: bool)
61166079 }
61176080 }
61186081
6119 const pl_offset: u32 = @intCast(errUnionPayloadOffset(payload_ty, zcu));
6082 const pl_offset = errUnionPayloadOffset(payload_ty, zcu);
61206083 if (op_is_ptr or isByRef(payload_ty, zcu, cg.target)) {
6121 break :result try cg.buildPointerOffset(operand, pl_offset, .new);
6084 break :result try cg.offsetPointer(operand, pl_offset);
61226085 } else {
61236086 assert(isByRef(eu_ty, zcu, cg.target));
61246087 break :result try cg.load(operand, payload_ty, .{ .offset = pl_offset });
......@@ -6169,15 +6132,14 @@ fn airWrapErrUnionPayload(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
61696132 }
61706133
61716134 const err_union = try cg.allocStack(err_ty);
6172 const payload_ptr = try cg.buildPointerOffset(err_union, @as(u32, @intCast(errUnionPayloadOffset(pl_ty, zcu))), .new);
6173 try cg.store(payload_ptr, operand, pl_ty, .{});
6135 try cg.store(err_union, operand, pl_ty, .{ .offset = errUnionPayloadOffset(pl_ty, zcu) });
61746136
61756137 // ensure we also write '0' to the error part, so any present stack value gets overwritten by it.
6176 try cg.emitWValue(err_union);
6138 try cg.emitMemBase(err_union);
61776139 try cg.addImm32(0);
6178 const err_val_offset: u32 = @intCast(errUnionErrorOffset(pl_ty, zcu));
6140 const err_val_offset = errUnionErrorOffset(pl_ty, zcu);
61796141 try cg.addMemArg(.i32_store16, .{
6180 .offset = err_union.offset() + err_val_offset,
6142 .offset = err_union.memOffset() + err_val_offset,
61816143 .alignment = .@"2",
61826144 });
61836145 break :result err_union;
......@@ -6203,7 +6165,7 @@ fn airWrapErrUnionErr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
62036165 try cg.store(err_union, operand, .anyerror, .{ .offset = errUnionErrorOffset(pl_ty, zcu) });
62046166
62056167 // write 'undefined' to the payload
6206 const payload_ptr = try cg.buildPointerOffset(err_union, @as(u32, @intCast(errUnionPayloadOffset(pl_ty, zcu))), .new);
6168 const payload_ptr = try cg.offsetPointer(err_union, errUnionPayloadOffset(pl_ty, zcu));
62076169 const len = @as(u32, @intCast(err_ty.errorUnionPayload(zcu).abiSize(zcu)));
62086170 try cg.memset(Type.u8, payload_ptr, .{ .imm32 = len }, .{ .imm32 = 0xaa });
62096171
......@@ -6228,7 +6190,6 @@ fn airIsNull(cg: *CodeGen, inst: Air.Inst.Index, opcode: std.wasm.Opcode, op_kin
62286190fn isNull(cg: *CodeGen, operand: WValue, op_ty: Type, opcode: std.wasm.Opcode, op_kind: OpKind) InnerError!WValue {
62296191 const pt = cg.pt;
62306192 const zcu = pt.zcu;
6231 try cg.emitWValue(operand);
62326193 const optional_ty = switch (op_kind) {
62336194 .value => op_ty,
62346195 .ptr => op_ty.childType(zcu),
......@@ -6241,19 +6202,27 @@ fn isNull(cg: *CodeGen, operand: WValue, op_ty: Type, opcode: std.wasm.Opcode, o
62416202 const offset = std.math.cast(u32, payload_ty.abiSize(zcu)) orelse {
62426203 return cg.fail("Optional type {f} too big to fit into stack frame", .{optional_ty.fmt(pt)});
62436204 };
6244 try cg.addMemArg(.i32_load8_u, .{ .offset = operand.offset() + offset, .alignment = .@"1" });
6205 try cg.emitMemBase(operand);
6206 try cg.addMemArg(.i32_load8_u, .{ .offset = operand.memOffset() + offset, .alignment = .@"1" });
6207 } else {
6208 try cg.emitWValue(operand);
62456209 }
62466210 } else if (payload_ty.isSlice(zcu)) {
6211 const base_offset = operand.memOffset();
6212 try cg.emitMemBase(operand);
62476213 switch (cg.ptr_size) {
6248 .wasm32 => try cg.addMemArg(.i32_load, .{ .offset = operand.offset(), .alignment = .@"4" }),
6249 .wasm64 => try cg.addMemArg(.i64_load, .{ .offset = operand.offset(), .alignment = .@"8" }),
6214 .wasm32 => try cg.addMemArg(.i32_load, .{ .offset = base_offset, .alignment = .@"4" }),
6215 .wasm64 => try cg.addMemArg(.i64_load, .{ .offset = base_offset, .alignment = .@"8" }),
62506216 }
62516217 } else {
62526218 if (op_kind == .ptr) {
6219 try cg.emitMemBase(operand);
62536220 try cg.addMemArg(.i32_load, .{
6254 .offset = operand.offset(),
6221 .offset = operand.memOffset(),
62556222 .alignment = .@"4",
62566223 });
6224 } else {
6225 try cg.emitWValue(operand);
62576226 }
62586227 }
62596228
......@@ -6278,7 +6247,7 @@ fn airOptionalPayload(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
62786247 if (opt_ty.optionalReprIsPayload(zcu)) break :result operand;
62796248
62806249 if (isByRef(payload_ty, zcu, cg.target)) {
6281 break :result try cg.buildPointerOffset(operand, 0, .new);
6250 break :result operand;
62826251 }
62836252
62846253 break :result try cg.load(operand, payload_ty, .{});
......@@ -6298,7 +6267,7 @@ fn airOptionalPayloadPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
62986267 break :result operand;
62996268 }
63006269
6301 break :result try cg.buildPointerOffset(operand, 0, .new);
6270 break :result operand;
63026271 };
63036272 return cg.finishAir(inst, result, &.{ty_op.operand});
63046273}
......@@ -6319,12 +6288,11 @@ fn airOptionalPayloadPtrSet(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void
63196288 return cg.fail("Optional type {f} too big to fit into stack frame", .{opt_ty.fmt(pt)});
63206289 };
63216290
6322 try cg.emitWValue(operand);
6291 try cg.emitMemBase(operand);
63236292 try cg.addImm32(1);
6324 try cg.addMemArg(.i32_store8, .{ .offset = operand.offset() + offset, .alignment = .@"1" });
6293 try cg.addMemArg(.i32_store8, .{ .offset = operand.memOffset() + offset, .alignment = .@"1" });
63256294
6326 const result = try cg.buildPointerOffset(operand, 0, .new);
6327 return cg.finishAir(inst, result, &.{ty_op.operand});
6295 return cg.finishAir(inst, operand, &.{ty_op.operand});
63286296}
63296297
63306298fn airWrapOptional(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
......@@ -6336,9 +6304,9 @@ fn airWrapOptional(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
63366304 const result = result: {
63376305 if (!payload_ty.hasRuntimeBits(zcu)) {
63386306 const non_null_bit = try cg.allocStack(Type.u1);
6339 try cg.emitWValue(non_null_bit);
6307 try cg.emitMemBase(non_null_bit);
63406308 try cg.addImm32(1);
6341 try cg.addMemArg(.i32_store8, .{ .offset = non_null_bit.offset(), .alignment = .@"1" });
6309 try cg.addMemArg(.i32_store8, .{ .offset = non_null_bit.memOffset(), .alignment = .@"1" });
63426310 break :result non_null_bit;
63436311 }
63446312
......@@ -6353,12 +6321,11 @@ fn airWrapOptional(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
63536321
63546322 // Create optional type, set the non-null bit, and store the operand inside the optional type
63556323 const result_ptr = try cg.allocStack(op_ty);
6356 try cg.emitWValue(result_ptr);
6324 try cg.emitMemBase(result_ptr);
63576325 try cg.addImm32(1);
6358 try cg.addMemArg(.i32_store8, .{ .offset = result_ptr.offset() + offset, .alignment = .@"1" });
6326 try cg.addMemArg(.i32_store8, .{ .offset = result_ptr.memOffset() + offset, .alignment = .@"1" });
63596327
6360 const payload_ptr = try cg.buildPointerOffset(result_ptr, 0, .new);
6361 try cg.store(payload_ptr, operand, payload_ty, .{});
6328 try cg.store(result_ptr, operand, payload_ty, .{});
63626329 break :result result_ptr;
63636330 };
63646331
......@@ -6483,7 +6450,7 @@ fn airPtrElemVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
64836450 if (ptr_ty.isSlice(zcu)) {
64846451 _ = try cg.load(ptr, .usize, .{});
64856452 } else {
6486 try cg.lowerToStack(ptr);
6453 try cg.emitWValue(ptr);
64876454 }
64886455
64896456 // calculate index into slice
......@@ -6513,7 +6480,7 @@ fn airPtrElemPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
65136480 if (ptr_ty.isSlice(zcu)) {
65146481 _ = try cg.load(ptr, .usize, .{});
65156482 } else {
6516 try cg.lowerToStack(ptr);
6483 try cg.emitWValue(ptr);
65176484 }
65186485
65196486 // calculate index into ptr
......@@ -6538,7 +6505,7 @@ fn airPtrBinOp(cg: *CodeGen, inst: Air.Inst.Index, op: enum { add, sub }) InnerE
65386505 else => ptr_ty.childType(zcu),
65396506 };
65406507
6541 try cg.lowerToStack(ptr);
6508 try cg.emitWValue(ptr);
65426509 try cg.emitWValue(offset);
65436510
65446511 switch (cg.ptr_size) {
......@@ -6620,7 +6587,7 @@ fn memset(cg: *CodeGen, elem_ty: Type, ptr: WValue, len: WValue, value: WValue)
66206587 try cg.addLabel(.br_if, 0);
66216588 }
66226589
6623 try cg.lowerToStack(ptr);
6590 try cg.emitWValue(ptr);
66246591 try cg.emitWValue(value);
66256592 try cg.emitWValue(len);
66266593 try cg.addExtended(.memory_fill);
......@@ -6655,11 +6622,13 @@ fn memset(cg: *CodeGen, elem_ty: Type, ptr: WValue, len: WValue, value: WValue)
66556622
66566623 var end_ptr = try cg.allocLocal(Type.usize);
66576624 defer end_ptr.free(cg);
6658 var new_ptr = try cg.buildPointerOffset(ptr, 0, .new);
6625 var new_ptr = try cg.allocLocal(Type.usize);
66596626 defer new_ptr.free(cg);
6627 try cg.emitWValue(ptr);
6628 try cg.addLocal(.local_set, new_ptr.local.value);
66606629
66616630 // get the loop conditional: if current pointer address equals final pointer's address
6662 try cg.lowerToStack(ptr);
6631 try cg.emitWValue(ptr);
66636632 try cg.emitWValue(final_len);
66646633 switch (cg.ptr_size) {
66656634 .wasm32 => try cg.addTag(.i32_add),
......@@ -6714,7 +6683,7 @@ fn airArrayElemVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
67146683 const elem_size = elem_ty.abiSize(zcu);
67156684
67166685 if (isByRef(array_ty, zcu, cg.target)) {
6717 try cg.lowerToStack(array);
6686 try cg.emitWValue(array);
67186687 try cg.emitWValue(index);
67196688 try cg.addImm32(@intCast(elem_size));
67206689 try cg.addTag(.i32_mul);
......@@ -6747,7 +6716,7 @@ fn airArrayElemVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
67476716 try cg.store(stack_vec, array, array_ty, .{});
67486717
67496718 // Is a non-unrolled vector (v128)
6750 try cg.lowerToStack(stack_vec);
6719 try cg.emitWValue(stack_vec);
67516720 try cg.emitWValue(index);
67526721 try cg.addImm32(@intCast(elem_size));
67536722 try cg.addTag(.i32_mul);
......@@ -6783,13 +6752,13 @@ fn airSplat(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
67836752 64 => @backingInt(std.wasm.SimdOpcode.v128_load64_splat),
67846753 else => break :blk, // Cannot make use of simd-instructions
67856754 };
6786 try cg.emitWValue(operand);
6755 try cg.emitMemBase(operand);
67876756 const extra_index: u32 = @intCast(cg.mir_extra.items.len);
67886757 // stores as := opcode, offset, alignment (opcode::memarg)
67896758 try cg.mir_extra.ensureUnusedCapacity(cg.gpa, 4);
67906759 cg.mir_extra.appendAssumeCapacity(opcode);
67916760 _ = try cg.addExtraAssumeCapacity(Mir.MemArg{
6792 .offset = operand.offset(),
6761 .offset = operand.memOffset(),
67936762 .alignment = elem_ty.abiAlignment(zcu).toStdMem(),
67946763 });
67956764 try cg.addInst(.{ .tag = .simd_prefix, .data = .{ .payload = extra_index } });
......@@ -6856,12 +6825,12 @@ fn airShuffleOne(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
68566825
68576826 const dest_alloc = try cg.allocStack(result_ty);
68586827 for (mask, 0..) |mask_elem, out_idx| {
6859 try cg.emitWValue(dest_alloc);
6828 try cg.emitMemBase(dest_alloc);
68606829 const elem_val = switch (mask_elem.unwrap()) {
68616830 .elem => |idx| try cg.load(operand, elem_ty, .{ .offset = elem_size * idx }),
68626831 .value => |val| try cg.lowerConstant(.fromInterned(val)),
68636832 };
6864 try cg.store(.stack, elem_val, elem_ty, .{ .offset = dest_alloc.offset() + elem_size * out_idx });
6833 try cg.store(.stack, elem_val, elem_ty, .{ .offset = dest_alloc.memOffset() + elem_size * out_idx });
68656834 }
68666835 return cg.finishAir(inst, dest_alloc, &.{unwrapped.operand});
68676836}
......@@ -6924,13 +6893,13 @@ fn airShuffleTwo(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
69246893
69256894 const dest_alloc = try cg.allocStack(result_ty);
69266895 for (mask, 0..) |mask_elem, out_idx| {
6927 try cg.emitWValue(dest_alloc);
6896 try cg.emitMemBase(dest_alloc);
69286897 const elem_val = switch (mask_elem.unwrap()) {
69296898 .a_elem => |idx| try cg.load(operand_a, elem_ty, .{ .offset = elem_size * idx }),
69306899 .b_elem => |idx| try cg.load(operand_b, elem_ty, .{ .offset = elem_size * idx }),
69316900 .undef => try cg.emitUndefined(elem_ty),
69326901 };
6933 try cg.store(.stack, elem_val, elem_ty, .{ .offset = dest_alloc.offset() + elem_size * out_idx });
6902 try cg.store(.stack, elem_val, elem_ty, .{ .offset = dest_alloc.memOffset() + elem_size * out_idx });
69346903 }
69356904 return cg.finishAir(inst, dest_alloc, &.{ unwrapped.operand_a, unwrapped.operand_b });
69366905}
......@@ -6956,39 +6925,18 @@ fn airAggregateInit(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
69566925 .array, .vector => {
69576926 const result = try cg.allocStack(result_ty);
69586927 const elem_ty = result_ty.childType(zcu);
6959 const elem_size = @as(u32, @intCast(elem_ty.abiSize(zcu)));
6928 const elem_size = elem_ty.abiSize(zcu);
69606929 const sentinel = result_ty.sentinel(zcu);
69616930
6962 // When the element type is by reference, we must copy the entire
6963 // value. It is therefore safer to move the offset pointer and store
6964 // each value individually, instead of using store offsets.
6965 if (isByRef(elem_ty, zcu, cg.target)) {
6966 // copy stack pointer into a temporary local, which is
6967 // moved for each element to store each value in the right position.
6968 const offset = try cg.buildPointerOffset(result, 0, .new);
6969 for (elements, 0..) |elem, elem_index| {
6970 const elem_val = try cg.resolveInst(elem);
6971 try cg.store(offset, elem_val, elem_ty, .{});
6972
6973 if (elem_index < elements.len - 1 or sentinel != null) {
6974 _ = try cg.buildPointerOffset(offset, elem_size, .modify);
6975 }
6976 }
6977 if (sentinel) |s| {
6978 const val = try cg.resolveValue(s);
6979 try cg.store(offset, val, elem_ty, .{});
6980 }
6981 } else {
6982 var offset: u32 = 0;
6983 for (elements) |elem| {
6984 const elem_val = try cg.resolveInst(elem);
6985 try cg.store(result, elem_val, elem_ty, .{ .offset = offset });
6986 offset += elem_size;
6987 }
6988 if (sentinel) |s| {
6989 const val = try cg.resolveValue(s);
6990 try cg.store(result, val, elem_ty, .{ .offset = offset });
6991 }
6931 var offset: u64 = 0;
6932 for (elements) |elem| {
6933 const elem_val = try cg.resolveInst(elem);
6934 try cg.store(result, elem_val, elem_ty, .{ .offset = offset });
6935 offset += elem_size;
6936 }
6937 if (sentinel) |s| {
6938 const val = try cg.resolveValue(s);
6939 try cg.store(result, val, elem_ty, .{ .offset = offset });
69926940 }
69936941 break :result_value result;
69946942 },
......@@ -7001,10 +6949,9 @@ fn airAggregateInit(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
70016949
70026950 const elem_ty = result_ty.fieldType(elem_index, zcu);
70036951 const field_offset = result_ty.structFieldOffset(elem_index, zcu);
7004 const offset = try cg.buildPointerOffset(result, field_offset, .new);
70056952
70066953 const value = try cg.resolveInst(elem);
7007 try cg.store(offset, value, elem_ty, .{});
6954 try cg.store(result, value, elem_ty, .{ .offset = field_offset });
70086955 }
70096956
70106957 break :result_value result;
......@@ -7051,12 +6998,7 @@ fn airUnionInit(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
70516998 const result_ptr = try cg.allocStack(union_ty);
70526999 const payload = try cg.resolveInst(extra.init);
70537000 if (layout.tag_align.compare(.gte, layout.payload_align)) {
7054 if (isByRef(field_ty, zcu, cg.target)) {
7055 const payload_ptr = try cg.buildPointerOffset(result_ptr, layout.tag_size, .new);
7056 try cg.store(payload_ptr, payload, field_ty, .{});
7057 } else {
7058 try cg.store(result_ptr, payload, field_ty, .{ .offset = layout.tag_size });
7059 }
7001 try cg.store(result_ptr, payload, field_ty, .{ .offset = layout.tag_size });
70607002
70617003 if (layout.tag_size > 0) {
70627004 try cg.store(result_ptr, tag_int, .fromInterned(union_obj.enum_tag_type), .{});
......@@ -7163,13 +7105,11 @@ fn airErrUnionPayloadPtrSet(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void
71637105 .{ .offset = errUnionErrorOffset(payload_ty, zcu) },
71647106 );
71657107
7166 const result = result: {
7167 if (!payload_ty.hasRuntimeBits(zcu)) {
7168 break :result operand;
7169 }
7108 if (!payload_ty.hasRuntimeBits(zcu)) {
7109 return cg.finishAir(inst, operand, &.{ty_op.operand});
7110 }
71707111
7171 break :result try cg.buildPointerOffset(operand, @as(u32, @intCast(errUnionPayloadOffset(payload_ty, zcu))), .new);
7172 };
7112 const result = try cg.offsetPointer(operand, errUnionPayloadOffset(payload_ty, zcu));
71737113 return cg.finishAir(inst, result, &.{ty_op.operand});
71747114}
71757115
......@@ -7194,16 +7134,23 @@ fn airFieldParentPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
71947134 },
71957135 };
71967136
7197 const result = if (field_offset != 0) result: {
7198 const base = try cg.buildPointerOffset(field_ptr, 0, .new);
7199 try cg.addLocal(.local_get, base.local.value);
7200 try cg.addImm32(@intCast(field_offset));
7201 try cg.addTag(.i32_sub);
7202 try cg.addLocal(.local_set, base.local.value);
7203 break :result base;
7204 } else field_ptr;
7137 if (field_offset == 0) {
7138 return cg.finishAir(inst, field_ptr, &.{extra.field_ptr});
7139 }
72057140
7206 return cg.finishAir(inst, result, &.{extra.field_ptr});
7141 // maybe use offsetPointer? sub looks better in disasm
7142 try cg.emitWValue(field_ptr);
7143 switch (cg.ptr_size) {
7144 .wasm32 => {
7145 try cg.addImm32(@intCast(field_offset));
7146 try cg.addTag(.i32_sub);
7147 },
7148 .wasm64 => {
7149 try cg.addImm64(field_offset);
7150 try cg.addTag(.i64_sub);
7151 },
7152 }
7153 return cg.finishAir(inst, .stack, &.{extra.field_ptr});
72077154}
72087155
72097156fn sliceOrArrayPtr(cg: *CodeGen, ptr: WValue, ptr_ty: Type) InnerError!WValue {
......@@ -7322,7 +7269,7 @@ fn airErrorName(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
73227269fn airPtrSliceFieldPtr(cg: *CodeGen, inst: Air.Inst.Index, offset: u32) InnerError!void {
73237270 const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
73247271 const slice_ptr = try cg.resolveInst(ty_op.operand);
7325 const result = try cg.buildPointerOffset(slice_ptr, offset, .new);
7272 const result = try cg.offsetPointer(slice_ptr, offset);
73267273 return cg.finishAir(inst, result, &.{ty_op.operand});
73277274}
73287275
......@@ -7392,13 +7339,15 @@ fn lowerTry(
73927339 try cg.startBlock(.block, .empty);
73937340
73947341 // check if the error tag is set for the error union.
7395 try cg.emitWValue(err_union);
73967342 if (pl_has_bits or operand_is_ptr) {
7397 const err_offset: u32 = @intCast(errUnionErrorOffset(pl_ty, zcu));
7343 try cg.emitMemBase(err_union);
7344 const err_offset = errUnionErrorOffset(pl_ty, zcu);
73987345 try cg.addMemArg(.i32_load16_u, .{
7399 .offset = err_union.offset() + err_offset,
7346 .offset = err_union.memOffset() + err_offset,
74007347 .alignment = Type.anyerror.abiAlignment(zcu).toStdMem(),
74017348 });
7349 } else {
7350 try cg.emitWValue(err_union);
74027351 }
74037352 try cg.addTag(.i32_eqz);
74047353 try cg.addLabel(.br_if, 0); // jump out of block when error is '0'
......@@ -7417,9 +7366,9 @@ fn lowerTry(
74177366 return .none;
74187367 }
74197368
7420 const pl_offset: u32 = @intCast(errUnionPayloadOffset(pl_ty, zcu));
7369 const pl_offset = errUnionPayloadOffset(pl_ty, zcu);
74217370 if (operand_is_ptr or isByRef(pl_ty, zcu, cg.target)) {
7422 return buildPointerOffset(cg, err_union, pl_offset, .new);
7371 return cg.offsetPointer(err_union, pl_offset);
74237372 }
74247373 const payload = try cg.load(err_union, pl_ty, .{ .offset = pl_offset });
74257374 return payload.toLocal(cg, pl_ty);
......@@ -7448,7 +7397,7 @@ fn callIntrinsic(
74487397 // and emit it as our first argument
74497398 const sret = if (want_sret_param) blk: {
74507399 const sret_local = try cg.allocStack(return_type);
7451 try cg.lowerToStack(sret_local);
7400 try cg.emitWValue(sret_local);
74527401 break :blk sret_local;
74537402 } else .none;
74547403
......@@ -7476,7 +7425,7 @@ fn airTagName(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
74767425 const enum_ty = cg.typeOf(un_op);
74777426
74787427 try cg.addInst(.{ .tag = .enum_tag_name_table_ref, .data = .{ .ip_index = enum_ty.toIntern() } });
7479 try cg.lowerToStack(operand);
7428 try cg.emitWValue(operand);
74807429 try cg.addInst(.{ .tag = .call_tag_index, .data = .{ .ip_index = enum_ty.toIntern() } });
74817430
74827431 switch (cg.ptr_size) {
......@@ -7500,7 +7449,7 @@ fn airIsNamedEnumValue(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
75007449 const operand = try cg.resolveInst(un_op);
75017450 const enum_ty = cg.typeOf(un_op);
75027451
7503 try cg.lowerToStack(operand);
7452 try cg.emitWValue(operand);
75047453 try cg.addInst(.{ .tag = .call_tag_index, .data = .{ .ip_index = enum_ty.toIntern() } });
75057454 try cg.addImm32(~@as(u32, 0));
75067455 try cg.addTag(.i32_ne);
......@@ -7614,9 +7563,9 @@ fn airCmpxchg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
76147563
76157564 const ptr_val = if (cg.useAtomicFeature()) val: {
76167565 const val_local = try cg.allocLocal(ty);
7617 try cg.emitWValue(ptr_operand);
7618 try cg.lowerToStack(expected_val);
7619 try cg.lowerToStack(new_val);
7566 try cg.emitMemBase(ptr_operand);
7567 try cg.emitWValue(expected_val);
7568 try cg.emitWValue(new_val);
76207569 try cg.addAtomicMemArg(switch (ty.abiSize(zcu)) {
76217570 1 => .i32_atomic_rmw8_cmpxchg_u,
76227571 2 => .i32_atomic_rmw16_cmpxchg_u,
......@@ -7624,7 +7573,7 @@ fn airCmpxchg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
76247573 8 => .i32_atomic_rmw_cmpxchg,
76257574 else => |size| return cg.fail("TODO: implement `@cmpxchg` for types with abi size '{d}'", .{size}),
76267575 }, .{
7627 .offset = ptr_operand.offset(),
7576 .offset = ptr_operand.memOffset(),
76287577 .alignment = ty.abiAlignment(zcu).toStdMem(),
76297578 });
76307579 try cg.addLocal(.local_tee, val_local.local.value);
......@@ -7637,13 +7586,13 @@ fn airCmpxchg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
76377586 }
76387587 const ptr_val = try WValue.toLocal(try cg.load(ptr_operand, ty, .{}), cg, ty);
76397588
7640 try cg.lowerToStack(ptr_operand);
7641 try cg.lowerToStack(new_val);
7589 try cg.emitMemBase(ptr_operand);
7590 try cg.emitWValue(new_val);
76427591 try cg.emitWValue(ptr_val);
76437592 _ = try cg.intCmp(int_ty, .eq, ptr_val, expected_val);
76447593 try cg.addLocal(.local_tee, cmp_result.local.value);
76457594 try cg.addTag(.select);
7646 try cg.store(.stack, .stack, ty, .{});
7595 try cg.store(.stack, .stack, ty, .{ .offset = ptr_operand.memOffset() });
76477596
76487597 break :val ptr_val;
76497598 };
......@@ -7684,9 +7633,9 @@ fn airAtomicLoad(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
76847633 8 => .i64_atomic_load,
76857634 else => |size| return cg.fail("TODO: @atomicLoad for types with abi size {d}", .{size}),
76867635 };
7687 try cg.emitWValue(ptr);
7636 try cg.emitMemBase(ptr);
76887637 try cg.addAtomicMemArg(tag, .{
7689 .offset = ptr.offset(),
7638 .offset = ptr.memOffset(),
76907639 .alignment = ty.abiAlignment(zcu).toStdMem(),
76917640 });
76927641 } else {
......@@ -7719,7 +7668,7 @@ fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
77197668 // create a loop to cmpxchg the new value
77207669 try cg.startBlock(.loop, .empty);
77217670
7722 try cg.emitWValue(ptr);
7671 try cg.emitMemBase(ptr);
77237672 try cg.emitWValue(value);
77247673 if (op == .Nand) {
77257674 const and_res = try cg.intAnd(int_ty, value, operand);
......@@ -7746,7 +7695,7 @@ fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
77467695 else => return cg.fail("TODO: implement `@atomicRmw` with operation `{s}` for types larger than 64 bits", .{@tagName(op)}),
77477696 },
77487697 .{
7749 .offset = ptr.offset(),
7698 .offset = ptr.memOffset(),
77507699 .alignment = ty.abiAlignment(zcu).toStdMem(),
77517700 },
77527701 );
......@@ -7764,7 +7713,7 @@ fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
77647713
77657714 // the other operations have their own instructions for Wasm.
77667715 else => {
7767 try cg.emitWValue(ptr);
7716 try cg.emitMemBase(ptr);
77687717 try cg.emitWValue(operand);
77697718 const tag: std.wasm.AtomicsOpcode = switch (ty.abiSize(zcu)) {
77707719 1 => switch (op) {
......@@ -7806,7 +7755,7 @@ fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
78067755 else => |size| return cg.fail("TODO: Implement `@atomicRmw` for types with abi size {d}", .{size}),
78077756 };
78087757 try cg.addAtomicMemArg(tag, .{
7809 .offset = ptr.offset(),
7758 .offset = ptr.memOffset(),
78107759 .alignment = ty.abiAlignment(zcu).toStdMem(),
78117760 });
78127761 return cg.finishAir(inst, .stack, &.{ pl_op.operand, extra.operand });
......@@ -7825,23 +7774,23 @@ fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
78257774 => {
78267775 if (ty.isAnyFloat()) {
78277776 const float_ty: FloatType = .fromType(cg, ty);
7828 try cg.emitWValue(ptr);
7777 try cg.emitMemBase(ptr);
78297778 _ = switch (op) {
78307779 .Add => try cg.floatAdd(float_ty, result, operand),
78317780 .Sub => try cg.floatSub(float_ty, result, operand),
78327781 else => unreachable,
78337782 };
7834 try cg.store(.stack, .stack, ty, .{ .offset = ptr.offset() });
7783 try cg.store(.stack, .stack, ty, .{ .offset = ptr.memOffset() });
78357784 } else {
78367785 const int_ty: IntType = .fromType(cg, ty);
7837 try cg.emitWValue(ptr);
7786 try cg.emitMemBase(ptr);
78387787 _ = switch (op) {
78397788 .Add => try cg.intAdd(int_ty, result, operand),
78407789 .Sub => try cg.intSub(int_ty, result, operand),
78417790 else => unreachable,
78427791 };
78437792 _ = try cg.intWrap(int_ty, .stack);
7844 try cg.store(.stack, .stack, ty, .{ .offset = ptr.offset() });
7793 try cg.store(.stack, .stack, ty, .{ .offset = ptr.memOffset() });
78457794 }
78467795 },
78477796 .And,
......@@ -7849,39 +7798,39 @@ fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
78497798 .Xor,
78507799 => {
78517800 const int_ty: IntType = .fromType(cg, ty);
7852 try cg.emitWValue(ptr);
7801 try cg.emitMemBase(ptr);
78537802 _ = switch (op) {
78547803 .And => try cg.intAnd(int_ty, result, operand),
78557804 .Or => try cg.intOr(int_ty, result, operand),
78567805 .Xor => try cg.intXor(int_ty, result, operand),
78577806 else => unreachable,
78587807 };
7859 try cg.store(.stack, .stack, ty, .{ .offset = ptr.offset() });
7808 try cg.store(.stack, .stack, ty, .{ .offset = ptr.memOffset() });
78607809 },
78617810 .Max,
78627811 .Min,
78637812 => {
78647813 if (ty.isAnyFloat()) {
78657814 const float_ty: FloatType = .fromType(cg, ty);
7866 try cg.emitWValue(ptr);
7815 try cg.emitMemBase(ptr);
78677816 try cg.emitWValue(result);
78687817 try cg.emitWValue(operand);
78697818 _ = try cg.floatCmp(float_ty, if (op == .Max) .gt else .lt, result, operand);
78707819 try cg.addTag(.select);
7871 try cg.store(.stack, .stack, ty, .{ .offset = ptr.offset() });
7820 try cg.store(.stack, .stack, ty, .{ .offset = ptr.memOffset() });
78727821 } else {
78737822 const int_ty: IntType = .fromType(cg, ty);
7874 try cg.emitWValue(ptr);
7823 try cg.emitMemBase(ptr);
78757824 try cg.emitWValue(result);
78767825 try cg.emitWValue(operand);
78777826 _ = try cg.intCmp(int_ty, if (op == .Max) .gt else .lt, result, operand);
78787827 try cg.addTag(.select);
7879 try cg.store(.stack, .stack, ty, .{ .offset = ptr.offset() });
7828 try cg.store(.stack, .stack, ty, .{ .offset = ptr.memOffset() });
78807829 }
78817830 },
78827831 .Nand => {
78837832 const int_ty: IntType = .fromType(cg, ty);
7884 try cg.emitWValue(ptr);
7833 try cg.emitMemBase(ptr);
78857834 const and_res = try cg.intAnd(int_ty, result, operand);
78867835 if (int_ty.bits <= 32) {
78877836 try cg.addImm32(~@as(u32, 0));
......@@ -7891,7 +7840,7 @@ fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
78917840 return cg.fail("TODO: `@atomicRmw` with operator `Nand` for types larger than 64 bits", .{});
78927841 }
78937842 _ = try cg.intXor(int_ty, and_res, .stack);
7894 try cg.store(.stack, .stack, ty, .{ .offset = ptr.offset() });
7843 try cg.store(.stack, .stack, ty, .{ .offset = ptr.memOffset() });
78957844 },
78967845 }
78977846
......@@ -7916,10 +7865,10 @@ fn airAtomicStore(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
79167865 8 => .i64_atomic_store,
79177866 else => |size| return cg.fail("TODO: @atomicLoad for types with abi size {d}", .{size}),
79187867 };
7919 try cg.emitWValue(ptr);
7920 try cg.lowerToStack(operand);
7868 try cg.emitMemBase(ptr);
7869 try cg.emitWValue(operand);
79217870 try cg.addAtomicMemArg(tag, .{
7922 .offset = ptr.offset(),
7871 .offset = ptr.memOffset(),
79237872 .alignment = ty.abiAlignment(zcu).toStdMem(),
79247873 });
79257874 } else {
......@@ -7996,7 +7945,7 @@ fn airAsm(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
79967945 return cg.fail("Self-hosted wasm backend requires input constraint to be equal \"r\"", .{});
79977946 }
79987947
7999 try cg.lowerToStack(operand);
7948 try cg.emitWValue(operand);
80007949 const op_local = try WValue.toLocal(.stack, cg, cg.typeOf(input.operand));
80017950
80027951 const gop = try local_map.getOrPutValue(cg.gpa, name, op_local.local.value);