authorgravatar for paul.verigo@gmail.comPavel Verigo <paul.verigo@gmail.com> 2026-08-21 14:36:33+02:00
committergravatar for paul.verigo@gmail.comPavel Verigo <paul.verigo@gmail.com> 2026-09-17 00:04:51+02:00
logb819da2d57316993c5bb77c93d50f626e32eea91
tree688b0efb2a4ce5d8bc5501de2dc4d4dd0e54f56c
parentd12892e8904bbb27aac7baa69f483348e516a30e

wasm: remove lowerToStack and buildOffsetPointer rework

1 files changed, 347 insertions(+), 398 deletions(-)

src/codegen/wasm/CodeGen.zig+347-398
...@@ -271,23 +271,16 @@ const WValue = union(enum) {...@@ -271,23 +271,16 @@ const WValue = union(enum) {
271 /// pointing to where the value lives.271 /// pointing to where the value lives.
272 stack_offset: struct {272 stack_offset: struct {
273 /// Contains the actual value of the offset273 /// Contains the actual value of the offset
274 value: u32,274 value: u64,
275 /// The amount of instructions referencing this `WValue`
276 references: u32,
277 },275 },
278276
279 /// Returns the offset from the bottom of the stack. This is useful when277 /// Returns the offset from the bottom of the stack.
280 /// we use the load or store instruction to ensure we retrieve the value278 fn memOffset(value: WValue) u64 {
281 /// from the correct position, rather than the value that lives at the279 return switch (value) {
282 /// bottom of the stack. For instances where `WValue` is not `stack_value`280 .stack_offset => |stack_offset| stack_offset.value,
283 /// this will return 0, which allows us to simply call this function for all
284 /// loads and stores without requiring checks everywhere.
285 fn offset(value: WValue) u32 {
286 switch (value) {
287 .stack_offset => |stack_offset| return stack_offset.value,
288 .dead => unreachable,281 .dead => unreachable,
289 else => return 0,282 else => 0,
290 }283 };
291 }284 }
292285
293 /// Promotes a `WValue` to a local when given value is on top of the stack.286 /// Promotes a `WValue` to a local when given value is on top of the stack.
...@@ -651,6 +644,30 @@ fn genBlockType(ty: Type, zcu: *const Zcu, target: *const std.Target) std.wasm.B...@@ -651,6 +644,30 @@ fn genBlockType(ty: Type, zcu: *const Zcu, target: *const std.Target) std.wasm.B
651 };644 };
652}645}
653646
647/// Returns `ptr` with `offset` applied.
648fn offsetPointer(cg: *CodeGen, ptr: WValue, offset: u64) InnerError!WValue {
649 if (offset == 0) return ptr;
650 return switch (ptr) {
651 .stack_offset => |stack_offset| .{ .stack_offset = .{
652 .value = stack_offset.value + offset,
653 } },
654 else => result: {
655 try cg.emitWValue(ptr);
656 switch (cg.ptr_size) {
657 .wasm32 => {
658 try cg.addImm32(@intCast(offset));
659 try cg.addTag(.i32_add);
660 },
661 .wasm64 => {
662 try cg.addImm64(offset);
663 try cg.addTag(.i64_add);
664 },
665 }
666 break :result .stack;
667 },
668 };
669}
670
654/// Writes the bytecode depending on the given `WValue` in `val`671/// Writes the bytecode depending on the given `WValue` in `val`
655fn emitWValue(cg: *CodeGen, value: WValue) InnerError!void {672fn emitWValue(cg: *CodeGen, value: WValue) InnerError!void {
656 switch (value) {673 switch (value) {
...@@ -718,7 +735,18 @@ fn emitWValue(cg: *CodeGen, value: WValue) InnerError!void {...@@ -718,7 +735,18 @@ fn emitWValue(cg: *CodeGen, value: WValue) InnerError!void {
718 });735 });
719 }736 }
720 },737 },
721 .stack_offset => try cg.addLocal(.local_get, cg.bottom_stack_value.local.value), // caller must ensure to address the offset738 .stack_offset => |stack_offset| {
739 try cg.emitWValue(cg.bottom_stack_value);
740 _ = try cg.offsetPointer(.stack, stack_offset.value);
741 },
742 }
743}
744
745/// Emits the base address for a memory instruction.
746fn emitMemBase(cg: *CodeGen, ptr: WValue) InnerError!void {
747 switch (ptr) {
748 .stack_offset => try cg.addLocal(.local_get, cg.bottom_stack_value.local.value),
749 else => try cg.emitWValue(ptr),
722 }750 }
723}751}
724752
...@@ -989,7 +1017,7 @@ pub fn firstParamSRet(...@@ -989,7 +1017,7 @@ pub fn firstParamSRet(
989/// it matches the ABI.1017/// it matches the ABI.
990fn lowerArg(cg: *CodeGen, cc: std.lang.CallingConvention, ty: Type, value: WValue) !void {1018fn lowerArg(cg: *CodeGen, cc: std.lang.CallingConvention, ty: Type, value: WValue) !void {
991 if (cc != .wasm_mvp) {1019 if (cc != .wasm_mvp) {
992 return cg.lowerToStack(value);1020 return cg.emitWValue(value);
993 }1021 }
9941022
995 const zcu = cg.pt.zcu;1023 const zcu = cg.pt.zcu;
...@@ -997,7 +1025,7 @@ fn lowerArg(cg: *CodeGen, cc: std.lang.CallingConvention, ty: Type, value: WValu...@@ -997,7 +1025,7 @@ fn lowerArg(cg: *CodeGen, cc: std.lang.CallingConvention, ty: Type, value: WValu
997 switch (abi.classifyType(ty, zcu, cg.target)) {1025 switch (abi.classifyType(ty, zcu, cg.target)) {
998 .direct => |scalar_ty| {1026 .direct => |scalar_ty| {
999 if (!isByRef(ty, zcu, cg.target)) {1027 if (!isByRef(ty, zcu, cg.target)) {
1000 return cg.lowerToStack(value);1028 return cg.emitWValue(value);
1001 } else {1029 } else {
1002 _ = try cg.load(value, scalar_ty, .{});1030 _ = try cg.load(value, scalar_ty, .{});
1003 }1031 }
...@@ -1005,15 +1033,16 @@ fn lowerArg(cg: *CodeGen, cc: std.lang.CallingConvention, ty: Type, value: WValu...@@ -1005,15 +1033,16 @@ fn lowerArg(cg: *CodeGen, cc: std.lang.CallingConvention, ty: Type, value: WValu
1005 .double_i64 => {1033 .double_i64 => {
1006 assert(ty.abiSize(zcu) == 16);1034 assert(ty.abiSize(zcu) == 16);
1007 // in this case we have an integer or float that must be lowered as 2 i64's.1035 // in this case we have an integer or float that must be lowered as 2 i64's.
1008 try cg.emitWValue(value);1036 const base_offset = value.memOffset();
1009 try cg.addMemArg(.i64_load, .{ .offset = value.offset(), .alignment = .@"8" });1037 try cg.emitMemBase(value);
1010 try cg.emitWValue(value);1038 try cg.addMemArg(.i64_load, .{ .offset = base_offset, .alignment = .@"8" });
1011 try cg.addMemArg(.i64_load, .{ .offset = value.offset() + 8, .alignment = .@"8" });1039 try cg.emitMemBase(value);
1040 try cg.addMemArg(.i64_load, .{ .offset = base_offset + 8, .alignment = .@"8" });
1012 },1041 },
1013 .indirect => {1042 .indirect => {
1014 const stack_copy = try cg.allocStack(ty);1043 const stack_copy = try cg.allocStack(ty);
1015 try cg.store(stack_copy, value, ty, .{});1044 try cg.store(stack_copy, value, ty, .{});
1016 return cg.lowerToStack(stack_copy);1045 return cg.emitWValue(stack_copy);
1017 },1046 },
1018 .unrolled => |vector| {1047 .unrolled => |vector| {
1019 const elem_size = vector.elem_type.abiSize(zcu);1048 const elem_size = vector.elem_type.abiSize(zcu);
...@@ -1024,30 +1053,6 @@ fn lowerArg(cg: *CodeGen, cc: std.lang.CallingConvention, ty: Type, value: WValu...@@ -1024,30 +1053,6 @@ fn lowerArg(cg: *CodeGen, cc: std.lang.CallingConvention, ty: Type, value: WValu
1024 }1053 }
1025}1054}
10261055
1027/// Lowers a `WValue` to the stack. This means when the `value` results in
1028/// `.stack_offset` we calculate the pointer of this offset and use that.
1029/// The value is left on the stack, and not stored in any temporary.
1030fn lowerToStack(cg: *CodeGen, value: WValue) !void {
1031 switch (value) {
1032 .stack_offset => |offset| {
1033 try cg.emitWValue(value);
1034 if (offset.value > 0) {
1035 switch (cg.ptr_size) {
1036 .wasm32 => {
1037 try cg.addImm32(offset.value);
1038 try cg.addTag(.i32_add);
1039 },
1040 .wasm64 => {
1041 try cg.addImm64(offset.value);
1042 try cg.addTag(.i64_add);
1043 },
1044 }
1045 }
1046 },
1047 else => try cg.emitWValue(value),
1048 }
1049}
1050
1051/// Creates a local for the initial stack value1056/// Creates a local for the initial stack value
1052/// Asserts `initial_stack_value` is `.none`1057/// Asserts `initial_stack_value` is `.none`
1053fn initializeStack(cg: *CodeGen) !void {1058fn initializeStack(cg: *CodeGen) !void {
...@@ -1113,7 +1118,7 @@ fn allocStackBytes(cg: *CodeGen, size: u32, alignment: Alignment) !WValue {...@@ -1113,7 +1118,7 @@ fn allocStackBytes(cg: *CodeGen, size: u32, alignment: Alignment) !WValue {
1113 const offset: u32 = @intCast(alignment.forward(cg.stack_size));1118 const offset: u32 = @intCast(alignment.forward(cg.stack_size));
1114 defer cg.stack_size = offset + size;1119 defer cg.stack_size = offset + size;
11151120
1116 return .{ .stack_offset = .{ .value = offset, .references = 1 } };1121 return .{ .stack_offset = .{ .value = offset } };
1117}1122}
11181123
1119/// From a given AIR instruction generates a pointer to the stack where1124/// From a given AIR instruction generates a pointer to the stack where
...@@ -1141,7 +1146,7 @@ fn allocStackPtr(cg: *CodeGen, inst: Air.Inst.Index) !WValue {...@@ -1141,7 +1146,7 @@ fn allocStackPtr(cg: *CodeGen, inst: Air.Inst.Index) !WValue {
1141 const offset: u32 = @intCast(abi_alignment.forward(cg.stack_size));1146 const offset: u32 = @intCast(abi_alignment.forward(cg.stack_size));
1142 defer cg.stack_size = offset + abi_size;1147 defer cg.stack_size = offset + abi_size;
11431148
1144 return .{ .stack_offset = .{ .value = offset, .references = 1 } };1149 return .{ .stack_offset = .{ .value = offset } };
1145}1150}
11461151
1147fn emitMemoryCopy(cg: *CodeGen, dst: WValue, src: WValue, len: WValue) !void {1152fn emitMemoryCopy(cg: *CodeGen, dst: WValue, src: WValue, len: WValue) !void {
...@@ -1171,8 +1176,8 @@ fn emitMemoryCopy(cg: *CodeGen, dst: WValue, src: WValue, len: WValue) !void {...@@ -1171,8 +1176,8 @@ fn emitMemoryCopy(cg: *CodeGen, dst: WValue, src: WValue, len: WValue) !void {
1171 try cg.addLabel(.br_if, 0);1176 try cg.addLabel(.br_if, 0);
1172 }1177 }
11731178
1174 try cg.lowerToStack(dst);1179 try cg.emitWValue(dst);
1175 try cg.lowerToStack(src);1180 try cg.emitWValue(src);
1176 try cg.emitWValue(len);1181 try cg.emitWValue(len);
1177 try cg.addExtended(.memory_copy);1182 try cg.addExtended(.memory_copy);
11781183
...@@ -1187,8 +1192,8 @@ fn memcpy(cg: *CodeGen, dst: WValue, src: WValue, len: WValue) !void {...@@ -1187,8 +1192,8 @@ fn memcpy(cg: *CodeGen, dst: WValue, src: WValue, len: WValue) !void {
1187 return;1192 return;
1188 }1193 }
11891194
1190 try cg.lowerToStack(dst);1195 try cg.emitWValue(dst);
1191 try cg.lowerToStack(src);1196 try cg.emitWValue(src);
1192 try cg.emitWValue(len);1197 try cg.emitWValue(len);
1193 try cg.addCallIntrinsic(.memcpy);1198 try cg.addCallIntrinsic(.memcpy);
1194 try cg.addTag(.drop);1199 try cg.addTag(.drop);
...@@ -1200,8 +1205,8 @@ fn memmove(cg: *CodeGen, dst: WValue, src: WValue, len: WValue) !void {...@@ -1200,8 +1205,8 @@ fn memmove(cg: *CodeGen, dst: WValue, src: WValue, len: WValue) !void {
1200 return;1205 return;
1201 }1206 }
12021207
1203 try cg.lowerToStack(dst);1208 try cg.emitWValue(dst);
1204 try cg.lowerToStack(src);1209 try cg.emitWValue(src);
1205 try cg.emitWValue(len);1210 try cg.emitWValue(len);
1206 try cg.addCallIntrinsic(.memmove);1211 try cg.addCallIntrinsic(.memmove);
1207 try cg.addTag(.drop);1212 try cg.addTag(.drop);
...@@ -1283,34 +1288,6 @@ pub fn determineSimdStoreStrategy(ty: Type, zcu: *const Zcu, target: *const std....@@ -1283,34 +1288,6 @@ pub fn determineSimdStoreStrategy(ty: Type, zcu: *const Zcu, target: *const std.
1283 return .unrolled;1288 return .unrolled;
1284}1289}
12851290
1286/// Creates a new local for a pointer that points to memory with given offset.
1287/// This can be used to get a pointer to a struct field, error payload, etc.
1288/// By providing `modify` as action, it will modify the given `ptr_value` instead of making a new
1289/// local value to store the pointer. This allows for local re-use and improves binary size.
1290fn buildPointerOffset(cg: *CodeGen, ptr_value: WValue, offset: u64, action: enum { modify, new }) InnerError!WValue {
1291 // do not perform arithmetic when offset is 0.
1292 if (offset == 0 and ptr_value.offset() == 0 and action == .modify) return ptr_value;
1293 const result_ptr: WValue = switch (action) {
1294 .new => try cg.ensureAllocLocal(Type.usize),
1295 .modify => ptr_value,
1296 };
1297 try cg.emitWValue(ptr_value);
1298 if (offset + ptr_value.offset() > 0) {
1299 switch (cg.ptr_size) {
1300 .wasm32 => {
1301 try cg.addImm32(@intCast(offset + ptr_value.offset()));
1302 try cg.addTag(.i32_add);
1303 },
1304 .wasm64 => {
1305 try cg.addImm64(offset + ptr_value.offset());
1306 try cg.addTag(.i64_add);
1307 },
1308 }
1309 }
1310 try cg.addLocal(.local_set, result_ptr.local.value);
1311 return result_ptr;
1312}
1313
1314fn genInst(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {1291fn genInst(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
1315 const zcu = cg.pt.zcu;1292 const zcu = cg.pt.zcu;
1316 const air_tags = cg.air.instructions.items(.tag);1293 const air_tags = cg.air.instructions.items(.tag);
...@@ -1331,7 +1308,7 @@ fn genInst(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -1331,7 +1308,7 @@ fn genInst(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
1331 const elem_ty = cg.typeOf(bin_op.rhs);1308 const elem_ty = cg.typeOf(bin_op.rhs);
1332 const elem_size = elem_ty.abiSize(zcu);1309 const elem_size = elem_ty.abiSize(zcu);
13331310
1334 try cg.lowerToStack(vec_ptr);1311 try cg.emitWValue(vec_ptr);
1335 try cg.emitWValue(elem_idx);1312 try cg.emitWValue(elem_idx);
1336 try cg.addImm32(@intCast(elem_size));1313 try cg.addImm32(@intCast(elem_size));
1337 try cg.addTag(.i32_mul);1314 try cg.addTag(.i32_mul);
...@@ -2108,7 +2085,7 @@ fn airCall(cg: *CodeGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier)...@@ -2108,7 +2085,7 @@ fn airCall(cg: *CodeGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier)
2108 } else .none;2085 } else .none;
21092086
2110 if (first_param_sret) {2087 if (first_param_sret) {
2111 try cg.lowerToStack(sret);2088 try cg.emitWValue(sret);
2112 }2089 }
21132090
2114 for (args, 0..) |arg, arg_i| {2091 for (args, 0..) |arg, arg_i| {
...@@ -2122,7 +2099,7 @@ fn airCall(cg: *CodeGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier)...@@ -2122,7 +2099,7 @@ fn airCall(cg: *CodeGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier)
2122 }2099 }
21232100
2124 if (fn_info.is_var_args) {2101 if (fn_info.is_var_args) {
2125 try cg.lowerToStack(varargs_buf);2102 try cg.emitWValue(varargs_buf);
2126 }2103 }
21272104
2128 if (callee) |nav_index| {2105 if (callee) |nav_index| {
...@@ -2243,8 +2220,8 @@ fn airVaArg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -2243,8 +2220,8 @@ fn airVaArg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
22432220
2244 try cg.addLocal(.local_set, arg_ptr.local.value);2221 try cg.addLocal(.local_set, arg_ptr.local.value);
22452222
2246 try cg.lowerToStack(operand);2223 try cg.emitWValue(operand);
2247 try cg.lowerToStack(arg_ptr);2224 try cg.emitWValue(arg_ptr);
2248 switch (cg.ptr_size) {2225 switch (cg.ptr_size) {
2249 .wasm32 => {2226 .wasm32 => {
2250 try cg.addImm32(abi_size);2227 try cg.addImm32(abi_size);
...@@ -2325,28 +2302,22 @@ fn store(cg: *CodeGen, lhs: WValue, rhs: WValue, ty: Type, memarg: MemArg) Inner...@@ -2325,28 +2302,22 @@ fn store(cg: *CodeGen, lhs: WValue, rhs: WValue, ty: Type, memarg: MemArg) Inner
2325 natural_alignment.minStrict(memarg.alignment);2302 natural_alignment.minStrict(memarg.alignment);
23262303
2327 if (isByRef(ty, zcu, cg.target)) {2304 if (isByRef(ty, zcu, cg.target)) {
2328 const offset_ptr: WValue = switch (memarg.offset + lhs.offset()) {2305 const offset_ptr = try cg.offsetPointer(lhs, memarg.offset);
2329 0 => lhs,
2330 else => |total_offset| ptr: {
2331 try cg.emitWValue(lhs);
2332 try cg.addImm32(@intCast(total_offset));
2333 try cg.addTag(.i32_add);
2334 break :ptr .stack;
2335 },
2336 };
2337 return cg.memcpy(offset_ptr, rhs, .{ .imm32 = @intCast(abi_size) });2306 return cg.memcpy(offset_ptr, rhs, .{ .imm32 = @intCast(abi_size) });
2338 }2307 }
23392308
2309 const base_offset = lhs.memOffset();
2310 try cg.emitMemBase(lhs);
2311
2340 if (ty.zigTypeTag(zcu) == .vector) {2312 if (ty.zigTypeTag(zcu) == .vector) {
2341 try cg.emitWValue(lhs);2313 try cg.emitWValue(rhs);
2342 try cg.lowerToStack(rhs);
2343 // TODO: Add helper functions for simd opcodes2314 // TODO: Add helper functions for simd opcodes
2344 const extra_index: u32 = @intCast(cg.mir_extra.items.len);2315 const extra_index: u32 = @intCast(cg.mir_extra.items.len);
2345 // stores as := opcode, offset, alignment (opcode::memarg)2316 // stores as := opcode, offset, alignment (opcode::memarg)
2346 try cg.mir_extra.ensureUnusedCapacity(cg.gpa, 4);2317 try cg.mir_extra.ensureUnusedCapacity(cg.gpa, 4);
2347 cg.mir_extra.appendAssumeCapacity(@backingInt(std.wasm.SimdOpcode.v128_store));2318 cg.mir_extra.appendAssumeCapacity(@backingInt(std.wasm.SimdOpcode.v128_store));
2348 _ = try cg.addExtraAssumeCapacity(Mir.MemArg{2319 _ = try cg.addExtraAssumeCapacity(Mir.MemArg{
2349 .offset = memarg.offset + lhs.offset(),2320 .offset = base_offset + memarg.offset,
2350 .alignment = alignment.toStdMem(),2321 .alignment = alignment.toStdMem(),
2351 });2322 });
2352 return cg.addInst(.{ .tag = .simd_prefix, .data = .{ .payload = extra_index } });2323 return cg.addInst(.{ .tag = .simd_prefix, .data = .{ .payload = extra_index } });
...@@ -2371,13 +2342,12 @@ fn store(cg: *CodeGen, lhs: WValue, rhs: WValue, ty: Type, memarg: MemArg) Inner...@@ -2371,13 +2342,12 @@ fn store(cg: *CodeGen, lhs: WValue, rhs: WValue, ty: Type, memarg: MemArg) Inner
2371 }2342 }
2372 };2343 };
23732344
2374 try cg.emitWValue(lhs);2345 try cg.emitWValue(rhs);
2375 try cg.lowerToStack(rhs);
23762346
2377 try cg.addMemArg(2347 try cg.addMemArg(
2378 store_opcode,2348 store_opcode,
2379 .{2349 .{
2380 .offset = memarg.offset + lhs.offset(),2350 .offset = base_offset + memarg.offset,
2381 .alignment = alignment.toStdMem(),2351 .alignment = alignment.toStdMem(),
2382 },2352 },
2383 );2353 );
...@@ -2421,23 +2391,18 @@ fn load(cg: *CodeGen, operand: WValue, ty: Type, memarg: MemArg) InnerError!WVal...@@ -2421,23 +2391,18 @@ fn load(cg: *CodeGen, operand: WValue, ty: Type, memarg: MemArg) InnerError!WVal
2421 natural_alignment.minStrict(memarg.alignment);2391 natural_alignment.minStrict(memarg.alignment);
24222392
2423 if (isByRef(ty, zcu, cg.target)) {2393 if (isByRef(ty, zcu, cg.target)) {
2424 const src_ptr_maybe_stack: WValue = switch (memarg.offset + operand.offset()) {2394 var src_ptr = try cg.offsetPointer(operand, memarg.offset);
2425 0 => operand,2395 const owns_src_ptr = src_ptr == .stack;
2426 else => |total_offset| ptr: {2396 src_ptr = try src_ptr.toLocal(cg, .usize);
2427 try cg.emitWValue(operand);2397 defer if (owns_src_ptr) src_ptr.free(cg);
2428 try cg.addImm32(@intCast(total_offset));2398
2429 try cg.addTag(.i32_add);
2430 break :ptr .stack;
2431 },
2432 };
2433 const src_ptr = try src_ptr_maybe_stack.toLocal(cg, .usize);
2434 const new_ptr = try cg.allocStack(ty);2399 const new_ptr = try cg.allocStack(ty);
2435 try cg.store(new_ptr, src_ptr, ty, .{});2400 try cg.memcpy(new_ptr, src_ptr, .{ .imm32 = @intCast(ty.abiSize(zcu)) });
2436 return new_ptr;2401 return new_ptr;
2437 }2402 }
24382403
2439 // load local's value from memory by its stack position2404 const base_offset = operand.memOffset();
2440 try cg.emitWValue(operand);2405 try cg.emitMemBase(operand);
24412406
2442 if (ty.zigTypeTag(zcu) == .vector) {2407 if (ty.zigTypeTag(zcu) == .vector) {
2443 // TODO: Add helper functions for simd opcodes2408 // TODO: Add helper functions for simd opcodes
...@@ -2446,7 +2411,7 @@ fn load(cg: *CodeGen, operand: WValue, ty: Type, memarg: MemArg) InnerError!WVal...@@ -2446,7 +2411,7 @@ fn load(cg: *CodeGen, operand: WValue, ty: Type, memarg: MemArg) InnerError!WVal
2446 try cg.mir_extra.ensureUnusedCapacity(cg.gpa, 4);2411 try cg.mir_extra.ensureUnusedCapacity(cg.gpa, 4);
2447 cg.mir_extra.appendAssumeCapacity(@backingInt(std.wasm.SimdOpcode.v128_load));2412 cg.mir_extra.appendAssumeCapacity(@backingInt(std.wasm.SimdOpcode.v128_load));
2448 _ = try cg.addExtraAssumeCapacity(Mir.MemArg{2413 _ = try cg.addExtraAssumeCapacity(Mir.MemArg{
2449 .offset = memarg.offset + operand.offset(),2414 .offset = base_offset + memarg.offset,
2450 .alignment = alignment.toStdMem(),2415 .alignment = alignment.toStdMem(),
2451 });2416 });
2452 try cg.addInst(.{ .tag = .simd_prefix, .data = .{ .payload = extra_index } });2417 try cg.addInst(.{ .tag = .simd_prefix, .data = .{ .payload = extra_index } });
...@@ -2477,7 +2442,7 @@ fn load(cg: *CodeGen, operand: WValue, ty: Type, memarg: MemArg) InnerError!WVal...@@ -2477,7 +2442,7 @@ fn load(cg: *CodeGen, operand: WValue, ty: Type, memarg: MemArg) InnerError!WVal
2477 try cg.addMemArg(2442 try cg.addMemArg(
2478 load_opcode,2443 load_opcode,
2479 .{2444 .{
2480 .offset = memarg.offset + operand.offset(),2445 .offset = base_offset + memarg.offset,
2481 .alignment = alignment.toStdMem(),2446 .alignment = alignment.toStdMem(),
2482 },2447 },
2483 );2448 );
...@@ -2647,9 +2612,9 @@ fn intAdd(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue...@@ -2647,9 +2612,9 @@ fn intAdd(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue
2647 else => {2612 else => {
2648 const result = try cg.allocInt(ty);2613 const result = try cg.allocInt(ty);
26492614
2650 try cg.lowerToStack(result);2615 try cg.emitWValue(result);
2651 try cg.lowerToStack(lhs);2616 try cg.emitWValue(lhs);
2652 try cg.lowerToStack(rhs);2617 try cg.emitWValue(rhs);
2653 try cg.addImm32(@intFromBool(ty.is_signed));2618 try cg.addImm32(@intFromBool(ty.is_signed));
2654 try cg.addImm32(ty.bits);2619 try cg.addImm32(ty.bits);
2655 try cg.addCallIntrinsic(.__addo_limb64);2620 try cg.addCallIntrinsic(.__addo_limb64);
...@@ -2700,9 +2665,9 @@ fn intSub(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue...@@ -2700,9 +2665,9 @@ fn intSub(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue
2700 else => {2665 else => {
2701 const result = try cg.allocInt(ty);2666 const result = try cg.allocInt(ty);
27022667
2703 try cg.lowerToStack(result);2668 try cg.emitWValue(result);
2704 try cg.lowerToStack(lhs);2669 try cg.emitWValue(lhs);
2705 try cg.lowerToStack(rhs);2670 try cg.emitWValue(rhs);
2706 try cg.addImm32(@intFromBool(ty.is_signed));2671 try cg.addImm32(@intFromBool(ty.is_signed));
2707 try cg.addImm32(ty.bits);2672 try cg.addImm32(ty.bits);
2708 try cg.addCallIntrinsic(.__subo_limb64);2673 try cg.addCallIntrinsic(.__subo_limb64);
...@@ -2732,9 +2697,9 @@ fn intMul(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue...@@ -2732,9 +2697,9 @@ fn intMul(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue
2732 else => {2697 else => {
2733 const result = try cg.allocInt(ty);2698 const result = try cg.allocInt(ty);
27342699
2735 try cg.lowerToStack(result);2700 try cg.emitWValue(result);
2736 try cg.lowerToStack(lhs);2701 try cg.emitWValue(lhs);
2737 try cg.lowerToStack(rhs);2702 try cg.emitWValue(rhs);
2738 try cg.addImm32(@intFromBool(ty.is_signed));2703 try cg.addImm32(@intFromBool(ty.is_signed));
2739 try cg.addImm32(ty.bits);2704 try cg.addImm32(ty.bits);
2740 try cg.addCallIntrinsic(.__mulo_limb64);2705 try cg.addCallIntrinsic(.__mulo_limb64);
...@@ -3041,9 +3006,9 @@ fn intAnd(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue...@@ -3041,9 +3006,9 @@ fn intAnd(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue
3041 else => {3006 else => {
3042 const result = try cg.allocInt(ty);3007 const result = try cg.allocInt(ty);
30433008
3044 try cg.lowerToStack(result);3009 try cg.emitWValue(result);
3045 try cg.lowerToStack(lhs);3010 try cg.emitWValue(lhs);
3046 try cg.lowerToStack(rhs);3011 try cg.emitWValue(rhs);
3047 try cg.addImm32(ty.bits);3012 try cg.addImm32(ty.bits);
3048 try cg.addCallIntrinsic(.__and_limb64);3013 try cg.addCallIntrinsic(.__and_limb64);
30493014
...@@ -3085,9 +3050,9 @@ fn intOr(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue...@@ -3085,9 +3050,9 @@ fn intOr(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue
3085 else => {3050 else => {
3086 const result = try cg.allocInt(ty);3051 const result = try cg.allocInt(ty);
30873052
3088 try cg.lowerToStack(result);3053 try cg.emitWValue(result);
3089 try cg.lowerToStack(lhs);3054 try cg.emitWValue(lhs);
3090 try cg.lowerToStack(rhs);3055 try cg.emitWValue(rhs);
3091 try cg.addImm32(ty.bits);3056 try cg.addImm32(ty.bits);
3092 try cg.addCallIntrinsic(.__or_limb64);3057 try cg.addCallIntrinsic(.__or_limb64);
30933058
...@@ -3129,9 +3094,9 @@ fn intXor(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue...@@ -3129,9 +3094,9 @@ fn intXor(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue
3129 else => {3094 else => {
3130 const result = try cg.allocInt(ty);3095 const result = try cg.allocInt(ty);
31313096
3132 try cg.lowerToStack(result);3097 try cg.emitWValue(result);
3133 try cg.lowerToStack(lhs);3098 try cg.emitWValue(lhs);
3134 try cg.lowerToStack(rhs);3099 try cg.emitWValue(rhs);
3135 try cg.addImm32(ty.bits);3100 try cg.addImm32(ty.bits);
3136 try cg.addCallIntrinsic(.__xor_limb64);3101 try cg.addCallIntrinsic(.__xor_limb64);
31373102
...@@ -3176,13 +3141,14 @@ fn intNot(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {...@@ -3176,13 +3141,14 @@ fn intNot(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3176 65...128 => {3141 65...128 => {
3177 const result = try cg.allocStack(Type.u128);3142 const result = try cg.allocStack(Type.u128);
31783143
3179 try cg.emitWValue(result);3144 const result_offset = result.memOffset();
3145 try cg.emitMemBase(result);
3180 _ = try cg.load(operand, .u64, .{});3146 _ = try cg.load(operand, .u64, .{});
3181 try cg.addImm64(~@as(u64, 0));3147 try cg.addImm64(~@as(u64, 0));
3182 try cg.addTag(.i64_xor);3148 try cg.addTag(.i64_xor);
3183 try cg.store(.stack, .stack, .u64, .{ .offset = result.offset() });3149 try cg.store(.stack, .stack, .u64, .{ .offset = result_offset });
31843150
3185 try cg.emitWValue(result);3151 try cg.emitMemBase(result);
3186 _ = try cg.load(operand, .u64, .{ .offset = 8 });3152 _ = try cg.load(operand, .u64, .{ .offset = 8 });
3187 const high_mask: u64 = if (ty.is_signed)3153 const high_mask: u64 = if (ty.is_signed)
3188 ~@as(u64, 0)3154 ~@as(u64, 0)
...@@ -3190,15 +3156,15 @@ fn intNot(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {...@@ -3190,15 +3156,15 @@ fn intNot(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3190 ~@as(u64, 0) >> @intCast(128 - ty.bits);3156 ~@as(u64, 0) >> @intCast(128 - ty.bits);
3191 try cg.addImm64(high_mask);3157 try cg.addImm64(high_mask);
3192 try cg.addTag(.i64_xor);3158 try cg.addTag(.i64_xor);
3193 try cg.store(.stack, .stack, .u64, .{ .offset = result.offset() + 8 });3159 try cg.store(.stack, .stack, .u64, .{ .offset = result_offset + 8 });
31943160
3195 return result;3161 return result;
3196 },3162 },
3197 else => {3163 else => {
3198 const result = try cg.allocInt(ty);3164 const result = try cg.allocInt(ty);
31993165
3200 try cg.lowerToStack(result);3166 try cg.emitWValue(result);
3201 try cg.lowerToStack(operand);3167 try cg.emitWValue(operand);
3202 try cg.addImm32(@intFromBool(ty.is_signed));3168 try cg.addImm32(@intFromBool(ty.is_signed));
3203 try cg.addImm32(ty.bits);3169 try cg.addImm32(ty.bits);
3204 try cg.addCallIntrinsic(.__not_limb64);3170 try cg.addCallIntrinsic(.__not_limb64);
...@@ -3230,9 +3196,9 @@ fn intShl(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue...@@ -3230,9 +3196,9 @@ fn intShl(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue
3230 else => {3196 else => {
3231 const result = try cg.allocInt(ty);3197 const result = try cg.allocInt(ty);
32323198
3233 try cg.lowerToStack(result);3199 try cg.emitWValue(result);
3234 try cg.lowerToStack(lhs);3200 try cg.emitWValue(lhs);
3235 try cg.lowerToStack(rhs);3201 try cg.emitWValue(rhs);
3236 try cg.addImm32(@intFromBool(ty.is_signed));3202 try cg.addImm32(@intFromBool(ty.is_signed));
3237 try cg.addImm32(ty.bits);3203 try cg.addImm32(ty.bits);
3238 try cg.addCallIntrinsic(.__shlo_limb64);3204 try cg.addCallIntrinsic(.__shlo_limb64);
...@@ -3270,9 +3236,9 @@ fn intShr(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue...@@ -3270,9 +3236,9 @@ fn intShr(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue
3270 else => {3236 else => {
3271 const result = try cg.allocInt(ty);3237 const result = try cg.allocInt(ty);
32723238
3273 try cg.lowerToStack(result);3239 try cg.emitWValue(result);
3274 try cg.lowerToStack(lhs);3240 try cg.emitWValue(lhs);
3275 try cg.lowerToStack(rhs);3241 try cg.emitWValue(rhs);
3276 try cg.addImm32(@intFromBool(ty.is_signed));3242 try cg.addImm32(@intFromBool(ty.is_signed));
3277 try cg.addImm32(ty.bits);3243 try cg.addImm32(ty.bits);
3278 try cg.addCallIntrinsic(.__shr_limb64);3244 try cg.addCallIntrinsic(.__shr_limb64);
...@@ -3320,8 +3286,9 @@ fn intAbs(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {...@@ -3320,8 +3286,9 @@ fn intAbs(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3320 const u128_ty: IntType = .{ .is_signed = false, .bits = 128 };3286 const u128_ty: IntType = .{ .is_signed = false, .bits = 128 };
33213287
3322 const mask = try cg.allocStack(Type.u128);3288 const mask = try cg.allocStack(Type.u128);
3323 try cg.emitWValue(mask);3289 const mask_offset = mask.memOffset();
3324 try cg.emitWValue(mask);3290 try cg.emitMemBase(mask);
3291 try cg.emitMemBase(mask);
33253292
3326 _ = try cg.load(operand, .u64, .{ .offset = 8 });3293 _ = try cg.load(operand, .u64, .{ .offset = 8 });
3327 try cg.addImm64(63);3294 try cg.addImm64(63);
...@@ -3330,9 +3297,9 @@ fn intAbs(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {...@@ -3330,9 +3297,9 @@ fn intAbs(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3330 var tmp = try cg.allocLocal(Type.u64);3297 var tmp = try cg.allocLocal(Type.u64);
3331 defer tmp.free(cg);3298 defer tmp.free(cg);
3332 try cg.addLocal(.local_tee, tmp.local.value);3299 try cg.addLocal(.local_tee, tmp.local.value);
3333 try cg.store(.stack, .stack, .u64, .{ .offset = mask.offset() + 0 });3300 try cg.store(.stack, .stack, .u64, .{ .offset = mask_offset });
3334 try cg.emitWValue(tmp);3301 try cg.emitWValue(tmp);
3335 try cg.store(.stack, .stack, .u64, .{ .offset = mask.offset() + 8 });3302 try cg.store(.stack, .stack, .u64, .{ .offset = mask_offset + 8 });
33363303
3337 const a = try cg.intXor(u128_ty, operand, mask);3304 const a = try cg.intXor(u128_ty, operand, mask);
3338 const b = try cg.intSub(u128_ty, a, mask);3305 const b = try cg.intSub(u128_ty, a, mask);
...@@ -3341,8 +3308,8 @@ fn intAbs(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {...@@ -3341,8 +3308,8 @@ fn intAbs(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3341 else => {3308 else => {
3342 const result = try cg.allocInt(ty);3309 const result = try cg.allocInt(ty);
33433310
3344 try cg.lowerToStack(result);3311 try cg.emitWValue(result);
3345 try cg.lowerToStack(operand);3312 try cg.emitWValue(operand);
3346 try cg.addImm32(ty.bits);3313 try cg.addImm32(ty.bits);
3347 try cg.addCallIntrinsic(.__abs_limb64);3314 try cg.addCallIntrinsic(.__abs_limb64);
33483315
...@@ -3352,16 +3319,16 @@ fn intAbs(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {...@@ -3352,16 +3319,16 @@ fn intAbs(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3352}3319}
33533320
3354fn intMax(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue {3321fn intMax(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue {
3355 try cg.lowerToStack(lhs);3322 try cg.emitWValue(lhs);
3356 try cg.lowerToStack(rhs);3323 try cg.emitWValue(rhs);
3357 _ = try cg.intCmp(ty, .gt, lhs, rhs);3324 _ = try cg.intCmp(ty, .gt, lhs, rhs);
3358 try cg.addTag(.select);3325 try cg.addTag(.select);
3359 return .stack;3326 return .stack;
3360}3327}
33613328
3362fn intMin(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue {3329fn intMin(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue {
3363 try cg.lowerToStack(lhs);3330 try cg.emitWValue(lhs);
3364 try cg.lowerToStack(rhs);3331 try cg.emitWValue(rhs);
3365 _ = try cg.intCmp(ty, .lt, lhs, rhs);3332 _ = try cg.intCmp(ty, .lt, lhs, rhs);
3366 try cg.addTag(.select);3333 try cg.addTag(.select);
3367 return .stack;3334 return .stack;
...@@ -3427,7 +3394,7 @@ fn intClz(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {...@@ -3427,7 +3394,7 @@ fn intClz(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3427 return .stack;3394 return .stack;
3428 },3395 },
3429 else => {3396 else => {
3430 try cg.lowerToStack(operand);3397 try cg.emitWValue(operand);
3431 try cg.addImm32(ty.bits);3398 try cg.addImm32(ty.bits);
3432 try cg.addCallIntrinsic(.__clz_limb64);3399 try cg.addCallIntrinsic(.__clz_limb64);
34333400
...@@ -3479,7 +3446,7 @@ fn intCtz(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {...@@ -3479,7 +3446,7 @@ fn intCtz(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3479 return .stack;3446 return .stack;
3480 },3447 },
3481 else => {3448 else => {
3482 try cg.lowerToStack(operand);3449 try cg.emitWValue(operand);
3483 try cg.addImm32(ty.bits);3450 try cg.addImm32(ty.bits);
3484 try cg.addCallIntrinsic(.__ctz_limb64);3451 try cg.addCallIntrinsic(.__ctz_limb64);
34853452
...@@ -3525,7 +3492,7 @@ fn intPopCount(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {...@@ -3525,7 +3492,7 @@ fn intPopCount(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3525 return .stack;3492 return .stack;
3526 },3493 },
3527 else => {3494 else => {
3528 try cg.lowerToStack(operand);3495 try cg.emitWValue(operand);
3529 try cg.addImm32(ty.bits);3496 try cg.addImm32(ty.bits);
3530 try cg.addCallIntrinsic(.__popcount_limb64);3497 try cg.addCallIntrinsic(.__popcount_limb64);
35313498
...@@ -3560,7 +3527,8 @@ fn intBitReverse(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {...@@ -3560,7 +3527,8 @@ fn intBitReverse(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3560 65...128 => {3527 65...128 => {
3561 const tmp = try cg.allocStack(Type.u128);3528 const tmp = try cg.allocStack(Type.u128);
35623529
3563 try cg.emitWValue(tmp);3530 const tmp_offset = tmp.memOffset();
3531 try cg.emitMemBase(tmp);
3564 const hi = try cg.load(operand, .u64, .{ .offset = 8 });3532 const hi = try cg.load(operand, .u64, .{ .offset = 8 });
3565 const hi_rev = try cg.callIntrinsic(3533 const hi_rev = try cg.callIntrinsic(
3566 .__bitreversedi2,3534 .__bitreversedi2,
...@@ -3569,9 +3537,9 @@ fn intBitReverse(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {...@@ -3569,9 +3537,9 @@ fn intBitReverse(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3569 &.{hi},3537 &.{hi},
3570 );3538 );
3571 try cg.emitWValue(hi_rev);3539 try cg.emitWValue(hi_rev);
3572 try cg.store(.stack, .stack, .u64, .{ .offset = tmp.offset() });3540 try cg.store(.stack, .stack, .u64, .{ .offset = tmp_offset });
35733541
3574 try cg.emitWValue(tmp);3542 try cg.emitMemBase(tmp);
3575 const lo = try cg.load(operand, .u64, .{});3543 const lo = try cg.load(operand, .u64, .{});
3576 const lo_rev = try cg.callIntrinsic(3544 const lo_rev = try cg.callIntrinsic(
3577 .__bitreversedi2,3545 .__bitreversedi2,
...@@ -3580,7 +3548,7 @@ fn intBitReverse(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {...@@ -3580,7 +3548,7 @@ fn intBitReverse(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3580 &.{lo},3548 &.{lo},
3581 );3549 );
3582 try cg.emitWValue(lo_rev);3550 try cg.emitWValue(lo_rev);
3583 try cg.store(.stack, .stack, .u64, .{ .offset = tmp.offset() + 8 });3551 try cg.store(.stack, .stack, .u64, .{ .offset = tmp_offset + 8 });
35843552
3585 if (ty.bits < 128) {3553 if (ty.bits < 128) {
3586 const shift_ty: IntType = .{ .is_signed = ty.is_signed, .bits = 128 };3554 const shift_ty: IntType = .{ .is_signed = ty.is_signed, .bits = 128 };
...@@ -3592,8 +3560,8 @@ fn intBitReverse(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {...@@ -3592,8 +3560,8 @@ fn intBitReverse(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3592 else => {3560 else => {
3593 const result = try cg.allocInt(ty);3561 const result = try cg.allocInt(ty);
35943562
3595 try cg.lowerToStack(result);3563 try cg.emitWValue(result);
3596 try cg.lowerToStack(operand);3564 try cg.emitWValue(operand);
3597 try cg.addImm32(@intFromBool(ty.is_signed));3565 try cg.addImm32(@intFromBool(ty.is_signed));
3598 try cg.addImm32(ty.bits);3566 try cg.addImm32(ty.bits);
3599 try cg.addCallIntrinsic(.__bitreverse_limb64);3567 try cg.addCallIntrinsic(.__bitreverse_limb64);
...@@ -3629,7 +3597,8 @@ fn intByteSwap(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {...@@ -3629,7 +3597,8 @@ fn intByteSwap(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3629 65...128 => {3597 65...128 => {
3630 const result = try cg.allocStack(Type.u128);3598 const result = try cg.allocStack(Type.u128);
36313599
3632 try cg.emitWValue(result);3600 const result_offset = result.memOffset();
3601 try cg.emitMemBase(result);
36333602
3634 const low = try cg.load(operand, .u64, .{});3603 const low = try cg.load(operand, .u64, .{});
3635 const swap_low = try cg.callIntrinsic(3604 const swap_low = try cg.callIntrinsic(
...@@ -3638,9 +3607,9 @@ fn intByteSwap(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {...@@ -3638,9 +3607,9 @@ fn intByteSwap(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3638 Type.u64,3607 Type.u64,
3639 &.{low},3608 &.{low},
3640 );3609 );
3641 try cg.store(.stack, swap_low, .u64, .{ .offset = result.offset() + 8 });3610 try cg.store(.stack, swap_low, .u64, .{ .offset = result_offset + 8 });
36423611
3643 try cg.emitWValue(result);3612 try cg.emitMemBase(result);
36443613
3645 const high = try cg.load(operand, .u64, .{ .offset = 8 });3614 const high = try cg.load(operand, .u64, .{ .offset = 8 });
3646 const swap_high = try cg.callIntrinsic(3615 const swap_high = try cg.callIntrinsic(
...@@ -3649,7 +3618,7 @@ fn intByteSwap(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {...@@ -3649,7 +3618,7 @@ fn intByteSwap(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3649 Type.u64,3618 Type.u64,
3650 &.{high},3619 &.{high},
3651 );3620 );
3652 try cg.store(.stack, swap_high, .u64, .{ .offset = result.offset() });3621 try cg.store(.stack, swap_high, .u64, .{ .offset = result_offset });
36533622
3654 if (ty.bits < 128) {3623 if (ty.bits < 128) {
3655 const shift_ty: IntType = .{ .is_signed = ty.is_signed, .bits = 128 };3624 const shift_ty: IntType = .{ .is_signed = ty.is_signed, .bits = 128 };
...@@ -3661,8 +3630,8 @@ fn intByteSwap(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {...@@ -3661,8 +3630,8 @@ fn intByteSwap(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3661 else => {3630 else => {
3662 const result = try cg.allocInt(ty);3631 const result = try cg.allocInt(ty);
36633632
3664 try cg.lowerToStack(result);3633 try cg.emitWValue(result);
3665 try cg.lowerToStack(operand);3634 try cg.emitWValue(operand);
3666 try cg.addImm32(@intFromBool(ty.is_signed));3635 try cg.addImm32(@intFromBool(ty.is_signed));
3667 try cg.addImm32(ty.bits);3636 try cg.addImm32(ty.bits);
3668 try cg.addCallIntrinsic(.__byteswap_limb64);3637 try cg.addCallIntrinsic(.__byteswap_limb64);
...@@ -3706,11 +3675,12 @@ fn intWrap(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {...@@ -3706,11 +3675,12 @@ fn intWrap(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3706 65...127 => {3675 65...127 => {
3707 const result = try cg.allocStack(Type.u128);3676 const result = try cg.allocStack(Type.u128);
37083677
3709 try cg.emitWValue(result);3678 const result_offset = result.memOffset();
3679 try cg.emitMemBase(result);
3710 _ = try cg.load(operand, .u64, .{});3680 _ = try cg.load(operand, .u64, .{});
3711 try cg.store(.stack, .stack, .u64, .{ .offset = result.offset() });3681 try cg.store(.stack, .stack, .u64, .{ .offset = result_offset });
37123682
3713 try cg.emitWValue(result);3683 try cg.emitMemBase(result);
3714 _ = try cg.load(operand, .u64, .{ .offset = 8 });3684 _ = try cg.load(operand, .u64, .{ .offset = 8 });
3715 if (ty.is_signed) {3685 if (ty.is_signed) {
3716 try cg.addImm64(128 - ty.bits);3686 try cg.addImm64(128 - ty.bits);
...@@ -3721,7 +3691,7 @@ fn intWrap(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {...@@ -3721,7 +3691,7 @@ fn intWrap(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3721 try cg.addImm64(~@as(u64, 0) >> @intCast(128 - ty.bits));3691 try cg.addImm64(~@as(u64, 0) >> @intCast(128 - ty.bits));
3722 try cg.addTag(.i64_and);3692 try cg.addTag(.i64_and);
3723 }3693 }
3724 try cg.store(.stack, .stack, .u64, .{ .offset = result.offset() + 8 });3694 try cg.store(.stack, .stack, .u64, .{ .offset = result_offset + 8 });
37253695
3726 return result;3696 return result;
3727 },3697 },
...@@ -3738,7 +3708,7 @@ fn intWrap(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {...@@ -3738,7 +3708,7 @@ fn intWrap(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3738 try cg.memcpy(result, operand, .{ .imm32 = used_len - 8 });3708 try cg.memcpy(result, operand, .{ .imm32 = used_len - 8 });
3739 const pad = 64 - ty.bits % 64;3709 const pad = 64 - ty.bits % 64;
37403710
3741 try cg.emitWValue(result);3711 try cg.emitMemBase(result);
3742 _ = try cg.load(operand, .u64, .{ .offset = used_len - 8 });3712 _ = try cg.load(operand, .u64, .{ .offset = used_len - 8 });
3743 if (ty.is_signed) {3713 if (ty.is_signed) {
3744 try cg.addImm64(pad);3714 try cg.addImm64(pad);
...@@ -3749,14 +3719,14 @@ fn intWrap(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {...@@ -3749,14 +3719,14 @@ fn intWrap(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3749 try cg.addImm64(~@as(u64, 0) >> @intCast(pad));3719 try cg.addImm64(~@as(u64, 0) >> @intCast(pad));
3750 try cg.addTag(.i64_and);3720 try cg.addTag(.i64_and);
3751 }3721 }
3752 try cg.store(.stack, .stack, .u64, .{ .offset = result.offset() + used_len - 8 });3722 try cg.store(.stack, .stack, .u64, .{ .offset = result.memOffset() + used_len - 8 });
3753 } else {3723 } else {
3754 try cg.memcpy(result, operand, .{ .imm32 = used_len });3724 try cg.memcpy(result, operand, .{ .imm32 = used_len });
3755 }3725 }
37563726
3757 const full_len = @divExact(bits, 8);3727 const full_len = @divExact(bits, 8);
3758 if (used_len + 8 == full_len) { // last limb needs sign extended3728 if (used_len + 8 == full_len) { // last limb needs sign extended
3759 try cg.emitWValue(result);3729 try cg.emitMemBase(result);
3760 if (ty.is_signed) {3730 if (ty.is_signed) {
3761 _ = try cg.load(result, .u64, .{ .offset = used_len - 8 });3731 _ = try cg.load(result, .u64, .{ .offset = used_len - 8 });
3762 try cg.addImm64(63);3732 try cg.addImm64(63);
...@@ -3764,7 +3734,7 @@ fn intWrap(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {...@@ -3764,7 +3734,7 @@ fn intWrap(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3764 } else {3734 } else {
3765 try cg.addImm64(0);3735 try cg.addImm64(0);
3766 }3736 }
3767 try cg.store(.stack, .stack, .u64, .{ .offset = result.offset() + used_len });3737 try cg.store(.stack, .stack, .u64, .{ .offset = result.memOffset() + used_len });
3768 }3738 }
37693739
3770 return result;3740 return result;
...@@ -3858,20 +3828,20 @@ fn intAddSat(cg: *CodeGen, int_ty: IntType, lhs: WValue, rhs: WValue) InnerError...@@ -3858,20 +3828,20 @@ fn intAddSat(cg: *CodeGen, int_ty: IntType, lhs: WValue, rhs: WValue) InnerError
3858 defer rhs_is_neg.free(cg);3828 defer rhs_is_neg.free(cg);
3859 const min_val = try cg.intMinValue(int_ty);3829 const min_val = try cg.intMinValue(int_ty);
38603830
3861 try cg.lowerToStack(min_val);3831 try cg.emitWValue(min_val);
3862 try cg.lowerToStack(max_val);3832 try cg.emitWValue(max_val);
3863 try cg.emitWValue(rhs_is_neg);3833 try cg.emitWValue(rhs_is_neg);
3864 try cg.addTag(.select);3834 try cg.addTag(.select);
38653835
3866 try cg.lowerToStack(op_val);3836 try cg.emitWValue(op_val);
3867 const overflow_cmp = try cg.intCmp(int_ty, .lt, op_val, lhs);3837 const overflow_cmp = try cg.intCmp(int_ty, .lt, op_val, lhs);
3868 const is_overflow = try cg.intCmp(.u32, .neq, rhs_is_neg, overflow_cmp);3838 const is_overflow = try cg.intCmp(.u32, .neq, rhs_is_neg, overflow_cmp);
3869 try cg.emitWValue(is_overflow);3839 try cg.emitWValue(is_overflow);
3870 try cg.addTag(.select);3840 try cg.addTag(.select);
3871 return .stack;3841 return .stack;
3872 } else {3842 } else {
3873 try cg.lowerToStack(max_val);3843 try cg.emitWValue(max_val);
3874 try cg.lowerToStack(op_val);3844 try cg.emitWValue(op_val);
38753845
3876 const is_overflow = try cg.intCmp(int_ty, .lt, op_val, lhs);3846 const is_overflow = try cg.intCmp(int_ty, .lt, op_val, lhs);
3877 try cg.emitWValue(is_overflow);3847 try cg.emitWValue(is_overflow);
...@@ -3892,12 +3862,12 @@ fn intSubSat(cg: *CodeGen, int_ty: IntType, lhs: WValue, rhs: WValue) InnerError...@@ -3892,12 +3862,12 @@ fn intSubSat(cg: *CodeGen, int_ty: IntType, lhs: WValue, rhs: WValue) InnerError
3892 const max_val = try cg.intMaxValue(int_ty);3862 const max_val = try cg.intMaxValue(int_ty);
3893 const min_val = try cg.intMinValue(int_ty);3863 const min_val = try cg.intMinValue(int_ty);
38943864
3895 try cg.lowerToStack(max_val);3865 try cg.emitWValue(max_val);
3896 try cg.lowerToStack(min_val);3866 try cg.emitWValue(min_val);
3897 try cg.emitWValue(rhs_is_neg);3867 try cg.emitWValue(rhs_is_neg);
3898 try cg.addTag(.select);3868 try cg.addTag(.select);
38993869
3900 try cg.lowerToStack(op_val);3870 try cg.emitWValue(op_val);
3901 const overflow_cmp = try cg.intCmp(int_ty, .gt, op_val, lhs);3871 const overflow_cmp = try cg.intCmp(int_ty, .gt, op_val, lhs);
3902 const is_overflow = try cg.intCmp(.u32, .neq, rhs_is_neg, overflow_cmp);3872 const is_overflow = try cg.intCmp(.u32, .neq, rhs_is_neg, overflow_cmp);
3903 try cg.emitWValue(is_overflow);3873 try cg.emitWValue(is_overflow);
...@@ -3906,8 +3876,8 @@ fn intSubSat(cg: *CodeGen, int_ty: IntType, lhs: WValue, rhs: WValue) InnerError...@@ -3906,8 +3876,8 @@ fn intSubSat(cg: *CodeGen, int_ty: IntType, lhs: WValue, rhs: WValue) InnerError
3906 } else {3876 } else {
3907 const zero = try cg.intZeroValue(int_ty);3877 const zero = try cg.intZeroValue(int_ty);
39083878
3909 try cg.lowerToStack(zero);3879 try cg.emitWValue(zero);
3910 try cg.lowerToStack(op_val);3880 try cg.emitWValue(op_val);
3911 const is_overflow = try cg.intCmp(int_ty, .lt, lhs, rhs);3881 const is_overflow = try cg.intCmp(int_ty, .lt, lhs, rhs);
3912 try cg.emitWValue(is_overflow);3882 try cg.emitWValue(is_overflow);
3913 try cg.addTag(.select);3883 try cg.addTag(.select);
...@@ -3931,10 +3901,10 @@ fn intMulSat(cg: *CodeGen, int_ty: IntType, lhs: WValue, rhs: WValue) InnerError...@@ -3931,10 +3901,10 @@ fn intMulSat(cg: *CodeGen, int_ty: IntType, lhs: WValue, rhs: WValue) InnerError
3931 if (int_ty.is_signed) {3901 if (int_ty.is_signed) {
3932 const min_val = try cg.intMinValue(int_ty);3902 const min_val = try cg.intMinValue(int_ty);
39333903
3934 try cg.lowerToStack(min_val);3904 try cg.emitWValue(min_val);
39353905
3936 try cg.lowerToStack(max_val);3906 try cg.emitWValue(max_val);
3937 try cg.lowerToStack(op_val);3907 try cg.emitWValue(op_val);
3938 const max_ext = try cg.intCast(ext_ty, int_ty, max_val);3908 const max_ext = try cg.intCast(ext_ty, int_ty, max_val);
3939 const ov_pos = try cg.intCmp(ext_ty, .lt, max_ext, mul_ext);3909 const ov_pos = try cg.intCmp(ext_ty, .lt, max_ext, mul_ext);
3940 try cg.emitWValue(ov_pos);3910 try cg.emitWValue(ov_pos);
...@@ -3942,12 +3912,12 @@ fn intMulSat(cg: *CodeGen, int_ty: IntType, lhs: WValue, rhs: WValue) InnerError...@@ -3942,12 +3912,12 @@ fn intMulSat(cg: *CodeGen, int_ty: IntType, lhs: WValue, rhs: WValue) InnerError
39423912
3943 const min_ext = try cg.intCast(ext_ty, int_ty, min_val);3913 const min_ext = try cg.intCast(ext_ty, int_ty, min_val);
3944 const ov_neg = try cg.intCmp(ext_ty, .gt, min_ext, mul_ext);3914 const ov_neg = try cg.intCmp(ext_ty, .gt, min_ext, mul_ext);
3945 try cg.lowerToStack(ov_neg);3915 try cg.emitWValue(ov_neg);
3946 try cg.addTag(.select);3916 try cg.addTag(.select);
3947 return .stack;3917 return .stack;
3948 } else {3918 } else {
3949 try cg.lowerToStack(max_val);3919 try cg.emitWValue(max_val);
3950 try cg.lowerToStack(op_val);3920 try cg.emitWValue(op_val);
3951 const max_ext = try cg.intCast(ext_ty, int_ty, max_val);3921 const max_ext = try cg.intCast(ext_ty, int_ty, max_val);
3952 const is_overflow = try cg.intCmp(ext_ty, .lt, max_ext, mul_ext);3922 const is_overflow = try cg.intCmp(ext_ty, .lt, max_ext, mul_ext);
3953 try cg.emitWValue(is_overflow);3923 try cg.emitWValue(is_overflow);
...@@ -3970,20 +3940,20 @@ fn intShlSat(cg: *CodeGen, int_ty: IntType, lhs: WValue, rhs: WValue) InnerError...@@ -3970,20 +3940,20 @@ fn intShlSat(cg: *CodeGen, int_ty: IntType, lhs: WValue, rhs: WValue) InnerError
3970 const zero = try cg.intZeroValue(int_ty);3940 const zero = try cg.intZeroValue(int_ty);
3971 const min_val = try cg.intMinValue(int_ty);3941 const min_val = try cg.intMinValue(int_ty);
39723942
3973 try cg.lowerToStack(min_val);3943 try cg.emitWValue(min_val);
3974 try cg.lowerToStack(max_val);3944 try cg.emitWValue(max_val);
3975 const lhs_is_neg = try cg.intCmp(int_ty, .lt, lhs, zero);3945 const lhs_is_neg = try cg.intCmp(int_ty, .lt, lhs, zero);
3976 try cg.emitWValue(lhs_is_neg);3946 try cg.emitWValue(lhs_is_neg);
3977 try cg.addTag(.select);3947 try cg.addTag(.select);
39783948
3979 try cg.lowerToStack(op_val);3949 try cg.emitWValue(op_val);
3980 const is_overflow = try cg.intCmp(int_ty, .neq, check_val, lhs);3950 const is_overflow = try cg.intCmp(int_ty, .neq, check_val, lhs);
3981 try cg.emitWValue(is_overflow);3951 try cg.emitWValue(is_overflow);
3982 try cg.addTag(.select);3952 try cg.addTag(.select);
3983 return .stack;3953 return .stack;
3984 } else {3954 } else {
3985 try cg.lowerToStack(max_val);3955 try cg.emitWValue(max_val);
3986 try cg.lowerToStack(op_val);3956 try cg.emitWValue(op_val);
3987 const is_overflow = try cg.intCmp(int_ty, .neq, check_val, lhs);3957 const is_overflow = try cg.intCmp(int_ty, .neq, check_val, lhs);
3988 try cg.emitWValue(is_overflow);3958 try cg.emitWValue(is_overflow);
3989 try cg.addTag(.select);3959 try cg.addTag(.select);
...@@ -4055,9 +4025,9 @@ fn intAddOverflow(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerErro...@@ -4055,9 +4025,9 @@ fn intAddOverflow(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerErro
4055 else => {4025 else => {
4056 const result = try cg.allocInt(ty);4026 const result = try cg.allocInt(ty);
40574027
4058 try cg.lowerToStack(result);4028 try cg.emitWValue(result);
4059 try cg.lowerToStack(lhs);4029 try cg.emitWValue(lhs);
4060 try cg.lowerToStack(rhs);4030 try cg.emitWValue(rhs);
4061 try cg.addImm32(@intFromBool(ty.is_signed));4031 try cg.addImm32(@intFromBool(ty.is_signed));
4062 try cg.addImm32(ty.bits);4032 try cg.addImm32(ty.bits);
4063 try cg.addCallIntrinsic(.__addo_limb64);4033 try cg.addCallIntrinsic(.__addo_limb64);
...@@ -4087,9 +4057,9 @@ fn intSubOverflow(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerErro...@@ -4087,9 +4057,9 @@ fn intSubOverflow(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerErro
4087 else => {4057 else => {
4088 const result = try cg.allocInt(ty);4058 const result = try cg.allocInt(ty);
40894059
4090 try cg.lowerToStack(result);4060 try cg.emitWValue(result);
4091 try cg.lowerToStack(lhs);4061 try cg.emitWValue(lhs);
4092 try cg.lowerToStack(rhs);4062 try cg.emitWValue(rhs);
4093 try cg.addImm32(@intFromBool(ty.is_signed));4063 try cg.addImm32(@intFromBool(ty.is_signed));
4094 try cg.addImm32(ty.bits);4064 try cg.addImm32(ty.bits);
4095 try cg.addCallIntrinsic(.__subo_limb64);4065 try cg.addCallIntrinsic(.__subo_limb64);
...@@ -4145,9 +4115,9 @@ fn intMulOverflow(cg: *CodeGen, int_ty: IntType, lhs: WValue, rhs: WValue) Inner...@@ -4145,9 +4115,9 @@ fn intMulOverflow(cg: *CodeGen, int_ty: IntType, lhs: WValue, rhs: WValue) Inner
4145 } else {4115 } else {
4146 const result = try cg.allocInt(int_ty);4116 const result = try cg.allocInt(int_ty);
41474117
4148 try cg.lowerToStack(result);4118 try cg.emitWValue(result);
4149 try cg.lowerToStack(lhs);4119 try cg.emitWValue(lhs);
4150 try cg.lowerToStack(rhs);4120 try cg.emitWValue(rhs);
4151 try cg.addImm32(@intFromBool(int_ty.is_signed));4121 try cg.addImm32(@intFromBool(int_ty.is_signed));
4152 try cg.addImm32(int_ty.bits);4122 try cg.addImm32(int_ty.bits);
4153 try cg.addCallIntrinsic(.__mulo_limb64);4123 try cg.addCallIntrinsic(.__mulo_limb64);
...@@ -4174,9 +4144,9 @@ fn intShlOverflow(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerErro...@@ -4174,9 +4144,9 @@ fn intShlOverflow(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerErro
4174 else => {4144 else => {
4175 const result = try cg.allocInt(ty);4145 const result = try cg.allocInt(ty);
41764146
4177 try cg.lowerToStack(result);4147 try cg.emitWValue(result);
4178 try cg.lowerToStack(lhs);4148 try cg.emitWValue(lhs);
4179 try cg.lowerToStack(rhs);4149 try cg.emitWValue(rhs);
4180 try cg.addImm32(@intFromBool(ty.is_signed));4150 try cg.addImm32(@intFromBool(ty.is_signed));
4181 try cg.addImm32(ty.bits);4151 try cg.addImm32(ty.bits);
4182 try cg.addCallIntrinsic(.__shlo_limb64);4152 try cg.addCallIntrinsic(.__shlo_limb64);
...@@ -4213,15 +4183,15 @@ fn intCast(cg: *CodeGen, dest_ty: IntType, src_ty: IntType, operand: WValue) Inn...@@ -4213,15 +4183,15 @@ fn intCast(cg: *CodeGen, dest_ty: IntType, src_ty: IntType, operand: WValue) Inn
4213 } else {4183 } else {
4214 var src_len: u32 = undefined;4184 var src_len: u32 = undefined;
4215 if (src_bits == 32) {4185 if (src_bits == 32) {
4216 try cg.emitWValue(result);4186 try cg.emitMemBase(result);
4217 try cg.emitWValue(operand);4187 try cg.emitWValue(operand);
4218 try cg.addTag(if (src_ty.is_signed) .i64_extend_i32_s else .i64_extend_i32_u);4188 try cg.addTag(if (src_ty.is_signed) .i64_extend_i32_s else .i64_extend_i32_u);
4219 try cg.store(.stack, .stack, .u64, .{ .offset = result.offset() });4189 try cg.store(.stack, .stack, .u64, .{ .offset = result.memOffset() });
4220 src_len = 8;4190 src_len = 8;
4221 } else if (src_bits == 64) {4191 } else if (src_bits == 64) {
4222 try cg.emitWValue(result);4192 try cg.emitMemBase(result);
4223 try cg.emitWValue(operand);4193 try cg.emitWValue(operand);
4224 try cg.store(.stack, .stack, .u64, .{ .offset = result.offset() });4194 try cg.store(.stack, .stack, .u64, .{ .offset = result.memOffset() });
4225 src_len = 8;4195 src_len = 8;
4226 } else {4196 } else {
4227 src_len = src_bits / 8;4197 src_len = src_bits / 8;
...@@ -4230,7 +4200,7 @@ fn intCast(cg: *CodeGen, dest_ty: IntType, src_ty: IntType, operand: WValue) Inn...@@ -4230,7 +4200,7 @@ fn intCast(cg: *CodeGen, dest_ty: IntType, src_ty: IntType, operand: WValue) Inn
42304200
4231 if (dest_bits == 128) {4201 if (dest_bits == 128) {
4232 if (src_ty.is_signed) {4202 if (src_ty.is_signed) {
4233 try cg.emitWValue(result);4203 try cg.emitMemBase(result);
4234 if (src_bits == 32) {4204 if (src_bits == 32) {
4235 try cg.emitWValue(operand);4205 try cg.emitWValue(operand);
4236 try cg.addTag(if (dest_ty.is_signed) .i64_extend_i32_s else .i64_extend_i32_u);4206 try cg.addTag(if (dest_ty.is_signed) .i64_extend_i32_s else .i64_extend_i32_u);
...@@ -4238,7 +4208,7 @@ fn intCast(cg: *CodeGen, dest_ty: IntType, src_ty: IntType, operand: WValue) Inn...@@ -4238,7 +4208,7 @@ fn intCast(cg: *CodeGen, dest_ty: IntType, src_ty: IntType, operand: WValue) Inn
4238 try cg.emitWValue(operand);4208 try cg.emitWValue(operand);
4239 } else unreachable;4209 } else unreachable;
4240 const shr = try cg.intShr(IntType.i64, .stack, .{ .imm32 = 63 });4210 const shr = try cg.intShr(IntType.i64, .stack, .{ .imm32 = 63 });
4241 try cg.store(.stack, shr, .u64, .{ .offset = 8 + result.offset() });4211 try cg.store(.stack, shr, .u64, .{ .offset = result.memOffset() + 8 });
4242 } else {4212 } else {
4243 try cg.store(result, .{ .imm64 = 0 }, .u64, .{ .offset = 8 });4213 try cg.store(result, .{ .imm64 = 0 }, .u64, .{ .offset = 8 });
4244 }4214 }
...@@ -4642,20 +4612,22 @@ fn floatNeg(cg: *CodeGen, ty: FloatType, arg: WValue) InnerError!WValue {...@@ -4642,20 +4612,22 @@ fn floatNeg(cg: *CodeGen, ty: FloatType, arg: WValue) InnerError!WValue {
4642 },4612 },
4643 .f80, .f128 => {4613 .f80, .f128 => {
4644 const result = try cg.allocStack(Type.f128);4614 const result = try cg.allocStack(Type.f128);
4645 try cg.emitWValue(result);4615 const result_offset = result.memOffset();
4646 try cg.emitWValue(arg);4616 const arg_offset = arg.memOffset();
4647 try cg.addMemArg(.i64_load, .{ .offset = 0 + arg.offset(), .alignment = .@"2" });4617 try cg.emitMemBase(result);
4648 try cg.addMemArg(.i64_store, .{ .offset = 0 + result.offset(), .alignment = .@"2" });4618 try cg.emitMemBase(arg);
4649 try cg.emitWValue(result);4619 try cg.addMemArg(.i64_load, .{ .offset = arg_offset, .alignment = .@"2" });
4650 try cg.emitWValue(arg);4620 try cg.addMemArg(.i64_store, .{ .offset = result_offset, .alignment = .@"2" });
4651 try cg.addMemArg(.i64_load, .{ .offset = 8 + arg.offset(), .alignment = .@"2" });4621 try cg.emitMemBase(result);
4622 try cg.emitMemBase(arg);
4623 try cg.addMemArg(.i64_load, .{ .offset = arg_offset + 8, .alignment = .@"2" });
4652 if (ty == .f80) {4624 if (ty == .f80) {
4653 try cg.addImm64(0x8000);4625 try cg.addImm64(0x8000);
4654 } else {4626 } else {
4655 try cg.addImm64(0x8000000000000000);4627 try cg.addImm64(0x8000000000000000);
4656 }4628 }
4657 try cg.addTag(.i64_xor);4629 try cg.addTag(.i64_xor);
4658 try cg.addMemArg(.i64_store, .{ .offset = 8 + result.offset(), .alignment = .@"2" });4630 try cg.addMemArg(.i64_store, .{ .offset = result_offset + 8, .alignment = .@"2" });
4659 return result;4631 return result;
4660 },4632 },
4661 }4633 }
...@@ -4681,20 +4653,22 @@ fn floatAbs(cg: *CodeGen, ty: FloatType, arg: WValue) InnerError!WValue {...@@ -4681,20 +4653,22 @@ fn floatAbs(cg: *CodeGen, ty: FloatType, arg: WValue) InnerError!WValue {
4681 },4653 },
4682 .f80, .f128 => {4654 .f80, .f128 => {
4683 const result = try cg.allocStack(Type.f128);4655 const result = try cg.allocStack(Type.f128);
4684 try cg.emitWValue(result);4656 const result_offset = result.memOffset();
4685 try cg.emitWValue(arg);4657 const arg_offset = arg.memOffset();
4686 try cg.addMemArg(.i64_load, .{ .offset = 0 + arg.offset(), .alignment = .@"2" });4658 try cg.emitMemBase(result);
4687 try cg.addMemArg(.i64_store, .{ .offset = 0 + result.offset(), .alignment = .@"2" });4659 try cg.emitMemBase(arg);
4688 try cg.emitWValue(result);4660 try cg.addMemArg(.i64_load, .{ .offset = arg_offset, .alignment = .@"2" });
4689 try cg.emitWValue(arg);4661 try cg.addMemArg(.i64_store, .{ .offset = result_offset, .alignment = .@"2" });
4690 try cg.addMemArg(.i64_load, .{ .offset = 8 + arg.offset(), .alignment = .@"2" });4662 try cg.emitMemBase(result);
4663 try cg.emitMemBase(arg);
4664 try cg.addMemArg(.i64_load, .{ .offset = arg_offset + 8, .alignment = .@"2" });
4691 if (ty == .f80) {4665 if (ty == .f80) {
4692 try cg.addImm64(0x7FFF);4666 try cg.addImm64(0x7FFF);
4693 } else {4667 } else {
4694 try cg.addImm64(0x7FFFFFFFFFFFFFFF);4668 try cg.addImm64(0x7FFFFFFFFFFFFFFF);
4695 }4669 }
4696 try cg.addTag(.i64_and);4670 try cg.addTag(.i64_and);
4697 try cg.addMemArg(.i64_store, .{ .offset = 8 + result.offset(), .alignment = .@"2" });4671 try cg.addMemArg(.i64_store, .{ .offset = result_offset + 8, .alignment = .@"2" });
4698 return result;4672 return result;
4699 },4673 },
4700 }4674 }
...@@ -5377,8 +5351,8 @@ fn intCmp(cg: *CodeGen, ty: IntType, op: std.math.CompareOperator, lhs: WValue,...@@ -5377,8 +5351,8 @@ fn intCmp(cg: *CodeGen, ty: IntType, op: std.math.CompareOperator, lhs: WValue,
5377 0 => unreachable,5351 0 => unreachable,
5378 1...32 => {5352 1...32 => {
5379 // lhs or rhs could be stack pointers5353 // lhs or rhs could be stack pointers
5380 try cg.lowerToStack(lhs);5354 try cg.emitWValue(lhs);
5381 try cg.lowerToStack(rhs);5355 try cg.emitWValue(rhs);
5382 const opcode: Mir.Inst.Tag = switch (op) {5356 const opcode: Mir.Inst.Tag = switch (op) {
5383 .eq => .i32_eq,5357 .eq => .i32_eq,
5384 .neq => .i32_ne,5358 .neq => .i32_ne,
...@@ -5392,8 +5366,8 @@ fn intCmp(cg: *CodeGen, ty: IntType, op: std.math.CompareOperator, lhs: WValue,...@@ -5392,8 +5366,8 @@ fn intCmp(cg: *CodeGen, ty: IntType, op: std.math.CompareOperator, lhs: WValue,
5392 },5366 },
5393 33...64 => {5367 33...64 => {
5394 // lhs or rhs could be stack pointers5368 // lhs or rhs could be stack pointers
5395 try cg.lowerToStack(lhs);5369 try cg.emitWValue(lhs);
5396 try cg.lowerToStack(rhs);5370 try cg.emitWValue(rhs);
5397 const opcode: Mir.Inst.Tag = switch (op) {5371 const opcode: Mir.Inst.Tag = switch (op) {
5398 .eq => .i64_eq,5372 .eq => .i64_eq,
5399 .neq => .i64_ne,5373 .neq => .i64_ne,
...@@ -5442,8 +5416,8 @@ fn intCmp(cg: *CodeGen, ty: IntType, op: std.math.CompareOperator, lhs: WValue,...@@ -5442,8 +5416,8 @@ fn intCmp(cg: *CodeGen, ty: IntType, op: std.math.CompareOperator, lhs: WValue,
5442 return .stack;5416 return .stack;
5443 },5417 },
5444 else => {5418 else => {
5445 try cg.lowerToStack(lhs);5419 try cg.emitWValue(lhs);
5446 try cg.lowerToStack(rhs);5420 try cg.emitWValue(rhs);
5447 try cg.addImm32(@intFromBool(ty.is_signed));5421 try cg.addImm32(@intFromBool(ty.is_signed));
5448 try cg.addImm32(ty.bits);5422 try cg.addImm32(ty.bits);
5449 try cg.addCallIntrinsic(.__cmp_limb64);5423 try cg.addCallIntrinsic(.__cmp_limb64);
...@@ -5554,7 +5528,7 @@ fn airBr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -5554,7 +5528,7 @@ fn airBr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
5554 // if operand has codegen bits we should break with a value5528 // if operand has codegen bits we should break with a value
5555 if (block.value != .none) {5529 if (block.value != .none) {
5556 const operand = try cg.resolveInst(br.operand);5530 const operand = try cg.resolveInst(br.operand);
5557 try cg.lowerToStack(operand);5531 try cg.emitWValue(operand);
5558 try cg.addLocal(.local_set, block.value.local.value);5532 try cg.addLocal(.local_set, block.value.local.value);
5559 }5533 }
55605534
...@@ -5618,11 +5592,12 @@ fn airIntFromPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -5618,11 +5592,12 @@ fn airIntFromPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
5618 assert(operand_ty.abiAlignment(zcu) == dest_ty.abiAlignment(zcu));5592 assert(operand_ty.abiAlignment(zcu) == dest_ty.abiAlignment(zcu));
56195593
5620 const operand = try cg.resolveInst(ty_op.operand);5594 const operand = try cg.resolveInst(ty_op.operand);
5621 const result = switch (operand) {5595 if (operand == .stack_offset) {
5622 .stack_offset => try cg.buildPointerOffset(operand, 0, .new),5596 // materialize operand, we assume precomputed value is more efficient
5623 else => operand,5597 try cg.emitWValue(operand);
5624 };5598 return cg.finishAir(inst, .stack, &.{ty_op.operand});
5625 return cg.finishAir(inst, result, &.{ty_op.operand});5599 }
5600 return cg.finishAir(inst, operand, &.{ty_op.operand});
5626}5601}
56275602
5628fn airUnionFromEnum(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {5603fn airUnionFromEnum(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
...@@ -5801,16 +5776,7 @@ fn structFieldPtr(...@@ -5801,16 +5776,7 @@ fn structFieldPtr(
5801 },5776 },
5802 else => struct_ty.structFieldOffset(index, zcu),5777 else => struct_ty.structFieldOffset(index, zcu),
5803 };5778 };
5804 // save a load and store when we can simply reuse the operand5779 return cg.offsetPointer(struct_ptr, offset);
5805 if (offset == 0) {
5806 return struct_ptr;
5807 }
5808 switch (struct_ptr) {
5809 .stack_offset => |stack_offset| {
5810 return .{ .stack_offset = .{ .value = stack_offset.value + @as(u32, @intCast(offset)), .references = 1 } };
5811 },
5812 else => return cg.buildPointerOffset(struct_ptr, offset, .new),
5813 }
5814}5780}
58155781
5816fn airAggFieldVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {5782fn airAggFieldVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
...@@ -5832,12 +5798,7 @@ fn airAggFieldVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -5832,12 +5798,7 @@ fn airAggFieldVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
5832 return cg.fail("Field type '{f}' too big to fit into stack frame", .{field_ty.fmt(pt)});5798 return cg.fail("Field type '{f}' too big to fit into stack frame", .{field_ty.fmt(pt)});
5833 };5799 };
5834 if (isByRef(field_ty, zcu, cg.target)) {5800 if (isByRef(field_ty, zcu, cg.target)) {
5835 switch (operand) {5801 break :result try cg.offsetPointer(operand, offset);
5836 .stack_offset => |stack_offset| {
5837 break :result .{ .stack_offset = .{ .value = stack_offset.value + offset, .references = 1 } };
5838 },
5839 else => break :result try cg.buildPointerOffset(operand, offset, .new),
5840 }
5841 }5802 }
5842 break :result try cg.load(operand, field_ty, .{ .offset = offset });5803 break :result try cg.load(operand, field_ty, .{ .offset = offset });
5843 },5804 },
...@@ -5859,7 +5820,7 @@ fn airSwitchBr(cg: *CodeGen, inst: Air.Inst.Index, is_dispatch_loop: bool) Inner...@@ -5859,7 +5820,7 @@ fn airSwitchBr(cg: *CodeGen, inst: Air.Inst.Index, is_dispatch_loop: bool) Inner
5859 const target = if (is_dispatch_loop) target: {5820 const target = if (is_dispatch_loop) target: {
5860 const initial_target = try cg.resolveInst(switch_br.operand);5821 const initial_target = try cg.resolveInst(switch_br.operand);
5861 const target: WValue = try cg.allocLocal(target_ty);5822 const target: WValue = try cg.allocLocal(target_ty);
5862 try cg.lowerToStack(initial_target);5823 try cg.emitWValue(initial_target);
5863 try cg.addLocal(.local_set, target.local.value);5824 try cg.addLocal(.local_set, target.local.value);
58645825
5865 try cg.startBlock(.loop, .empty); // dispatch loop start5826 try cg.startBlock(.loop, .empty); // dispatch loop start
...@@ -6052,7 +6013,7 @@ fn airSwitchDispatch(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -6052,7 +6013,7 @@ fn airSwitchDispatch(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6052 const switch_loop = cg.blocks.get(br.block_inst).?;6013 const switch_loop = cg.blocks.get(br.block_inst).?;
60536014
6054 const operand = try cg.resolveInst(br.operand);6015 const operand = try cg.resolveInst(br.operand);
6055 try cg.lowerToStack(operand);6016 try cg.emitWValue(operand);
6056 try cg.addLocal(.local_set, switch_loop.value.local.value);6017 try cg.addLocal(.local_set, switch_loop.value.local.value);
60576018
6058 const idx: u32 = cg.block_depth - switch_loop.label;6019 const idx: u32 = cg.block_depth - switch_loop.label;
...@@ -6080,12 +6041,14 @@ fn airIsErr(cg: *CodeGen, inst: Air.Inst.Index, opcode: std.wasm.Opcode, op_kind...@@ -6080,12 +6041,14 @@ fn airIsErr(cg: *CodeGen, inst: Air.Inst.Index, opcode: std.wasm.Opcode, op_kind
6080 }6041 }
6081 }6042 }
60826043
6083 try cg.emitWValue(operand);
6084 if (op_kind == .ptr or pl_ty.hasRuntimeBits(zcu)) {6044 if (op_kind == .ptr or pl_ty.hasRuntimeBits(zcu)) {
6045 try cg.emitMemBase(operand);
6085 try cg.addMemArg(.i32_load16_u, .{6046 try cg.addMemArg(.i32_load16_u, .{
6086 .offset = operand.offset() + @as(u32, @intCast(errUnionErrorOffset(pl_ty, zcu))),6047 .offset = operand.memOffset() + errUnionErrorOffset(pl_ty, zcu),
6087 .alignment = Type.anyerror.abiAlignment(zcu).toStdMem(),6048 .alignment = Type.anyerror.abiAlignment(zcu).toStdMem(),
6088 });6049 });
6050 } else {
6051 try cg.emitWValue(operand);
6089 }6052 }
60906053
6091 // Compare the error value with '0'6054 // Compare the error value with '0'
...@@ -6116,9 +6079,9 @@ fn airUnwrapErrUnionPayload(cg: *CodeGen, inst: Air.Inst.Index, op_is_ptr: bool)...@@ -6116,9 +6079,9 @@ fn airUnwrapErrUnionPayload(cg: *CodeGen, inst: Air.Inst.Index, op_is_ptr: bool)
6116 }6079 }
6117 }6080 }
61186081
6119 const pl_offset: u32 = @intCast(errUnionPayloadOffset(payload_ty, zcu));6082 const pl_offset = errUnionPayloadOffset(payload_ty, zcu);
6120 if (op_is_ptr or isByRef(payload_ty, zcu, cg.target)) {6083 if (op_is_ptr or isByRef(payload_ty, zcu, cg.target)) {
6121 break :result try cg.buildPointerOffset(operand, pl_offset, .new);6084 break :result try cg.offsetPointer(operand, pl_offset);
6122 } else {6085 } else {
6123 assert(isByRef(eu_ty, zcu, cg.target));6086 assert(isByRef(eu_ty, zcu, cg.target));
6124 break :result try cg.load(operand, payload_ty, .{ .offset = pl_offset });6087 break :result try cg.load(operand, payload_ty, .{ .offset = pl_offset });
...@@ -6169,15 +6132,14 @@ fn airWrapErrUnionPayload(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -6169,15 +6132,14 @@ fn airWrapErrUnionPayload(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6169 }6132 }
61706133
6171 const err_union = try cg.allocStack(err_ty);6134 const err_union = try cg.allocStack(err_ty);
6172 const payload_ptr = try cg.buildPointerOffset(err_union, @as(u32, @intCast(errUnionPayloadOffset(pl_ty, zcu))), .new);6135 try cg.store(err_union, operand, pl_ty, .{ .offset = errUnionPayloadOffset(pl_ty, zcu) });
6173 try cg.store(payload_ptr, operand, pl_ty, .{});
61746136
6175 // ensure we also write '0' to the error part, so any present stack value gets overwritten by it.6137 // ensure we also write '0' to the error part, so any present stack value gets overwritten by it.
6176 try cg.emitWValue(err_union);6138 try cg.emitMemBase(err_union);
6177 try cg.addImm32(0);6139 try cg.addImm32(0);
6178 const err_val_offset: u32 = @intCast(errUnionErrorOffset(pl_ty, zcu));6140 const err_val_offset = errUnionErrorOffset(pl_ty, zcu);
6179 try cg.addMemArg(.i32_store16, .{6141 try cg.addMemArg(.i32_store16, .{
6180 .offset = err_union.offset() + err_val_offset,6142 .offset = err_union.memOffset() + err_val_offset,
6181 .alignment = .@"2",6143 .alignment = .@"2",
6182 });6144 });
6183 break :result err_union;6145 break :result err_union;
...@@ -6203,7 +6165,7 @@ fn airWrapErrUnionErr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -6203,7 +6165,7 @@ fn airWrapErrUnionErr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6203 try cg.store(err_union, operand, .anyerror, .{ .offset = errUnionErrorOffset(pl_ty, zcu) });6165 try cg.store(err_union, operand, .anyerror, .{ .offset = errUnionErrorOffset(pl_ty, zcu) });
62046166
6205 // write 'undefined' to the payload6167 // write 'undefined' to the payload
6206 const payload_ptr = try cg.buildPointerOffset(err_union, @as(u32, @intCast(errUnionPayloadOffset(pl_ty, zcu))), .new);6168 const payload_ptr = try cg.offsetPointer(err_union, errUnionPayloadOffset(pl_ty, zcu));
6207 const len = @as(u32, @intCast(err_ty.errorUnionPayload(zcu).abiSize(zcu)));6169 const len = @as(u32, @intCast(err_ty.errorUnionPayload(zcu).abiSize(zcu)));
6208 try cg.memset(Type.u8, payload_ptr, .{ .imm32 = len }, .{ .imm32 = 0xaa });6170 try cg.memset(Type.u8, payload_ptr, .{ .imm32 = len }, .{ .imm32 = 0xaa });
62096171
...@@ -6228,7 +6190,6 @@ fn airIsNull(cg: *CodeGen, inst: Air.Inst.Index, opcode: std.wasm.Opcode, op_kin...@@ -6228,7 +6190,6 @@ fn airIsNull(cg: *CodeGen, inst: Air.Inst.Index, opcode: std.wasm.Opcode, op_kin
6228fn isNull(cg: *CodeGen, operand: WValue, op_ty: Type, opcode: std.wasm.Opcode, op_kind: OpKind) InnerError!WValue {6190fn isNull(cg: *CodeGen, operand: WValue, op_ty: Type, opcode: std.wasm.Opcode, op_kind: OpKind) InnerError!WValue {
6229 const pt = cg.pt;6191 const pt = cg.pt;
6230 const zcu = pt.zcu;6192 const zcu = pt.zcu;
6231 try cg.emitWValue(operand);
6232 const optional_ty = switch (op_kind) {6193 const optional_ty = switch (op_kind) {
6233 .value => op_ty,6194 .value => op_ty,
6234 .ptr => op_ty.childType(zcu),6195 .ptr => op_ty.childType(zcu),
...@@ -6241,19 +6202,27 @@ fn isNull(cg: *CodeGen, operand: WValue, op_ty: Type, opcode: std.wasm.Opcode, o...@@ -6241,19 +6202,27 @@ fn isNull(cg: *CodeGen, operand: WValue, op_ty: Type, opcode: std.wasm.Opcode, o
6241 const offset = std.math.cast(u32, payload_ty.abiSize(zcu)) orelse {6202 const offset = std.math.cast(u32, payload_ty.abiSize(zcu)) orelse {
6242 return cg.fail("Optional type {f} too big to fit into stack frame", .{optional_ty.fmt(pt)});6203 return cg.fail("Optional type {f} too big to fit into stack frame", .{optional_ty.fmt(pt)});
6243 };6204 };
6244 try cg.addMemArg(.i32_load8_u, .{ .offset = operand.offset() + offset, .alignment = .@"1" });6205 try cg.emitMemBase(operand);
6206 try cg.addMemArg(.i32_load8_u, .{ .offset = operand.memOffset() + offset, .alignment = .@"1" });
6207 } else {
6208 try cg.emitWValue(operand);
6245 }6209 }
6246 } else if (payload_ty.isSlice(zcu)) {6210 } else if (payload_ty.isSlice(zcu)) {
6211 const base_offset = operand.memOffset();
6212 try cg.emitMemBase(operand);
6247 switch (cg.ptr_size) {6213 switch (cg.ptr_size) {
6248 .wasm32 => try cg.addMemArg(.i32_load, .{ .offset = operand.offset(), .alignment = .@"4" }),6214 .wasm32 => try cg.addMemArg(.i32_load, .{ .offset = base_offset, .alignment = .@"4" }),
6249 .wasm64 => try cg.addMemArg(.i64_load, .{ .offset = operand.offset(), .alignment = .@"8" }),6215 .wasm64 => try cg.addMemArg(.i64_load, .{ .offset = base_offset, .alignment = .@"8" }),
6250 }6216 }
6251 } else {6217 } else {
6252 if (op_kind == .ptr) {6218 if (op_kind == .ptr) {
6219 try cg.emitMemBase(operand);
6253 try cg.addMemArg(.i32_load, .{6220 try cg.addMemArg(.i32_load, .{
6254 .offset = operand.offset(),6221 .offset = operand.memOffset(),
6255 .alignment = .@"4",6222 .alignment = .@"4",
6256 });6223 });
6224 } else {
6225 try cg.emitWValue(operand);
6257 }6226 }
6258 }6227 }
62596228
...@@ -6278,7 +6247,7 @@ fn airOptionalPayload(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -6278,7 +6247,7 @@ fn airOptionalPayload(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6278 if (opt_ty.optionalReprIsPayload(zcu)) break :result operand;6247 if (opt_ty.optionalReprIsPayload(zcu)) break :result operand;
62796248
6280 if (isByRef(payload_ty, zcu, cg.target)) {6249 if (isByRef(payload_ty, zcu, cg.target)) {
6281 break :result try cg.buildPointerOffset(operand, 0, .new);6250 break :result operand;
6282 }6251 }
62836252
6284 break :result try cg.load(operand, payload_ty, .{});6253 break :result try cg.load(operand, payload_ty, .{});
...@@ -6298,7 +6267,7 @@ fn airOptionalPayloadPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -6298,7 +6267,7 @@ fn airOptionalPayloadPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6298 break :result operand;6267 break :result operand;
6299 }6268 }
63006269
6301 break :result try cg.buildPointerOffset(operand, 0, .new);6270 break :result operand;
6302 };6271 };
6303 return cg.finishAir(inst, result, &.{ty_op.operand});6272 return cg.finishAir(inst, result, &.{ty_op.operand});
6304}6273}
...@@ -6319,12 +6288,11 @@ fn airOptionalPayloadPtrSet(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void...@@ -6319,12 +6288,11 @@ fn airOptionalPayloadPtrSet(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void
6319 return cg.fail("Optional type {f} too big to fit into stack frame", .{opt_ty.fmt(pt)});6288 return cg.fail("Optional type {f} too big to fit into stack frame", .{opt_ty.fmt(pt)});
6320 };6289 };
63216290
6322 try cg.emitWValue(operand);6291 try cg.emitMemBase(operand);
6323 try cg.addImm32(1);6292 try cg.addImm32(1);
6324 try cg.addMemArg(.i32_store8, .{ .offset = operand.offset() + offset, .alignment = .@"1" });6293 try cg.addMemArg(.i32_store8, .{ .offset = operand.memOffset() + offset, .alignment = .@"1" });
63256294
6326 const result = try cg.buildPointerOffset(operand, 0, .new);6295 return cg.finishAir(inst, operand, &.{ty_op.operand});
6327 return cg.finishAir(inst, result, &.{ty_op.operand});
6328}6296}
63296297
6330fn airWrapOptional(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {6298fn airWrapOptional(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
...@@ -6336,9 +6304,9 @@ fn airWrapOptional(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -6336,9 +6304,9 @@ fn airWrapOptional(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6336 const result = result: {6304 const result = result: {
6337 if (!payload_ty.hasRuntimeBits(zcu)) {6305 if (!payload_ty.hasRuntimeBits(zcu)) {
6338 const non_null_bit = try cg.allocStack(Type.u1);6306 const non_null_bit = try cg.allocStack(Type.u1);
6339 try cg.emitWValue(non_null_bit);6307 try cg.emitMemBase(non_null_bit);
6340 try cg.addImm32(1);6308 try cg.addImm32(1);
6341 try cg.addMemArg(.i32_store8, .{ .offset = non_null_bit.offset(), .alignment = .@"1" });6309 try cg.addMemArg(.i32_store8, .{ .offset = non_null_bit.memOffset(), .alignment = .@"1" });
6342 break :result non_null_bit;6310 break :result non_null_bit;
6343 }6311 }
63446312
...@@ -6353,12 +6321,11 @@ fn airWrapOptional(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -6353,12 +6321,11 @@ fn airWrapOptional(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
63536321
6354 // Create optional type, set the non-null bit, and store the operand inside the optional type6322 // Create optional type, set the non-null bit, and store the operand inside the optional type
6355 const result_ptr = try cg.allocStack(op_ty);6323 const result_ptr = try cg.allocStack(op_ty);
6356 try cg.emitWValue(result_ptr);6324 try cg.emitMemBase(result_ptr);
6357 try cg.addImm32(1);6325 try cg.addImm32(1);
6358 try cg.addMemArg(.i32_store8, .{ .offset = result_ptr.offset() + offset, .alignment = .@"1" });6326 try cg.addMemArg(.i32_store8, .{ .offset = result_ptr.memOffset() + offset, .alignment = .@"1" });
63596327
6360 const payload_ptr = try cg.buildPointerOffset(result_ptr, 0, .new);6328 try cg.store(result_ptr, operand, payload_ty, .{});
6361 try cg.store(payload_ptr, operand, payload_ty, .{});
6362 break :result result_ptr;6329 break :result result_ptr;
6363 };6330 };
63646331
...@@ -6483,7 +6450,7 @@ fn airPtrElemVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -6483,7 +6450,7 @@ fn airPtrElemVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6483 if (ptr_ty.isSlice(zcu)) {6450 if (ptr_ty.isSlice(zcu)) {
6484 _ = try cg.load(ptr, .usize, .{});6451 _ = try cg.load(ptr, .usize, .{});
6485 } else {6452 } else {
6486 try cg.lowerToStack(ptr);6453 try cg.emitWValue(ptr);
6487 }6454 }
64886455
6489 // calculate index into slice6456 // calculate index into slice
...@@ -6513,7 +6480,7 @@ fn airPtrElemPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -6513,7 +6480,7 @@ fn airPtrElemPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6513 if (ptr_ty.isSlice(zcu)) {6480 if (ptr_ty.isSlice(zcu)) {
6514 _ = try cg.load(ptr, .usize, .{});6481 _ = try cg.load(ptr, .usize, .{});
6515 } else {6482 } else {
6516 try cg.lowerToStack(ptr);6483 try cg.emitWValue(ptr);
6517 }6484 }
65186485
6519 // calculate index into ptr6486 // calculate index into ptr
...@@ -6538,7 +6505,7 @@ fn airPtrBinOp(cg: *CodeGen, inst: Air.Inst.Index, op: enum { add, sub }) InnerE...@@ -6538,7 +6505,7 @@ fn airPtrBinOp(cg: *CodeGen, inst: Air.Inst.Index, op: enum { add, sub }) InnerE
6538 else => ptr_ty.childType(zcu),6505 else => ptr_ty.childType(zcu),
6539 };6506 };
65406507
6541 try cg.lowerToStack(ptr);6508 try cg.emitWValue(ptr);
6542 try cg.emitWValue(offset);6509 try cg.emitWValue(offset);
65436510
6544 switch (cg.ptr_size) {6511 switch (cg.ptr_size) {
...@@ -6620,7 +6587,7 @@ fn memset(cg: *CodeGen, elem_ty: Type, ptr: WValue, len: WValue, value: WValue)...@@ -6620,7 +6587,7 @@ fn memset(cg: *CodeGen, elem_ty: Type, ptr: WValue, len: WValue, value: WValue)
6620 try cg.addLabel(.br_if, 0);6587 try cg.addLabel(.br_if, 0);
6621 }6588 }
66226589
6623 try cg.lowerToStack(ptr);6590 try cg.emitWValue(ptr);
6624 try cg.emitWValue(value);6591 try cg.emitWValue(value);
6625 try cg.emitWValue(len);6592 try cg.emitWValue(len);
6626 try cg.addExtended(.memory_fill);6593 try cg.addExtended(.memory_fill);
...@@ -6655,11 +6622,13 @@ fn memset(cg: *CodeGen, elem_ty: Type, ptr: WValue, len: WValue, value: WValue)...@@ -6655,11 +6622,13 @@ fn memset(cg: *CodeGen, elem_ty: Type, ptr: WValue, len: WValue, value: WValue)
66556622
6656 var end_ptr = try cg.allocLocal(Type.usize);6623 var end_ptr = try cg.allocLocal(Type.usize);
6657 defer end_ptr.free(cg);6624 defer end_ptr.free(cg);
6658 var new_ptr = try cg.buildPointerOffset(ptr, 0, .new);6625 var new_ptr = try cg.allocLocal(Type.usize);
6659 defer new_ptr.free(cg);6626 defer new_ptr.free(cg);
6627 try cg.emitWValue(ptr);
6628 try cg.addLocal(.local_set, new_ptr.local.value);
66606629
6661 // get the loop conditional: if current pointer address equals final pointer's address6630 // get the loop conditional: if current pointer address equals final pointer's address
6662 try cg.lowerToStack(ptr);6631 try cg.emitWValue(ptr);
6663 try cg.emitWValue(final_len);6632 try cg.emitWValue(final_len);
6664 switch (cg.ptr_size) {6633 switch (cg.ptr_size) {
6665 .wasm32 => try cg.addTag(.i32_add),6634 .wasm32 => try cg.addTag(.i32_add),
...@@ -6714,7 +6683,7 @@ fn airArrayElemVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -6714,7 +6683,7 @@ fn airArrayElemVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6714 const elem_size = elem_ty.abiSize(zcu);6683 const elem_size = elem_ty.abiSize(zcu);
67156684
6716 if (isByRef(array_ty, zcu, cg.target)) {6685 if (isByRef(array_ty, zcu, cg.target)) {
6717 try cg.lowerToStack(array);6686 try cg.emitWValue(array);
6718 try cg.emitWValue(index);6687 try cg.emitWValue(index);
6719 try cg.addImm32(@intCast(elem_size));6688 try cg.addImm32(@intCast(elem_size));
6720 try cg.addTag(.i32_mul);6689 try cg.addTag(.i32_mul);
...@@ -6747,7 +6716,7 @@ fn airArrayElemVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -6747,7 +6716,7 @@ fn airArrayElemVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6747 try cg.store(stack_vec, array, array_ty, .{});6716 try cg.store(stack_vec, array, array_ty, .{});
67486717
6749 // Is a non-unrolled vector (v128)6718 // Is a non-unrolled vector (v128)
6750 try cg.lowerToStack(stack_vec);6719 try cg.emitWValue(stack_vec);
6751 try cg.emitWValue(index);6720 try cg.emitWValue(index);
6752 try cg.addImm32(@intCast(elem_size));6721 try cg.addImm32(@intCast(elem_size));
6753 try cg.addTag(.i32_mul);6722 try cg.addTag(.i32_mul);
...@@ -6783,13 +6752,13 @@ fn airSplat(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -6783,13 +6752,13 @@ fn airSplat(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6783 64 => @backingInt(std.wasm.SimdOpcode.v128_load64_splat),6752 64 => @backingInt(std.wasm.SimdOpcode.v128_load64_splat),
6784 else => break :blk, // Cannot make use of simd-instructions6753 else => break :blk, // Cannot make use of simd-instructions
6785 };6754 };
6786 try cg.emitWValue(operand);6755 try cg.emitMemBase(operand);
6787 const extra_index: u32 = @intCast(cg.mir_extra.items.len);6756 const extra_index: u32 = @intCast(cg.mir_extra.items.len);
6788 // stores as := opcode, offset, alignment (opcode::memarg)6757 // stores as := opcode, offset, alignment (opcode::memarg)
6789 try cg.mir_extra.ensureUnusedCapacity(cg.gpa, 4);6758 try cg.mir_extra.ensureUnusedCapacity(cg.gpa, 4);
6790 cg.mir_extra.appendAssumeCapacity(opcode);6759 cg.mir_extra.appendAssumeCapacity(opcode);
6791 _ = try cg.addExtraAssumeCapacity(Mir.MemArg{6760 _ = try cg.addExtraAssumeCapacity(Mir.MemArg{
6792 .offset = operand.offset(),6761 .offset = operand.memOffset(),
6793 .alignment = elem_ty.abiAlignment(zcu).toStdMem(),6762 .alignment = elem_ty.abiAlignment(zcu).toStdMem(),
6794 });6763 });
6795 try cg.addInst(.{ .tag = .simd_prefix, .data = .{ .payload = extra_index } });6764 try cg.addInst(.{ .tag = .simd_prefix, .data = .{ .payload = extra_index } });
...@@ -6856,12 +6825,12 @@ fn airShuffleOne(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -6856,12 +6825,12 @@ fn airShuffleOne(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
68566825
6857 const dest_alloc = try cg.allocStack(result_ty);6826 const dest_alloc = try cg.allocStack(result_ty);
6858 for (mask, 0..) |mask_elem, out_idx| {6827 for (mask, 0..) |mask_elem, out_idx| {
6859 try cg.emitWValue(dest_alloc);6828 try cg.emitMemBase(dest_alloc);
6860 const elem_val = switch (mask_elem.unwrap()) {6829 const elem_val = switch (mask_elem.unwrap()) {
6861 .elem => |idx| try cg.load(operand, elem_ty, .{ .offset = elem_size * idx }),6830 .elem => |idx| try cg.load(operand, elem_ty, .{ .offset = elem_size * idx }),
6862 .value => |val| try cg.lowerConstant(.fromInterned(val)),6831 .value => |val| try cg.lowerConstant(.fromInterned(val)),
6863 };6832 };
6864 try cg.store(.stack, elem_val, elem_ty, .{ .offset = dest_alloc.offset() + elem_size * out_idx });6833 try cg.store(.stack, elem_val, elem_ty, .{ .offset = dest_alloc.memOffset() + elem_size * out_idx });
6865 }6834 }
6866 return cg.finishAir(inst, dest_alloc, &.{unwrapped.operand});6835 return cg.finishAir(inst, dest_alloc, &.{unwrapped.operand});
6867}6836}
...@@ -6924,13 +6893,13 @@ fn airShuffleTwo(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -6924,13 +6893,13 @@ fn airShuffleTwo(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
69246893
6925 const dest_alloc = try cg.allocStack(result_ty);6894 const dest_alloc = try cg.allocStack(result_ty);
6926 for (mask, 0..) |mask_elem, out_idx| {6895 for (mask, 0..) |mask_elem, out_idx| {
6927 try cg.emitWValue(dest_alloc);6896 try cg.emitMemBase(dest_alloc);
6928 const elem_val = switch (mask_elem.unwrap()) {6897 const elem_val = switch (mask_elem.unwrap()) {
6929 .a_elem => |idx| try cg.load(operand_a, elem_ty, .{ .offset = elem_size * idx }),6898 .a_elem => |idx| try cg.load(operand_a, elem_ty, .{ .offset = elem_size * idx }),
6930 .b_elem => |idx| try cg.load(operand_b, elem_ty, .{ .offset = elem_size * idx }),6899 .b_elem => |idx| try cg.load(operand_b, elem_ty, .{ .offset = elem_size * idx }),
6931 .undef => try cg.emitUndefined(elem_ty),6900 .undef => try cg.emitUndefined(elem_ty),
6932 };6901 };
6933 try cg.store(.stack, elem_val, elem_ty, .{ .offset = dest_alloc.offset() + elem_size * out_idx });6902 try cg.store(.stack, elem_val, elem_ty, .{ .offset = dest_alloc.memOffset() + elem_size * out_idx });
6934 }6903 }
6935 return cg.finishAir(inst, dest_alloc, &.{ unwrapped.operand_a, unwrapped.operand_b });6904 return cg.finishAir(inst, dest_alloc, &.{ unwrapped.operand_a, unwrapped.operand_b });
6936}6905}
...@@ -6956,39 +6925,18 @@ fn airAggregateInit(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -6956,39 +6925,18 @@ fn airAggregateInit(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6956 .array, .vector => {6925 .array, .vector => {
6957 const result = try cg.allocStack(result_ty);6926 const result = try cg.allocStack(result_ty);
6958 const elem_ty = result_ty.childType(zcu);6927 const elem_ty = result_ty.childType(zcu);
6959 const elem_size = @as(u32, @intCast(elem_ty.abiSize(zcu)));6928 const elem_size = elem_ty.abiSize(zcu);
6960 const sentinel = result_ty.sentinel(zcu);6929 const sentinel = result_ty.sentinel(zcu);
69616930
6962 // When the element type is by reference, we must copy the entire6931 var offset: u64 = 0;
6963 // value. It is therefore safer to move the offset pointer and store6932 for (elements) |elem| {
6964 // each value individually, instead of using store offsets.6933 const elem_val = try cg.resolveInst(elem);
6965 if (isByRef(elem_ty, zcu, cg.target)) {6934 try cg.store(result, elem_val, elem_ty, .{ .offset = offset });
6966 // copy stack pointer into a temporary local, which is6935 offset += elem_size;
6967 // moved for each element to store each value in the right position.6936 }
6968 const offset = try cg.buildPointerOffset(result, 0, .new);6937 if (sentinel) |s| {
6969 for (elements, 0..) |elem, elem_index| {6938 const val = try cg.resolveValue(s);
6970 const elem_val = try cg.resolveInst(elem);6939 try cg.store(result, val, elem_ty, .{ .offset = offset });
6971 try cg.store(offset, elem_val, elem_ty, .{});
6972
6973 if (elem_index < elements.len - 1 or sentinel != null) {
6974 _ = try cg.buildPointerOffset(offset, elem_size, .modify);
6975 }
6976 }
6977 if (sentinel) |s| {
6978 const val = try cg.resolveValue(s);
6979 try cg.store(offset, val, elem_ty, .{});
6980 }
6981 } else {
6982 var offset: u32 = 0;
6983 for (elements) |elem| {
6984 const elem_val = try cg.resolveInst(elem);
6985 try cg.store(result, elem_val, elem_ty, .{ .offset = offset });
6986 offset += elem_size;
6987 }
6988 if (sentinel) |s| {
6989 const val = try cg.resolveValue(s);
6990 try cg.store(result, val, elem_ty, .{ .offset = offset });
6991 }
6992 }6940 }
6993 break :result_value result;6941 break :result_value result;
6994 },6942 },
...@@ -7001,10 +6949,9 @@ fn airAggregateInit(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -7001,10 +6949,9 @@ fn airAggregateInit(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
70016949
7002 const elem_ty = result_ty.fieldType(elem_index, zcu);6950 const elem_ty = result_ty.fieldType(elem_index, zcu);
7003 const field_offset = result_ty.structFieldOffset(elem_index, zcu);6951 const field_offset = result_ty.structFieldOffset(elem_index, zcu);
7004 const offset = try cg.buildPointerOffset(result, field_offset, .new);
70056952
7006 const value = try cg.resolveInst(elem);6953 const value = try cg.resolveInst(elem);
7007 try cg.store(offset, value, elem_ty, .{});6954 try cg.store(result, value, elem_ty, .{ .offset = field_offset });
7008 }6955 }
70096956
7010 break :result_value result;6957 break :result_value result;
...@@ -7051,12 +6998,7 @@ fn airUnionInit(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -7051,12 +6998,7 @@ fn airUnionInit(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7051 const result_ptr = try cg.allocStack(union_ty);6998 const result_ptr = try cg.allocStack(union_ty);
7052 const payload = try cg.resolveInst(extra.init);6999 const payload = try cg.resolveInst(extra.init);
7053 if (layout.tag_align.compare(.gte, layout.payload_align)) {7000 if (layout.tag_align.compare(.gte, layout.payload_align)) {
7054 if (isByRef(field_ty, zcu, cg.target)) {7001 try cg.store(result_ptr, payload, field_ty, .{ .offset = layout.tag_size });
7055 const payload_ptr = try cg.buildPointerOffset(result_ptr, layout.tag_size, .new);
7056 try cg.store(payload_ptr, payload, field_ty, .{});
7057 } else {
7058 try cg.store(result_ptr, payload, field_ty, .{ .offset = layout.tag_size });
7059 }
70607002
7061 if (layout.tag_size > 0) {7003 if (layout.tag_size > 0) {
7062 try cg.store(result_ptr, tag_int, .fromInterned(union_obj.enum_tag_type), .{});7004 try cg.store(result_ptr, tag_int, .fromInterned(union_obj.enum_tag_type), .{});
...@@ -7163,13 +7105,11 @@ fn airErrUnionPayloadPtrSet(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void...@@ -7163,13 +7105,11 @@ fn airErrUnionPayloadPtrSet(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void
7163 .{ .offset = errUnionErrorOffset(payload_ty, zcu) },7105 .{ .offset = errUnionErrorOffset(payload_ty, zcu) },
7164 );7106 );
71657107
7166 const result = result: {7108 if (!payload_ty.hasRuntimeBits(zcu)) {
7167 if (!payload_ty.hasRuntimeBits(zcu)) {7109 return cg.finishAir(inst, operand, &.{ty_op.operand});
7168 break :result operand;7110 }
7169 }
71707111
7171 break :result try cg.buildPointerOffset(operand, @as(u32, @intCast(errUnionPayloadOffset(payload_ty, zcu))), .new);7112 const result = try cg.offsetPointer(operand, errUnionPayloadOffset(payload_ty, zcu));
7172 };
7173 return cg.finishAir(inst, result, &.{ty_op.operand});7113 return cg.finishAir(inst, result, &.{ty_op.operand});
7174}7114}
71757115
...@@ -7194,16 +7134,23 @@ fn airFieldParentPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -7194,16 +7134,23 @@ fn airFieldParentPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7194 },7134 },
7195 };7135 };
71967136
7197 const result = if (field_offset != 0) result: {7137 if (field_offset == 0) {
7198 const base = try cg.buildPointerOffset(field_ptr, 0, .new);7138 return cg.finishAir(inst, field_ptr, &.{extra.field_ptr});
7199 try cg.addLocal(.local_get, base.local.value);7139 }
7200 try cg.addImm32(@intCast(field_offset));
7201 try cg.addTag(.i32_sub);
7202 try cg.addLocal(.local_set, base.local.value);
7203 break :result base;
7204 } else field_ptr;
72057140
7206 return cg.finishAir(inst, result, &.{extra.field_ptr});7141 // maybe use offsetPointer? sub looks better in disasm
7142 try cg.emitWValue(field_ptr);
7143 switch (cg.ptr_size) {
7144 .wasm32 => {
7145 try cg.addImm32(@intCast(field_offset));
7146 try cg.addTag(.i32_sub);
7147 },
7148 .wasm64 => {
7149 try cg.addImm64(field_offset);
7150 try cg.addTag(.i64_sub);
7151 },
7152 }
7153 return cg.finishAir(inst, .stack, &.{extra.field_ptr});
7207}7154}
72087155
7209fn sliceOrArrayPtr(cg: *CodeGen, ptr: WValue, ptr_ty: Type) InnerError!WValue {7156fn sliceOrArrayPtr(cg: *CodeGen, ptr: WValue, ptr_ty: Type) InnerError!WValue {
...@@ -7322,7 +7269,7 @@ fn airErrorName(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -7322,7 +7269,7 @@ fn airErrorName(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7322fn airPtrSliceFieldPtr(cg: *CodeGen, inst: Air.Inst.Index, offset: u32) InnerError!void {7269fn airPtrSliceFieldPtr(cg: *CodeGen, inst: Air.Inst.Index, offset: u32) InnerError!void {
7323 const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;7270 const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
7324 const slice_ptr = try cg.resolveInst(ty_op.operand);7271 const slice_ptr = try cg.resolveInst(ty_op.operand);
7325 const result = try cg.buildPointerOffset(slice_ptr, offset, .new);7272 const result = try cg.offsetPointer(slice_ptr, offset);
7326 return cg.finishAir(inst, result, &.{ty_op.operand});7273 return cg.finishAir(inst, result, &.{ty_op.operand});
7327}7274}
73287275
...@@ -7392,13 +7339,15 @@ fn lowerTry(...@@ -7392,13 +7339,15 @@ fn lowerTry(
7392 try cg.startBlock(.block, .empty);7339 try cg.startBlock(.block, .empty);
73937340
7394 // check if the error tag is set for the error union.7341 // check if the error tag is set for the error union.
7395 try cg.emitWValue(err_union);
7396 if (pl_has_bits or operand_is_ptr) {7342 if (pl_has_bits or operand_is_ptr) {
7397 const err_offset: u32 = @intCast(errUnionErrorOffset(pl_ty, zcu));7343 try cg.emitMemBase(err_union);
7344 const err_offset = errUnionErrorOffset(pl_ty, zcu);
7398 try cg.addMemArg(.i32_load16_u, .{7345 try cg.addMemArg(.i32_load16_u, .{
7399 .offset = err_union.offset() + err_offset,7346 .offset = err_union.memOffset() + err_offset,
7400 .alignment = Type.anyerror.abiAlignment(zcu).toStdMem(),7347 .alignment = Type.anyerror.abiAlignment(zcu).toStdMem(),
7401 });7348 });
7349 } else {
7350 try cg.emitWValue(err_union);
7402 }7351 }
7403 try cg.addTag(.i32_eqz);7352 try cg.addTag(.i32_eqz);
7404 try cg.addLabel(.br_if, 0); // jump out of block when error is '0'7353 try cg.addLabel(.br_if, 0); // jump out of block when error is '0'
...@@ -7417,9 +7366,9 @@ fn lowerTry(...@@ -7417,9 +7366,9 @@ fn lowerTry(
7417 return .none;7366 return .none;
7418 }7367 }
74197368
7420 const pl_offset: u32 = @intCast(errUnionPayloadOffset(pl_ty, zcu));7369 const pl_offset = errUnionPayloadOffset(pl_ty, zcu);
7421 if (operand_is_ptr or isByRef(pl_ty, zcu, cg.target)) {7370 if (operand_is_ptr or isByRef(pl_ty, zcu, cg.target)) {
7422 return buildPointerOffset(cg, err_union, pl_offset, .new);7371 return cg.offsetPointer(err_union, pl_offset);
7423 }7372 }
7424 const payload = try cg.load(err_union, pl_ty, .{ .offset = pl_offset });7373 const payload = try cg.load(err_union, pl_ty, .{ .offset = pl_offset });
7425 return payload.toLocal(cg, pl_ty);7374 return payload.toLocal(cg, pl_ty);
...@@ -7448,7 +7397,7 @@ fn callIntrinsic(...@@ -7448,7 +7397,7 @@ fn callIntrinsic(
7448 // and emit it as our first argument7397 // and emit it as our first argument
7449 const sret = if (want_sret_param) blk: {7398 const sret = if (want_sret_param) blk: {
7450 const sret_local = try cg.allocStack(return_type);7399 const sret_local = try cg.allocStack(return_type);
7451 try cg.lowerToStack(sret_local);7400 try cg.emitWValue(sret_local);
7452 break :blk sret_local;7401 break :blk sret_local;
7453 } else .none;7402 } else .none;
74547403
...@@ -7476,7 +7425,7 @@ fn airTagName(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -7476,7 +7425,7 @@ fn airTagName(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7476 const enum_ty = cg.typeOf(un_op);7425 const enum_ty = cg.typeOf(un_op);
74777426
7478 try cg.addInst(.{ .tag = .enum_tag_name_table_ref, .data = .{ .ip_index = enum_ty.toIntern() } });7427 try cg.addInst(.{ .tag = .enum_tag_name_table_ref, .data = .{ .ip_index = enum_ty.toIntern() } });
7479 try cg.lowerToStack(operand);7428 try cg.emitWValue(operand);
7480 try cg.addInst(.{ .tag = .call_tag_index, .data = .{ .ip_index = enum_ty.toIntern() } });7429 try cg.addInst(.{ .tag = .call_tag_index, .data = .{ .ip_index = enum_ty.toIntern() } });
74817430
7482 switch (cg.ptr_size) {7431 switch (cg.ptr_size) {
...@@ -7500,7 +7449,7 @@ fn airIsNamedEnumValue(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -7500,7 +7449,7 @@ fn airIsNamedEnumValue(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7500 const operand = try cg.resolveInst(un_op);7449 const operand = try cg.resolveInst(un_op);
7501 const enum_ty = cg.typeOf(un_op);7450 const enum_ty = cg.typeOf(un_op);
75027451
7503 try cg.lowerToStack(operand);7452 try cg.emitWValue(operand);
7504 try cg.addInst(.{ .tag = .call_tag_index, .data = .{ .ip_index = enum_ty.toIntern() } });7453 try cg.addInst(.{ .tag = .call_tag_index, .data = .{ .ip_index = enum_ty.toIntern() } });
7505 try cg.addImm32(~@as(u32, 0));7454 try cg.addImm32(~@as(u32, 0));
7506 try cg.addTag(.i32_ne);7455 try cg.addTag(.i32_ne);
...@@ -7614,9 +7563,9 @@ fn airCmpxchg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -7614,9 +7563,9 @@ fn airCmpxchg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
76147563
7615 const ptr_val = if (cg.useAtomicFeature()) val: {7564 const ptr_val = if (cg.useAtomicFeature()) val: {
7616 const val_local = try cg.allocLocal(ty);7565 const val_local = try cg.allocLocal(ty);
7617 try cg.emitWValue(ptr_operand);7566 try cg.emitMemBase(ptr_operand);
7618 try cg.lowerToStack(expected_val);7567 try cg.emitWValue(expected_val);
7619 try cg.lowerToStack(new_val);7568 try cg.emitWValue(new_val);
7620 try cg.addAtomicMemArg(switch (ty.abiSize(zcu)) {7569 try cg.addAtomicMemArg(switch (ty.abiSize(zcu)) {
7621 1 => .i32_atomic_rmw8_cmpxchg_u,7570 1 => .i32_atomic_rmw8_cmpxchg_u,
7622 2 => .i32_atomic_rmw16_cmpxchg_u,7571 2 => .i32_atomic_rmw16_cmpxchg_u,
...@@ -7624,7 +7573,7 @@ fn airCmpxchg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -7624,7 +7573,7 @@ fn airCmpxchg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7624 8 => .i32_atomic_rmw_cmpxchg,7573 8 => .i32_atomic_rmw_cmpxchg,
7625 else => |size| return cg.fail("TODO: implement `@cmpxchg` for types with abi size '{d}'", .{size}),7574 else => |size| return cg.fail("TODO: implement `@cmpxchg` for types with abi size '{d}'", .{size}),
7626 }, .{7575 }, .{
7627 .offset = ptr_operand.offset(),7576 .offset = ptr_operand.memOffset(),
7628 .alignment = ty.abiAlignment(zcu).toStdMem(),7577 .alignment = ty.abiAlignment(zcu).toStdMem(),
7629 });7578 });
7630 try cg.addLocal(.local_tee, val_local.local.value);7579 try cg.addLocal(.local_tee, val_local.local.value);
...@@ -7637,13 +7586,13 @@ fn airCmpxchg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -7637,13 +7586,13 @@ fn airCmpxchg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7637 }7586 }
7638 const ptr_val = try WValue.toLocal(try cg.load(ptr_operand, ty, .{}), cg, ty);7587 const ptr_val = try WValue.toLocal(try cg.load(ptr_operand, ty, .{}), cg, ty);
76397588
7640 try cg.lowerToStack(ptr_operand);7589 try cg.emitMemBase(ptr_operand);
7641 try cg.lowerToStack(new_val);7590 try cg.emitWValue(new_val);
7642 try cg.emitWValue(ptr_val);7591 try cg.emitWValue(ptr_val);
7643 _ = try cg.intCmp(int_ty, .eq, ptr_val, expected_val);7592 _ = try cg.intCmp(int_ty, .eq, ptr_val, expected_val);
7644 try cg.addLocal(.local_tee, cmp_result.local.value);7593 try cg.addLocal(.local_tee, cmp_result.local.value);
7645 try cg.addTag(.select);7594 try cg.addTag(.select);
7646 try cg.store(.stack, .stack, ty, .{});7595 try cg.store(.stack, .stack, ty, .{ .offset = ptr_operand.memOffset() });
76477596
7648 break :val ptr_val;7597 break :val ptr_val;
7649 };7598 };
...@@ -7684,9 +7633,9 @@ fn airAtomicLoad(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -7684,9 +7633,9 @@ fn airAtomicLoad(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7684 8 => .i64_atomic_load,7633 8 => .i64_atomic_load,
7685 else => |size| return cg.fail("TODO: @atomicLoad for types with abi size {d}", .{size}),7634 else => |size| return cg.fail("TODO: @atomicLoad for types with abi size {d}", .{size}),
7686 };7635 };
7687 try cg.emitWValue(ptr);7636 try cg.emitMemBase(ptr);
7688 try cg.addAtomicMemArg(tag, .{7637 try cg.addAtomicMemArg(tag, .{
7689 .offset = ptr.offset(),7638 .offset = ptr.memOffset(),
7690 .alignment = ty.abiAlignment(zcu).toStdMem(),7639 .alignment = ty.abiAlignment(zcu).toStdMem(),
7691 });7640 });
7692 } else {7641 } else {
...@@ -7719,7 +7668,7 @@ fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -7719,7 +7668,7 @@ fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7719 // create a loop to cmpxchg the new value7668 // create a loop to cmpxchg the new value
7720 try cg.startBlock(.loop, .empty);7669 try cg.startBlock(.loop, .empty);
77217670
7722 try cg.emitWValue(ptr);7671 try cg.emitMemBase(ptr);
7723 try cg.emitWValue(value);7672 try cg.emitWValue(value);
7724 if (op == .Nand) {7673 if (op == .Nand) {
7725 const and_res = try cg.intAnd(int_ty, value, operand);7674 const and_res = try cg.intAnd(int_ty, value, operand);
...@@ -7746,7 +7695,7 @@ fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -7746,7 +7695,7 @@ fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7746 else => return cg.fail("TODO: implement `@atomicRmw` with operation `{s}` for types larger than 64 bits", .{@tagName(op)}),7695 else => return cg.fail("TODO: implement `@atomicRmw` with operation `{s}` for types larger than 64 bits", .{@tagName(op)}),
7747 },7696 },
7748 .{7697 .{
7749 .offset = ptr.offset(),7698 .offset = ptr.memOffset(),
7750 .alignment = ty.abiAlignment(zcu).toStdMem(),7699 .alignment = ty.abiAlignment(zcu).toStdMem(),
7751 },7700 },
7752 );7701 );
...@@ -7764,7 +7713,7 @@ fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -7764,7 +7713,7 @@ fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
77647713
7765 // the other operations have their own instructions for Wasm.7714 // the other operations have their own instructions for Wasm.
7766 else => {7715 else => {
7767 try cg.emitWValue(ptr);7716 try cg.emitMemBase(ptr);
7768 try cg.emitWValue(operand);7717 try cg.emitWValue(operand);
7769 const tag: std.wasm.AtomicsOpcode = switch (ty.abiSize(zcu)) {7718 const tag: std.wasm.AtomicsOpcode = switch (ty.abiSize(zcu)) {
7770 1 => switch (op) {7719 1 => switch (op) {
...@@ -7806,7 +7755,7 @@ fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -7806,7 +7755,7 @@ fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7806 else => |size| return cg.fail("TODO: Implement `@atomicRmw` for types with abi size {d}", .{size}),7755 else => |size| return cg.fail("TODO: Implement `@atomicRmw` for types with abi size {d}", .{size}),
7807 };7756 };
7808 try cg.addAtomicMemArg(tag, .{7757 try cg.addAtomicMemArg(tag, .{
7809 .offset = ptr.offset(),7758 .offset = ptr.memOffset(),
7810 .alignment = ty.abiAlignment(zcu).toStdMem(),7759 .alignment = ty.abiAlignment(zcu).toStdMem(),
7811 });7760 });
7812 return cg.finishAir(inst, .stack, &.{ pl_op.operand, extra.operand });7761 return cg.finishAir(inst, .stack, &.{ pl_op.operand, extra.operand });
...@@ -7825,23 +7774,23 @@ fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -7825,23 +7774,23 @@ fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7825 => {7774 => {
7826 if (ty.isAnyFloat()) {7775 if (ty.isAnyFloat()) {
7827 const float_ty: FloatType = .fromType(cg, ty);7776 const float_ty: FloatType = .fromType(cg, ty);
7828 try cg.emitWValue(ptr);7777 try cg.emitMemBase(ptr);
7829 _ = switch (op) {7778 _ = switch (op) {
7830 .Add => try cg.floatAdd(float_ty, result, operand),7779 .Add => try cg.floatAdd(float_ty, result, operand),
7831 .Sub => try cg.floatSub(float_ty, result, operand),7780 .Sub => try cg.floatSub(float_ty, result, operand),
7832 else => unreachable,7781 else => unreachable,
7833 };7782 };
7834 try cg.store(.stack, .stack, ty, .{ .offset = ptr.offset() });7783 try cg.store(.stack, .stack, ty, .{ .offset = ptr.memOffset() });
7835 } else {7784 } else {
7836 const int_ty: IntType = .fromType(cg, ty);7785 const int_ty: IntType = .fromType(cg, ty);
7837 try cg.emitWValue(ptr);7786 try cg.emitMemBase(ptr);
7838 _ = switch (op) {7787 _ = switch (op) {
7839 .Add => try cg.intAdd(int_ty, result, operand),7788 .Add => try cg.intAdd(int_ty, result, operand),
7840 .Sub => try cg.intSub(int_ty, result, operand),7789 .Sub => try cg.intSub(int_ty, result, operand),
7841 else => unreachable,7790 else => unreachable,
7842 };7791 };
7843 _ = try cg.intWrap(int_ty, .stack);7792 _ = try cg.intWrap(int_ty, .stack);
7844 try cg.store(.stack, .stack, ty, .{ .offset = ptr.offset() });7793 try cg.store(.stack, .stack, ty, .{ .offset = ptr.memOffset() });
7845 }7794 }
7846 },7795 },
7847 .And,7796 .And,
...@@ -7849,39 +7798,39 @@ fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -7849,39 +7798,39 @@ fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7849 .Xor,7798 .Xor,
7850 => {7799 => {
7851 const int_ty: IntType = .fromType(cg, ty);7800 const int_ty: IntType = .fromType(cg, ty);
7852 try cg.emitWValue(ptr);7801 try cg.emitMemBase(ptr);
7853 _ = switch (op) {7802 _ = switch (op) {
7854 .And => try cg.intAnd(int_ty, result, operand),7803 .And => try cg.intAnd(int_ty, result, operand),
7855 .Or => try cg.intOr(int_ty, result, operand),7804 .Or => try cg.intOr(int_ty, result, operand),
7856 .Xor => try cg.intXor(int_ty, result, operand),7805 .Xor => try cg.intXor(int_ty, result, operand),
7857 else => unreachable,7806 else => unreachable,
7858 };7807 };
7859 try cg.store(.stack, .stack, ty, .{ .offset = ptr.offset() });7808 try cg.store(.stack, .stack, ty, .{ .offset = ptr.memOffset() });
7860 },7809 },
7861 .Max,7810 .Max,
7862 .Min,7811 .Min,
7863 => {7812 => {
7864 if (ty.isAnyFloat()) {7813 if (ty.isAnyFloat()) {
7865 const float_ty: FloatType = .fromType(cg, ty);7814 const float_ty: FloatType = .fromType(cg, ty);
7866 try cg.emitWValue(ptr);7815 try cg.emitMemBase(ptr);
7867 try cg.emitWValue(result);7816 try cg.emitWValue(result);
7868 try cg.emitWValue(operand);7817 try cg.emitWValue(operand);
7869 _ = try cg.floatCmp(float_ty, if (op == .Max) .gt else .lt, result, operand);7818 _ = try cg.floatCmp(float_ty, if (op == .Max) .gt else .lt, result, operand);
7870 try cg.addTag(.select);7819 try cg.addTag(.select);
7871 try cg.store(.stack, .stack, ty, .{ .offset = ptr.offset() });7820 try cg.store(.stack, .stack, ty, .{ .offset = ptr.memOffset() });
7872 } else {7821 } else {
7873 const int_ty: IntType = .fromType(cg, ty);7822 const int_ty: IntType = .fromType(cg, ty);
7874 try cg.emitWValue(ptr);7823 try cg.emitMemBase(ptr);
7875 try cg.emitWValue(result);7824 try cg.emitWValue(result);
7876 try cg.emitWValue(operand);7825 try cg.emitWValue(operand);
7877 _ = try cg.intCmp(int_ty, if (op == .Max) .gt else .lt, result, operand);7826 _ = try cg.intCmp(int_ty, if (op == .Max) .gt else .lt, result, operand);
7878 try cg.addTag(.select);7827 try cg.addTag(.select);
7879 try cg.store(.stack, .stack, ty, .{ .offset = ptr.offset() });7828 try cg.store(.stack, .stack, ty, .{ .offset = ptr.memOffset() });
7880 }7829 }
7881 },7830 },
7882 .Nand => {7831 .Nand => {
7883 const int_ty: IntType = .fromType(cg, ty);7832 const int_ty: IntType = .fromType(cg, ty);
7884 try cg.emitWValue(ptr);7833 try cg.emitMemBase(ptr);
7885 const and_res = try cg.intAnd(int_ty, result, operand);7834 const and_res = try cg.intAnd(int_ty, result, operand);
7886 if (int_ty.bits <= 32) {7835 if (int_ty.bits <= 32) {
7887 try cg.addImm32(~@as(u32, 0));7836 try cg.addImm32(~@as(u32, 0));
...@@ -7891,7 +7840,7 @@ fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -7891,7 +7840,7 @@ fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7891 return cg.fail("TODO: `@atomicRmw` with operator `Nand` for types larger than 64 bits", .{});7840 return cg.fail("TODO: `@atomicRmw` with operator `Nand` for types larger than 64 bits", .{});
7892 }7841 }
7893 _ = try cg.intXor(int_ty, and_res, .stack);7842 _ = try cg.intXor(int_ty, and_res, .stack);
7894 try cg.store(.stack, .stack, ty, .{ .offset = ptr.offset() });7843 try cg.store(.stack, .stack, ty, .{ .offset = ptr.memOffset() });
7895 },7844 },
7896 }7845 }
78977846
...@@ -7916,10 +7865,10 @@ fn airAtomicStore(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -7916,10 +7865,10 @@ fn airAtomicStore(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7916 8 => .i64_atomic_store,7865 8 => .i64_atomic_store,
7917 else => |size| return cg.fail("TODO: @atomicLoad for types with abi size {d}", .{size}),7866 else => |size| return cg.fail("TODO: @atomicLoad for types with abi size {d}", .{size}),
7918 };7867 };
7919 try cg.emitWValue(ptr);7868 try cg.emitMemBase(ptr);
7920 try cg.lowerToStack(operand);7869 try cg.emitWValue(operand);
7921 try cg.addAtomicMemArg(tag, .{7870 try cg.addAtomicMemArg(tag, .{
7922 .offset = ptr.offset(),7871 .offset = ptr.memOffset(),
7923 .alignment = ty.abiAlignment(zcu).toStdMem(),7872 .alignment = ty.abiAlignment(zcu).toStdMem(),
7924 });7873 });
7925 } else {7874 } else {
...@@ -7996,7 +7945,7 @@ fn airAsm(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -7996,7 +7945,7 @@ fn airAsm(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7996 return cg.fail("Self-hosted wasm backend requires input constraint to be equal \"r\"", .{});7945 return cg.fail("Self-hosted wasm backend requires input constraint to be equal \"r\"", .{});
7997 }7946 }
79987947
7999 try cg.lowerToStack(operand);7948 try cg.emitWValue(operand);
8000 const op_local = try WValue.toLocal(.stack, cg, cg.typeOf(input.operand));7949 const op_local = try WValue.toLocal(.stack, cg, cg.typeOf(input.operand));
80017950
8002 const gop = try local_map.getOrPutValue(cg.gpa, name, op_local.local.value);7951 const gop = try local_map.getOrPutValue(cg.gpa, name, op_local.local.value);