authorgravatar for alex@alexrp.comAlex Rønne Petersen <alex@alexrp.com> 2026-09-17 00:09:47+02:00
committergravatar for alex@alexrp.comAlex Rønne Petersen <alex@alexrp.com> 2026-09-17 00:09:47+02:00
logd8147ba476217797d600b7e5f46f0a67a5b96188
tree3b72c66376a962ddf75ff319fecd8489633e8fb2
parent39512c3e5547b411294369c8eec8093045f74d64
parent5119f0a70de2565f22e1edc5d43d49b655c6f7a7

Merge pull request 'wasm: codegen improvements + better liveness handling' (#36846) from pavelverigo/zig:wasm-codegen-imporvements into master

Reviewed-on: https://codeberg.org/ziglang/zig/pulls/36846

4 files changed, 1080 insertions(+), 1036 deletions(-)

src/codegen/wasm/CodeGen.zig+1057-1020
...@@ -136,13 +136,10 @@ air: Air,...@@ -136,13 +136,10 @@ air: Air,
136liveness: Air.Liveness,136liveness: Air.Liveness,
137gpa: mem.Allocator,137gpa: mem.Allocator,
138func_index: InternPool.Index,138func_index: InternPool.Index,
139/// Contains a list of current branches.139/// Contains a list of current branches. Stores LocalPool before entering branch.
140/// When we return from a branch, the branch will be popped from this list,140branches: std.ArrayList(LocalPool) = .empty,
141/// which means branches can only contain references from within its own branch,
142/// or a branch higher (lower index) in the tree.
143branches: std.ArrayList(Branch) = .empty,
144/// Table to save `WValue`'s generated by an `Air.Inst`141/// Table to save `WValue`'s generated by an `Air.Inst`
145// values: ValueTable,142values: ValueTable = .empty,
146/// Mapping from Air.Inst.Index to block ids143/// Mapping from Air.Inst.Index to block ids
147blocks: std.array_hash_map.Auto(Air.Inst.Index, struct {144blocks: std.array_hash_map.Auto(Air.Inst.Index, struct {
148 label: u32,145 label: u32,
...@@ -150,10 +147,6 @@ blocks: std.array_hash_map.Auto(Air.Inst.Index, struct {...@@ -150,10 +147,6 @@ blocks: std.array_hash_map.Auto(Air.Inst.Index, struct {
150}) = .{},147}) = .{},
151/// Maps `loop` instructions to their label. `br` to here repeats the loop.148/// Maps `loop` instructions to their label. `br` to here repeats the loop.
152loops: std.AutoHashMapUnmanaged(Air.Inst.Index, u32) = .empty,149loops: std.AutoHashMapUnmanaged(Air.Inst.Index, u32) = .empty,
153/// The index the next local generated will have
154/// NOTE: arguments share the index with locals therefore the first variable
155/// will have the index that comes after the last argument's index
156local_index: u32,
157/// The index of the current argument.150/// The index of the current argument.
158/// Used to track which argument is being referenced in `airArg`.151/// Used to track which argument is being referenced in `airArg`.
159arg_index: u32 = 0,152arg_index: u32 = 0,
...@@ -207,23 +200,8 @@ stack_size: u32 = 0,...@@ -207,23 +200,8 @@ stack_size: u32 = 0,
207/// However, local variables or the usage of `incoming_stack_alignment` in a `CallingConvention` can overwrite this default.200/// However, local variables or the usage of `incoming_stack_alignment` in a `CallingConvention` can overwrite this default.
208stack_alignment: Alignment = .@"16",201stack_alignment: Alignment = .@"16",
209202
210// For each individual Wasm valtype we store a seperate free list which203local_pool: LocalPool = .{},
211// allows us to re-use locals that are no longer used. e.g. a temporary local.204wasm_param_count: u32 = undefined,
212/// A list of indexes which represents a local of valtype `i32`.
213/// It is illegal to store a non-i32 valtype in this list.
214free_locals_i32: std.ArrayList(u32) = .empty,
215/// A list of indexes which represents a local of valtype `i64`.
216/// It is illegal to store a non-i64 valtype in this list.
217free_locals_i64: std.ArrayList(u32) = .empty,
218/// A list of indexes which represents a local of valtype `f32`.
219/// It is illegal to store a non-f32 valtype in this list.
220free_locals_f32: std.ArrayList(u32) = .empty,
221/// A list of indexes which represents a local of valtype `f64`.
222/// It is illegal to store a non-f64 valtype in this list.
223free_locals_f64: std.ArrayList(u32) = .empty,
224/// A list of indexes which represents a local of valtype `v127`.
225/// It is illegal to store a non-v128 valtype in this list.
226free_locals_v128: std.ArrayList(u32) = .empty,
227205
228/// When in debug mode, this tracks if no `finishAir` was missed.206/// When in debug mode, this tracks if no `finishAir` was missed.
229/// Forgetting to call `finishAir` will cause the result to not be207/// Forgetting to call `finishAir` will cause the result to not be
...@@ -239,13 +217,7 @@ const WValue = union(enum) {...@@ -239,13 +217,7 @@ const WValue = union(enum) {
239 none: void,217 none: void,
240 /// The value lives on top of the stack218 /// The value lives on top of the stack
241 stack: void,219 stack: void,
242 /// Index of the local220 local: u32, // index
243 local: struct {
244 /// Contains the index to the local
245 value: u32,
246 /// The amount of instructions referencing this `WValue`
247 references: u32,
248 },
249 /// An immediate 32bit value221 /// An immediate 32bit value
250 imm32: u32,222 imm32: u32,
251 /// An immediate 64bit value223 /// An immediate 64bit value
...@@ -271,23 +243,16 @@ const WValue = union(enum) {...@@ -271,23 +243,16 @@ const WValue = union(enum) {
271 /// pointing to where the value lives.243 /// pointing to where the value lives.
272 stack_offset: struct {244 stack_offset: struct {
273 /// Contains the actual value of the offset245 /// Contains the actual value of the offset
274 value: u32,246 value: u64,
275 /// The amount of instructions referencing this `WValue`
276 references: u32,
277 },247 },
278248
279 /// Returns the offset from the bottom of the stack. This is useful when249 /// Returns the offset from the bottom of the stack.
280 /// we use the load or store instruction to ensure we retrieve the value250 fn memOffset(value: WValue) u64 {
281 /// from the correct position, rather than the value that lives at the251 return switch (value) {
282 /// bottom of the stack. For instances where `WValue` is not `stack_value`252 .stack_offset => |stack_offset| stack_offset.value,
283 /// this will return 0, which allows us to simply call this function for all
284 /// loads and stores without requiring checks everywhere.
285 fn offset(value: WValue) u32 {
286 switch (value) {
287 .stack_offset => |stack_offset| return stack_offset.value,
288 .dead => unreachable,253 .dead => unreachable,
289 else => return 0,254 else => 0,
290 }255 };
291 }256 }
292257
293 /// Promotes a `WValue` to a local when given value is on top of the stack.258 /// Promotes a `WValue` to a local when given value is on top of the stack.
...@@ -297,7 +262,7 @@ const WValue = union(enum) {...@@ -297,7 +262,7 @@ const WValue = union(enum) {
297 switch (value) {262 switch (value) {
298 .stack => {263 .stack => {
299 const new_local = try gen.allocLocal(ty);264 const new_local = try gen.allocLocal(ty);
300 try gen.addLocal(.local_set, new_local.local.value);265 try gen.addLocal(.local_set, new_local.local);
301 return new_local;266 return new_local;
302 },267 },
303 else => return value,268 else => return value,
...@@ -306,23 +271,9 @@ const WValue = union(enum) {...@@ -306,23 +271,9 @@ const WValue = union(enum) {
306271
307 /// Marks a local as no longer being referenced and essentially allows272 /// Marks a local as no longer being referenced and essentially allows
308 /// us to re-use it somewhere else within the function.273 /// us to re-use it somewhere else within the function.
309 /// The valtype of the local is deducted by using the index of the given `WValue`.274 fn free(value: *WValue, cg: *CodeGen) void {
310 fn free(value: *WValue, gen: *CodeGen) void {
311 if (value.* != .local) return;275 if (value.* != .local) return;
312 const local_value = value.local.value;276 cg.releaseLocal(value.local);
313 const reserved = gen.args.len + @intFromBool(gen.return_value != .none) + @intFromBool(gen.varargs != .none);
314 if (local_value < reserved + 2) return; // reserved locals may never be re-used. Also accounts for 2 stack locals.
315
316 const index = local_value - reserved;
317 const valtype = gen.mir_locals.items[index];
318 switch (valtype) {
319 .i32 => gen.free_locals_i32.append(gen.gpa, local_value) catch return, // It's ok to fail any of those, a new local can be allocated instead
320 .i64 => gen.free_locals_i64.append(gen.gpa, local_value) catch return,
321 .f32 => gen.free_locals_f32.append(gen.gpa, local_value) catch return,
322 .f64 => gen.free_locals_f64.append(gen.gpa, local_value) catch return,
323 .v128 => gen.free_locals_v128.append(gen.gpa, local_value) catch return,
324 }
325 log.debug("freed local ({d}) of type {}", .{ local_value, valtype });
326 value.* = .dead;277 value.* = .dead;
327 }278 }
328};279};
...@@ -330,6 +281,77 @@ const WValue = union(enum) {...@@ -330,6 +281,77 @@ const WValue = union(enum) {
330/// Hashmap to store generated `WValue` for each `Air.Inst.Ref`281/// Hashmap to store generated `WValue` for each `Air.Inst.Ref`
331const ValueTable = std.array_hash_map.Auto(Air.Inst.Ref, WValue);282const ValueTable = std.array_hash_map.Auto(Air.Inst.Ref, WValue);
332283
284const LocalPool = struct {
285 const OptionalIndex = enum(u32) {
286 none = math.maxInt(u32),
287 _,
288
289 fn init(index: u32) OptionalIndex {
290 const result: OptionalIndex = @fromBackingInt(index);
291 assert(result != .none);
292 return result;
293 }
294
295 fn unwrap(index: OptionalIndex) ?u32 {
296 return if (index == .none) null else @backingInt(index);
297 }
298 };
299
300 const Local = union {
301 ref_count: u32,
302 next_free: OptionalIndex,
303 };
304
305 locals: std.ArrayList(Local) = .empty,
306 free_i32: OptionalIndex = .none,
307 free_i64: OptionalIndex = .none,
308 free_f32: OptionalIndex = .none,
309 free_f64: OptionalIndex = .none,
310 free_v128: OptionalIndex = .none,
311
312 fn deinit(pool: *LocalPool, gpa: Allocator) void {
313 pool.locals.deinit(gpa);
314 pool.* = undefined;
315 }
316
317 fn clone(pool: *LocalPool, gpa: Allocator) Allocator.Error!LocalPool {
318 return .{
319 .locals = try pool.locals.clone(gpa),
320 .free_i32 = pool.free_i32,
321 .free_i64 = pool.free_i64,
322 .free_f32 = pool.free_f32,
323 .free_f64 = pool.free_f64,
324 .free_v128 = pool.free_v128,
325 };
326 }
327
328 fn freeList(pool: *LocalPool, valtype: std.wasm.Valtype) *OptionalIndex {
329 return switch (valtype) {
330 .i32 => &pool.free_i32,
331 .i64 => &pool.free_i64,
332 .f32 => &pool.free_f32,
333 .f64 => &pool.free_f64,
334 .v128 => &pool.free_v128,
335 };
336 }
337
338 fn restore(snapshot: *LocalPool, cg: *CodeGen) void {
339 const local_pool = &cg.local_pool;
340 const old_local_count = snapshot.locals.items.len;
341 assert(local_pool.locals.items.len >= old_local_count);
342 @memcpy(local_pool.locals.items[0..old_local_count], snapshot.locals.items);
343 local_pool.free_i32 = snapshot.free_i32;
344 local_pool.free_i64 = snapshot.free_i64;
345 local_pool.free_f32 = snapshot.free_f32;
346 local_pool.free_f64 = snapshot.free_f64;
347 local_pool.free_v128 = snapshot.free_v128;
348
349 for (old_local_count..local_pool.locals.items.len) |index| {
350 cg.appendFreeLocal(cg.wasm_param_count + @as(u32, @intCast(index)));
351 }
352 }
353};
354
333const bookkeeping_init = if (std.debug.runtime_safety) @as(usize, 0) else {};355const bookkeeping_init = if (std.debug.runtime_safety) @as(usize, 0) else {};
334356
335const InnerError = Error || error{357const InnerError = Error || error{
...@@ -343,14 +365,11 @@ pub fn deinit(cg: *CodeGen) void {...@@ -343,14 +365,11 @@ pub fn deinit(cg: *CodeGen) void {
343 const gpa = cg.gpa;365 const gpa = cg.gpa;
344 for (cg.branches.items) |*branch| branch.deinit(gpa);366 for (cg.branches.items) |*branch| branch.deinit(gpa);
345 cg.branches.deinit(gpa);367 cg.branches.deinit(gpa);
368 cg.values.deinit(gpa);
346 cg.blocks.deinit(gpa);369 cg.blocks.deinit(gpa);
347 cg.loops.deinit(gpa);370 cg.loops.deinit(gpa);
348 cg.simd_immediates.deinit(gpa);371 cg.simd_immediates.deinit(gpa);
349 cg.free_locals_i32.deinit(gpa);372 cg.local_pool.deinit(gpa);
350 cg.free_locals_i64.deinit(gpa);
351 cg.free_locals_f32.deinit(gpa);
352 cg.free_locals_f64.deinit(gpa);
353 cg.free_locals_v128.deinit(gpa);
354 cg.mir_instructions.deinit(gpa);373 cg.mir_instructions.deinit(gpa);
355 cg.mir_extra.deinit(gpa);374 cg.mir_extra.deinit(gpa);
356 cg.mir_locals.deinit(gpa);375 cg.mir_locals.deinit(gpa);
...@@ -369,19 +388,11 @@ pub fn fail(cg: *CodeGen, comptime fmt: []const u8, args: anytype) Error {...@@ -369,19 +388,11 @@ pub fn fail(cg: *CodeGen, comptime fmt: []const u8, args: anytype) Error {
369/// Resolves the `WValue` for the given instruction `inst`388/// Resolves the `WValue` for the given instruction `inst`
370/// When the given instruction has a `Value`, it returns a constant instead389/// When the given instruction has a `Value`, it returns a constant instead
371fn resolveInst(cg: *CodeGen, ref: Air.Inst.Ref) InnerError!WValue {390fn resolveInst(cg: *CodeGen, ref: Air.Inst.Ref) InnerError!WValue {
372 var branch_index = cg.branches.items.len;391 if (cg.values.get(ref)) |value| return value;
373 while (branch_index > 0) : (branch_index -= 1) {
374 const branch = cg.branches.items[branch_index - 1];
375 if (branch.values.get(ref)) |value| {
376 return value;
377 }
378 }
379392
380 // when we did not find an existing instruction, it393 // when we did not find an existing instruction, it
381 // means we must generate it from a constant.394 // means we must generate it from a constant.
382 // We always store constants in the most outer branch as they must never395 const gop = try cg.values.getOrPut(cg.gpa, ref);
383 // be removed. The most outer branch is always at index 0.
384 const gop = try cg.branches.items[0].values.getOrPut(cg.gpa, ref);
385 assert(!gop.found_existing);396 assert(!gop.found_existing);
386397
387 const pt = cg.pt;398 const pt = cg.pt;
...@@ -420,7 +431,34 @@ fn resolveValue(cg: *CodeGen, val: Value) InnerError!WValue {...@@ -420,7 +431,34 @@ fn resolveValue(cg: *CodeGen, val: Value) InnerError!WValue {
420431
421/// NOTE: if result == .stack, it will be stored in .local432/// NOTE: if result == .stack, it will be stored in .local
422fn finishAir(cg: *CodeGen, inst: Air.Inst.Index, result: WValue, operands: []const Air.Inst.Ref) InnerError!void {433fn finishAir(cg: *CodeGen, inst: Air.Inst.Index, result: WValue, operands: []const Air.Inst.Ref) InnerError!void {
434 return cg.finishAirWithBranchDeaths(inst, result, operands, &.{});
435}
436
437/// Takes `branch_deaths` from continuing branch and calls `processDeath` after result retaining logic.
438fn finishAirWithBranchDeaths(
439 cg: *CodeGen,
440 inst: Air.Inst.Index,
441 result: WValue,
442 operands: []const Air.Inst.Ref,
443 branch_deaths: []const Air.Inst.Index,
444) InnerError!void {
423 assert(operands.len <= Air.Liveness.bpi - 1);445 assert(operands.len <= Air.Liveness.bpi - 1);
446
447 if (result == .local) aliased: {
448 for (operands) |operand| {
449 if (operand.toIndex() == null) continue;
450 const resolved_operand = cg.values.getPtr(operand) orelse unreachable;
451 if (resolved_operand.* != .local or resolved_operand.local != result.local) continue;
452
453 if (result.local >= cg.wasm_param_count) {
454 const local = &cg.local_pool.locals.items[result.local - cg.wasm_param_count];
455 assert(local.ref_count > 0);
456 local.ref_count += 1;
457 }
458 break :aliased;
459 }
460 }
461
424 var tomb_bits = cg.liveness.getTombBits(inst);462 var tomb_bits = cg.liveness.getTombBits(inst);
425 for (operands) |operand| {463 for (operands) |operand| {
426 const dies = @as(u1, @truncate(tomb_bits)) != 0;464 const dies = @as(u1, @truncate(tomb_bits)) != 0;
...@@ -428,18 +466,27 @@ fn finishAir(cg: *CodeGen, inst: Air.Inst.Index, result: WValue, operands: []con...@@ -428,18 +466,27 @@ fn finishAir(cg: *CodeGen, inst: Air.Inst.Index, result: WValue, operands: []con
428 if (!dies) continue;466 if (!dies) continue;
429 processDeath(cg, operand);467 processDeath(cg, operand);
430 }468 }
469 for (branch_deaths) |death| cg.processDeath(death.toRef());
431 try cg.finishAirResult(inst, result);470 try cg.finishAirResult(inst, result);
432}471}
433472
434fn finishAirResult(cg: *CodeGen, inst: Air.Inst.Index, result: WValue) InnerError!void {473fn finishAirResult(cg: *CodeGen, inst: Air.Inst.Index, result: WValue) InnerError!void {
435 // results of `none` can never be referenced.474 // results of `none` can never be referenced.
436 if (result != .none) {475 if (result != .none) {
437 const trackable_result = if (result != .stack)476 if (cg.liveness.isUnused(inst)) {
438 result477 if (result == .stack) {
439 else478 try cg.addTag(.drop);
440 try result.toLocal(cg, cg.typeOfIndex(inst));479 } else {
441 const branch = cg.currentBranch();480 var unused_result = result;
442 branch.values.putAssumeCapacityNoClobber(inst.toRef(), trackable_result);481 unused_result.free(cg);
482 }
483 } else {
484 const trackable_result = if (result != .stack)
485 result
486 else
487 try result.toLocal(cg, cg.typeOfIndex(inst));
488 try cg.values.putNoClobber(cg.gpa, inst.toRef(), trackable_result);
489 }
443 }490 }
444491
445 if (std.debug.runtime_safety) {492 if (std.debug.runtime_safety) {
...@@ -447,17 +494,16 @@ fn finishAirResult(cg: *CodeGen, inst: Air.Inst.Index, result: WValue) InnerErro...@@ -447,17 +494,16 @@ fn finishAirResult(cg: *CodeGen, inst: Air.Inst.Index, result: WValue) InnerErro
447 }494 }
448}495}
449496
450const Branch = struct {497fn pushBranch(cg: *CodeGen) InnerError!void {
451 values: ValueTable = .{},498 var snapshot = try cg.local_pool.clone(cg.gpa);
452499 errdefer snapshot.deinit(cg.gpa);
453 fn deinit(branch: *Branch, gpa: Allocator) void {500 try cg.branches.append(cg.gpa, snapshot);
454 branch.values.deinit(gpa);501}
455 branch.* = undefined;
456 }
457};
458502
459inline fn currentBranch(cg: *CodeGen) *Branch {503fn popBranch(cg: *CodeGen) void {
460 return &cg.branches.items[cg.branches.items.len - 1];504 var snapshot = cg.branches.pop().?;
505 defer snapshot.deinit(cg.gpa);
506 snapshot.restore(cg);
461}507}
462508
463fn feed(cg: *CodeGen, bt: *Air.Liveness.BigTomb, operand: Air.Inst.Ref) void {509fn feed(cg: *CodeGen, bt: *Air.Liveness.BigTomb, operand: Air.Inst.Ref) void {
...@@ -468,20 +514,30 @@ fn feed(cg: *CodeGen, bt: *Air.Liveness.BigTomb, operand: Air.Inst.Ref) void {...@@ -468,20 +514,30 @@ fn feed(cg: *CodeGen, bt: *Air.Liveness.BigTomb, operand: Air.Inst.Ref) void {
468514
469fn processDeath(cg: *CodeGen, ref: Air.Inst.Ref) void {515fn processDeath(cg: *CodeGen, ref: Air.Inst.Ref) void {
470 if (ref.toIndex() == null) return;516 if (ref.toIndex() == null) return;
471 // Branches are currently only allowed to free locals allocated517 const value = cg.values.getPtr(ref) orelse unreachable;
472 // within their own branch.
473 // TODO: Upon branch consolidation free any locals if needed.
474 const value = cg.currentBranch().values.getPtr(ref) orelse return;
475 if (value.* != .local) return;518 if (value.* != .local) return;
476 const reserved_indexes = cg.args.len + @intFromBool(cg.return_value != .none);519 cg.releaseLocal(value.local);
477 if (value.local.value < reserved_indexes) {520}
478 return; // function arguments can never be re-used521
479 }522fn verifyLocalPool(cg: *CodeGen) void {
480 log.debug("Decreasing reference for ref: %{d}, using local '{d}'", .{ @backingInt(ref.toIndex().?), value.local.value });523 if (true) return; // TODO: test more
481 value.local.references -= 1; // if this panics, a call to `reuseOperand` was forgotten by the developer524 if (!std.debug.runtime_safety) return;
482 if (value.local.references == 0) {525
483 value.free(cg);526 var free_count: usize = 0;
527 for ([_]std.wasm.Valtype{ .i32, .i64, .f32, .f64, .v128 }) |valtype| {
528 var next = cg.local_pool.freeList(valtype).*;
529 while (next.unwrap()) |index| {
530 assert(index >= cg.wasm_param_count);
531 const pool_index = index - cg.wasm_param_count;
532 assert(pool_index < cg.local_pool.locals.items.len);
533 assert(index != cg.initial_stack_value.local);
534 assert(index != cg.bottom_stack_value.local);
535 assert(cg.mir_locals.items[pool_index] == valtype);
536 free_count += 1;
537 next = cg.local_pool.locals.items[pool_index].next_free;
538 }
484 }539 }
540 assert(free_count + 2 == cg.local_pool.locals.items.len);
485}541}
486542
487pub fn addInst(cg: *CodeGen, inst: Mir.Inst) error{OutOfMemory}!void {543pub fn addInst(cg: *CodeGen, inst: Mir.Inst) error{OutOfMemory}!void {
...@@ -569,7 +625,7 @@ fn addCallIntrinsic(cg: *CodeGen, intrinsic: Mir.Intrinsic) error{OutOfMemory}!v...@@ -569,7 +625,7 @@ fn addCallIntrinsic(cg: *CodeGen, intrinsic: Mir.Intrinsic) error{OutOfMemory}!v
569/// Returns the index into `mir_extra`625/// Returns the index into `mir_extra`
570fn addExtra(cg: *CodeGen, extra: anytype) error{OutOfMemory}!u32 {626fn addExtra(cg: *CodeGen, extra: anytype) error{OutOfMemory}!u32 {
571 const field_count = @typeInfo(@TypeOf(extra)).@"struct".field_names.len;627 const field_count = @typeInfo(@TypeOf(extra)).@"struct".field_names.len;
572 try cg.mir_extra.ensureUnusedCapacity(cg.gpa, field_count);628 try cg.mir_extra.ensureUnusedCapacity(cg.gpa, field_count * 2);
573 return cg.addExtraAssumeCapacity(extra);629 return cg.addExtraAssumeCapacity(extra);
574}630}
575631
...@@ -579,14 +635,22 @@ fn addExtraAssumeCapacity(cg: *CodeGen, extra: anytype) error{OutOfMemory}!u32 {...@@ -579,14 +635,22 @@ fn addExtraAssumeCapacity(cg: *CodeGen, extra: anytype) error{OutOfMemory}!u32 {
579 const info = @typeInfo(@TypeOf(extra)).@"struct";635 const info = @typeInfo(@TypeOf(extra)).@"struct";
580 const result: u32 = @intCast(cg.mir_extra.items.len);636 const result: u32 = @intCast(cg.mir_extra.items.len);
581 inline for (info.field_names, info.field_types) |field_name, field_type| {637 inline for (info.field_names, info.field_types) |field_name, field_type| {
582 cg.mir_extra.appendAssumeCapacity(switch (field_type) {638 switch (field_type) {
583 u32 => @field(extra, field_name),639 u64 => {
584 i32 => @bitCast(@field(extra, field_name)),640 const value = @field(extra, field_name);
585 InternPool.Index,641 cg.mir_extra.appendAssumeCapacity(@truncate(value));
586 InternPool.Nav.Index,642 cg.mir_extra.appendAssumeCapacity(@truncate(value >> 32));
587 => @backingInt(@field(extra, field_name)),643 },
588 else => @compileError("Unsupported field type " ++ @typeName(field_type)),644 else => cg.mir_extra.appendAssumeCapacity(switch (field_type) {
589 });645 u32 => @field(extra, field_name),
646 i32 => @bitCast(@field(extra, field_name)),
647 InternPool.Index,
648 InternPool.Nav.Index,
649 Alignment,
650 => @backingInt(@field(extra, field_name)),
651 else => @compileError("Unsupported field type " ++ @typeName(field_type)),
652 }),
653 }
590 }654 }
591 return result;655 return result;
592}656}
...@@ -622,18 +686,75 @@ pub fn typeToValtype(ty: Type, zcu: *const Zcu, target: *const std.Target) std.w...@@ -622,18 +686,75 @@ pub fn typeToValtype(ty: Type, zcu: *const Zcu, target: *const std.Target) std.w
622/// Differently from `typeToValtype` this also allows `void` to create a block686/// Differently from `typeToValtype` this also allows `void` to create a block
623/// with no return type687/// with no return type
624fn genBlockType(ty: Type, zcu: *const Zcu, target: *const std.Target) std.wasm.BlockType {688fn genBlockType(ty: Type, zcu: *const Zcu, target: *const std.Target) std.wasm.BlockType {
625 return switch (ty.ip_index) {689 if (!ty.hasRuntimeBits(zcu)) return .empty;
626 .void_type, .noreturn_type => .empty,690 return .fromValtype(typeToValtype(ty, zcu, target));
627 else => .fromValtype(typeToValtype(ty, zcu, target)),691}
692
693/// Returns `ptr` with `offset` applied.
694fn offsetPointer(cg: *CodeGen, ptr: WValue, offset: u64) InnerError!WValue {
695 if (offset == 0) return ptr;
696 return switch (ptr) {
697 .stack_offset => |stack_offset| .{ .stack_offset = .{
698 .value = stack_offset.value + offset,
699 } },
700 else => result: {
701 try cg.emitWValue(ptr);
702 switch (cg.ptr_size) {
703 .wasm32 => {
704 try cg.addImm32(@intCast(offset));
705 try cg.addTag(.i32_add);
706 },
707 .wasm64 => {
708 try cg.addImm64(offset);
709 try cg.addTag(.i64_add);
710 },
711 }
712 break :result .stack;
713 },
628 };714 };
629}715}
630716
717const PointerOffsetOp = enum { add, sub };
718
719/// The offset may only be known at runtime. Base pointer must be already on the stack. Result is left on it also.
720fn emitScaledPointerOffset(cg: *CodeGen, offset: WValue, scale: u64, op: PointerOffsetOp) InnerError!void {
721 if (scale == 0 or switch (offset) {
722 .imm32 => |imm| imm == 0,
723 .imm64 => |imm| imm == 0,
724 else => false,
725 }) return;
726
727 try cg.emitWValue(offset);
728 switch (cg.ptr_size) {
729 .wasm32 => {
730 if (scale != 1) {
731 try cg.addImm32(@intCast(scale));
732 try cg.addTag(.i32_mul);
733 }
734 try cg.addTag(switch (op) {
735 .add => .i32_add,
736 .sub => .i32_sub,
737 });
738 },
739 .wasm64 => {
740 if (scale != 1) {
741 try cg.addImm64(scale);
742 try cg.addTag(.i64_mul);
743 }
744 try cg.addTag(switch (op) {
745 .add => .i64_add,
746 .sub => .i64_sub,
747 });
748 },
749 }
750}
751
631/// Writes the bytecode depending on the given `WValue` in `val`752/// Writes the bytecode depending on the given `WValue` in `val`
632fn emitWValue(cg: *CodeGen, value: WValue) InnerError!void {753fn emitWValue(cg: *CodeGen, value: WValue) InnerError!void {
633 switch (value) {754 switch (value) {
634 .dead => unreachable, // reference to free'd `WValue` (missing reuseOperand?)755 .dead => unreachable, // reference to free'd `WValue`
635 .none, .stack => {}, // no-op756 .none, .stack => {}, // no-op
636 .local => |idx| try cg.addLocal(.local_get, idx.value),757 .local => |idx| try cg.addLocal(.local_get, idx),
637 .imm32 => |val| try cg.addImm32(val),758 .imm32 => |val| try cg.addImm32(val),
638 .imm64 => |val| try cg.addImm64(val),759 .imm64 => |val| try cg.addImm64(val),
639 .imm128 => |val| try cg.addImm128(val),760 .imm128 => |val| try cg.addImm128(val),
...@@ -695,38 +816,38 @@ fn emitWValue(cg: *CodeGen, value: WValue) InnerError!void {...@@ -695,38 +816,38 @@ fn emitWValue(cg: *CodeGen, value: WValue) InnerError!void {
695 });816 });
696 }817 }
697 },818 },
698 .stack_offset => try cg.addLocal(.local_get, cg.bottom_stack_value.local.value), // caller must ensure to address the offset819 .stack_offset => |stack_offset| {
820 try cg.emitWValue(cg.bottom_stack_value);
821 _ = try cg.offsetPointer(.stack, stack_offset.value);
822 },
699 }823 }
700}824}
701825
702/// If given a local or stack-offset, increases the reference count by 1.826/// Emits the base address for a memory instruction.
703/// The old `WValue` found at instruction `ref` is then replaced by the827fn emitMemBase(cg: *CodeGen, ptr: WValue) InnerError!void {
704/// modified `WValue` and returned. When given a non-local or non-stack-offset,828 switch (ptr) {
705/// returns the given `operand` itfunc instead.829 .stack_offset => try cg.addLocal(.local_get, cg.bottom_stack_value.local),
706fn reuseOperand(cg: *CodeGen, ref: Air.Inst.Ref, operand: WValue) WValue {830 else => try cg.emitWValue(ptr),
707 if (operand != .local and operand != .stack_offset) return operand;
708 var new_value = operand;
709 switch (new_value) {
710 .local => |*local| local.references += 1,
711 .stack_offset => |*stack_offset| stack_offset.references += 1,
712 else => unreachable,
713 }831 }
714 const old_value = cg.getResolvedInst(ref);
715 old_value.* = new_value;
716 return new_value;
717}832}
718833
719/// From a reference, returns its resolved `WValue`.834fn appendFreeLocal(cg: *CodeGen, index: u32) void {
720/// It's illegal to provide a `Air.Inst.Ref` that hasn't been resolved yet.835 assert(index >= cg.wasm_param_count);
721fn getResolvedInst(cg: *CodeGen, ref: Air.Inst.Ref) *WValue {836 const mir_index = index - cg.wasm_param_count;
722 var index = cg.branches.items.len;837 const valtype = cg.mir_locals.items[mir_index];
723 while (index > 0) : (index -= 1) {838 const free_list = cg.local_pool.freeList(valtype);
724 const branch = cg.branches.items[index - 1];839 cg.local_pool.locals.items[mir_index] = .{ .next_free = free_list.* };
725 if (branch.values.getPtr(ref)) |value| {840 free_list.* = .init(index);
726 return value;841 log.debug("freed local ({d}) of type {}", .{ index, valtype });
727 }842}
728 }843
729 unreachable; // developer-error: This can only be called on resolved instructions. Use `resolveInst` instead.844fn releaseLocal(cg: *CodeGen, index: u32) void {
845 if (index < cg.wasm_param_count) return;
846 const local = &cg.local_pool.locals.items[index - cg.wasm_param_count];
847 assert(local.ref_count > 0);
848 local.ref_count -= 1;
849 if (local.ref_count > 0) return;
850 cg.appendFreeLocal(index);
730}851}
731852
732/// Creates one locals for a given `Type`.853/// Creates one locals for a given `Type`.
...@@ -734,16 +855,13 @@ fn getResolvedInst(cg: *CodeGen, ref: Air.Inst.Ref) *WValue {...@@ -734,16 +855,13 @@ fn getResolvedInst(cg: *CodeGen, ref: Air.Inst.Ref) *WValue {
734fn allocLocal(cg: *CodeGen, ty: Type) InnerError!WValue {855fn allocLocal(cg: *CodeGen, ty: Type) InnerError!WValue {
735 const zcu = cg.pt.zcu;856 const zcu = cg.pt.zcu;
736 const valtype = typeToValtype(ty, zcu, cg.target);857 const valtype = typeToValtype(ty, zcu, cg.target);
737 const index_or_null = switch (valtype) {858 const free_list = cg.local_pool.freeList(valtype);
738 .i32 => cg.free_locals_i32.pop(),859 if (free_list.unwrap()) |index| {
739 .i64 => cg.free_locals_i64.pop(),860 const local = &cg.local_pool.locals.items[index - cg.wasm_param_count];
740 .f32 => cg.free_locals_f32.pop(),861 free_list.* = local.next_free;
741 .f64 => cg.free_locals_f64.pop(),862 local.* = .{ .ref_count = 1 };
742 .v128 => cg.free_locals_v128.pop(),
743 };
744 if (index_or_null) |index| {
745 log.debug("reusing local ({d}) of type {}", .{ index, valtype });863 log.debug("reusing local ({d}) of type {}", .{ index, valtype });
746 return .{ .local = .{ .value = index, .references = 1 } };864 return .{ .local = index };
747 }865 }
748 log.debug("new local of type {}", .{valtype});866 log.debug("new local of type {}", .{valtype});
749 return cg.ensureAllocLocal(ty);867 return cg.ensureAllocLocal(ty);
...@@ -753,10 +871,13 @@ fn allocLocal(cg: *CodeGen, ty: Type) InnerError!WValue {...@@ -753,10 +871,13 @@ fn allocLocal(cg: *CodeGen, ty: Type) InnerError!WValue {
753/// to use a zero-initialized local.871/// to use a zero-initialized local.
754fn ensureAllocLocal(cg: *CodeGen, ty: Type) InnerError!WValue {872fn ensureAllocLocal(cg: *CodeGen, ty: Type) InnerError!WValue {
755 const zcu = cg.pt.zcu;873 const zcu = cg.pt.zcu;
756 try cg.mir_locals.append(cg.gpa, typeToValtype(ty, zcu, cg.target));874 try cg.mir_locals.ensureUnusedCapacity(cg.gpa, 1);
757 const initial_index = cg.local_index;875 try cg.local_pool.locals.ensureUnusedCapacity(cg.gpa, 1);
758 cg.local_index += 1;876 assert(cg.mir_locals.items.len == cg.local_pool.locals.items.len);
759 return .{ .local = .{ .value = initial_index, .references = 1 } };877 const initial_index = cg.wasm_param_count + @as(u32, @intCast(cg.local_pool.locals.items.len));
878 cg.mir_locals.appendAssumeCapacity(typeToValtype(ty, zcu, cg.target));
879 cg.local_pool.locals.appendAssumeCapacity(.{ .ref_count = 1 });
880 return .{ .local = initial_index };
760}881}
761882
762pub const Error = codegen.Error;883pub const Error = codegen.Error;
...@@ -798,7 +919,6 @@ pub fn generate(...@@ -798,7 +919,6 @@ pub fn generate(
798 .args = cc_result.args,919 .args = cc_result.args,
799 .return_value = cc_result.return_value,920 .return_value = cc_result.return_value,
800 .varargs = cc_result.varargs,921 .varargs = cc_result.varargs,
801 .local_index = cc_result.local_index,
802 .mir_instructions = .empty,922 .mir_instructions = .empty,
803 .mir_extra = .empty,923 .mir_extra = .empty,
804 .mir_locals = .empty,924 .mir_locals = .empty,
...@@ -821,17 +941,10 @@ pub fn generate(...@@ -821,17 +941,10 @@ pub fn generate(
821941
822fn generateInner(cg: *CodeGen, any_returns: bool) InnerError!Mir {942fn generateInner(cg: *CodeGen, any_returns: bool) InnerError!Mir {
823 const zcu = cg.pt.zcu;943 const zcu = cg.pt.zcu;
824 // branch used for const values944 cg.wasm_param_count = @intCast(cg.args.len + @intFromBool(cg.return_value != .none) + @intFromBool(cg.varargs != .none));
825 try cg.branches.append(cg.gpa, .{});945 cg.initial_stack_value = try cg.ensureAllocLocal(Type.usize);
826 // func scope branch946 cg.bottom_stack_value = try cg.ensureAllocLocal(Type.usize);
827 try cg.branches.append(cg.gpa, .{});947 defer assert(cg.branches.items.len == 0);
828 defer {
829 var func_branch = cg.branches.pop().?;
830 func_branch.deinit(cg.gpa);
831 var const_branch = cg.branches.pop().?;
832 const_branch.deinit(cg.gpa);
833 assert(cg.branches.items.len == 0); // missing branch merge
834 }
835 // Generate MIR for function body948 // Generate MIR for function body
836 try cg.genBody(cg.air.getMainBody());949 try cg.genBody(cg.air.getMainBody());
837950
...@@ -856,11 +969,11 @@ fn generateInner(cg: *CodeGen, any_returns: bool) InnerError!Mir {...@@ -856,11 +969,11 @@ fn generateInner(cg: *CodeGen, any_returns: bool) InnerError!Mir {
856 .instructions = cg.mir_instructions.toOwnedSlice(),969 .instructions = cg.mir_instructions.toOwnedSlice(),
857 .extra = cg.mir_extra.toOwnedSliceAssert(),970 .extra = cg.mir_extra.toOwnedSliceAssert(),
858 .locals = cg.mir_locals.toOwnedSliceAssert(),971 .locals = cg.mir_locals.toOwnedSliceAssert(),
859 .prologue = if (cg.initial_stack_value == .none) .none else .{972 .prologue = .{
860 .sp_local = cg.initial_stack_value.local.value,973 .sp_local = cg.initial_stack_value.local,
861 .flags = .{ .stack_alignment = cg.stack_alignment },974 .flags = .{ .stack_alignment = cg.stack_alignment },
862 .stack_size = cg.stack_size,975 .stack_size = cg.stack_size,
863 .bottom_stack_local = cg.bottom_stack_value.local.value,976 .bottom_stack_local = cg.bottom_stack_value.local,
864 },977 },
865 .uavs = cg.mir_uavs.move(),978 .uavs = cg.mir_uavs.move(),
866 .indirect_function_set = cg.mir_indirect_function_set.move(),979 .indirect_function_set = cg.mir_indirect_function_set.move(),
...@@ -907,7 +1020,7 @@ fn resolveCallingConventionValues(...@@ -907,7 +1020,7 @@ fn resolveCallingConventionValues(
907 if (firstParamSRet(fn_info.cc, Type.fromInterned(fn_info.return_type), zcu, target)) {1020 if (firstParamSRet(fn_info.cc, Type.fromInterned(fn_info.return_type), zcu, target)) {
908 // the sret arg will be passed as first argument, therefore we1021 // the sret arg will be passed as first argument, therefore we
909 // set the `return_value` before allocating locals for regular args.1022 // set the `return_value` before allocating locals for regular args.
910 result.return_value = .{ .local = .{ .value = result.local_index, .references = 1 } };1023 result.return_value = .{ .local = result.local_index };
911 result.local_index += 1;1024 result.local_index += 1;
912 }1025 }
9131026
...@@ -918,7 +1031,7 @@ fn resolveCallingConventionValues(...@@ -918,7 +1031,7 @@ fn resolveCallingConventionValues(
918 continue;1031 continue;
919 }1032 }
9201033
921 try args.append(.{ .local = .{ .value = result.local_index, .references = 1 } });1034 try args.append(.{ .local = result.local_index });
922 result.local_index += 1;1035 result.local_index += 1;
923 }1036 }
924 },1037 },
...@@ -931,17 +1044,17 @@ fn resolveCallingConventionValues(...@@ -931,17 +1044,17 @@ fn resolveCallingConventionValues(
9311044
932 switch (abi.classifyType(param_ty, zcu, target)) {1045 switch (abi.classifyType(param_ty, zcu, target)) {
933 .direct, .indirect => {1046 .direct, .indirect => {
934 try args.append(.{ .local = .{ .value = result.local_index, .references = 1 } });1047 try args.append(.{ .local = result.local_index });
935 result.local_index += 1;1048 result.local_index += 1;
936 },1049 },
937 .double_i64 => {1050 .double_i64 => {
938 try args.append(.{ .local = .{ .value = result.local_index, .references = 1 } });1051 try args.append(.{ .local = result.local_index });
939 try args.append(.{ .local = .{ .value = result.local_index + 1, .references = 1 } });1052 try args.append(.{ .local = result.local_index + 1 });
940 result.local_index += 2;1053 result.local_index += 2;
941 },1054 },
942 .unrolled => |vector| {1055 .unrolled => |vector| {
943 for (0..vector.len) |_| {1056 for (0..vector.len) |_| {
944 try args.append(.{ .local = .{ .value = result.local_index, .references = 1 } });1057 try args.append(.{ .local = result.local_index });
945 result.local_index += 1;1058 result.local_index += 1;
946 }1059 }
947 },1060 },
...@@ -952,7 +1065,7 @@ fn resolveCallingConventionValues(...@@ -952,7 +1065,7 @@ fn resolveCallingConventionValues(
952 }1065 }
9531066
954 if (fn_info.is_var_args) {1067 if (fn_info.is_var_args) {
955 result.varargs = .{ .local = .{ .value = result.local_index, .references = 1 } };1068 result.varargs = .{ .local = result.local_index };
956 result.local_index += 1;1069 result.local_index += 1;
957 }1070 }
9581071
...@@ -983,7 +1096,7 @@ pub fn firstParamSRet(...@@ -983,7 +1096,7 @@ pub fn firstParamSRet(
983/// it matches the ABI.1096/// it matches the ABI.
984fn lowerArg(cg: *CodeGen, cc: std.lang.CallingConvention, ty: Type, value: WValue) !void {1097fn lowerArg(cg: *CodeGen, cc: std.lang.CallingConvention, ty: Type, value: WValue) !void {
985 if (cc != .wasm_mvp) {1098 if (cc != .wasm_mvp) {
986 return cg.lowerToStack(value);1099 return cg.emitWValue(value);
987 }1100 }
9881101
989 const zcu = cg.pt.zcu;1102 const zcu = cg.pt.zcu;
...@@ -991,74 +1104,37 @@ fn lowerArg(cg: *CodeGen, cc: std.lang.CallingConvention, ty: Type, value: WValu...@@ -991,74 +1104,37 @@ fn lowerArg(cg: *CodeGen, cc: std.lang.CallingConvention, ty: Type, value: WValu
991 switch (abi.classifyType(ty, zcu, cg.target)) {1104 switch (abi.classifyType(ty, zcu, cg.target)) {
992 .direct => |scalar_ty| {1105 .direct => |scalar_ty| {
993 if (!isByRef(ty, zcu, cg.target)) {1106 if (!isByRef(ty, zcu, cg.target)) {
994 return cg.lowerToStack(value);1107 return cg.emitWValue(value);
995 } else {1108 } else {
996 _ = try cg.load(value, scalar_ty, 0);1109 _ = try cg.load(value, scalar_ty, .{});
997 }1110 }
998 },1111 },
999 .double_i64 => {1112 .double_i64 => {
1000 assert(ty.abiSize(zcu) == 16);1113 assert(ty.abiSize(zcu) == 16);
1001 // in this case we have an integer or float that must be lowered as 2 i64's.1114 // in this case we have an integer or float that must be lowered as 2 i64's.
1002 try cg.emitWValue(value);1115 const base_offset = value.memOffset();
1003 try cg.addMemArg(.i64_load, .{ .offset = value.offset(), .alignment = 8 });1116 try cg.emitMemBase(value);
1004 try cg.emitWValue(value);1117 try cg.addMemArg(.i64_load, .{ .offset = base_offset, .alignment = .@"8" });
1005 try cg.addMemArg(.i64_load, .{ .offset = value.offset() + 8, .alignment = 8 });1118 try cg.emitMemBase(value);
1119 try cg.addMemArg(.i64_load, .{ .offset = base_offset + 8, .alignment = .@"8" });
1006 },1120 },
1007 .indirect => {1121 .indirect => {
1008 const stack_copy = try cg.allocStack(ty);1122 const stack_copy = try cg.allocStack(ty);
1009 try cg.store(stack_copy, value, ty, 0);1123 try cg.store(stack_copy, value, ty, .{});
1010 return cg.lowerToStack(stack_copy);1124 return cg.emitWValue(stack_copy);
1011 },1125 },
1012 .unrolled => |vector| {1126 .unrolled => |vector| {
1013 const elem_size: u32 = @intCast(vector.elem_type.abiSize(zcu));1127 const elem_size = vector.elem_type.abiSize(zcu);
1014 for (0..vector.len) |index| {1128 for (0..vector.len) |index| {
1015 _ = try cg.load(value, vector.elem_type, @intCast(index * elem_size));1129 _ = try cg.load(value, vector.elem_type, .{ .offset = index * elem_size });
1016 }
1017 },
1018 }
1019}
1020
1021/// Lowers a `WValue` to the stack. This means when the `value` results in
1022/// `.stack_offset` we calculate the pointer of this offset and use that.
1023/// The value is left on the stack, and not stored in any temporary.
1024fn lowerToStack(cg: *CodeGen, value: WValue) !void {
1025 switch (value) {
1026 .stack_offset => |offset| {
1027 try cg.emitWValue(value);
1028 if (offset.value > 0) {
1029 switch (cg.ptr_size) {
1030 .wasm32 => {
1031 try cg.addImm32(offset.value);
1032 try cg.addTag(.i32_add);
1033 },
1034 .wasm64 => {
1035 try cg.addImm64(offset.value);
1036 try cg.addTag(.i64_add);
1037 },
1038 }
1039 }1130 }
1040 },1131 },
1041 else => try cg.emitWValue(value),
1042 }1132 }
1043}1133}
10441134
1045/// Creates a local for the initial stack value
1046/// Asserts `initial_stack_value` is `.none`
1047fn initializeStack(cg: *CodeGen) !void {
1048 assert(cg.initial_stack_value == .none);
1049 // Reserve a local to store the current stack pointer
1050 // We can later use this local to set the stack pointer back to the value
1051 // we have stored here.
1052 cg.initial_stack_value = try cg.ensureAllocLocal(Type.usize);
1053 // Also reserve a local to store the bottom stack value
1054 cg.bottom_stack_value = try cg.ensureAllocLocal(Type.usize);
1055}
1056
1057/// Reads the stack pointer from `Context.initial_stack_value` and writes it1135/// Reads the stack pointer from `Context.initial_stack_value` and writes it
1058/// to the global stack pointer variable1136/// to the global stack pointer variable
1059fn restoreStackPointer(cg: *CodeGen) !void {1137fn restoreStackPointer(cg: *CodeGen) !void {
1060 // only restore the pointer if it was initialized
1061 if (cg.initial_stack_value == .none) return;
1062 // Get the original stack pointer's value1138 // Get the original stack pointer's value
1063 try cg.emitWValue(cg.initial_stack_value);1139 try cg.emitWValue(cg.initial_stack_value);
10641140
...@@ -1098,16 +1174,12 @@ fn allocInt(cg: *CodeGen, int_ty: IntType) !WValue {...@@ -1098,16 +1174,12 @@ fn allocInt(cg: *CodeGen, int_ty: IntType) !WValue {
1098fn allocStackBytes(cg: *CodeGen, size: u32, alignment: Alignment) !WValue {1174fn allocStackBytes(cg: *CodeGen, size: u32, alignment: Alignment) !WValue {
1099 assert(size > 0);1175 assert(size > 0);
11001176
1101 if (cg.initial_stack_value == .none) {
1102 try cg.initializeStack();
1103 }
1104
1105 cg.stack_alignment = cg.stack_alignment.max(alignment);1177 cg.stack_alignment = cg.stack_alignment.max(alignment);
11061178
1107 const offset: u32 = @intCast(alignment.forward(cg.stack_size));1179 const offset: u32 = @intCast(alignment.forward(cg.stack_size));
1108 defer cg.stack_size = offset + size;1180 defer cg.stack_size = offset + size;
11091181
1110 return .{ .stack_offset = .{ .value = offset, .references = 1 } };1182 return .{ .stack_offset = .{ .value = offset } };
1111}1183}
11121184
1113/// From a given AIR instruction generates a pointer to the stack where1185/// From a given AIR instruction generates a pointer to the stack where
...@@ -1120,10 +1192,6 @@ fn allocStackPtr(cg: *CodeGen, inst: Air.Inst.Index) !WValue {...@@ -1120,10 +1192,6 @@ fn allocStackPtr(cg: *CodeGen, inst: Air.Inst.Index) !WValue {
1120 const ptr_ty = cg.typeOfIndex(inst);1192 const ptr_ty = cg.typeOfIndex(inst);
1121 const pointee_ty = ptr_ty.childType(zcu);1193 const pointee_ty = ptr_ty.childType(zcu);
11221194
1123 if (cg.initial_stack_value == .none) {
1124 try cg.initializeStack();
1125 }
1126
1127 const abi_alignment = ptr_ty.ptrAlignment(zcu);1195 const abi_alignment = ptr_ty.ptrAlignment(zcu);
1128 const abi_size = std.math.cast(u32, pointee_ty.abiSize(zcu)) orelse {1196 const abi_size = std.math.cast(u32, pointee_ty.abiSize(zcu)) orelse {
1129 return cg.fail("Type {f} with ABI size of {d} exceeds stack frame size", .{1197 return cg.fail("Type {f} with ABI size of {d} exceeds stack frame size", .{
...@@ -1135,7 +1203,7 @@ fn allocStackPtr(cg: *CodeGen, inst: Air.Inst.Index) !WValue {...@@ -1135,7 +1203,7 @@ fn allocStackPtr(cg: *CodeGen, inst: Air.Inst.Index) !WValue {
1135 const offset: u32 = @intCast(abi_alignment.forward(cg.stack_size));1203 const offset: u32 = @intCast(abi_alignment.forward(cg.stack_size));
1136 defer cg.stack_size = offset + abi_size;1204 defer cg.stack_size = offset + abi_size;
11371205
1138 return .{ .stack_offset = .{ .value = offset, .references = 1 } };1206 return .{ .stack_offset = .{ .value = offset } };
1139}1207}
11401208
1141fn emitMemoryCopy(cg: *CodeGen, dst: WValue, src: WValue, len: WValue) !void {1209fn emitMemoryCopy(cg: *CodeGen, dst: WValue, src: WValue, len: WValue) !void {
...@@ -1165,8 +1233,8 @@ fn emitMemoryCopy(cg: *CodeGen, dst: WValue, src: WValue, len: WValue) !void {...@@ -1165,8 +1233,8 @@ fn emitMemoryCopy(cg: *CodeGen, dst: WValue, src: WValue, len: WValue) !void {
1165 try cg.addLabel(.br_if, 0);1233 try cg.addLabel(.br_if, 0);
1166 }1234 }
11671235
1168 try cg.lowerToStack(dst);1236 try cg.emitWValue(dst);
1169 try cg.lowerToStack(src);1237 try cg.emitWValue(src);
1170 try cg.emitWValue(len);1238 try cg.emitWValue(len);
1171 try cg.addExtended(.memory_copy);1239 try cg.addExtended(.memory_copy);
11721240
...@@ -1176,26 +1244,26 @@ fn emitMemoryCopy(cg: *CodeGen, dst: WValue, src: WValue, len: WValue) !void {...@@ -1176,26 +1244,26 @@ fn emitMemoryCopy(cg: *CodeGen, dst: WValue, src: WValue, len: WValue) !void {
1176}1244}
11771245
1178fn memcpy(cg: *CodeGen, dst: WValue, src: WValue, len: WValue) !void {1246fn memcpy(cg: *CodeGen, dst: WValue, src: WValue, len: WValue) !void {
1179 if (cg.target.cpu.has(.wasm, .bulk_memory)) {1247 if (cg.target.cpu.has(.wasm, .bulk_memory_opt)) {
1180 try cg.emitMemoryCopy(dst, src, len);1248 try cg.emitMemoryCopy(dst, src, len);
1181 return;1249 return;
1182 }1250 }
11831251
1184 try cg.lowerToStack(dst);1252 try cg.emitWValue(dst);
1185 try cg.lowerToStack(src);1253 try cg.emitWValue(src);
1186 try cg.emitWValue(len);1254 try cg.emitWValue(len);
1187 try cg.addCallIntrinsic(.memcpy);1255 try cg.addCallIntrinsic(.memcpy);
1188 try cg.addTag(.drop);1256 try cg.addTag(.drop);
1189}1257}
11901258
1191fn memmove(cg: *CodeGen, dst: WValue, src: WValue, len: WValue) !void {1259fn memmove(cg: *CodeGen, dst: WValue, src: WValue, len: WValue) !void {
1192 if (cg.target.cpu.has(.wasm, .bulk_memory)) {1260 if (cg.target.cpu.has(.wasm, .bulk_memory_opt)) {
1193 try cg.emitMemoryCopy(dst, src, len);1261 try cg.emitMemoryCopy(dst, src, len);
1194 return;1262 return;
1195 }1263 }
11961264
1197 try cg.lowerToStack(dst);1265 try cg.emitWValue(dst);
1198 try cg.lowerToStack(src);1266 try cg.emitWValue(src);
1199 try cg.emitWValue(len);1267 try cg.emitWValue(len);
1200 try cg.addCallIntrinsic(.memmove);1268 try cg.addCallIntrinsic(.memmove);
1201 try cg.addTag(.drop);1269 try cg.addTag(.drop);
...@@ -1277,34 +1345,6 @@ pub fn determineSimdStoreStrategy(ty: Type, zcu: *const Zcu, target: *const std....@@ -1277,34 +1345,6 @@ pub fn determineSimdStoreStrategy(ty: Type, zcu: *const Zcu, target: *const std.
1277 return .unrolled;1345 return .unrolled;
1278}1346}
12791347
1280/// Creates a new local for a pointer that points to memory with given offset.
1281/// This can be used to get a pointer to a struct field, error payload, etc.
1282/// By providing `modify` as action, it will modify the given `ptr_value` instead of making a new
1283/// local value to store the pointer. This allows for local re-use and improves binary size.
1284fn buildPointerOffset(cg: *CodeGen, ptr_value: WValue, offset: u64, action: enum { modify, new }) InnerError!WValue {
1285 // do not perform arithmetic when offset is 0.
1286 if (offset == 0 and ptr_value.offset() == 0 and action == .modify) return ptr_value;
1287 const result_ptr: WValue = switch (action) {
1288 .new => try cg.ensureAllocLocal(Type.usize),
1289 .modify => ptr_value,
1290 };
1291 try cg.emitWValue(ptr_value);
1292 if (offset + ptr_value.offset() > 0) {
1293 switch (cg.ptr_size) {
1294 .wasm32 => {
1295 try cg.addImm32(@intCast(offset + ptr_value.offset()));
1296 try cg.addTag(.i32_add);
1297 },
1298 .wasm64 => {
1299 try cg.addImm64(offset + ptr_value.offset());
1300 try cg.addTag(.i64_add);
1301 },
1302 }
1303 }
1304 try cg.addLocal(.local_set, result_ptr.local.value);
1305 return result_ptr;
1306}
1307
1308fn genInst(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {1348fn genInst(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
1309 const zcu = cg.pt.zcu;1349 const zcu = cg.pt.zcu;
1310 const air_tags = cg.air.instructions.items(.tag);1350 const air_tags = cg.air.instructions.items(.tag);
...@@ -1325,14 +1365,12 @@ fn genInst(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -1325,14 +1365,12 @@ fn genInst(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
1325 const elem_ty = cg.typeOf(bin_op.rhs);1365 const elem_ty = cg.typeOf(bin_op.rhs);
1326 const elem_size = elem_ty.abiSize(zcu);1366 const elem_size = elem_ty.abiSize(zcu);
13271367
1328 try cg.lowerToStack(vec_ptr);1368 try cg.emitWValue(vec_ptr);
1329 try cg.emitWValue(elem_idx);1369 try cg.emitScaledPointerOffset(elem_idx, elem_size, .add);
1330 try cg.addImm32(@intCast(elem_size));1370 var ptr = try WValue.toLocal(.stack, cg, Type.usize);
1331 try cg.addTag(.i32_mul);1371 defer ptr.free(cg);
1332 try cg.addTag(.i32_add);
1333 const ptr = try WValue.toLocal(.stack, cg, Type.usize);
13341372
1335 try cg.store(ptr, elem_val, elem_ty, 0);1373 try cg.store(ptr, elem_val, elem_ty, .{});
13361374
1337 return cg.finishAir(inst, .none, &.{ pl_op.operand, bin_op.lhs, bin_op.rhs });1375 return cg.finishAir(inst, .none, &.{ pl_op.operand, bin_op.lhs, bin_op.rhs });
1338 },1376 },
...@@ -1509,8 +1547,8 @@ fn genInst(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -1509,8 +1547,8 @@ fn genInst(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
1509 const result = try cg.allocStack(cg.typeOfIndex(inst));1547 const result = try cg.allocStack(cg.typeOfIndex(inst));
1510 const offset: u32 = @intCast(ty.abiSize(cg.pt.zcu));1548 const offset: u32 = @intCast(ty.abiSize(cg.pt.zcu));
15111549
1512 try cg.store(result, res_tmp, ty, 0);1550 try cg.store(result, res_tmp, ty, .{});
1513 try cg.store(result, ov_tmp, Type.u1, offset);1551 try cg.store(result, ov_tmp, .u1, .{ .offset = offset });
15141552
1515 try cg.finishAir(inst, result, &.{ extra.lhs, extra.rhs });1553 try cg.finishAir(inst, result, &.{ extra.lhs, extra.rhs });
1516 },1554 },
...@@ -1598,17 +1636,7 @@ fn genInst(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -1598,17 +1636,7 @@ fn genInst(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
1598 const src_int_ty: IntType = .fromType(cg, src_ty);1636 const src_int_ty: IntType = .fromType(cg, src_ty);
1599 const dest_int_ty: IntType = .fromType(cg, dest_ty);1637 const dest_int_ty: IntType = .fromType(cg, dest_ty);
16001638
1601 const src_bits = src_int_ty.bits;1639 const result = try cg.intCast(dest_int_ty, src_int_ty, operand);
1602 const dest_bits = dest_int_ty.bits;
1603
1604 const same_class: bool = (src_bits <= 32 and dest_bits <= 32) or
1605 (src_bits >= 33 and src_bits <= 64 and dest_bits >= 33 and dest_bits <= 64) or
1606 (src_bits >= 65 and src_bits <= 128 and dest_bits >= 65 and dest_bits <= 128);
1607
1608 const result = if (same_class)
1609 cg.reuseOperand(ty_op.operand, operand)
1610 else
1611 try cg.intCast(dest_int_ty, src_int_ty, operand);
16121640
1613 try cg.finishAir(inst, result, &.{ty_op.operand});1641 try cg.finishAir(inst, result, &.{ty_op.operand});
1614 },1642 },
...@@ -1625,12 +1653,7 @@ fn genInst(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -1625,12 +1653,7 @@ fn genInst(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
16251653
1626 const src_int_ty: IntType = .fromType(cg, src_ty);1654 const src_int_ty: IntType = .fromType(cg, src_ty);
1627 const dest_int_ty: IntType = .fromType(cg, dest_ty);1655 const dest_int_ty: IntType = .fromType(cg, dest_ty);
16281656 const result = try cg.intTrunc(dest_int_ty, src_int_ty, operand);
1629 const result = if (src_int_ty.bits == dest_int_ty.bits)
1630 cg.reuseOperand(ty_op.operand, operand)
1631 else blk: {
1632 break :blk try cg.intTrunc(dest_int_ty, src_int_ty, operand);
1633 };
16341657
1635 try cg.finishAir(inst, result, &.{ty_op.operand});1658 try cg.finishAir(inst, result, &.{ty_op.operand});
1636 },1659 },
...@@ -1934,7 +1957,6 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -1934,7 +1957,6 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
1934 continue;1957 continue;
1935 }1958 }
1936 const old_bookkeeping_value = cg.air_bookkeeping;1959 const old_bookkeeping_value = cg.air_bookkeeping;
1937 try cg.currentBranch().values.ensureUnusedCapacity(cg.gpa, 1);
1938 try cg.genInst(inst);1960 try cg.genInst(inst);
19391961
1940 if (std.debug.runtime_safety and cg.air_bookkeeping < old_bookkeeping_value + 1) {1962 if (std.debug.runtime_safety and cg.air_bookkeeping < old_bookkeeping_value + 1) {
...@@ -1956,20 +1978,20 @@ fn airRet(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -1956,20 +1978,20 @@ fn airRet(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
1956 // result must be stored in the stack and we return a pointer1978 // result must be stored in the stack and we return a pointer
1957 // to the stack instead1979 // to the stack instead
1958 if (cg.return_value != .none) {1980 if (cg.return_value != .none) {
1959 try cg.store(cg.return_value, operand, ret_ty, 0);1981 try cg.store(cg.return_value, operand, ret_ty, .{});
1960 } else if (fn_info.cc == .wasm_mvp and ret_ty.hasRuntimeBits(zcu)) {1982 } else if (fn_info.cc == .wasm_mvp and ret_ty.hasRuntimeBits(zcu)) {
1961 switch (abi.classifyType(ret_ty, zcu, cg.target)) {1983 switch (abi.classifyType(ret_ty, zcu, cg.target)) {
1962 .direct => |scalar_type| {1984 .direct => |scalar_type| {
1963 if (!isByRef(ret_ty, zcu, cg.target)) {1985 if (!isByRef(ret_ty, zcu, cg.target)) {
1964 try cg.emitWValue(operand);1986 try cg.emitWValue(operand);
1965 } else {1987 } else {
1966 _ = try cg.load(operand, scalar_type, 0);1988 _ = try cg.load(operand, scalar_type, .{});
1967 }1989 }
1968 },1990 },
1969 .double_i64, .indirect => unreachable,1991 .double_i64, .indirect => unreachable,
1970 .unrolled => |vector| {1992 .unrolled => |vector| {
1971 assert(vector.len == 1);1993 assert(vector.len == 1);
1972 _ = try cg.load(operand, vector.elem_type, 0);1994 _ = try cg.load(operand, vector.elem_type, .{});
1973 },1995 },
1974 }1996 }
1975 } else {1997 } else {
...@@ -1982,7 +2004,8 @@ fn airRet(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -1982,7 +2004,8 @@ fn airRet(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
1982 try cg.restoreStackPointer();2004 try cg.restoreStackPointer();
1983 try cg.addTag(.@"return");2005 try cg.addTag(.@"return");
19842006
1985 return cg.finishAir(inst, .none, &.{un_op});2007 try cg.finishAir(inst, .none, &.{un_op});
2008 cg.verifyLocalPool();
1986}2009}
19872010
1988fn airRetPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {2011fn airRetPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
...@@ -2019,21 +2042,22 @@ fn airRetLoad(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -2019,21 +2042,22 @@ fn airRetLoad(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
2019 } else if (!firstParamSRet(fn_info.cc, Type.fromInterned(fn_info.return_type), zcu, cg.target)) {2042 } else if (!firstParamSRet(fn_info.cc, Type.fromInterned(fn_info.return_type), zcu, cg.target)) {
2020 if (fn_info.cc == .wasm_mvp) {2043 if (fn_info.cc == .wasm_mvp) {
2021 switch (abi.classifyType(ret_ty, zcu, cg.target)) {2044 switch (abi.classifyType(ret_ty, zcu, cg.target)) {
2022 .direct => |scalar_type| _ = try cg.load(operand, scalar_type, 0),2045 .direct => |scalar_type| _ = try cg.load(operand, scalar_type, .{}),
2023 .double_i64, .indirect => unreachable,2046 .double_i64, .indirect => unreachable,
2024 .unrolled => |vector| {2047 .unrolled => |vector| {
2025 assert(vector.len == 1);2048 assert(vector.len == 1);
2026 _ = try cg.load(operand, vector.elem_type, 0);2049 _ = try cg.load(operand, vector.elem_type, .{});
2027 },2050 },
2028 }2051 }
2029 } else {2052 } else {
2030 _ = try cg.load(operand, ret_ty, 0);2053 _ = try cg.load(operand, ret_ty, .{});
2031 }2054 }
2032 }2055 }
20332056
2034 try cg.restoreStackPointer();2057 try cg.restoreStackPointer();
2035 try cg.addTag(.@"return");2058 try cg.addTag(.@"return");
2036 return cg.finishAir(inst, .none, &.{un_op});2059 try cg.finishAir(inst, .none, &.{un_op});
2060 cg.verifyLocalPool();
2037}2061}
20382062
2039fn airCall(cg: *CodeGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier) InnerError!void {2063fn airCall(cg: *CodeGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier) InnerError!void {
...@@ -2109,7 +2133,7 @@ fn airCall(cg: *CodeGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier)...@@ -2109,7 +2133,7 @@ fn airCall(cg: *CodeGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier)
2109 const arg_align = arg_ty.abiAlignment(zcu);2133 const arg_align = arg_ty.abiAlignment(zcu);
21102134
2111 offset = @intCast(arg_align.forward(offset));2135 offset = @intCast(arg_align.forward(offset));
2112 try cg.store(buffer, arg_val, arg_ty, offset);2136 try cg.store(buffer, arg_val, arg_ty, .{ .offset = offset });
2113 offset += arg_size;2137 offset += arg_size;
2114 }2138 }
21152139
...@@ -2117,7 +2141,7 @@ fn airCall(cg: *CodeGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier)...@@ -2117,7 +2141,7 @@ fn airCall(cg: *CodeGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier)
2117 } else .none;2141 } else .none;
21182142
2119 if (first_param_sret) {2143 if (first_param_sret) {
2120 try cg.lowerToStack(sret);2144 try cg.emitWValue(sret);
2121 }2145 }
21222146
2123 for (args, 0..) |arg, arg_i| {2147 for (args, 0..) |arg, arg_i| {
...@@ -2131,7 +2155,7 @@ fn airCall(cg: *CodeGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier)...@@ -2131,7 +2155,7 @@ fn airCall(cg: *CodeGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier)
2131 }2155 }
21322156
2133 if (fn_info.is_var_args) {2157 if (fn_info.is_var_args) {
2134 try cg.lowerToStack(varargs_buf);2158 try cg.emitWValue(varargs_buf);
2135 }2159 }
21362160
2137 if (callee) |nav_index| {2161 if (callee) |nav_index| {
...@@ -2160,30 +2184,32 @@ fn airCall(cg: *CodeGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier)...@@ -2160,30 +2184,32 @@ fn airCall(cg: *CodeGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier)
2160 .direct => |scalar_type| {2184 .direct => |scalar_type| {
2161 if (!isByRef(ret_ty, zcu, cg.target)) {2185 if (!isByRef(ret_ty, zcu, cg.target)) {
2162 const result_local = try cg.allocLocal(ret_ty);2186 const result_local = try cg.allocLocal(ret_ty);
2163 try cg.addLocal(.local_set, result_local.local.value);2187 try cg.addLocal(.local_set, result_local.local);
2164 break :result_value result_local;2188 break :result_value result_local;
2165 } else {2189 } else {
2166 const result_local = try cg.allocLocal(scalar_type);2190 var result_local = try cg.allocLocal(scalar_type);
2167 try cg.addLocal(.local_set, result_local.local.value);2191 defer result_local.free(cg);
2192 try cg.addLocal(.local_set, result_local.local);
2168 const result = try cg.allocStack(ret_ty);2193 const result = try cg.allocStack(ret_ty);
2169 try cg.store(result, result_local, scalar_type, 0);2194 try cg.store(result, result_local, scalar_type, .{});
2170 break :result_value result;2195 break :result_value result;
2171 }2196 }
2172 },2197 },
2173 .double_i64, .indirect => unreachable,2198 .double_i64, .indirect => unreachable,
2174 .unrolled => |vector| {2199 .unrolled => |vector| {
2175 assert(vector.len == 1);2200 assert(vector.len == 1);
2176 const result_local = try cg.allocLocal(vector.elem_type);2201 var result_local = try cg.allocLocal(vector.elem_type);
2202 defer result_local.free(cg);
2177 // save call result from operand stack2203 // save call result from operand stack
2178 try cg.addLocal(.local_set, result_local.local.value);2204 try cg.addLocal(.local_set, result_local.local);
2179 const result = try cg.allocStack(ret_ty);2205 const result = try cg.allocStack(ret_ty);
2180 try cg.store(result, result_local, vector.elem_type, 0);2206 try cg.store(result, result_local, vector.elem_type, .{});
2181 break :result_value result;2207 break :result_value result;
2182 },2208 },
2183 }2209 }
2184 } else {2210 } else {
2185 const result_local = try cg.allocLocal(ret_ty);2211 const result_local = try cg.allocLocal(ret_ty);
2186 try cg.addLocal(.local_set, result_local.local.value);2212 try cg.addLocal(.local_set, result_local.local);
2187 break :result_value result_local;2213 break :result_value result_local;
2188 }2214 }
2189 };2215 };
...@@ -2208,7 +2234,7 @@ fn airVaCopy(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -2208,7 +2234,7 @@ fn airVaCopy(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
2208 const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;2234 const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
2209 const operand = try cg.resolveInst(ty_op.operand);2235 const operand = try cg.resolveInst(ty_op.operand);
22102236
2211 const result = try cg.load(operand, .usize, 0);2237 const result = try cg.load(operand, .usize, .{});
22122238
2213 return cg.finishAir(inst, result, &.{ty_op.operand});2239 return cg.finishAir(inst, result, &.{ty_op.operand});
2214}2240}
...@@ -2230,8 +2256,9 @@ fn airVaArg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -2230,8 +2256,9 @@ fn airVaArg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
2230 const abi_size: u32 = @intCast(load_ty.abiSize(zcu));2256 const abi_size: u32 = @intCast(load_ty.abiSize(zcu));
2231 const abi_align: u32 = @intCast(load_ty.abiAlignment(zcu).toByteUnits().?);2257 const abi_align: u32 = @intCast(load_ty.abiAlignment(zcu).toByteUnits().?);
22322258
2233 const arg_ptr = try cg.allocLocal(.usize);2259 var arg_ptr = try cg.allocLocal(.usize);
2234 _ = try cg.load(operand, .usize, 0);2260 defer arg_ptr.free(cg);
2261 _ = try cg.load(operand, .usize, .{});
22352262
2236 if (abi_align > 1) {2263 if (abi_align > 1) {
2237 switch (cg.ptr_size) {2264 switch (cg.ptr_size) {
...@@ -2250,10 +2277,10 @@ fn airVaArg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -2250,10 +2277,10 @@ fn airVaArg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
2250 }2277 }
2251 }2278 }
22522279
2253 try cg.addLocal(.local_set, arg_ptr.local.value);2280 try cg.addLocal(.local_set, arg_ptr.local);
22542281
2255 try cg.lowerToStack(operand);2282 try cg.emitWValue(operand);
2256 try cg.lowerToStack(arg_ptr);2283 try cg.emitWValue(arg_ptr);
2257 switch (cg.ptr_size) {2284 switch (cg.ptr_size) {
2258 .wasm32 => {2285 .wasm32 => {
2259 try cg.addImm32(abi_size);2286 try cg.addImm32(abi_size);
...@@ -2264,17 +2291,17 @@ fn airVaArg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -2264,17 +2291,17 @@ fn airVaArg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
2264 try cg.addTag(.i64_add);2291 try cg.addTag(.i64_add);
2265 },2292 },
2266 }2293 }
2267 try cg.store(.stack, .stack, .usize, 0);2294 try cg.store(.stack, .stack, .usize, .{});
22682295
2269 const result = if (is_f32_va_arg) result: {2296 const result = if (is_f32_va_arg) result: {
2270 const promoted = try cg.load(arg_ptr, Type.f64, 0);2297 const promoted = try cg.load(arg_ptr, .f64, .{});
2271 try cg.emitWValue(promoted);2298 try cg.emitWValue(promoted);
2272 try cg.addTag(.f32_demote_f64);2299 try cg.addTag(.f32_demote_f64);
22732300
2274 const result_local = try cg.allocLocal(Type.f32);2301 const result_local = try cg.allocLocal(Type.f32);
2275 try cg.addLocal(.local_set, result_local.local.value);2302 try cg.addLocal(.local_set, result_local.local);
2276 break :result result_local;2303 break :result result_local;
2277 } else try cg.load(arg_ptr, ty, 0);2304 } else try cg.load(arg_ptr, ty, .{});
22782305
2279 return cg.finishAir(inst, result, &.{ty_op.operand});2306 return cg.finishAir(inst, result, &.{ty_op.operand});
2280}2307}
...@@ -2290,27 +2317,44 @@ fn airStore(cg: *CodeGen, inst: Air.Inst.Index, safety: bool) InnerError!void {...@@ -2290,27 +2317,44 @@ fn airStore(cg: *CodeGen, inst: Air.Inst.Index, safety: bool) InnerError!void {
2290 const bin_op = cg.air.instructions.items(.data)[@backingInt(inst)].bin_op;2317 const bin_op = cg.air.instructions.items(.data)[@backingInt(inst)].bin_op;
22912318
2292 const lhs = try cg.resolveInst(bin_op.lhs);2319 const lhs = try cg.resolveInst(bin_op.lhs);
2293 const rhs = try cg.resolveInst(bin_op.rhs);
2294 const ptr_ty = cg.typeOf(bin_op.lhs);2320 const ptr_ty = cg.typeOf(bin_op.lhs);
2295 const ptr_info = ptr_ty.ptrInfo(zcu);2321 const ptr_info = ptr_ty.ptrInfo(zcu);
2296 const elem_ty = ptr_ty.childType(zcu);2322 const elem_ty = ptr_ty.childType(zcu);
2323 const rhs_is_undef = if (bin_op.rhs.toInterned()) |i| Value.fromInterned(i).isUndef(zcu) else false;
22972324
2298 if (!safety and bin_op.rhs == .undef) {2325 if (!safety and rhs_is_undef) {
2299 return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });2326 return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });
2300 }2327 }
23012328
2302 const offset: u32 = switch (ptr_info.flags.vector_index) {2329 const rhs = try cg.resolveInst(bin_op.rhs);
2330
2331 const offset: u64 = switch (ptr_info.flags.vector_index) {
2303 .none => offset: {2332 .none => offset: {
2304 assert(ptr_info.packed_offset.host_size == 0); // legalize .expand_packed_store2333 assert(ptr_info.packed_offset.host_size == 0); // legalize .expand_packed_store
2305 break :offset 0;2334 break :offset 0;
2306 },2335 },
2307 else => |index| @intCast(@backingInt(index) * elem_ty.abiSize(zcu)),2336 else => |index| @backingInt(index) * elem_ty.abiSize(zcu),
2308 };2337 };
2309 try cg.store(lhs, rhs, elem_ty, offset);2338
2339 if (safety and rhs_is_undef and isByRef(elem_ty, zcu, cg.target)) {
2340 const dst = try cg.offsetPointer(lhs, offset);
2341 try cg.memset(.u8, dst, .{ .imm32 = @intCast(elem_ty.abiSize(zcu)) }, .{ .imm32 = 0xaa });
2342 return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });
2343 }
2344
2345 try cg.store(lhs, rhs, elem_ty, .{
2346 .offset = offset,
2347 .alignment = ptr_ty.ptrAlignment(zcu),
2348 });
2310 return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });2349 return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });
2311}2350}
23122351
2313fn store(cg: *CodeGen, lhs: WValue, rhs: WValue, ty: Type, offset: u32) InnerError!void {2352const MemArg = struct {
2353 offset: u64 = 0,
2354 alignment: Alignment = .none,
2355};
2356
2357fn store(cg: *CodeGen, lhs: WValue, rhs: WValue, ty: Type, memarg: MemArg) InnerError!void {
2314 assert(!(lhs != .stack and rhs == .stack));2358 assert(!(lhs != .stack and rhs == .stack));
2315 const pt = cg.pt;2359 const pt = cg.pt;
2316 const zcu = pt.zcu;2360 const zcu = pt.zcu;
...@@ -2318,29 +2362,31 @@ fn store(cg: *CodeGen, lhs: WValue, rhs: WValue, ty: Type, offset: u32) InnerErr...@@ -2318,29 +2362,31 @@ fn store(cg: *CodeGen, lhs: WValue, rhs: WValue, ty: Type, offset: u32) InnerErr
23182362
2319 if (!ty.hasRuntimeBits(zcu)) return;2363 if (!ty.hasRuntimeBits(zcu)) return;
23202364
2365 const natural_alignment = ty.abiAlignment(zcu);
2366 // wasm memory instruction alignment is <= access width
2367 const alignment = if (memarg.alignment == .none)
2368 natural_alignment
2369 else
2370 natural_alignment.minStrict(memarg.alignment);
2371
2321 if (isByRef(ty, zcu, cg.target)) {2372 if (isByRef(ty, zcu, cg.target)) {
2322 const offset_ptr: WValue = switch (offset + lhs.offset()) {2373 const offset_ptr = try cg.offsetPointer(lhs, memarg.offset);
2323 0 => lhs,
2324 else => |total_offset| ptr: {
2325 try cg.emitWValue(lhs);
2326 try cg.addImm32(total_offset);
2327 try cg.addTag(.i32_add);
2328 break :ptr .stack;
2329 },
2330 };
2331 return cg.memcpy(offset_ptr, rhs, .{ .imm32 = @intCast(abi_size) });2374 return cg.memcpy(offset_ptr, rhs, .{ .imm32 = @intCast(abi_size) });
2332 }2375 }
23332376
2377 const base_offset = lhs.memOffset();
2378 try cg.emitMemBase(lhs);
2379
2334 if (ty.zigTypeTag(zcu) == .vector) {2380 if (ty.zigTypeTag(zcu) == .vector) {
2335 try cg.emitWValue(lhs);2381 try cg.emitWValue(rhs);
2336 try cg.lowerToStack(rhs);
2337 // TODO: Add helper functions for simd opcodes2382 // TODO: Add helper functions for simd opcodes
2338 const extra_index: u32 = @intCast(cg.mir_extra.items.len);2383 const extra_index: u32 = @intCast(cg.mir_extra.items.len);
2339 // stores as := opcode, offset, alignment (opcode::memarg)2384 // stores as := opcode, offset, alignment (opcode::memarg)
2340 try cg.mir_extra.appendSlice(cg.gpa, &[_]u32{2385 try cg.mir_extra.ensureUnusedCapacity(cg.gpa, 4);
2341 @backingInt(std.wasm.SimdOpcode.v128_store),2386 cg.mir_extra.appendAssumeCapacity(@backingInt(std.wasm.SimdOpcode.v128_store));
2342 offset + lhs.offset(),2387 _ = try cg.addExtraAssumeCapacity(Mir.MemArg{
2343 @intCast(ty.abiAlignment(zcu).toByteUnits().?),2388 .offset = base_offset + memarg.offset,
2389 .alignment = alignment,
2344 });2390 });
2345 return cg.addInst(.{ .tag = .simd_prefix, .data = .{ .payload = extra_index } });2391 return cg.addInst(.{ .tag = .simd_prefix, .data = .{ .payload = extra_index } });
2346 }2392 }
...@@ -2364,14 +2410,13 @@ fn store(cg: *CodeGen, lhs: WValue, rhs: WValue, ty: Type, offset: u32) InnerErr...@@ -2364,14 +2410,13 @@ fn store(cg: *CodeGen, lhs: WValue, rhs: WValue, ty: Type, offset: u32) InnerErr
2364 }2410 }
2365 };2411 };
23662412
2367 try cg.emitWValue(lhs);2413 try cg.emitWValue(rhs);
2368 try cg.lowerToStack(rhs);
23692414
2370 try cg.addMemArg(2415 try cg.addMemArg(
2371 store_opcode,2416 store_opcode,
2372 .{2417 .{
2373 .offset = offset + lhs.offset(),2418 .offset = base_offset + memarg.offset,
2374 .alignment = @intCast(ty.abiAlignment(zcu).toByteUnits().?),2419 .alignment = alignment,
2375 },2420 },
2376 );2421 );
2377}2422}
...@@ -2387,48 +2432,55 @@ fn airLoad(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -2387,48 +2432,55 @@ fn airLoad(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
23872432
2388 assert(elem_ty.hasRuntimeBits(zcu));2433 assert(elem_ty.hasRuntimeBits(zcu));
23892434
2390 const offset: u32 = switch (ptr_info.flags.vector_index) {2435 const offset: u64 = switch (ptr_info.flags.vector_index) {
2391 .none => offset: {2436 .none => offset: {
2392 assert(ptr_info.packed_offset.host_size == 0); // legalize .expand_packed_load2437 assert(ptr_info.packed_offset.host_size == 0); // legalize .expand_packed_load
2393 break :offset 0;2438 break :offset 0;
2394 },2439 },
2395 else => |index| @intCast(@backingInt(index) * elem_ty.abiSize(zcu)),2440 else => |index| @backingInt(index) * elem_ty.abiSize(zcu),
2396 };2441 };
2397 const result = try cg.load(operand, elem_ty, offset);2442 const result = try cg.load(operand, elem_ty, .{
2443 .offset = offset,
2444 .alignment = ptr_ty.ptrAlignment(zcu),
2445 });
2398 return cg.finishAir(inst, result, &.{ty_op.operand});2446 return cg.finishAir(inst, result, &.{ty_op.operand});
2399}2447}
24002448
2401/// Loads an operand from the linear memory section.2449/// Loads an operand from the linear memory section.
2402/// NOTE: Leaves the value on the stack, if isByRef == false.2450/// NOTE: Leaves the value on the stack, if isByRef == false.
2403fn load(cg: *CodeGen, operand: WValue, ty: Type, offset: u32) InnerError!WValue {2451fn load(cg: *CodeGen, operand: WValue, ty: Type, memarg: MemArg) InnerError!WValue {
2404 const zcu = cg.pt.zcu;2452 const zcu = cg.pt.zcu;
2453
2454 const natural_alignment = ty.abiAlignment(zcu);
2455 // wasm memory instruction alignment is <= access width
2456 const alignment = if (memarg.alignment == .none)
2457 natural_alignment
2458 else
2459 natural_alignment.minStrict(memarg.alignment);
2460
2405 if (isByRef(ty, zcu, cg.target)) {2461 if (isByRef(ty, zcu, cg.target)) {
2406 const src_ptr_maybe_stack: WValue = switch (offset + operand.offset()) {2462 var src_ptr = try cg.offsetPointer(operand, memarg.offset);
2407 0 => operand,2463 const owns_src_ptr = src_ptr == .stack;
2408 else => |total_offset| ptr: {2464 src_ptr = try src_ptr.toLocal(cg, .usize);
2409 try cg.emitWValue(operand);2465 defer if (owns_src_ptr) src_ptr.free(cg);
2410 try cg.addImm32(total_offset);2466
2411 try cg.addTag(.i32_add);
2412 break :ptr .stack;
2413 },
2414 };
2415 const src_ptr = try src_ptr_maybe_stack.toLocal(cg, .usize);
2416 const new_ptr = try cg.allocStack(ty);2467 const new_ptr = try cg.allocStack(ty);
2417 try cg.store(new_ptr, src_ptr, ty, 0);2468 try cg.memcpy(new_ptr, src_ptr, .{ .imm32 = @intCast(ty.abiSize(zcu)) });
2418 return new_ptr;2469 return new_ptr;
2419 }2470 }
24202471
2421 // load local's value from memory by its stack position2472 const base_offset = operand.memOffset();
2422 try cg.emitWValue(operand);2473 try cg.emitMemBase(operand);
24232474
2424 if (ty.zigTypeTag(zcu) == .vector) {2475 if (ty.zigTypeTag(zcu) == .vector) {
2425 // TODO: Add helper functions for simd opcodes2476 // TODO: Add helper functions for simd opcodes
2426 const extra_index: u32 = @intCast(cg.mir_extra.items.len);2477 const extra_index: u32 = @intCast(cg.mir_extra.items.len);
2427 // stores as := opcode, offset, alignment (opcode::memarg)2478 // stores as := opcode, offset, alignment (opcode::memarg)
2428 try cg.mir_extra.appendSlice(cg.gpa, &[_]u32{2479 try cg.mir_extra.ensureUnusedCapacity(cg.gpa, 4);
2429 @backingInt(std.wasm.SimdOpcode.v128_load),2480 cg.mir_extra.appendAssumeCapacity(@backingInt(std.wasm.SimdOpcode.v128_load));
2430 offset + operand.offset(),2481 _ = try cg.addExtraAssumeCapacity(Mir.MemArg{
2431 @intCast(ty.abiAlignment(zcu).toByteUnits().?),2482 .offset = base_offset + memarg.offset,
2483 .alignment = alignment,
2432 });2484 });
2433 try cg.addInst(.{ .tag = .simd_prefix, .data = .{ .payload = extra_index } });2485 try cg.addInst(.{ .tag = .simd_prefix, .data = .{ .payload = extra_index } });
2434 return .stack;2486 return .stack;
...@@ -2458,8 +2510,8 @@ fn load(cg: *CodeGen, operand: WValue, ty: Type, offset: u32) InnerError!WValue...@@ -2458,8 +2510,8 @@ fn load(cg: *CodeGen, operand: WValue, ty: Type, offset: u32) InnerError!WValue
2458 try cg.addMemArg(2510 try cg.addMemArg(
2459 load_opcode,2511 load_opcode,
2460 .{2512 .{
2461 .offset = offset + operand.offset(),2513 .offset = base_offset + memarg.offset,
2462 .alignment = @intCast(ty.abiAlignment(zcu).toByteUnits().?),2514 .alignment = alignment,
2463 },2515 },
2464 );2516 );
24652517
...@@ -2487,7 +2539,7 @@ fn airArg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -2487,7 +2539,7 @@ fn airArg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
2487 cg.arg_index += 1;2539 cg.arg_index += 1;
2488 if (isByRef(arg_ty, zcu, cg.target)) {2540 if (isByRef(arg_ty, zcu, cg.target)) {
2489 const result = try cg.allocStack(arg_ty);2541 const result = try cg.allocStack(arg_ty);
2490 try cg.store(result, arg, scalar_type, 0);2542 try cg.store(result, arg, scalar_type, .{});
2491 return cg.finishAir(inst, result, &.{});2543 return cg.finishAir(inst, result, &.{});
2492 }2544 }
2493 },2545 },
...@@ -2495,15 +2547,15 @@ fn airArg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -2495,15 +2547,15 @@ fn airArg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
2495 .double_i64 => {2547 .double_i64 => {
2496 cg.arg_index += 2;2548 cg.arg_index += 2;
2497 const result = try cg.allocStack(arg_ty);2549 const result = try cg.allocStack(arg_ty);
2498 try cg.store(result, arg, Type.u64, 0);2550 try cg.store(result, arg, .u64, .{});
2499 try cg.store(result, cg.args[arg_index + 1], Type.u64, 8);2551 try cg.store(result, cg.args[arg_index + 1], .u64, .{ .offset = 8 });
2500 return cg.finishAir(inst, result, &.{});2552 return cg.finishAir(inst, result, &.{});
2501 },2553 },
2502 .unrolled => |vector| {2554 .unrolled => |vector| {
2503 const result = try cg.allocStack(arg_ty);2555 const result = try cg.allocStack(arg_ty);
2504 const elem_size: u32 = @intCast(vector.elem_type.abiSize(zcu));2556 const elem_size = vector.elem_type.abiSize(zcu);
2505 for (0..vector.len) |index| {2557 for (0..vector.len) |index| {
2506 try cg.store(result, cg.args[cg.arg_index], vector.elem_type, @intCast(index * elem_size));2558 try cg.store(result, cg.args[cg.arg_index], vector.elem_type, .{ .offset = index * elem_size });
2507 cg.arg_index += 1;2559 cg.arg_index += 1;
2508 }2560 }
2509 return cg.finishAir(inst, result, &.{});2561 return cg.finishAir(inst, result, &.{});
...@@ -2605,15 +2657,15 @@ fn intAdd(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue...@@ -2605,15 +2657,15 @@ fn intAdd(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue
2605 65...128 => {2657 65...128 => {
2606 const result = try cg.allocStack(Type.u128);2658 const result = try cg.allocStack(Type.u128);
26072659
2608 var lhs_lsb = try (try cg.load(lhs, Type.u64, 0)).toLocal(cg, Type.u64);2660 var lhs_lsb = try (try cg.load(lhs, .u64, .{})).toLocal(cg, Type.u64);
2609 defer lhs_lsb.free(cg);2661 defer lhs_lsb.free(cg);
2610 var rhs_lsb = try (try cg.load(rhs, Type.u64, 0)).toLocal(cg, Type.u64);2662 var rhs_lsb = try (try cg.load(rhs, .u64, .{})).toLocal(cg, Type.u64);
2611 defer rhs_lsb.free(cg);2663 defer rhs_lsb.free(cg);
2612 var op_lsb = try (try cg.intAdd(.u64, lhs_lsb, rhs_lsb)).toLocal(cg, Type.u64);2664 var op_lsb = try (try cg.intAdd(.u64, lhs_lsb, rhs_lsb)).toLocal(cg, Type.u64);
2613 defer op_lsb.free(cg);2665 defer op_lsb.free(cg);
26142666
2615 const lhs_msb = try cg.load(lhs, Type.u64, 8);2667 const lhs_msb = try cg.load(lhs, .u64, .{ .offset = 8 });
2616 const rhs_msb = try cg.load(rhs, Type.u64, 8);2668 const rhs_msb = try cg.load(rhs, .u64, .{ .offset = 8 });
2617 const op_msb = try cg.intAdd(.u64, lhs_msb, rhs_msb);2669 const op_msb = try cg.intAdd(.u64, lhs_msb, rhs_msb);
26182670
2619 const lt = try cg.intCmp(.u64, .lt, op_lsb, rhs_lsb);2671 const lt = try cg.intCmp(.u64, .lt, op_lsb, rhs_lsb);
...@@ -2621,16 +2673,16 @@ fn intAdd(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue...@@ -2621,16 +2673,16 @@ fn intAdd(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue
2621 var tmp_op = try (try cg.intAdd(.u64, op_msb, tmp)).toLocal(cg, Type.u64);2673 var tmp_op = try (try cg.intAdd(.u64, op_msb, tmp)).toLocal(cg, Type.u64);
2622 defer tmp_op.free(cg);2674 defer tmp_op.free(cg);
26232675
2624 try cg.store(result, op_lsb, Type.u64, 0);2676 try cg.store(result, op_lsb, .u64, .{});
2625 try cg.store(result, tmp_op, Type.u64, 8);2677 try cg.store(result, tmp_op, .u64, .{ .offset = 8 });
2626 return result;2678 return result;
2627 },2679 },
2628 else => {2680 else => {
2629 const result = try cg.allocInt(ty);2681 const result = try cg.allocInt(ty);
26302682
2631 try cg.lowerToStack(result);2683 try cg.emitWValue(result);
2632 try cg.lowerToStack(lhs);2684 try cg.emitWValue(lhs);
2633 try cg.lowerToStack(rhs);2685 try cg.emitWValue(rhs);
2634 try cg.addImm32(@intFromBool(ty.is_signed));2686 try cg.addImm32(@intFromBool(ty.is_signed));
2635 try cg.addImm32(ty.bits);2687 try cg.addImm32(ty.bits);
2636 try cg.addCallIntrinsic(.__addo_limb64);2688 try cg.addCallIntrinsic(.__addo_limb64);
...@@ -2658,15 +2710,15 @@ fn intSub(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue...@@ -2658,15 +2710,15 @@ fn intSub(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue
2658 65...128 => {2710 65...128 => {
2659 const result = try cg.allocStack(Type.u128);2711 const result = try cg.allocStack(Type.u128);
26602712
2661 var lhs_lsb = try (try cg.load(lhs, Type.u64, 0)).toLocal(cg, Type.u64);2713 var lhs_lsb = try (try cg.load(lhs, .u64, .{})).toLocal(cg, Type.u64);
2662 defer lhs_lsb.free(cg);2714 defer lhs_lsb.free(cg);
2663 var rhs_lsb = try (try cg.load(rhs, Type.u64, 0)).toLocal(cg, Type.u64);2715 var rhs_lsb = try (try cg.load(rhs, .u64, .{})).toLocal(cg, Type.u64);
2664 defer rhs_lsb.free(cg);2716 defer rhs_lsb.free(cg);
2665 var op_lsb = try (try cg.intSub(.u64, lhs_lsb, rhs_lsb)).toLocal(cg, Type.u64);2717 var op_lsb = try (try cg.intSub(.u64, lhs_lsb, rhs_lsb)).toLocal(cg, Type.u64);
2666 defer op_lsb.free(cg);2718 defer op_lsb.free(cg);
26672719
2668 const lhs_msb = try cg.load(lhs, Type.u64, 8);2720 const lhs_msb = try cg.load(lhs, .u64, .{ .offset = 8 });
2669 const rhs_msb = try cg.load(rhs, Type.u64, 8);2721 const rhs_msb = try cg.load(rhs, .u64, .{ .offset = 8 });
2670 const op_msb = try cg.intSub(.u64, lhs_msb, rhs_msb);2722 const op_msb = try cg.intSub(.u64, lhs_msb, rhs_msb);
26712723
2672 const lt = try cg.intCmp(.u64, .lt, lhs_lsb, rhs_lsb);2724 const lt = try cg.intCmp(.u64, .lt, lhs_lsb, rhs_lsb);
...@@ -2674,16 +2726,16 @@ fn intSub(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue...@@ -2674,16 +2726,16 @@ fn intSub(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue
2674 var tmp_op = try (try cg.intSub(.u64, op_msb, tmp)).toLocal(cg, Type.u64);2726 var tmp_op = try (try cg.intSub(.u64, op_msb, tmp)).toLocal(cg, Type.u64);
2675 defer tmp_op.free(cg);2727 defer tmp_op.free(cg);
26762728
2677 try cg.store(result, op_lsb, Type.u64, 0);2729 try cg.store(result, op_lsb, .u64, .{});
2678 try cg.store(result, tmp_op, Type.u64, 8);2730 try cg.store(result, tmp_op, .u64, .{ .offset = 8 });
2679 return result;2731 return result;
2680 },2732 },
2681 else => {2733 else => {
2682 const result = try cg.allocInt(ty);2734 const result = try cg.allocInt(ty);
26832735
2684 try cg.lowerToStack(result);2736 try cg.emitWValue(result);
2685 try cg.lowerToStack(lhs);2737 try cg.emitWValue(lhs);
2686 try cg.lowerToStack(rhs);2738 try cg.emitWValue(rhs);
2687 try cg.addImm32(@intFromBool(ty.is_signed));2739 try cg.addImm32(@intFromBool(ty.is_signed));
2688 try cg.addImm32(ty.bits);2740 try cg.addImm32(ty.bits);
2689 try cg.addCallIntrinsic(.__subo_limb64);2741 try cg.addCallIntrinsic(.__subo_limb64);
...@@ -2713,9 +2765,9 @@ fn intMul(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue...@@ -2713,9 +2765,9 @@ fn intMul(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue
2713 else => {2765 else => {
2714 const result = try cg.allocInt(ty);2766 const result = try cg.allocInt(ty);
27152767
2716 try cg.lowerToStack(result);2768 try cg.emitWValue(result);
2717 try cg.lowerToStack(lhs);2769 try cg.emitWValue(lhs);
2718 try cg.lowerToStack(rhs);2770 try cg.emitWValue(rhs);
2719 try cg.addImm32(@intFromBool(ty.is_signed));2771 try cg.addImm32(@intFromBool(ty.is_signed));
2720 try cg.addImm32(ty.bits);2772 try cg.addImm32(ty.bits);
2721 try cg.addCallIntrinsic(.__mulo_limb64);2773 try cg.addCallIntrinsic(.__mulo_limb64);
...@@ -3007,24 +3059,26 @@ fn intAnd(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue...@@ -3007,24 +3059,26 @@ fn intAnd(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue
3007 65...128 => {3059 65...128 => {
3008 const result = try cg.allocStack(Type.u128);3060 const result = try cg.allocStack(Type.u128);
30093061
3010 const lhs_lsb = try cg.load(lhs, Type.u64, 0);3062 const lhs_lsb = try cg.load(lhs, .u64, .{});
3011 const rhs_lsb = try cg.load(rhs, Type.u64, 0);3063 const rhs_lsb = try cg.load(rhs, .u64, .{});
3012 const and_lsb = try (try cg.intAnd(.u64, lhs_lsb, rhs_lsb)).toLocal(cg, Type.u64);3064 var and_lsb = try (try cg.intAnd(.u64, lhs_lsb, rhs_lsb)).toLocal(cg, Type.u64);
3013 try cg.store(result, and_lsb, Type.u64, 0);3065 defer and_lsb.free(cg);
3066 try cg.store(result, and_lsb, .u64, .{});
30143067
3015 const lhs_msb = try cg.load(lhs, Type.u64, 8);3068 const lhs_msb = try cg.load(lhs, .u64, .{ .offset = 8 });
3016 const rhs_msb = try cg.load(rhs, Type.u64, 8);3069 const rhs_msb = try cg.load(rhs, .u64, .{ .offset = 8 });
3017 const and_msb = try (try cg.intAnd(.u64, lhs_msb, rhs_msb)).toLocal(cg, Type.u64);3070 var and_msb = try (try cg.intAnd(.u64, lhs_msb, rhs_msb)).toLocal(cg, Type.u64);
3018 try cg.store(result, and_msb, Type.u64, 8);3071 defer and_msb.free(cg);
3072 try cg.store(result, and_msb, .u64, .{ .offset = 8 });
30193073
3020 return result;3074 return result;
3021 },3075 },
3022 else => {3076 else => {
3023 const result = try cg.allocInt(ty);3077 const result = try cg.allocInt(ty);
30243078
3025 try cg.lowerToStack(result);3079 try cg.emitWValue(result);
3026 try cg.lowerToStack(lhs);3080 try cg.emitWValue(lhs);
3027 try cg.lowerToStack(rhs);3081 try cg.emitWValue(rhs);
3028 try cg.addImm32(ty.bits);3082 try cg.addImm32(ty.bits);
3029 try cg.addCallIntrinsic(.__and_limb64);3083 try cg.addCallIntrinsic(.__and_limb64);
30303084
...@@ -3051,24 +3105,26 @@ fn intOr(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue...@@ -3051,24 +3105,26 @@ fn intOr(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue
3051 65...128 => {3105 65...128 => {
3052 const result = try cg.allocStack(Type.u128);3106 const result = try cg.allocStack(Type.u128);
30533107
3054 const lhs_lsb = try cg.load(lhs, Type.u64, 0);3108 const lhs_lsb = try cg.load(lhs, .u64, .{});
3055 const rhs_lsb = try cg.load(rhs, Type.u64, 0);3109 const rhs_lsb = try cg.load(rhs, .u64, .{});
3056 const or_lsb = try (try cg.intOr(.u64, lhs_lsb, rhs_lsb)).toLocal(cg, Type.u64);3110 var or_lsb = try (try cg.intOr(.u64, lhs_lsb, rhs_lsb)).toLocal(cg, Type.u64);
3057 try cg.store(result, or_lsb, Type.u64, 0);3111 defer or_lsb.free(cg);
3112 try cg.store(result, or_lsb, .u64, .{});
30583113
3059 const lhs_msb = try cg.load(lhs, Type.u64, 8);3114 const lhs_msb = try cg.load(lhs, .u64, .{ .offset = 8 });
3060 const rhs_msb = try cg.load(rhs, Type.u64, 8);3115 const rhs_msb = try cg.load(rhs, .u64, .{ .offset = 8 });
3061 const or_msb = try (try cg.intOr(.u64, lhs_msb, rhs_msb)).toLocal(cg, Type.u64);3116 var or_msb = try (try cg.intOr(.u64, lhs_msb, rhs_msb)).toLocal(cg, Type.u64);
3062 try cg.store(result, or_msb, Type.u64, 8);3117 defer or_msb.free(cg);
3118 try cg.store(result, or_msb, .u64, .{ .offset = 8 });
30633119
3064 return result;3120 return result;
3065 },3121 },
3066 else => {3122 else => {
3067 const result = try cg.allocInt(ty);3123 const result = try cg.allocInt(ty);
30683124
3069 try cg.lowerToStack(result);3125 try cg.emitWValue(result);
3070 try cg.lowerToStack(lhs);3126 try cg.emitWValue(lhs);
3071 try cg.lowerToStack(rhs);3127 try cg.emitWValue(rhs);
3072 try cg.addImm32(ty.bits);3128 try cg.addImm32(ty.bits);
3073 try cg.addCallIntrinsic(.__or_limb64);3129 try cg.addCallIntrinsic(.__or_limb64);
30743130
...@@ -3095,24 +3151,26 @@ fn intXor(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue...@@ -3095,24 +3151,26 @@ fn intXor(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue
3095 65...128 => {3151 65...128 => {
3096 const result = try cg.allocStack(Type.u128);3152 const result = try cg.allocStack(Type.u128);
30973153
3098 const lhs_lsb = try cg.load(lhs, Type.u64, 0);3154 const lhs_lsb = try cg.load(lhs, .u64, .{});
3099 const rhs_lsb = try cg.load(rhs, Type.u64, 0);3155 const rhs_lsb = try cg.load(rhs, .u64, .{});
3100 const xor_lsb = try (try cg.intXor(.u64, lhs_lsb, rhs_lsb)).toLocal(cg, Type.u64);3156 var xor_lsb = try (try cg.intXor(.u64, lhs_lsb, rhs_lsb)).toLocal(cg, Type.u64);
3101 try cg.store(result, xor_lsb, Type.u64, 0);3157 defer xor_lsb.free(cg);
3158 try cg.store(result, xor_lsb, .u64, .{});
31023159
3103 const lhs_msb = try cg.load(lhs, Type.u64, 8);3160 const lhs_msb = try cg.load(lhs, .u64, .{ .offset = 8 });
3104 const rhs_msb = try cg.load(rhs, Type.u64, 8);3161 const rhs_msb = try cg.load(rhs, .u64, .{ .offset = 8 });
3105 const xor_msb = try (try cg.intXor(.u64, lhs_msb, rhs_msb)).toLocal(cg, Type.u64);3162 var xor_msb = try (try cg.intXor(.u64, lhs_msb, rhs_msb)).toLocal(cg, Type.u64);
3106 try cg.store(result, xor_msb, Type.u64, 8);3163 defer xor_msb.free(cg);
3164 try cg.store(result, xor_msb, .u64, .{ .offset = 8 });
31073165
3108 return result;3166 return result;
3109 },3167 },
3110 else => {3168 else => {
3111 const result = try cg.allocInt(ty);3169 const result = try cg.allocInt(ty);
31123170
3113 try cg.lowerToStack(result);3171 try cg.emitWValue(result);
3114 try cg.lowerToStack(lhs);3172 try cg.emitWValue(lhs);
3115 try cg.lowerToStack(rhs);3173 try cg.emitWValue(rhs);
3116 try cg.addImm32(ty.bits);3174 try cg.addImm32(ty.bits);
3117 try cg.addCallIntrinsic(.__xor_limb64);3175 try cg.addCallIntrinsic(.__xor_limb64);
31183176
...@@ -3157,29 +3215,30 @@ fn intNot(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {...@@ -3157,29 +3215,30 @@ fn intNot(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3157 65...128 => {3215 65...128 => {
3158 const result = try cg.allocStack(Type.u128);3216 const result = try cg.allocStack(Type.u128);
31593217
3160 try cg.emitWValue(result);3218 const result_offset = result.memOffset();
3161 _ = try cg.load(operand, Type.u64, 0);3219 try cg.emitMemBase(result);
3220 _ = try cg.load(operand, .u64, .{});
3162 try cg.addImm64(~@as(u64, 0));3221 try cg.addImm64(~@as(u64, 0));
3163 try cg.addTag(.i64_xor);3222 try cg.addTag(.i64_xor);
3164 try cg.store(.stack, .stack, Type.u64, result.offset());3223 try cg.store(.stack, .stack, .u64, .{ .offset = result_offset });
31653224
3166 try cg.emitWValue(result);3225 try cg.emitMemBase(result);
3167 _ = try cg.load(operand, Type.u64, 8);3226 _ = try cg.load(operand, .u64, .{ .offset = 8 });
3168 const high_mask: u64 = if (ty.is_signed)3227 const high_mask: u64 = if (ty.is_signed)
3169 ~@as(u64, 0)3228 ~@as(u64, 0)
3170 else3229 else
3171 ~@as(u64, 0) >> @intCast(128 - ty.bits);3230 ~@as(u64, 0) >> @intCast(128 - ty.bits);
3172 try cg.addImm64(high_mask);3231 try cg.addImm64(high_mask);
3173 try cg.addTag(.i64_xor);3232 try cg.addTag(.i64_xor);
3174 try cg.store(.stack, .stack, Type.u64, result.offset() + 8);3233 try cg.store(.stack, .stack, .u64, .{ .offset = result_offset + 8 });
31753234
3176 return result;3235 return result;
3177 },3236 },
3178 else => {3237 else => {
3179 const result = try cg.allocInt(ty);3238 const result = try cg.allocInt(ty);
31803239
3181 try cg.lowerToStack(result);3240 try cg.emitWValue(result);
3182 try cg.lowerToStack(operand);3241 try cg.emitWValue(operand);
3183 try cg.addImm32(@intFromBool(ty.is_signed));3242 try cg.addImm32(@intFromBool(ty.is_signed));
3184 try cg.addImm32(ty.bits);3243 try cg.addImm32(ty.bits);
3185 try cg.addCallIntrinsic(.__not_limb64);3244 try cg.addCallIntrinsic(.__not_limb64);
...@@ -3211,9 +3270,9 @@ fn intShl(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue...@@ -3211,9 +3270,9 @@ fn intShl(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue
3211 else => {3270 else => {
3212 const result = try cg.allocInt(ty);3271 const result = try cg.allocInt(ty);
32133272
3214 try cg.lowerToStack(result);3273 try cg.emitWValue(result);
3215 try cg.lowerToStack(lhs);3274 try cg.emitWValue(lhs);
3216 try cg.lowerToStack(rhs);3275 try cg.emitWValue(rhs);
3217 try cg.addImm32(@intFromBool(ty.is_signed));3276 try cg.addImm32(@intFromBool(ty.is_signed));
3218 try cg.addImm32(ty.bits);3277 try cg.addImm32(ty.bits);
3219 try cg.addCallIntrinsic(.__shlo_limb64);3278 try cg.addCallIntrinsic(.__shlo_limb64);
...@@ -3251,9 +3310,9 @@ fn intShr(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue...@@ -3251,9 +3310,9 @@ fn intShr(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue
3251 else => {3310 else => {
3252 const result = try cg.allocInt(ty);3311 const result = try cg.allocInt(ty);
32533312
3254 try cg.lowerToStack(result);3313 try cg.emitWValue(result);
3255 try cg.lowerToStack(lhs);3314 try cg.emitWValue(lhs);
3256 try cg.lowerToStack(rhs);3315 try cg.emitWValue(rhs);
3257 try cg.addImm32(@intFromBool(ty.is_signed));3316 try cg.addImm32(@intFromBool(ty.is_signed));
3258 try cg.addImm32(ty.bits);3317 try cg.addImm32(ty.bits);
3259 try cg.addCallIntrinsic(.__shr_limb64);3318 try cg.addCallIntrinsic(.__shr_limb64);
...@@ -3274,7 +3333,7 @@ fn intAbs(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {...@@ -3274,7 +3333,7 @@ fn intAbs(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
32743333
3275 var mask = try cg.allocLocal(Type.i32);3334 var mask = try cg.allocLocal(Type.i32);
3276 defer mask.free(cg);3335 defer mask.free(cg);
3277 try cg.addLocal(.local_tee, mask.local.value);3336 try cg.addLocal(.local_tee, mask.local);
32783337
3279 try cg.emitWValue(operand);3338 try cg.emitWValue(operand);
3280 try cg.addTag(.i32_xor);3339 try cg.addTag(.i32_xor);
...@@ -3289,7 +3348,7 @@ fn intAbs(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {...@@ -3289,7 +3348,7 @@ fn intAbs(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
32893348
3290 var mask = try cg.allocLocal(Type.i64);3349 var mask = try cg.allocLocal(Type.i64);
3291 defer mask.free(cg);3350 defer mask.free(cg);
3292 try cg.addLocal(.local_tee, mask.local.value);3351 try cg.addLocal(.local_tee, mask.local);
32933352
3294 try cg.emitWValue(operand);3353 try cg.emitWValue(operand);
3295 try cg.addTag(.i64_xor);3354 try cg.addTag(.i64_xor);
...@@ -3301,19 +3360,20 @@ fn intAbs(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {...@@ -3301,19 +3360,20 @@ fn intAbs(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3301 const u128_ty: IntType = .{ .is_signed = false, .bits = 128 };3360 const u128_ty: IntType = .{ .is_signed = false, .bits = 128 };
33023361
3303 const mask = try cg.allocStack(Type.u128);3362 const mask = try cg.allocStack(Type.u128);
3304 try cg.emitWValue(mask);3363 const mask_offset = mask.memOffset();
3305 try cg.emitWValue(mask);3364 try cg.emitMemBase(mask);
3365 try cg.emitMemBase(mask);
33063366
3307 _ = try cg.load(operand, Type.u64, 8);3367 _ = try cg.load(operand, .u64, .{ .offset = 8 });
3308 try cg.addImm64(63);3368 try cg.addImm64(63);
3309 try cg.addTag(.i64_shr_s);3369 try cg.addTag(.i64_shr_s);
33103370
3311 var tmp = try cg.allocLocal(Type.u64);3371 var tmp = try cg.allocLocal(Type.u64);
3312 defer tmp.free(cg);3372 defer tmp.free(cg);
3313 try cg.addLocal(.local_tee, tmp.local.value);3373 try cg.addLocal(.local_tee, tmp.local);
3314 try cg.store(.stack, .stack, Type.u64, mask.offset() + 0);3374 try cg.store(.stack, .stack, .u64, .{ .offset = mask_offset });
3315 try cg.emitWValue(tmp);3375 try cg.emitWValue(tmp);
3316 try cg.store(.stack, .stack, Type.u64, mask.offset() + 8);3376 try cg.store(.stack, .stack, .u64, .{ .offset = mask_offset + 8 });
33173377
3318 const a = try cg.intXor(u128_ty, operand, mask);3378 const a = try cg.intXor(u128_ty, operand, mask);
3319 const b = try cg.intSub(u128_ty, a, mask);3379 const b = try cg.intSub(u128_ty, a, mask);
...@@ -3322,8 +3382,8 @@ fn intAbs(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {...@@ -3322,8 +3382,8 @@ fn intAbs(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3322 else => {3382 else => {
3323 const result = try cg.allocInt(ty);3383 const result = try cg.allocInt(ty);
33243384
3325 try cg.lowerToStack(result);3385 try cg.emitWValue(result);
3326 try cg.lowerToStack(operand);3386 try cg.emitWValue(operand);
3327 try cg.addImm32(ty.bits);3387 try cg.addImm32(ty.bits);
3328 try cg.addCallIntrinsic(.__abs_limb64);3388 try cg.addCallIntrinsic(.__abs_limb64);
33293389
...@@ -3333,16 +3393,16 @@ fn intAbs(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {...@@ -3333,16 +3393,16 @@ fn intAbs(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3333}3393}
33343394
3335fn intMax(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue {3395fn intMax(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue {
3336 try cg.lowerToStack(lhs);3396 try cg.emitWValue(lhs);
3337 try cg.lowerToStack(rhs);3397 try cg.emitWValue(rhs);
3338 _ = try cg.intCmp(ty, .gt, lhs, rhs);3398 _ = try cg.intCmp(ty, .gt, lhs, rhs);
3339 try cg.addTag(.select);3399 try cg.addTag(.select);
3340 return .stack;3400 return .stack;
3341}3401}
33423402
3343fn intMin(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue {3403fn intMin(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue {
3344 try cg.lowerToStack(lhs);3404 try cg.emitWValue(lhs);
3345 try cg.lowerToStack(rhs);3405 try cg.emitWValue(rhs);
3346 _ = try cg.intCmp(ty, .lt, lhs, rhs);3406 _ = try cg.intCmp(ty, .lt, lhs, rhs);
3347 try cg.addTag(.select);3407 try cg.addTag(.select);
3348 return .stack;3408 return .stack;
...@@ -3381,7 +3441,7 @@ fn intClz(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {...@@ -3381,7 +3441,7 @@ fn intClz(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3381 return .stack;3441 return .stack;
3382 },3442 },
3383 65...128 => {3443 65...128 => {
3384 var msb = try (try cg.load(operand, Type.u64, 8)).toLocal(cg, Type.u64);3444 var msb = try (try cg.load(operand, .u64, .{ .offset = 8 })).toLocal(cg, Type.u64);
3385 defer msb.free(cg);3445 defer msb.free(cg);
33863446
3387 if (ty.is_signed and ty.bits < 128) {3447 if (ty.is_signed and ty.bits < 128) {
...@@ -3392,7 +3452,7 @@ fn intClz(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {...@@ -3392,7 +3452,7 @@ fn intClz(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3392 }3452 }
33933453
3394 try cg.addTag(.i64_clz);3454 try cg.addTag(.i64_clz);
3395 _ = try cg.load(operand, Type.u64, 0);3455 _ = try cg.load(operand, .u64, .{});
3396 try cg.addTag(.i64_clz);3456 try cg.addTag(.i64_clz);
3397 try cg.emitWValue(.{ .imm64 = 64 });3457 try cg.emitWValue(.{ .imm64 = 64 });
3398 try cg.addTag(.i64_add);3458 try cg.addTag(.i64_add);
...@@ -3408,7 +3468,7 @@ fn intClz(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {...@@ -3408,7 +3468,7 @@ fn intClz(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3408 return .stack;3468 return .stack;
3409 },3469 },
3410 else => {3470 else => {
3411 try cg.lowerToStack(operand);3471 try cg.emitWValue(operand);
3412 try cg.addImm32(ty.bits);3472 try cg.addImm32(ty.bits);
3413 try cg.addCallIntrinsic(.__clz_limb64);3473 try cg.addCallIntrinsic(.__clz_limb64);
34143474
...@@ -3440,13 +3500,13 @@ fn intCtz(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {...@@ -3440,13 +3500,13 @@ fn intCtz(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3440 return .stack;3500 return .stack;
3441 },3501 },
3442 65...128 => {3502 65...128 => {
3443 var lsb = try (try cg.load(operand, Type.u64, 0)).toLocal(cg, Type.u64);3503 var lsb = try (try cg.load(operand, .u64, .{})).toLocal(cg, Type.u64);
3444 defer lsb.free(cg);3504 defer lsb.free(cg);
34453505
3446 try cg.emitWValue(lsb);3506 try cg.emitWValue(lsb);
3447 try cg.addTag(.i64_ctz);3507 try cg.addTag(.i64_ctz);
34483508
3449 _ = try cg.load(operand, Type.u64, 8);3509 _ = try cg.load(operand, .u64, .{ .offset = 8 });
3450 if (ty.bits < 128) {3510 if (ty.bits < 128) {
3451 try cg.addImm64(@as(u64, 1) << @intCast(ty.bits - 64));3511 try cg.addImm64(@as(u64, 1) << @intCast(ty.bits - 64));
3452 try cg.addTag(.i64_or);3512 try cg.addTag(.i64_or);
...@@ -3460,7 +3520,7 @@ fn intCtz(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {...@@ -3460,7 +3520,7 @@ fn intCtz(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3460 return .stack;3520 return .stack;
3461 },3521 },
3462 else => {3522 else => {
3463 try cg.lowerToStack(operand);3523 try cg.emitWValue(operand);
3464 try cg.addImm32(ty.bits);3524 try cg.addImm32(ty.bits);
3465 try cg.addCallIntrinsic(.__ctz_limb64);3525 try cg.addCallIntrinsic(.__ctz_limb64);
34663526
...@@ -3492,9 +3552,9 @@ fn intPopCount(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {...@@ -3492,9 +3552,9 @@ fn intPopCount(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3492 return .stack;3552 return .stack;
3493 },3553 },
3494 65...128 => {3554 65...128 => {
3495 _ = try cg.load(operand, Type.u64, 0);3555 _ = try cg.load(operand, .u64, .{});
3496 try cg.addTag(.i64_popcnt);3556 try cg.addTag(.i64_popcnt);
3497 _ = try cg.load(operand, Type.u64, 8);3557 _ = try cg.load(operand, .u64, .{ .offset = 8 });
3498 if (ty.is_signed and ty.bits < 128) {3558 if (ty.is_signed and ty.bits < 128) {
3499 try cg.addImm64(128 - ty.bits);3559 try cg.addImm64(128 - ty.bits);
3500 try cg.addTag(.i64_shl);3560 try cg.addTag(.i64_shl);
...@@ -3506,7 +3566,7 @@ fn intPopCount(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {...@@ -3506,7 +3566,7 @@ fn intPopCount(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3506 return .stack;3566 return .stack;
3507 },3567 },
3508 else => {3568 else => {
3509 try cg.lowerToStack(operand);3569 try cg.emitWValue(operand);
3510 try cg.addImm32(ty.bits);3570 try cg.addImm32(ty.bits);
3511 try cg.addCallIntrinsic(.__popcount_limb64);3571 try cg.addCallIntrinsic(.__popcount_limb64);
35123572
...@@ -3541,8 +3601,9 @@ fn intBitReverse(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {...@@ -3541,8 +3601,9 @@ fn intBitReverse(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3541 65...128 => {3601 65...128 => {
3542 const tmp = try cg.allocStack(Type.u128);3602 const tmp = try cg.allocStack(Type.u128);
35433603
3544 try cg.emitWValue(tmp);3604 const tmp_offset = tmp.memOffset();
3545 const hi = try cg.load(operand, Type.u64, 8);3605 try cg.emitMemBase(tmp);
3606 const hi = try cg.load(operand, .u64, .{ .offset = 8 });
3546 const hi_rev = try cg.callIntrinsic(3607 const hi_rev = try cg.callIntrinsic(
3547 .__bitreversedi2,3608 .__bitreversedi2,
3548 &.{.u64_type},3609 &.{.u64_type},
...@@ -3550,10 +3611,10 @@ fn intBitReverse(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {...@@ -3550,10 +3611,10 @@ fn intBitReverse(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3550 &.{hi},3611 &.{hi},
3551 );3612 );
3552 try cg.emitWValue(hi_rev);3613 try cg.emitWValue(hi_rev);
3553 try cg.store(.stack, .stack, Type.u64, tmp.offset());3614 try cg.store(.stack, .stack, .u64, .{ .offset = tmp_offset });
35543615
3555 try cg.emitWValue(tmp);3616 try cg.emitMemBase(tmp);
3556 const lo = try cg.load(operand, Type.u64, 0);3617 const lo = try cg.load(operand, .u64, .{});
3557 const lo_rev = try cg.callIntrinsic(3618 const lo_rev = try cg.callIntrinsic(
3558 .__bitreversedi2,3619 .__bitreversedi2,
3559 &.{.u64_type},3620 &.{.u64_type},
...@@ -3561,7 +3622,7 @@ fn intBitReverse(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {...@@ -3561,7 +3622,7 @@ fn intBitReverse(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3561 &.{lo},3622 &.{lo},
3562 );3623 );
3563 try cg.emitWValue(lo_rev);3624 try cg.emitWValue(lo_rev);
3564 try cg.store(.stack, .stack, Type.u64, tmp.offset() + 8);3625 try cg.store(.stack, .stack, .u64, .{ .offset = tmp_offset + 8 });
35653626
3566 if (ty.bits < 128) {3627 if (ty.bits < 128) {
3567 const shift_ty: IntType = .{ .is_signed = ty.is_signed, .bits = 128 };3628 const shift_ty: IntType = .{ .is_signed = ty.is_signed, .bits = 128 };
...@@ -3573,8 +3634,8 @@ fn intBitReverse(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {...@@ -3573,8 +3634,8 @@ fn intBitReverse(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3573 else => {3634 else => {
3574 const result = try cg.allocInt(ty);3635 const result = try cg.allocInt(ty);
35753636
3576 try cg.lowerToStack(result);3637 try cg.emitWValue(result);
3577 try cg.lowerToStack(operand);3638 try cg.emitWValue(operand);
3578 try cg.addImm32(@intFromBool(ty.is_signed));3639 try cg.addImm32(@intFromBool(ty.is_signed));
3579 try cg.addImm32(ty.bits);3640 try cg.addImm32(ty.bits);
3580 try cg.addCallIntrinsic(.__bitreverse_limb64);3641 try cg.addCallIntrinsic(.__bitreverse_limb64);
...@@ -3610,27 +3671,28 @@ fn intByteSwap(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {...@@ -3610,27 +3671,28 @@ fn intByteSwap(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3610 65...128 => {3671 65...128 => {
3611 const result = try cg.allocStack(Type.u128);3672 const result = try cg.allocStack(Type.u128);
36123673
3613 try cg.emitWValue(result);3674 const result_offset = result.memOffset();
3675 try cg.emitMemBase(result);
36143676
3615 const low = try cg.load(operand, Type.u64, 0);3677 const low = try cg.load(operand, .u64, .{});
3616 const swap_low = try cg.callIntrinsic(3678 const swap_low = try cg.callIntrinsic(
3617 .__bswapdi2,3679 .__bswapdi2,
3618 &.{.u64_type},3680 &.{.u64_type},
3619 Type.u64,3681 Type.u64,
3620 &.{low},3682 &.{low},
3621 );3683 );
3622 try cg.store(.stack, swap_low, Type.u64, result.offset() + 8);3684 try cg.store(.stack, swap_low, .u64, .{ .offset = result_offset + 8 });
36233685
3624 try cg.emitWValue(result);3686 try cg.emitMemBase(result);
36253687
3626 const high = try cg.load(operand, Type.u64, 8);3688 const high = try cg.load(operand, .u64, .{ .offset = 8 });
3627 const swap_high = try cg.callIntrinsic(3689 const swap_high = try cg.callIntrinsic(
3628 .__bswapdi2,3690 .__bswapdi2,
3629 &.{.u64_type},3691 &.{.u64_type},
3630 Type.u64,3692 Type.u64,
3631 &.{high},3693 &.{high},
3632 );3694 );
3633 try cg.store(.stack, swap_high, Type.u64, result.offset());3695 try cg.store(.stack, swap_high, .u64, .{ .offset = result_offset });
36343696
3635 if (ty.bits < 128) {3697 if (ty.bits < 128) {
3636 const shift_ty: IntType = .{ .is_signed = ty.is_signed, .bits = 128 };3698 const shift_ty: IntType = .{ .is_signed = ty.is_signed, .bits = 128 };
...@@ -3642,8 +3704,8 @@ fn intByteSwap(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {...@@ -3642,8 +3704,8 @@ fn intByteSwap(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3642 else => {3704 else => {
3643 const result = try cg.allocInt(ty);3705 const result = try cg.allocInt(ty);
36443706
3645 try cg.lowerToStack(result);3707 try cg.emitWValue(result);
3646 try cg.lowerToStack(operand);3708 try cg.emitWValue(operand);
3647 try cg.addImm32(@intFromBool(ty.is_signed));3709 try cg.addImm32(@intFromBool(ty.is_signed));
3648 try cg.addImm32(ty.bits);3710 try cg.addImm32(ty.bits);
3649 try cg.addCallIntrinsic(.__byteswap_limb64);3711 try cg.addCallIntrinsic(.__byteswap_limb64);
...@@ -3687,12 +3749,13 @@ fn intWrap(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {...@@ -3687,12 +3749,13 @@ fn intWrap(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3687 65...127 => {3749 65...127 => {
3688 const result = try cg.allocStack(Type.u128);3750 const result = try cg.allocStack(Type.u128);
36893751
3690 try cg.emitWValue(result);3752 const result_offset = result.memOffset();
3691 _ = try cg.load(operand, Type.u64, 0);3753 try cg.emitMemBase(result);
3692 try cg.store(.stack, .stack, Type.u64, result.offset());3754 _ = try cg.load(operand, .u64, .{});
3755 try cg.store(.stack, .stack, .u64, .{ .offset = result_offset });
36933756
3694 try cg.emitWValue(result);3757 try cg.emitMemBase(result);
3695 _ = try cg.load(operand, Type.u64, 8);3758 _ = try cg.load(operand, .u64, .{ .offset = 8 });
3696 if (ty.is_signed) {3759 if (ty.is_signed) {
3697 try cg.addImm64(128 - ty.bits);3760 try cg.addImm64(128 - ty.bits);
3698 try cg.addTag(.i64_shl);3761 try cg.addTag(.i64_shl);
...@@ -3702,7 +3765,7 @@ fn intWrap(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {...@@ -3702,7 +3765,7 @@ fn intWrap(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3702 try cg.addImm64(~@as(u64, 0) >> @intCast(128 - ty.bits));3765 try cg.addImm64(~@as(u64, 0) >> @intCast(128 - ty.bits));
3703 try cg.addTag(.i64_and);3766 try cg.addTag(.i64_and);
3704 }3767 }
3705 try cg.store(.stack, .stack, Type.u64, result.offset() + 8);3768 try cg.store(.stack, .stack, .u64, .{ .offset = result_offset + 8 });
37063769
3707 return result;3770 return result;
3708 },3771 },
...@@ -3719,8 +3782,8 @@ fn intWrap(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {...@@ -3719,8 +3782,8 @@ fn intWrap(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3719 try cg.memcpy(result, operand, .{ .imm32 = used_len - 8 });3782 try cg.memcpy(result, operand, .{ .imm32 = used_len - 8 });
3720 const pad = 64 - ty.bits % 64;3783 const pad = 64 - ty.bits % 64;
37213784
3722 try cg.emitWValue(result);3785 try cg.emitMemBase(result);
3723 _ = try cg.load(operand, Type.u64, used_len - 8);3786 _ = try cg.load(operand, .u64, .{ .offset = used_len - 8 });
3724 if (ty.is_signed) {3787 if (ty.is_signed) {
3725 try cg.addImm64(pad);3788 try cg.addImm64(pad);
3726 try cg.addTag(.i64_shl);3789 try cg.addTag(.i64_shl);
...@@ -3730,22 +3793,22 @@ fn intWrap(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {...@@ -3730,22 +3793,22 @@ fn intWrap(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3730 try cg.addImm64(~@as(u64, 0) >> @intCast(pad));3793 try cg.addImm64(~@as(u64, 0) >> @intCast(pad));
3731 try cg.addTag(.i64_and);3794 try cg.addTag(.i64_and);
3732 }3795 }
3733 try cg.store(.stack, .stack, Type.u64, result.offset() + used_len - 8);3796 try cg.store(.stack, .stack, .u64, .{ .offset = result.memOffset() + used_len - 8 });
3734 } else {3797 } else {
3735 try cg.memcpy(result, operand, .{ .imm32 = used_len });3798 try cg.memcpy(result, operand, .{ .imm32 = used_len });
3736 }3799 }
37373800
3738 const full_len = @divExact(bits, 8);3801 const full_len = @divExact(bits, 8);
3739 if (used_len + 8 == full_len) { // last limb needs sign extended3802 if (used_len + 8 == full_len) { // last limb needs sign extended
3740 try cg.emitWValue(result);3803 try cg.emitMemBase(result);
3741 if (ty.is_signed) {3804 if (ty.is_signed) {
3742 _ = try cg.load(result, Type.u64, used_len - 8);3805 _ = try cg.load(result, .u64, .{ .offset = used_len - 8 });
3743 try cg.addImm64(63);3806 try cg.addImm64(63);
3744 try cg.addTag(.i64_shr_s);3807 try cg.addTag(.i64_shr_s);
3745 } else {3808 } else {
3746 try cg.addImm64(0);3809 try cg.addImm64(0);
3747 }3810 }
3748 try cg.store(.stack, .stack, Type.u64, result.offset() + used_len);3811 try cg.store(.stack, .stack, .u64, .{ .offset = result.memOffset() + used_len });
3749 }3812 }
37503813
3751 return result;3814 return result;
...@@ -3768,12 +3831,12 @@ fn intMaxValue(cg: *CodeGen, int_ty: IntType) InnerError!WValue {...@@ -3768,12 +3831,12 @@ fn intMaxValue(cg: *CodeGen, int_ty: IntType) InnerError!WValue {
3768 }3831 }
3769 } else if (int_ty.bits <= 128) {3832 } else if (int_ty.bits <= 128) {
3770 const result = try cg.allocInt(int_ty);3833 const result = try cg.allocInt(int_ty);
3771 try cg.store(result, .{ .imm64 = ~@as(u64, 0) }, Type.u64, 0);3834 try cg.store(result, .{ .imm64 = ~@as(u64, 0) }, .u64, .{});
37723835
3773 if (int_ty.is_signed) {3836 if (int_ty.is_signed) {
3774 try cg.store(result, .{ .imm64 = (~@as(u64, 0) >> @intCast(128 - int_ty.bits)) >> 1 }, Type.u64, 8);3837 try cg.store(result, .{ .imm64 = (~@as(u64, 0) >> @intCast(128 - int_ty.bits)) >> 1 }, .u64, .{ .offset = 8 });
3775 } else {3838 } else {
3776 try cg.store(result, .{ .imm64 = ~@as(u64, 0) >> @intCast(128 - int_ty.bits) }, Type.u64, 8);3839 try cg.store(result, .{ .imm64 = ~@as(u64, 0) >> @intCast(128 - int_ty.bits) }, .u64, .{ .offset = 8 });
3777 }3840 }
3778 return result;3841 return result;
3779 } else {3842 } else {
...@@ -3784,13 +3847,13 @@ fn intMaxValue(cg: *CodeGen, int_ty: IntType) InnerError!WValue {...@@ -3784,13 +3847,13 @@ fn intMaxValue(cg: *CodeGen, int_ty: IntType) InnerError!WValue {
3784 try cg.memset(Type.u8, result, .{ .imm32 = used_len - 8 }, .{ .imm32 = 0xFF });3847 try cg.memset(Type.u8, result, .{ .imm32 = used_len - 8 }, .{ .imm32 = 0xFF });
37853848
3786 if (int_ty.is_signed) {3849 if (int_ty.is_signed) {
3787 try cg.store(result, .{ .imm64 = (~@as(u64, 0) >> @intCast(used_len * 8 - int_ty.bits)) >> 1 }, Type.u64, used_len - 8);3850 try cg.store(result, .{ .imm64 = (~@as(u64, 0) >> @intCast(used_len * 8 - int_ty.bits)) >> 1 }, .u64, .{ .offset = used_len - 8 });
3788 } else {3851 } else {
3789 try cg.store(result, .{ .imm64 = ~@as(u64, 0) >> @intCast(used_len * 8 - int_ty.bits) }, Type.u64, used_len - 8);3852 try cg.store(result, .{ .imm64 = ~@as(u64, 0) >> @intCast(used_len * 8 - int_ty.bits) }, .u64, .{ .offset = used_len - 8 });
3790 }3853 }
37913854
3792 if (used_len + 8 == full_len) {3855 if (used_len + 8 == full_len) {
3793 try cg.store(result, .{ .imm64 = 0 }, Type.u64, full_len - 8);3856 try cg.store(result, .{ .imm64 = 0 }, .u64, .{ .offset = full_len - 8 });
3794 }3857 }
37953858
3796 return result;3859 return result;
...@@ -3807,8 +3870,8 @@ fn intMinValue(cg: *CodeGen, int_ty: IntType) InnerError!WValue {...@@ -3807,8 +3870,8 @@ fn intMinValue(cg: *CodeGen, int_ty: IntType) InnerError!WValue {
3807 return .{ .imm64 = ~@as(u64, 0) << @intCast(int_ty.bits - 1) };3870 return .{ .imm64 = ~@as(u64, 0) << @intCast(int_ty.bits - 1) };
3808 } else if (int_ty.bits <= 128) {3871 } else if (int_ty.bits <= 128) {
3809 const result = try cg.allocInt(int_ty);3872 const result = try cg.allocInt(int_ty);
3810 try cg.store(result, .{ .imm64 = 0 }, Type.u64, 0);3873 try cg.store(result, .{ .imm64 = 0 }, .u64, .{});
3811 try cg.store(result, .{ .imm64 = ~@as(u64, 0) << @intCast(int_ty.bits - 65) }, Type.u64, 8);3874 try cg.store(result, .{ .imm64 = ~@as(u64, 0) << @intCast(int_ty.bits - 65) }, .u64, .{ .offset = 8 });
3812 return result;3875 return result;
3813 } else {3876 } else {
3814 const result = try cg.allocInt(int_ty);3877 const result = try cg.allocInt(int_ty);
...@@ -3816,10 +3879,10 @@ fn intMinValue(cg: *CodeGen, int_ty: IntType) InnerError!WValue {...@@ -3816,10 +3879,10 @@ fn intMinValue(cg: *CodeGen, int_ty: IntType) InnerError!WValue {
3816 const used_len = @divCeil(int_ty.bits, 64) * 8;3879 const used_len = @divCeil(int_ty.bits, 64) * 8;
38173880
3818 try cg.memset(Type.u8, result, .{ .imm32 = used_len - 8 }, .{ .imm32 = 0 });3881 try cg.memset(Type.u8, result, .{ .imm32 = used_len - 8 }, .{ .imm32 = 0 });
3819 try cg.store(result, .{ .imm64 = ~@as(u64, 0) << @intCast(int_ty.bits - (used_len - 8) * 8 - 1) }, Type.u64, used_len - 8);3882 try cg.store(result, .{ .imm64 = ~@as(u64, 0) << @intCast(int_ty.bits - (used_len - 8) * 8 - 1) }, .u64, .{ .offset = used_len - 8 });
38203883
3821 if (used_len + 8 == full_len) {3884 if (used_len + 8 == full_len) {
3822 try cg.store(result, .{ .imm64 = ~@as(u64, 0) }, Type.u64, full_len - 8);3885 try cg.store(result, .{ .imm64 = ~@as(u64, 0) }, .u64, .{ .offset = full_len - 8 });
3823 }3886 }
38243887
3825 return result;3888 return result;
...@@ -3839,20 +3902,20 @@ fn intAddSat(cg: *CodeGen, int_ty: IntType, lhs: WValue, rhs: WValue) InnerError...@@ -3839,20 +3902,20 @@ fn intAddSat(cg: *CodeGen, int_ty: IntType, lhs: WValue, rhs: WValue) InnerError
3839 defer rhs_is_neg.free(cg);3902 defer rhs_is_neg.free(cg);
3840 const min_val = try cg.intMinValue(int_ty);3903 const min_val = try cg.intMinValue(int_ty);
38413904
3842 try cg.lowerToStack(min_val);3905 try cg.emitWValue(min_val);
3843 try cg.lowerToStack(max_val);3906 try cg.emitWValue(max_val);
3844 try cg.emitWValue(rhs_is_neg);3907 try cg.emitWValue(rhs_is_neg);
3845 try cg.addTag(.select);3908 try cg.addTag(.select);
38463909
3847 try cg.lowerToStack(op_val);3910 try cg.emitWValue(op_val);
3848 const overflow_cmp = try cg.intCmp(int_ty, .lt, op_val, lhs);3911 const overflow_cmp = try cg.intCmp(int_ty, .lt, op_val, lhs);
3849 const is_overflow = try cg.intCmp(.u32, .neq, rhs_is_neg, overflow_cmp);3912 const is_overflow = try cg.intCmp(.u32, .neq, rhs_is_neg, overflow_cmp);
3850 try cg.emitWValue(is_overflow);3913 try cg.emitWValue(is_overflow);
3851 try cg.addTag(.select);3914 try cg.addTag(.select);
3852 return .stack;3915 return .stack;
3853 } else {3916 } else {
3854 try cg.lowerToStack(max_val);3917 try cg.emitWValue(max_val);
3855 try cg.lowerToStack(op_val);3918 try cg.emitWValue(op_val);
38563919
3857 const is_overflow = try cg.intCmp(int_ty, .lt, op_val, lhs);3920 const is_overflow = try cg.intCmp(int_ty, .lt, op_val, lhs);
3858 try cg.emitWValue(is_overflow);3921 try cg.emitWValue(is_overflow);
...@@ -3873,12 +3936,12 @@ fn intSubSat(cg: *CodeGen, int_ty: IntType, lhs: WValue, rhs: WValue) InnerError...@@ -3873,12 +3936,12 @@ fn intSubSat(cg: *CodeGen, int_ty: IntType, lhs: WValue, rhs: WValue) InnerError
3873 const max_val = try cg.intMaxValue(int_ty);3936 const max_val = try cg.intMaxValue(int_ty);
3874 const min_val = try cg.intMinValue(int_ty);3937 const min_val = try cg.intMinValue(int_ty);
38753938
3876 try cg.lowerToStack(max_val);3939 try cg.emitWValue(max_val);
3877 try cg.lowerToStack(min_val);3940 try cg.emitWValue(min_val);
3878 try cg.emitWValue(rhs_is_neg);3941 try cg.emitWValue(rhs_is_neg);
3879 try cg.addTag(.select);3942 try cg.addTag(.select);
38803943
3881 try cg.lowerToStack(op_val);3944 try cg.emitWValue(op_val);
3882 const overflow_cmp = try cg.intCmp(int_ty, .gt, op_val, lhs);3945 const overflow_cmp = try cg.intCmp(int_ty, .gt, op_val, lhs);
3883 const is_overflow = try cg.intCmp(.u32, .neq, rhs_is_neg, overflow_cmp);3946 const is_overflow = try cg.intCmp(.u32, .neq, rhs_is_neg, overflow_cmp);
3884 try cg.emitWValue(is_overflow);3947 try cg.emitWValue(is_overflow);
...@@ -3887,8 +3950,8 @@ fn intSubSat(cg: *CodeGen, int_ty: IntType, lhs: WValue, rhs: WValue) InnerError...@@ -3887,8 +3950,8 @@ fn intSubSat(cg: *CodeGen, int_ty: IntType, lhs: WValue, rhs: WValue) InnerError
3887 } else {3950 } else {
3888 const zero = try cg.intZeroValue(int_ty);3951 const zero = try cg.intZeroValue(int_ty);
38893952
3890 try cg.lowerToStack(zero);3953 try cg.emitWValue(zero);
3891 try cg.lowerToStack(op_val);3954 try cg.emitWValue(op_val);
3892 const is_overflow = try cg.intCmp(int_ty, .lt, lhs, rhs);3955 const is_overflow = try cg.intCmp(int_ty, .lt, lhs, rhs);
3893 try cg.emitWValue(is_overflow);3956 try cg.emitWValue(is_overflow);
3894 try cg.addTag(.select);3957 try cg.addTag(.select);
...@@ -3912,10 +3975,10 @@ fn intMulSat(cg: *CodeGen, int_ty: IntType, lhs: WValue, rhs: WValue) InnerError...@@ -3912,10 +3975,10 @@ fn intMulSat(cg: *CodeGen, int_ty: IntType, lhs: WValue, rhs: WValue) InnerError
3912 if (int_ty.is_signed) {3975 if (int_ty.is_signed) {
3913 const min_val = try cg.intMinValue(int_ty);3976 const min_val = try cg.intMinValue(int_ty);
39143977
3915 try cg.lowerToStack(min_val);3978 try cg.emitWValue(min_val);
39163979
3917 try cg.lowerToStack(max_val);3980 try cg.emitWValue(max_val);
3918 try cg.lowerToStack(op_val);3981 try cg.emitWValue(op_val);
3919 const max_ext = try cg.intCast(ext_ty, int_ty, max_val);3982 const max_ext = try cg.intCast(ext_ty, int_ty, max_val);
3920 const ov_pos = try cg.intCmp(ext_ty, .lt, max_ext, mul_ext);3983 const ov_pos = try cg.intCmp(ext_ty, .lt, max_ext, mul_ext);
3921 try cg.emitWValue(ov_pos);3984 try cg.emitWValue(ov_pos);
...@@ -3923,12 +3986,12 @@ fn intMulSat(cg: *CodeGen, int_ty: IntType, lhs: WValue, rhs: WValue) InnerError...@@ -3923,12 +3986,12 @@ fn intMulSat(cg: *CodeGen, int_ty: IntType, lhs: WValue, rhs: WValue) InnerError
39233986
3924 const min_ext = try cg.intCast(ext_ty, int_ty, min_val);3987 const min_ext = try cg.intCast(ext_ty, int_ty, min_val);
3925 const ov_neg = try cg.intCmp(ext_ty, .gt, min_ext, mul_ext);3988 const ov_neg = try cg.intCmp(ext_ty, .gt, min_ext, mul_ext);
3926 try cg.lowerToStack(ov_neg);3989 try cg.emitWValue(ov_neg);
3927 try cg.addTag(.select);3990 try cg.addTag(.select);
3928 return .stack;3991 return .stack;
3929 } else {3992 } else {
3930 try cg.lowerToStack(max_val);3993 try cg.emitWValue(max_val);
3931 try cg.lowerToStack(op_val);3994 try cg.emitWValue(op_val);
3932 const max_ext = try cg.intCast(ext_ty, int_ty, max_val);3995 const max_ext = try cg.intCast(ext_ty, int_ty, max_val);
3933 const is_overflow = try cg.intCmp(ext_ty, .lt, max_ext, mul_ext);3996 const is_overflow = try cg.intCmp(ext_ty, .lt, max_ext, mul_ext);
3934 try cg.emitWValue(is_overflow);3997 try cg.emitWValue(is_overflow);
...@@ -3951,20 +4014,20 @@ fn intShlSat(cg: *CodeGen, int_ty: IntType, lhs: WValue, rhs: WValue) InnerError...@@ -3951,20 +4014,20 @@ fn intShlSat(cg: *CodeGen, int_ty: IntType, lhs: WValue, rhs: WValue) InnerError
3951 const zero = try cg.intZeroValue(int_ty);4014 const zero = try cg.intZeroValue(int_ty);
3952 const min_val = try cg.intMinValue(int_ty);4015 const min_val = try cg.intMinValue(int_ty);
39534016
3954 try cg.lowerToStack(min_val);4017 try cg.emitWValue(min_val);
3955 try cg.lowerToStack(max_val);4018 try cg.emitWValue(max_val);
3956 const lhs_is_neg = try cg.intCmp(int_ty, .lt, lhs, zero);4019 const lhs_is_neg = try cg.intCmp(int_ty, .lt, lhs, zero);
3957 try cg.emitWValue(lhs_is_neg);4020 try cg.emitWValue(lhs_is_neg);
3958 try cg.addTag(.select);4021 try cg.addTag(.select);
39594022
3960 try cg.lowerToStack(op_val);4023 try cg.emitWValue(op_val);
3961 const is_overflow = try cg.intCmp(int_ty, .neq, check_val, lhs);4024 const is_overflow = try cg.intCmp(int_ty, .neq, check_val, lhs);
3962 try cg.emitWValue(is_overflow);4025 try cg.emitWValue(is_overflow);
3963 try cg.addTag(.select);4026 try cg.addTag(.select);
3964 return .stack;4027 return .stack;
3965 } else {4028 } else {
3966 try cg.lowerToStack(max_val);4029 try cg.emitWValue(max_val);
3967 try cg.lowerToStack(op_val);4030 try cg.emitWValue(op_val);
3968 const is_overflow = try cg.intCmp(int_ty, .neq, check_val, lhs);4031 const is_overflow = try cg.intCmp(int_ty, .neq, check_val, lhs);
3969 try cg.emitWValue(is_overflow);4032 try cg.emitWValue(is_overflow);
3970 try cg.addTag(.select);4033 try cg.addTag(.select);
...@@ -3979,8 +4042,8 @@ fn intZeroValue(cg: *CodeGen, int_ty: IntType) InnerError!WValue {...@@ -3979,8 +4042,8 @@ fn intZeroValue(cg: *CodeGen, int_ty: IntType) InnerError!WValue {
3979 33...64 => return .{ .imm64 = 0 },4042 33...64 => return .{ .imm64 = 0 },
3980 65...128 => {4043 65...128 => {
3981 const result = try cg.allocInt(int_ty);4044 const result = try cg.allocInt(int_ty);
3982 try cg.store(result, .{ .imm64 = 0 }, Type.u64, 0);4045 try cg.store(result, .{ .imm64 = 0 }, .u64, .{});
3983 try cg.store(result, .{ .imm64 = 0 }, Type.u64, 8);4046 try cg.store(result, .{ .imm64 = 0 }, .u64, .{ .offset = 8 });
3984 return result;4047 return result;
3985 },4048 },
3986 else => {4049 else => {
...@@ -4003,7 +4066,7 @@ fn toLocalInt(cg: *CodeGen, value: WValue, int_ty: IntType) InnerError!WValue {...@@ -4003,7 +4066,7 @@ fn toLocalInt(cg: *CodeGen, value: WValue, int_ty: IntType) InnerError!WValue {
4003 else => return cg.fail("TODO: Support toLocalInt for integer bitsize: {d}", .{int_ty.bits}),4066 else => return cg.fail("TODO: Support toLocalInt for integer bitsize: {d}", .{int_ty.bits}),
4004 };4067 };
4005 const new_local = try cg.allocLocal(ty);4068 const new_local = try cg.allocLocal(ty);
4006 try cg.addLocal(.local_set, new_local.local.value);4069 try cg.addLocal(.local_set, new_local.local);
4007 return new_local;4070 return new_local;
4008 },4071 },
4009 .local, .stack_offset => return value,4072 .local, .stack_offset => return value,
...@@ -4036,9 +4099,9 @@ fn intAddOverflow(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerErro...@@ -4036,9 +4099,9 @@ fn intAddOverflow(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerErro
4036 else => {4099 else => {
4037 const result = try cg.allocInt(ty);4100 const result = try cg.allocInt(ty);
40384101
4039 try cg.lowerToStack(result);4102 try cg.emitWValue(result);
4040 try cg.lowerToStack(lhs);4103 try cg.emitWValue(lhs);
4041 try cg.lowerToStack(rhs);4104 try cg.emitWValue(rhs);
4042 try cg.addImm32(@intFromBool(ty.is_signed));4105 try cg.addImm32(@intFromBool(ty.is_signed));
4043 try cg.addImm32(ty.bits);4106 try cg.addImm32(ty.bits);
4044 try cg.addCallIntrinsic(.__addo_limb64);4107 try cg.addCallIntrinsic(.__addo_limb64);
...@@ -4068,9 +4131,9 @@ fn intSubOverflow(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerErro...@@ -4068,9 +4131,9 @@ fn intSubOverflow(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerErro
4068 else => {4131 else => {
4069 const result = try cg.allocInt(ty);4132 const result = try cg.allocInt(ty);
40704133
4071 try cg.lowerToStack(result);4134 try cg.emitWValue(result);
4072 try cg.lowerToStack(lhs);4135 try cg.emitWValue(lhs);
4073 try cg.lowerToStack(rhs);4136 try cg.emitWValue(rhs);
4074 try cg.addImm32(@intFromBool(ty.is_signed));4137 try cg.addImm32(@intFromBool(ty.is_signed));
4075 try cg.addImm32(ty.bits);4138 try cg.addImm32(ty.bits);
4076 try cg.addCallIntrinsic(.__subo_limb64);4139 try cg.addCallIntrinsic(.__subo_limb64);
...@@ -4080,39 +4143,55 @@ fn intSubOverflow(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerErro...@@ -4080,39 +4143,55 @@ fn intSubOverflow(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerErro
4080}4143}
40814144
4082fn intMulOverflow(cg: *CodeGen, int_ty: IntType, lhs: WValue, rhs: WValue) InnerError!OverflowResult {4145fn intMulOverflow(cg: *CodeGen, int_ty: IntType, lhs: WValue, rhs: WValue) InnerError!OverflowResult {
4146 if (int_ty.bits > 64 and (int_ty.bits != 128 or !int_ty.is_signed)) {
4147 const result = try cg.allocInt(int_ty);
4148
4149 try cg.emitWValue(result);
4150 try cg.emitWValue(lhs);
4151 try cg.emitWValue(rhs);
4152 try cg.addImm32(@intFromBool(int_ty.is_signed));
4153 try cg.addImm32(int_ty.bits);
4154 try cg.addCallIntrinsic(.__mulo_limb64);
4155
4156 return .{ .result = result, .ov = .stack };
4157 }
4158
4083 const overflow_bit = try cg.allocLocal(Type.u32);4159 const overflow_bit = try cg.allocLocal(Type.u32);
4084 try cg.addImm32(0);4160 try cg.addImm32(0);
4085 try cg.addLocal(.local_set, overflow_bit.local.value);4161 try cg.addLocal(.local_set, overflow_bit.local);
40864162
4087 const result_val = if (int_ty.bits <= 32) blk: {4163 const result_val = if (int_ty.bits <= 32) blk: {
4088 const new_ty: IntType = .{ .is_signed = int_ty.is_signed, .bits = 64 };4164 const new_ty: IntType = .{ .is_signed = int_ty.is_signed, .bits = 64 };
4089 const lhs_upcast = try cg.intCast(new_ty, int_ty, lhs);4165 const lhs_upcast = try cg.intCast(new_ty, int_ty, lhs);
4090 const rhs_upcast = try cg.intCast(new_ty, int_ty, rhs);4166 const rhs_upcast = try cg.intCast(new_ty, int_ty, rhs);
4091 const mul_raw = try cg.intMul(new_ty, lhs_upcast, rhs_upcast);4167 const mul_raw = try cg.intMul(new_ty, lhs_upcast, rhs_upcast);
4092 const bin_op = try cg.toLocalInt(mul_raw, new_ty);4168 var bin_op = try cg.toLocalInt(mul_raw, new_ty);
4169 defer bin_op.free(cg);
40934170
4094 const res = try cg.intTrunc(int_ty, new_ty, bin_op);4171 const res = try cg.intTrunc(int_ty, new_ty, bin_op);
4095 const res_tmp = try cg.toLocalInt(res, int_ty);4172 const res_tmp = try cg.toLocalInt(res, int_ty);
40964173
4097 const res_upcast = try cg.intCast(new_ty, int_ty, res_tmp);4174 const res_upcast = try cg.intCast(new_ty, int_ty, res_tmp);
4098 _ = try cg.intCmp(new_ty, .neq, res_upcast, bin_op);4175 _ = try cg.intCmp(new_ty, .neq, res_upcast, bin_op);
4099 try cg.addLocal(.local_set, overflow_bit.local.value);4176 try cg.addLocal(.local_set, overflow_bit.local);
4100 break :blk res_tmp;4177 break :blk res_tmp;
4101 } else if (int_ty.bits <= 64) blk: {4178 } else if (int_ty.bits <= 64) blk: {
4102 const new_ty: IntType = .{ .is_signed = int_ty.is_signed, .bits = 128 };4179 const new_ty: IntType = .{ .is_signed = int_ty.is_signed, .bits = 128 };
4103 const lhs_upcast = try cg.intCast(new_ty, int_ty, lhs);4180 const lhs_upcast = try cg.intCast(new_ty, int_ty, lhs);
4104 const rhs_upcast = try cg.intCast(new_ty, int_ty, rhs);4181 const rhs_upcast = try cg.intCast(new_ty, int_ty, rhs);
4105 const mul_raw = try cg.intMul(new_ty, lhs_upcast, rhs_upcast);4182 const mul_raw = try cg.intMul(new_ty, lhs_upcast, rhs_upcast);
4106 const bin_op = try cg.toLocalInt(mul_raw, new_ty);4183 var bin_op = try cg.toLocalInt(mul_raw, new_ty);
4184 defer bin_op.free(cg);
41074185
4108 const res = try cg.intTrunc(int_ty, new_ty, bin_op);4186 const res = try cg.intTrunc(int_ty, new_ty, bin_op);
4109 const res_tmp = try cg.toLocalInt(res, int_ty);4187 const res_tmp = try cg.toLocalInt(res, int_ty);
41104188
4111 const res_upcast = try cg.intCast(new_ty, int_ty, res_tmp);4189 const res_upcast = try cg.intCast(new_ty, int_ty, res_tmp);
4112 _ = try cg.intCmp(new_ty, .neq, res_upcast, bin_op);4190 _ = try cg.intCmp(new_ty, .neq, res_upcast, bin_op);
4113 try cg.addLocal(.local_set, overflow_bit.local.value);4191 try cg.addLocal(.local_set, overflow_bit.local);
4114 break :blk res_tmp;4192 break :blk res_tmp;
4115 } else if (int_ty.bits == 128 and int_ty.is_signed) blk: {4193 } else blk: {
4194 assert(int_ty.bits == 128 and int_ty.is_signed);
4116 const overflow_ret = try cg.allocStack(Type.i32);4195 const overflow_ret = try cg.allocStack(Type.i32);
4117 const res = try cg.callIntrinsic(4196 const res = try cg.callIntrinsic(
4118 .__muloti4,4197 .__muloti4,
...@@ -4120,20 +4199,9 @@ fn intMulOverflow(cg: *CodeGen, int_ty: IntType, lhs: WValue, rhs: WValue) Inner...@@ -4120,20 +4199,9 @@ fn intMulOverflow(cg: *CodeGen, int_ty: IntType, lhs: WValue, rhs: WValue) Inner
4120 Type.i128,4199 Type.i128,
4121 &.{ lhs, rhs, overflow_ret },4200 &.{ lhs, rhs, overflow_ret },
4122 );4201 );
4123 _ = try cg.load(overflow_ret, Type.i32, 0);4202 _ = try cg.load(overflow_ret, .i32, .{});
4124 try cg.addLocal(.local_set, overflow_bit.local.value);4203 try cg.addLocal(.local_set, overflow_bit.local);
4125 break :blk res;4204 break :blk res;
4126 } else {
4127 const result = try cg.allocInt(int_ty);
4128
4129 try cg.lowerToStack(result);
4130 try cg.lowerToStack(lhs);
4131 try cg.lowerToStack(rhs);
4132 try cg.addImm32(@intFromBool(int_ty.is_signed));
4133 try cg.addImm32(int_ty.bits);
4134 try cg.addCallIntrinsic(.__mulo_limb64);
4135
4136 return .{ .result = result, .ov = .stack };
4137 };4205 };
41384206
4139 return .{ .result = result_val, .ov = .{ .local = overflow_bit.local } };4207 return .{ .result = result_val, .ov = .{ .local = overflow_bit.local } };
...@@ -4155,9 +4223,9 @@ fn intShlOverflow(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerErro...@@ -4155,9 +4223,9 @@ fn intShlOverflow(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerErro
4155 else => {4223 else => {
4156 const result = try cg.allocInt(ty);4224 const result = try cg.allocInt(ty);
41574225
4158 try cg.lowerToStack(result);4226 try cg.emitWValue(result);
4159 try cg.lowerToStack(lhs);4227 try cg.emitWValue(lhs);
4160 try cg.lowerToStack(rhs);4228 try cg.emitWValue(rhs);
4161 try cg.addImm32(@intFromBool(ty.is_signed));4229 try cg.addImm32(@intFromBool(ty.is_signed));
4162 try cg.addImm32(ty.bits);4230 try cg.addImm32(ty.bits);
4163 try cg.addCallIntrinsic(.__shlo_limb64);4231 try cg.addCallIntrinsic(.__shlo_limb64);
...@@ -4194,15 +4262,15 @@ fn intCast(cg: *CodeGen, dest_ty: IntType, src_ty: IntType, operand: WValue) Inn...@@ -4194,15 +4262,15 @@ fn intCast(cg: *CodeGen, dest_ty: IntType, src_ty: IntType, operand: WValue) Inn
4194 } else {4262 } else {
4195 var src_len: u32 = undefined;4263 var src_len: u32 = undefined;
4196 if (src_bits == 32) {4264 if (src_bits == 32) {
4197 try cg.emitWValue(result);4265 try cg.emitMemBase(result);
4198 try cg.emitWValue(operand);4266 try cg.emitWValue(operand);
4199 try cg.addTag(if (src_ty.is_signed) .i64_extend_i32_s else .i64_extend_i32_u);4267 try cg.addTag(if (src_ty.is_signed) .i64_extend_i32_s else .i64_extend_i32_u);
4200 try cg.store(.stack, .stack, Type.u64, result.offset());4268 try cg.store(.stack, .stack, .u64, .{ .offset = result.memOffset() });
4201 src_len = 8;4269 src_len = 8;
4202 } else if (src_bits == 64) {4270 } else if (src_bits == 64) {
4203 try cg.emitWValue(result);4271 try cg.emitMemBase(result);
4204 try cg.emitWValue(operand);4272 try cg.emitWValue(operand);
4205 try cg.store(.stack, .stack, Type.u64, result.offset());4273 try cg.store(.stack, .stack, .u64, .{ .offset = result.memOffset() });
4206 src_len = 8;4274 src_len = 8;
4207 } else {4275 } else {
4208 src_len = src_bits / 8;4276 src_len = src_bits / 8;
...@@ -4211,7 +4279,7 @@ fn intCast(cg: *CodeGen, dest_ty: IntType, src_ty: IntType, operand: WValue) Inn...@@ -4211,7 +4279,7 @@ fn intCast(cg: *CodeGen, dest_ty: IntType, src_ty: IntType, operand: WValue) Inn
42114279
4212 if (dest_bits == 128) {4280 if (dest_bits == 128) {
4213 if (src_ty.is_signed) {4281 if (src_ty.is_signed) {
4214 try cg.emitWValue(result);4282 try cg.emitMemBase(result);
4215 if (src_bits == 32) {4283 if (src_bits == 32) {
4216 try cg.emitWValue(operand);4284 try cg.emitWValue(operand);
4217 try cg.addTag(if (dest_ty.is_signed) .i64_extend_i32_s else .i64_extend_i32_u);4285 try cg.addTag(if (dest_ty.is_signed) .i64_extend_i32_s else .i64_extend_i32_u);
...@@ -4219,9 +4287,9 @@ fn intCast(cg: *CodeGen, dest_ty: IntType, src_ty: IntType, operand: WValue) Inn...@@ -4219,9 +4287,9 @@ fn intCast(cg: *CodeGen, dest_ty: IntType, src_ty: IntType, operand: WValue) Inn
4219 try cg.emitWValue(operand);4287 try cg.emitWValue(operand);
4220 } else unreachable;4288 } else unreachable;
4221 const shr = try cg.intShr(IntType.i64, .stack, .{ .imm32 = 63 });4289 const shr = try cg.intShr(IntType.i64, .stack, .{ .imm32 = 63 });
4222 try cg.store(.stack, shr, Type.u64, 8 + result.offset());4290 try cg.store(.stack, shr, .u64, .{ .offset = result.memOffset() + 8 });
4223 } else {4291 } else {
4224 try cg.store(result, .{ .imm64 = 0 }, Type.u64, 8);4292 try cg.store(result, .{ .imm64 = 0 }, .u64, .{ .offset = 8 });
4225 }4293 }
4226 } else {4294 } else {
4227 var pad = result;4295 var pad = result;
...@@ -4236,7 +4304,7 @@ fn intCast(cg: *CodeGen, dest_ty: IntType, src_ty: IntType, operand: WValue) Inn...@@ -4236,7 +4304,7 @@ fn intCast(cg: *CodeGen, dest_ty: IntType, src_ty: IntType, operand: WValue) Inn
4236 _ = try cg.intShr(IntType.i64, .stack, .{ .imm32 = 63 });4304 _ = try cg.intShr(IntType.i64, .stack, .{ .imm32 = 63 });
4237 try cg.addTag(.i32_wrap_i64);4305 try cg.addTag(.i32_wrap_i64);
4238 } else {4306 } else {
4239 _ = try cg.load(operand, Type.u64, src_len - 8);4307 _ = try cg.load(operand, .u64, .{ .offset = src_len - 8 });
4240 _ = try cg.intShr(IntType.i64, .stack, .{ .imm32 = 63 });4308 _ = try cg.intShr(IntType.i64, .stack, .{ .imm32 = 63 });
4241 try cg.addTag(.i32_wrap_i64);4309 try cg.addTag(.i32_wrap_i64);
4242 }4310 }
...@@ -4254,11 +4322,12 @@ fn intCast(cg: *CodeGen, dest_ty: IntType, src_ty: IntType, operand: WValue) Inn...@@ -4254,11 +4322,12 @@ fn intCast(cg: *CodeGen, dest_ty: IntType, src_ty: IntType, operand: WValue) Inn
4254 assert(dest_bits <= 64);4322 assert(dest_bits <= 64);
4255 assert(src_bits >= 128);4323 assert(src_bits >= 128);
4256 const load_ty = if (dest_bits == 32) Type.u32 else Type.u64;4324 const load_ty = if (dest_bits == 32) Type.u32 else Type.u64;
4257 return cg.load(operand, load_ty, 0);4325 return cg.load(operand, load_ty, .{});
4258 }4326 }
4259}4327}
42604328
4261fn intTrunc(cg: *CodeGen, dest_ty: IntType, src_ty: IntType, operand: WValue) InnerError!WValue {4329fn intTrunc(cg: *CodeGen, dest_ty: IntType, src_ty: IntType, operand: WValue) InnerError!WValue {
4330 assert(src_ty.bits > dest_ty.bits);
4262 var result = try cg.intCast(dest_ty, src_ty, operand);4331 var result = try cg.intCast(dest_ty, src_ty, operand);
42634332
4264 const dest_wasm_bits = cg.intBackingBits(dest_ty.bits);4333 const dest_wasm_bits = cg.intBackingBits(dest_ty.bits);
...@@ -4622,20 +4691,22 @@ fn floatNeg(cg: *CodeGen, ty: FloatType, arg: WValue) InnerError!WValue {...@@ -4622,20 +4691,22 @@ fn floatNeg(cg: *CodeGen, ty: FloatType, arg: WValue) InnerError!WValue {
4622 },4691 },
4623 .f80, .f128 => {4692 .f80, .f128 => {
4624 const result = try cg.allocStack(Type.f128);4693 const result = try cg.allocStack(Type.f128);
4625 try cg.emitWValue(result);4694 const result_offset = result.memOffset();
4626 try cg.emitWValue(arg);4695 const arg_offset = arg.memOffset();
4627 try cg.addMemArg(.i64_load, .{ .offset = 0 + arg.offset(), .alignment = 2 });4696 try cg.emitMemBase(result);
4628 try cg.addMemArg(.i64_store, .{ .offset = 0 + result.offset(), .alignment = 2 });4697 try cg.emitMemBase(arg);
4629 try cg.emitWValue(result);4698 try cg.addMemArg(.i64_load, .{ .offset = arg_offset, .alignment = .@"2" });
4630 try cg.emitWValue(arg);4699 try cg.addMemArg(.i64_store, .{ .offset = result_offset, .alignment = .@"2" });
4631 try cg.addMemArg(.i64_load, .{ .offset = 8 + arg.offset(), .alignment = 2 });4700 try cg.emitMemBase(result);
4701 try cg.emitMemBase(arg);
4702 try cg.addMemArg(.i64_load, .{ .offset = arg_offset + 8, .alignment = .@"2" });
4632 if (ty == .f80) {4703 if (ty == .f80) {
4633 try cg.addImm64(0x8000);4704 try cg.addImm64(0x8000);
4634 } else {4705 } else {
4635 try cg.addImm64(0x8000000000000000);4706 try cg.addImm64(0x8000000000000000);
4636 }4707 }
4637 try cg.addTag(.i64_xor);4708 try cg.addTag(.i64_xor);
4638 try cg.addMemArg(.i64_store, .{ .offset = 8 + result.offset(), .alignment = 2 });4709 try cg.addMemArg(.i64_store, .{ .offset = result_offset + 8, .alignment = .@"2" });
4639 return result;4710 return result;
4640 },4711 },
4641 }4712 }
...@@ -4661,20 +4732,22 @@ fn floatAbs(cg: *CodeGen, ty: FloatType, arg: WValue) InnerError!WValue {...@@ -4661,20 +4732,22 @@ fn floatAbs(cg: *CodeGen, ty: FloatType, arg: WValue) InnerError!WValue {
4661 },4732 },
4662 .f80, .f128 => {4733 .f80, .f128 => {
4663 const result = try cg.allocStack(Type.f128);4734 const result = try cg.allocStack(Type.f128);
4664 try cg.emitWValue(result);4735 const result_offset = result.memOffset();
4665 try cg.emitWValue(arg);4736 const arg_offset = arg.memOffset();
4666 try cg.addMemArg(.i64_load, .{ .offset = 0 + arg.offset(), .alignment = 2 });4737 try cg.emitMemBase(result);
4667 try cg.addMemArg(.i64_store, .{ .offset = 0 + result.offset(), .alignment = 2 });4738 try cg.emitMemBase(arg);
4668 try cg.emitWValue(result);4739 try cg.addMemArg(.i64_load, .{ .offset = arg_offset, .alignment = .@"2" });
4669 try cg.emitWValue(arg);4740 try cg.addMemArg(.i64_store, .{ .offset = result_offset, .alignment = .@"2" });
4670 try cg.addMemArg(.i64_load, .{ .offset = 8 + arg.offset(), .alignment = 2 });4741 try cg.emitMemBase(result);
4742 try cg.emitMemBase(arg);
4743 try cg.addMemArg(.i64_load, .{ .offset = arg_offset + 8, .alignment = .@"2" });
4671 if (ty == .f80) {4744 if (ty == .f80) {
4672 try cg.addImm64(0x7FFF);4745 try cg.addImm64(0x7FFF);
4673 } else {4746 } else {
4674 try cg.addImm64(0x7FFFFFFFFFFFFFFF);4747 try cg.addImm64(0x7FFFFFFFFFFFFFFF);
4675 }4748 }
4676 try cg.addTag(.i64_and);4749 try cg.addTag(.i64_and);
4677 try cg.addMemArg(.i64_store, .{ .offset = 8 + result.offset(), .alignment = 2 });4750 try cg.addMemArg(.i64_store, .{ .offset = result_offset + 8, .alignment = .@"2" });
4678 return result;4751 return result;
4679 },4752 },
4680 }4753 }
...@@ -5184,30 +5257,36 @@ fn airBlock(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -5184,30 +5257,36 @@ fn airBlock(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
51845257
5185fn lowerBlock(cg: *CodeGen, inst: Air.Inst.Index, block_ty: Type, body: []const Air.Inst.Index) InnerError!void {5258fn lowerBlock(cg: *CodeGen, inst: Air.Inst.Index, block_ty: Type, body: []const Air.Inst.Index) InnerError!void {
5186 const zcu = cg.pt.zcu;5259 const zcu = cg.pt.zcu;
5187 // if wasm_block_ty is non-empty, we create a register to store the temporary value5260
5261 if (block_ty.isNoReturn(zcu)) {
5262 assert(cg.liveness.getBlock(inst).deaths.len == 0);
5263 try cg.genBody(body);
5264 return cg.finishAir(inst, .none, &.{});
5265 }
5266
5188 const block_result: WValue = if (block_ty.hasRuntimeBits(zcu))5267 const block_result: WValue = if (block_ty.hasRuntimeBits(zcu))
5189 try cg.allocLocal(block_ty)5268 .stack
5190 else5269 else
5191 .none;5270 .none;
51925271
5193 try cg.startBlock(.block, .empty);5272 try cg.startBlock(.block, genBlockType(block_ty, zcu, cg.target));
5194 // Here we set the current block idx, so breaks know the depth to jump5273 // Here we set the current block idx, so breaks know the depth to jump
5195 // to when breaking out.5274 // to when breaking out.
5196 try cg.blocks.putNoClobber(cg.gpa, inst, .{5275 try cg.blocks.putNoClobber(cg.gpa, inst, .{
5197 .label = cg.block_depth,5276 .label = cg.block_depth,
5198 .value = block_result,5277 .value = block_result,
5199 });5278 });
5279 defer assert(cg.blocks.swapRemove(inst));
52005280
5201 {5281 {
5202 try cg.branches.append(cg.gpa, .{});5282 try cg.pushBranch();
5203 defer {5283 defer cg.popBranch();
5204 var branch = cg.branches.pop().?;
5205 branch.deinit(cg.gpa);
5206 }
5207 try cg.genBody(body);5284 try cg.genBody(body);
5208 try cg.endBlock();5285 try cg.endBlock();
5209 }5286 }
52105287
5288 for (cg.liveness.getBlock(inst).deaths) |death| cg.processDeath(death.toRef());
5289
5211 return cg.finishAir(inst, block_result, &.{});5290 return cg.finishAir(inst, block_result, &.{});
5212}5291}
52135292
...@@ -5238,6 +5317,7 @@ fn airLoop(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -5238,6 +5317,7 @@ fn airLoop(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
52385317
5239 try cg.genBody(block.body);5318 try cg.genBody(block.body);
5240 try cg.endBlock();5319 try cg.endBlock();
5320 try cg.addTag(.@"unreachable");
52415321
5242 return cg.finishAir(inst, .none, &.{});5322 return cg.finishAir(inst, .none, &.{});
5243}5323}
...@@ -5247,6 +5327,7 @@ fn airCondBr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -5247,6 +5327,7 @@ fn airCondBr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
5247 const condition = try cg.resolveInst(cond_br.condition);5327 const condition = try cg.resolveInst(cond_br.condition);
5248 const then_body = cond_br.then_body;5328 const then_body = cond_br.then_body;
5249 const else_body = cond_br.else_body;5329 const else_body = cond_br.else_body;
5330 const branch_liveness = cg.liveness.getCondBr(inst);
52505331
5251 // result type is always noreturn, so use `block_empty` as type.5332 // result type is always noreturn, so use `block_empty` as type.
5252 try cg.startBlock(.block, .empty);5333 try cg.startBlock(.block, .empty);
...@@ -5258,24 +5339,21 @@ fn airCondBr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -5258,24 +5339,21 @@ fn airCondBr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
5258 // and continue with the then codepath5339 // and continue with the then codepath
5259 try cg.addLabel(.br_if, 0);5340 try cg.addLabel(.br_if, 0);
52605341
5261 try cg.branches.ensureUnusedCapacity(cg.gpa, 2);5342 if (cg.liveness.operandDies(inst, 0)) cg.processDeath(cond_br.condition);
5343
5262 {5344 {
5263 cg.branches.appendAssumeCapacity(.{});5345 try cg.pushBranch();
5264 defer {5346 defer cg.popBranch();
5265 var else_stack = cg.branches.pop().?;5347 for (branch_liveness.else_deaths) |death| cg.processDeath(death.toRef());
5266 else_stack.deinit(cg.gpa);
5267 }
5268 try cg.genBody(else_body);5348 try cg.genBody(else_body);
5269 try cg.endBlock();5349 try cg.endBlock();
5270 }5350 }
52715351
5272 // Outer block that matches the condition5352 // Outer block that matches the condition
5273 {5353 {
5274 cg.branches.appendAssumeCapacity(.{});5354 try cg.pushBranch();
5275 defer {5355 defer cg.popBranch();
5276 var then_stack = cg.branches.pop().?;5356 for (branch_liveness.then_deaths) |death| cg.processDeath(death.toRef());
5277 then_stack.deinit(cg.gpa);
5278 }
5279 try cg.genBody(then_body);5357 try cg.genBody(then_body);
5280 }5358 }
52815359
...@@ -5311,22 +5389,22 @@ fn airCmp(cg: *CodeGen, inst: Air.Inst.Index, op: std.math.CompareOperator) Inne...@@ -5311,22 +5389,22 @@ fn airCmp(cg: *CodeGen, inst: Air.Inst.Index, op: std.math.CompareOperator) Inne
5311 try cg.startBlock(.block, .empty);5389 try cg.startBlock(.block, .empty);
53125390
5313 try cg.addImm32(if (op == .eq) 0 else 1);5391 try cg.addImm32(if (op == .eq) 0 else 1);
5314 try cg.addLocal(.local_set, result.local.value);5392 try cg.addLocal(.local_set, result.local);
53155393
5316 _ = try cg.isNull(lhs, operand_ty, .i32_eq, .value);5394 _ = try cg.isNull(lhs, operand_ty, .i32_eq, .value);
5317 try cg.addLocal(.local_tee, lhs_null.local.value);5395 try cg.addLocal(.local_tee, lhs_null.local);
5318 _ = try cg.isNull(rhs, operand_ty, .i32_eq, .value);5396 _ = try cg.isNull(rhs, operand_ty, .i32_eq, .value);
5319 try cg.addTag(.i32_ne);5397 try cg.addTag(.i32_ne);
5320 try cg.addLabel(.br_if, 0);5398 try cg.addLabel(.br_if, 0);
53215399
5322 try cg.addImm32(if (op == .eq) 1 else 0);5400 try cg.addImm32(if (op == .eq) 1 else 0);
5323 try cg.addLocal(.local_set, result.local.value);5401 try cg.addLocal(.local_set, result.local);
53245402
5325 try cg.addLocal(.local_get, lhs_null.local.value);5403 try cg.addLocal(.local_get, lhs_null.local);
5326 try cg.addLabel(.br_if, 0);5404 try cg.addLabel(.br_if, 0);
53275405
5328 _ = try cg.load(lhs, payload_ty, 0);5406 _ = try cg.load(lhs, payload_ty, .{});
5329 _ = try cg.load(rhs, payload_ty, 0);5407 _ = try cg.load(rhs, payload_ty, .{});
53305408
5331 if (payload_ty.isAnyFloat()) {5409 if (payload_ty.isAnyFloat()) {
5332 _ = try cg.floatCmp(.fromType(cg, payload_ty), op, .stack, .stack);5410 _ = try cg.floatCmp(.fromType(cg, payload_ty), op, .stack, .stack);
...@@ -5334,10 +5412,10 @@ fn airCmp(cg: *CodeGen, inst: Air.Inst.Index, op: std.math.CompareOperator) Inne...@@ -5334,10 +5412,10 @@ fn airCmp(cg: *CodeGen, inst: Air.Inst.Index, op: std.math.CompareOperator) Inne
5334 _ = try cg.intCmp(.fromType(cg, payload_ty), op, .stack, .stack);5412 _ = try cg.intCmp(.fromType(cg, payload_ty), op, .stack, .stack);
5335 }5413 }
53365414
5337 try cg.addLocal(.local_set, result.local.value);5415 try cg.addLocal(.local_set, result.local);
5338 try cg.endBlock();5416 try cg.endBlock();
53395417
5340 try cg.addLocal(.local_get, result.local.value);5418 try cg.addLocal(.local_get, result.local);
5341 try cg.finishAir(inst, .stack, &.{ bin_op.lhs, bin_op.rhs });5419 try cg.finishAir(inst, .stack, &.{ bin_op.lhs, bin_op.rhs });
5342 } else {5420 } else {
5343 const result = try cg.intCmp(.fromType(cg, operand_ty), op, lhs, rhs);5421 const result = try cg.intCmp(.fromType(cg, operand_ty), op, lhs, rhs);
...@@ -5357,8 +5435,8 @@ fn intCmp(cg: *CodeGen, ty: IntType, op: std.math.CompareOperator, lhs: WValue,...@@ -5357,8 +5435,8 @@ fn intCmp(cg: *CodeGen, ty: IntType, op: std.math.CompareOperator, lhs: WValue,
5357 0 => unreachable,5435 0 => unreachable,
5358 1...32 => {5436 1...32 => {
5359 // lhs or rhs could be stack pointers5437 // lhs or rhs could be stack pointers
5360 try cg.lowerToStack(lhs);5438 try cg.emitWValue(lhs);
5361 try cg.lowerToStack(rhs);5439 try cg.emitWValue(rhs);
5362 const opcode: Mir.Inst.Tag = switch (op) {5440 const opcode: Mir.Inst.Tag = switch (op) {
5363 .eq => .i32_eq,5441 .eq => .i32_eq,
5364 .neq => .i32_ne,5442 .neq => .i32_ne,
...@@ -5372,8 +5450,8 @@ fn intCmp(cg: *CodeGen, ty: IntType, op: std.math.CompareOperator, lhs: WValue,...@@ -5372,8 +5450,8 @@ fn intCmp(cg: *CodeGen, ty: IntType, op: std.math.CompareOperator, lhs: WValue,
5372 },5450 },
5373 33...64 => {5451 33...64 => {
5374 // lhs or rhs could be stack pointers5452 // lhs or rhs could be stack pointers
5375 try cg.lowerToStack(lhs);5453 try cg.emitWValue(lhs);
5376 try cg.lowerToStack(rhs);5454 try cg.emitWValue(rhs);
5377 const opcode: Mir.Inst.Tag = switch (op) {5455 const opcode: Mir.Inst.Tag = switch (op) {
5378 .eq => .i64_eq,5456 .eq => .i64_eq,
5379 .neq => .i64_ne,5457 .neq => .i64_ne,
...@@ -5386,16 +5464,16 @@ fn intCmp(cg: *CodeGen, ty: IntType, op: std.math.CompareOperator, lhs: WValue,...@@ -5386,16 +5464,16 @@ fn intCmp(cg: *CodeGen, ty: IntType, op: std.math.CompareOperator, lhs: WValue,
5386 return .stack;5464 return .stack;
5387 },5465 },
5388 65...128 => {5466 65...128 => {
5389 var lhs_msb = try (try cg.load(lhs, Type.u64, 8)).toLocal(cg, Type.u64);5467 var lhs_msb = try (try cg.load(lhs, .u64, .{ .offset = 8 })).toLocal(cg, Type.u64);
5390 defer lhs_msb.free(cg);5468 defer lhs_msb.free(cg);
5391 var rhs_msb = try (try cg.load(rhs, Type.u64, 8)).toLocal(cg, Type.u64);5469 var rhs_msb = try (try cg.load(rhs, .u64, .{ .offset = 8 })).toLocal(cg, Type.u64);
5392 defer rhs_msb.free(cg);5470 defer rhs_msb.free(cg);
53935471
5394 switch (op) {5472 switch (op) {
5395 .eq, .neq => {5473 .eq, .neq => {
5396 const xor_high = try cg.intXor(.u64, lhs_msb, rhs_msb);5474 const xor_high = try cg.intXor(.u64, lhs_msb, rhs_msb);
5397 const lhs_lsb = try cg.load(lhs, Type.u64, 0);5475 const lhs_lsb = try cg.load(lhs, .u64, .{});
5398 const rhs_lsb = try cg.load(rhs, Type.u64, 0);5476 const rhs_lsb = try cg.load(rhs, .u64, .{});
5399 const xor_low = try cg.intXor(.u64, lhs_lsb, rhs_lsb);5477 const xor_low = try cg.intXor(.u64, lhs_lsb, rhs_lsb);
5400 const or_result = try cg.intOr(.u64, xor_high, xor_low);5478 const or_result = try cg.intOr(.u64, xor_high, xor_low);
54015479
...@@ -5408,8 +5486,8 @@ fn intCmp(cg: *CodeGen, ty: IntType, op: std.math.CompareOperator, lhs: WValue,...@@ -5408,8 +5486,8 @@ fn intCmp(cg: *CodeGen, ty: IntType, op: std.math.CompareOperator, lhs: WValue,
5408 else => {5486 else => {
5409 const word_int_ty: IntType = if (ty.is_signed) .i64 else .u64;5487 const word_int_ty: IntType = if (ty.is_signed) .i64 else .u64;
54105488
5411 const lhs_lsb = try cg.load(lhs, Type.u64, 0);5489 const lhs_lsb = try cg.load(lhs, .u64, .{});
5412 const rhs_lsb = try cg.load(rhs, Type.u64, 0);5490 const rhs_lsb = try cg.load(rhs, .u64, .{});
54135491
5414 // leave values on stack for 'select'5492 // leave values on stack for 'select'
5415 _ = try cg.intCmp(.u64, op, lhs_lsb, rhs_lsb);5493 _ = try cg.intCmp(.u64, op, lhs_lsb, rhs_lsb);
...@@ -5422,8 +5500,8 @@ fn intCmp(cg: *CodeGen, ty: IntType, op: std.math.CompareOperator, lhs: WValue,...@@ -5422,8 +5500,8 @@ fn intCmp(cg: *CodeGen, ty: IntType, op: std.math.CompareOperator, lhs: WValue,
5422 return .stack;5500 return .stack;
5423 },5501 },
5424 else => {5502 else => {
5425 try cg.lowerToStack(lhs);5503 try cg.emitWValue(lhs);
5426 try cg.lowerToStack(rhs);5504 try cg.emitWValue(rhs);
5427 try cg.addImm32(@intFromBool(ty.is_signed));5505 try cg.addImm32(@intFromBool(ty.is_signed));
5428 try cg.addImm32(ty.bits);5506 try cg.addImm32(ty.bits);
5429 try cg.addCallIntrinsic(.__cmp_limb64);5507 try cg.addCallIntrinsic(.__cmp_limb64);
...@@ -5531,11 +5609,13 @@ fn airBr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -5531,11 +5609,13 @@ fn airBr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
5531 const br = cg.air.instructions.items(.data)[@backingInt(inst)].br;5609 const br = cg.air.instructions.items(.data)[@backingInt(inst)].br;
5532 const block = cg.blocks.get(br.block_inst).?;5610 const block = cg.blocks.get(br.block_inst).?;
55335611
5534 // if operand has codegen bits we should break with a value5612 switch (block.value) {
5535 if (block.value != .none) {5613 .none => {},
5536 const operand = try cg.resolveInst(br.operand);5614 .stack => {
5537 try cg.lowerToStack(operand);5615 const operand = try cg.resolveInst(br.operand);
5538 try cg.addLocal(.local_set, block.value.local.value);5616 try cg.emitWValue(operand);
5617 },
5618 else => unreachable,
5539 }5619 }
55405620
5541 // We map every block to its block index.5621 // We map every block to its block index.
...@@ -5558,7 +5638,8 @@ fn airRepeat(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -5558,7 +5638,8 @@ fn airRepeat(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
55585638
5559fn airTrap(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {5639fn airTrap(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
5560 try cg.addTag(.@"unreachable");5640 try cg.addTag(.@"unreachable");
5561 return cg.finishAir(inst, .none, &.{});5641 try cg.finishAir(inst, .none, &.{});
5642 cg.verifyLocalPool();
5562}5643}
55635644
5564fn airBreakpoint(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {5645fn airBreakpoint(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
...@@ -5570,7 +5651,8 @@ fn airBreakpoint(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -5570,7 +5651,8 @@ fn airBreakpoint(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
55705651
5571fn airUnreachable(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {5652fn airUnreachable(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
5572 try cg.addTag(.@"unreachable");5653 try cg.addTag(.@"unreachable");
5573 return cg.finishAir(inst, .none, &.{});5654 try cg.finishAir(inst, .none, &.{});
5655 cg.verifyLocalPool();
5574}5656}
55755657
5576fn airNopCast(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {5658fn airNopCast(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
...@@ -5584,8 +5666,7 @@ fn airNopCast(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -5584,8 +5666,7 @@ fn airNopCast(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
5584 assert(operand_ty.abiAlignment(zcu) == dest_ty.abiAlignment(zcu));5666 assert(operand_ty.abiAlignment(zcu) == dest_ty.abiAlignment(zcu));
55855667
5586 const operand = try cg.resolveInst(ty_op.operand);5668 const operand = try cg.resolveInst(ty_op.operand);
5587 const result = cg.reuseOperand(ty_op.operand, operand);5669 return cg.finishAir(inst, operand, &.{ty_op.operand});
5588 return cg.finishAir(inst, result, &.{ty_op.operand});
5589}5670}
55905671
5591fn airIntFromPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {5672fn airIntFromPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
...@@ -5599,11 +5680,12 @@ fn airIntFromPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -5599,11 +5680,12 @@ fn airIntFromPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
5599 assert(operand_ty.abiAlignment(zcu) == dest_ty.abiAlignment(zcu));5680 assert(operand_ty.abiAlignment(zcu) == dest_ty.abiAlignment(zcu));
56005681
5601 const operand = try cg.resolveInst(ty_op.operand);5682 const operand = try cg.resolveInst(ty_op.operand);
5602 const result = switch (operand) {5683 if (operand == .stack_offset) {
5603 .stack_offset => try cg.buildPointerOffset(operand, 0, .new),5684 // materialize operand, we assume precomputed value is more efficient
5604 else => cg.reuseOperand(ty_op.operand, operand),5685 try cg.emitWValue(operand);
5605 };5686 return cg.finishAir(inst, .stack, &.{ty_op.operand});
5606 return cg.finishAir(inst, result, &.{ty_op.operand});5687 }
5688 return cg.finishAir(inst, operand, &.{ty_op.operand});
5607}5689}
56085690
5609fn airUnionFromEnum(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {5691fn airUnionFromEnum(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
...@@ -5616,7 +5698,7 @@ fn airUnionFromEnum(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -5616,7 +5698,7 @@ fn airUnionFromEnum(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
56165698
5617 const enum_value = try cg.resolveInst(ty_op.operand);5699 const enum_value = try cg.resolveInst(ty_op.operand);
5618 const result = try cg.allocStack(union_ty);5700 const result = try cg.allocStack(union_ty);
5619 try cg.store(result, enum_value, enum_ty, @intCast(layout.tagOffset()));5701 try cg.store(result, enum_value, enum_ty, .{ .offset = layout.tagOffset() });
56205702
5621 return cg.finishAir(inst, result, &.{ty_op.operand});5703 return cg.finishAir(inst, result, &.{ty_op.operand});
5622}5704}
...@@ -5627,7 +5709,7 @@ fn airBitcast(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -5627,7 +5709,7 @@ fn airBitcast(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
5627 const dest_ty = cg.typeOfIndex(inst);5709 const dest_ty = cg.typeOfIndex(inst);
5628 const src_ty = cg.typeOf(ty_op.operand);5710 const src_ty = cg.typeOf(ty_op.operand);
56295711
5630 const result = (try cg.bitcast(dest_ty, src_ty, operand)) orelse cg.reuseOperand(ty_op.operand, operand);5712 const result = try cg.bitcast(dest_ty, src_ty, operand);
56315713
5632 return cg.finishAir(inst, result, &.{ty_op.operand});5714 return cg.finishAir(inst, result, &.{ty_op.operand});
5633}5715}
...@@ -5654,8 +5736,8 @@ fn bitcastClass(cg: *CodeGen, ty: Type) BitcastClass {...@@ -5654,8 +5736,8 @@ fn bitcastClass(cg: *CodeGen, ty: Type) BitcastClass {
5654 };5736 };
5655}5737}
56565738
5657fn bitcast(cg: *CodeGen, dest_ty: Type, src_ty: Type, operand: WValue) InnerError!?WValue {5739fn bitcast(cg: *CodeGen, dest_ty: Type, src_ty: Type, operand: WValue) InnerError!WValue {
5658 if (dest_ty.eql(src_ty)) return null;5740 if (dest_ty.eql(src_ty)) return operand;
56595741
5660 const zcu = cg.pt.zcu;5742 const zcu = cg.pt.zcu;
5661 const src_class = cg.bitcastClass(src_ty);5743 const src_class = cg.bitcastClass(src_ty);
...@@ -5674,17 +5756,17 @@ fn bitcast(cg: *CodeGen, dest_ty: Type, src_ty: Type, operand: WValue) InnerErro...@@ -5674,17 +5756,17 @@ fn bitcast(cg: *CodeGen, dest_ty: Type, src_ty: Type, operand: WValue) InnerErro
56745756
5675 if (src_by_ref and dest_by_ref) {5757 if (src_by_ref and dest_by_ref) {
5676 if (needs_wrapping) return try cg.intWrap(dest_class.int, operand);5758 if (needs_wrapping) return try cg.intWrap(dest_class.int, operand);
5677 return null;5759 return operand;
5678 }5760 }
56795761
5680 if (dest_by_ref) {5762 if (dest_by_ref) {
5681 const result = try cg.allocStack(src_ty);5763 const result = try cg.allocStack(src_ty);
5682 try cg.store(result, operand, src_ty, 0);5764 try cg.store(result, operand, src_ty, .{});
5683 return result;5765 return result;
5684 }5766 }
56855767
5686 if (src_by_ref) {5768 if (src_by_ref) {
5687 return try cg.load(operand, dest_ty, 0);5769 return try cg.load(operand, dest_ty, .{});
5688 }5770 }
56895771
5690 switch (src_class) {5772 switch (src_class) {
...@@ -5708,7 +5790,7 @@ fn bitcast(cg: *CodeGen, dest_ty: Type, src_ty: Type, operand: WValue) InnerErro...@@ -5708,7 +5790,7 @@ fn bitcast(cg: *CodeGen, dest_ty: Type, src_ty: Type, operand: WValue) InnerErro
57085790
5709 switch (dest_class) {5791 switch (dest_class) {
5710 .float => |float_ty| switch (float_ty) {5792 .float => |float_ty| switch (float_ty) {
5711 .f16 => return null,5793 .f16 => return operand,
5712 .f32 => {5794 .f32 => {
5713 try cg.emitWValue(operand);5795 try cg.emitWValue(operand);
5714 try cg.addTag(.f32_reinterpret_i32);5796 try cg.addTag(.f32_reinterpret_i32);
...@@ -5729,7 +5811,7 @@ fn bitcast(cg: *CodeGen, dest_ty: Type, src_ty: Type, operand: WValue) InnerErro...@@ -5729,7 +5811,7 @@ fn bitcast(cg: *CodeGen, dest_ty: Type, src_ty: Type, operand: WValue) InnerErro
5729 return try cg.intWrap(dest_class.int, operand);5811 return try cg.intWrap(dest_class.int, operand);
5730 }5812 }
57315813
5732 return null;5814 return operand;
5733}5815}
57345816
5735fn airStructFieldPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {5817fn airStructFieldPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
...@@ -5740,7 +5822,7 @@ fn airStructFieldPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -5740,7 +5822,7 @@ fn airStructFieldPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
5740 const struct_ptr = try cg.resolveInst(extra.data.struct_operand);5822 const struct_ptr = try cg.resolveInst(extra.data.struct_operand);
5741 const struct_ptr_ty = cg.typeOf(extra.data.struct_operand);5823 const struct_ptr_ty = cg.typeOf(extra.data.struct_operand);
5742 const struct_ty = struct_ptr_ty.childType(zcu);5824 const struct_ty = struct_ptr_ty.childType(zcu);
5743 const result = try cg.structFieldPtr(inst, extra.data.struct_operand, struct_ptr, struct_ptr_ty, struct_ty, extra.data.field_index);5825 const result = try cg.structFieldPtr(inst, struct_ptr, struct_ptr_ty, struct_ty, extra.data.field_index);
5744 return cg.finishAir(inst, result, &.{extra.data.struct_operand});5826 return cg.finishAir(inst, result, &.{extra.data.struct_operand});
5745}5827}
57465828
...@@ -5751,14 +5833,13 @@ fn airStructFieldPtrIndex(cg: *CodeGen, inst: Air.Inst.Index, index: u32) InnerE...@@ -5751,14 +5833,13 @@ fn airStructFieldPtrIndex(cg: *CodeGen, inst: Air.Inst.Index, index: u32) InnerE
5751 const struct_ptr_ty = cg.typeOf(ty_op.operand);5833 const struct_ptr_ty = cg.typeOf(ty_op.operand);
5752 const struct_ty = struct_ptr_ty.childType(zcu);5834 const struct_ty = struct_ptr_ty.childType(zcu);
57535835
5754 const result = try cg.structFieldPtr(inst, ty_op.operand, struct_ptr, struct_ptr_ty, struct_ty, index);5836 const result = try cg.structFieldPtr(inst, struct_ptr, struct_ptr_ty, struct_ty, index);
5755 return cg.finishAir(inst, result, &.{ty_op.operand});5837 return cg.finishAir(inst, result, &.{ty_op.operand});
5756}5838}
57575839
5758fn structFieldPtr(5840fn structFieldPtr(
5759 cg: *CodeGen,5841 cg: *CodeGen,
5760 inst: Air.Inst.Index,5842 inst: Air.Inst.Index,
5761 ref: Air.Inst.Ref,
5762 struct_ptr: WValue,5843 struct_ptr: WValue,
5763 struct_ptr_ty: Type,5844 struct_ptr_ty: Type,
5764 struct_ty: Type,5845 struct_ty: Type,
...@@ -5783,16 +5864,7 @@ fn structFieldPtr(...@@ -5783,16 +5864,7 @@ fn structFieldPtr(
5783 },5864 },
5784 else => struct_ty.structFieldOffset(index, zcu),5865 else => struct_ty.structFieldOffset(index, zcu),
5785 };5866 };
5786 // save a load and store when we can simply reuse the operand5867 return cg.offsetPointer(struct_ptr, offset);
5787 if (offset == 0) {
5788 return cg.reuseOperand(ref, struct_ptr);
5789 }
5790 switch (struct_ptr) {
5791 .stack_offset => |stack_offset| {
5792 return .{ .stack_offset = .{ .value = stack_offset.value + @as(u32, @intCast(offset)), .references = 1 } };
5793 },
5794 else => return cg.buildPointerOffset(struct_ptr, offset, .new),
5795 }
5796}5868}
57975869
5798fn airAggFieldVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {5870fn airAggFieldVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
...@@ -5814,44 +5886,45 @@ fn airAggFieldVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -5814,44 +5886,45 @@ fn airAggFieldVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
5814 return cg.fail("Field type '{f}' too big to fit into stack frame", .{field_ty.fmt(pt)});5886 return cg.fail("Field type '{f}' too big to fit into stack frame", .{field_ty.fmt(pt)});
5815 };5887 };
5816 if (isByRef(field_ty, zcu, cg.target)) {5888 if (isByRef(field_ty, zcu, cg.target)) {
5817 switch (operand) {5889 break :result try cg.offsetPointer(operand, offset);
5818 .stack_offset => |stack_offset| {
5819 break :result .{ .stack_offset = .{ .value = stack_offset.value + offset, .references = 1 } };
5820 },
5821 else => break :result try cg.buildPointerOffset(operand, offset, .new),
5822 }
5823 }5890 }
5824 break :result try cg.load(operand, field_ty, offset);5891 break :result try cg.load(operand, field_ty, .{ .offset = offset });
5825 },5892 },
5826 };5893 };
58275894
5828 return cg.finishAir(inst, result, &.{struct_field.struct_operand});5895 return cg.finishAir(inst, result, &.{struct_field.struct_operand});
5829}5896}
58305897
5898// Both `.switch_br` and `.loop_switch_br` are noreturn.
5831fn airSwitchBr(cg: *CodeGen, inst: Air.Inst.Index, is_dispatch_loop: bool) InnerError!void {5899fn airSwitchBr(cg: *CodeGen, inst: Air.Inst.Index, is_dispatch_loop: bool) InnerError!void {
5832 const pt = cg.pt;5900 const pt = cg.pt;
5833 const zcu = pt.zcu;5901 const zcu = pt.zcu;
58345902
5835 const switch_br = cg.air.unwrapSwitch(inst);5903 const switch_br = cg.air.unwrapSwitch(inst);
5836 const target_ty = cg.typeOf(switch_br.operand);5904 const target_ty = cg.typeOf(switch_br.operand);
5905 const branch_liveness = try cg.liveness.getSwitchBr(cg.gpa, inst, switch_br.cases_len + 1);
5906 defer cg.gpa.free(branch_liveness.deaths);
58375907
5838 assert(target_ty.hasRuntimeBits(zcu));5908 assert(target_ty.hasRuntimeBits(zcu));
58395909
5840 // swap target value with placeholder local, for dispatching5910 const operand_dies = cg.liveness.operandDies(inst, 0);
5911 var owned_target: WValue = .none;
5912 errdefer owned_target.free(cg);
5841 const target = if (is_dispatch_loop) target: {5913 const target = if (is_dispatch_loop) target: {
5842 const initial_target = try cg.resolveInst(switch_br.operand);5914 const initial_target = try cg.resolveInst(switch_br.operand);
5843 const target: WValue = try cg.allocLocal(target_ty);5915 owned_target = try cg.allocLocal(target_ty);
5844 try cg.lowerToStack(initial_target);5916 try cg.emitWValue(initial_target);
5845 try cg.addLocal(.local_set, target.local.value);5917 try cg.addLocal(.local_set, owned_target.local);
58465918
5847 try cg.startBlock(.loop, .empty); // dispatch loop start5919 try cg.startBlock(.loop, .empty); // dispatch loop start
5848 try cg.blocks.putNoClobber(cg.gpa, inst, .{5920 try cg.blocks.putNoClobber(cg.gpa, inst, .{
5849 .label = cg.block_depth,5921 .label = cg.block_depth,
5850 .value = target,5922 .value = owned_target,
5851 });5923 });
58525924
5853 break :target target;5925 break :target owned_target;
5854 } else try cg.resolveInst(switch_br.operand);5926 } else try cg.resolveInst(switch_br.operand);
5927 defer if (is_dispatch_loop) assert(cg.blocks.swapRemove(inst));
58555928
5856 const has_else_body = switch_br.else_body_len != 0;5929 const has_else_body = switch_br.else_body_len != 0;
5857 const branch_count = switch_br.cases_len + 1; // if else branch is missing, we trap when failing all conditions5930 const branch_count = switch_br.cases_len + 1; // if else branch is missing, we trap when failing all conditions
...@@ -5863,15 +5936,19 @@ fn airSwitchBr(cg: *CodeGen, inst: Air.Inst.Index, is_dispatch_loop: bool) Inner...@@ -5863,15 +5936,19 @@ fn airSwitchBr(cg: *CodeGen, inst: Air.Inst.Index, is_dispatch_loop: bool) Inner
5863 var it = switch_br.iterateCases();5936 var it = switch_br.iterateCases();
5864 const else_body = it.elseBody();5937 const else_body = it.elseBody();
58655938
5866 cg.branches.appendAssumeCapacity(.{});5939 if (operand_dies) cg.processDeath(switch_br.operand);
5867 defer {5940 owned_target.free(cg);
5868 var else_branch = cg.branches.pop().?;5941
5869 else_branch.deinit(cg.gpa);5942 {
5943 try cg.pushBranch();
5944 defer cg.popBranch();
5945 for (branch_liveness.deaths[0]) |death| cg.processDeath(death.toRef());
5946 try cg.genBody(else_body);
5870 }5947 }
5871 try cg.genBody(else_body);
58725948
5873 if (is_dispatch_loop) {5949 if (is_dispatch_loop) {
5874 try cg.endBlock(); // dispatch loop end5950 try cg.endBlock(); // dispatch loop end
5951 try cg.addTag(.@"unreachable");
5875 }5952 }
5876 return cg.finishAir(inst, .none, &.{});5953 return cg.finishAir(inst, .none, &.{});
5877 }5954 }
...@@ -5916,8 +5993,6 @@ fn airSwitchBr(cg: *CodeGen, inst: Air.Inst.Index, is_dispatch_loop: bool) Inner...@@ -5916,8 +5993,6 @@ fn airSwitchBr(cg: *CodeGen, inst: Air.Inst.Index, is_dispatch_loop: bool) Inner
5916 break :width width_bigint.toConst().toInt(u32) catch null;5993 break :width width_bigint.toConst().toInt(u32) catch null;
5917 };5994 };
59185995
5919 try cg.startBlock(.block, .empty); // whole switch block start
5920
5921 for (0..branch_count) |_| {5996 for (0..branch_count) |_| {
5922 try cg.startBlock(.block, .empty);5997 try cg.startBlock(.block, .empty);
5923 }5998 }
...@@ -5992,38 +6067,38 @@ fn airSwitchBr(cg: *CodeGen, inst: Air.Inst.Index, is_dispatch_loop: bool) Inner...@@ -5992,38 +6067,38 @@ fn airSwitchBr(cg: *CodeGen, inst: Air.Inst.Index, is_dispatch_loop: bool) Inner
5992 try cg.addLabel(.br, branch_count - 1);6067 try cg.addLabel(.br, branch_count - 1);
5993 }6068 }
59946069
6070 if (operand_dies) cg.processDeath(switch_br.operand);
6071 owned_target.free(cg);
6072
5995 var cases_it = switch_br.iterateCases();6073 var cases_it = switch_br.iterateCases();
5996 while (cases_it.next()) |case| {6074 while (cases_it.next()) |case| {
5997 try cg.endBlock();6075 try cg.endBlock();
59986076
5999 cg.branches.appendAssumeCapacity(.{});6077 {
6000 defer {6078 try cg.pushBranch();
6001 var case_branch = cg.branches.pop().?;6079 defer cg.popBranch();
6002 case_branch.deinit(cg.gpa);6080 for (branch_liveness.deaths[case.idx]) |death| cg.processDeath(death.toRef());
6081 try cg.genBody(case.body);
6003 }6082 }
6004 try cg.genBody(case.body);
6005
6006 try cg.addLabel(.br, branch_count - case.idx - 1); // matching case found and executed => exit switch
6007 }6083 }
60086084
6009 try cg.endBlock();6085 try cg.endBlock();
6010 if (has_else_body) {6086 if (has_else_body) {
6011 const else_body = cases_it.elseBody();6087 const else_body = cases_it.elseBody();
60126088
6013 cg.branches.appendAssumeCapacity(.{});6089 {
6014 defer {6090 try cg.pushBranch();
6015 var else_branch = cg.branches.pop().?;6091 defer cg.popBranch();
6016 else_branch.deinit(cg.gpa);6092 for (branch_liveness.deaths[branch_liveness.deaths.len - 1]) |death| cg.processDeath(death.toRef());
6093 try cg.genBody(else_body);
6017 }6094 }
6018 try cg.genBody(else_body);
6019 } else {6095 } else {
6020 try cg.addTag(.@"unreachable");6096 try cg.addTag(.@"unreachable");
6021 }6097 }
60226098
6023 try cg.endBlock(); // whole switch block end
6024
6025 if (is_dispatch_loop) {6099 if (is_dispatch_loop) {
6026 try cg.endBlock(); // dispatch loop end6100 try cg.endBlock(); // dispatch loop end
6101 try cg.addTag(.@"unreachable");
6027 }6102 }
60286103
6029 return cg.finishAir(inst, .none, &.{});6104 return cg.finishAir(inst, .none, &.{});
...@@ -6034,8 +6109,8 @@ fn airSwitchDispatch(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -6034,8 +6109,8 @@ fn airSwitchDispatch(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6034 const switch_loop = cg.blocks.get(br.block_inst).?;6109 const switch_loop = cg.blocks.get(br.block_inst).?;
60356110
6036 const operand = try cg.resolveInst(br.operand);6111 const operand = try cg.resolveInst(br.operand);
6037 try cg.lowerToStack(operand);6112 try cg.emitWValue(operand);
6038 try cg.addLocal(.local_set, switch_loop.value.local.value);6113 try cg.addLocal(.local_set, switch_loop.value.local);
60396114
6040 const idx: u32 = cg.block_depth - switch_loop.label;6115 const idx: u32 = cg.block_depth - switch_loop.label;
6041 try cg.addLabel(.br, idx);6116 try cg.addLabel(.br, idx);
...@@ -6062,12 +6137,14 @@ fn airIsErr(cg: *CodeGen, inst: Air.Inst.Index, opcode: std.wasm.Opcode, op_kind...@@ -6062,12 +6137,14 @@ fn airIsErr(cg: *CodeGen, inst: Air.Inst.Index, opcode: std.wasm.Opcode, op_kind
6062 }6137 }
6063 }6138 }
60646139
6065 try cg.emitWValue(operand);
6066 if (op_kind == .ptr or pl_ty.hasRuntimeBits(zcu)) {6140 if (op_kind == .ptr or pl_ty.hasRuntimeBits(zcu)) {
6141 try cg.emitMemBase(operand);
6067 try cg.addMemArg(.i32_load16_u, .{6142 try cg.addMemArg(.i32_load16_u, .{
6068 .offset = operand.offset() + @as(u32, @intCast(errUnionErrorOffset(pl_ty, zcu))),6143 .offset = operand.memOffset() + errUnionErrorOffset(pl_ty, zcu),
6069 .alignment = @intCast(Type.anyerror.abiAlignment(zcu).toByteUnits().?),6144 .alignment = Type.anyerror.abiAlignment(zcu),
6070 });6145 });
6146 } else {
6147 try cg.emitWValue(operand);
6071 }6148 }
60726149
6073 // Compare the error value with '0'6150 // Compare the error value with '0'
...@@ -6092,18 +6169,18 @@ fn airUnwrapErrUnionPayload(cg: *CodeGen, inst: Air.Inst.Index, op_is_ptr: bool)...@@ -6092,18 +6169,18 @@ fn airUnwrapErrUnionPayload(cg: *CodeGen, inst: Air.Inst.Index, op_is_ptr: bool)
6092 const result: WValue = result: {6169 const result: WValue = result: {
6093 if (!payload_ty.hasRuntimeBits(zcu)) {6170 if (!payload_ty.hasRuntimeBits(zcu)) {
6094 if (op_is_ptr) {6171 if (op_is_ptr) {
6095 break :result cg.reuseOperand(ty_op.operand, operand);6172 break :result operand;
6096 } else {6173 } else {
6097 break :result .none;6174 break :result .none;
6098 }6175 }
6099 }6176 }
61006177
6101 const pl_offset: u32 = @intCast(errUnionPayloadOffset(payload_ty, zcu));6178 const pl_offset = errUnionPayloadOffset(payload_ty, zcu);
6102 if (op_is_ptr or isByRef(payload_ty, zcu, cg.target)) {6179 if (op_is_ptr or isByRef(payload_ty, zcu, cg.target)) {
6103 break :result try cg.buildPointerOffset(operand, pl_offset, .new);6180 break :result try cg.offsetPointer(operand, pl_offset);
6104 } else {6181 } else {
6105 assert(isByRef(eu_ty, zcu, cg.target));6182 assert(isByRef(eu_ty, zcu, cg.target));
6106 break :result try cg.load(operand, payload_ty, pl_offset);6183 break :result try cg.load(operand, payload_ty, .{ .offset = pl_offset });
6107 }6184 }
6108 };6185 };
6109 return cg.finishAir(inst, result, &.{ty_op.operand});6186 return cg.finishAir(inst, result, &.{ty_op.operand});
...@@ -6128,10 +6205,10 @@ fn airUnwrapErrUnionError(cg: *CodeGen, inst: Air.Inst.Index, op_is_ptr: bool) I...@@ -6128,10 +6205,10 @@ fn airUnwrapErrUnionError(cg: *CodeGen, inst: Air.Inst.Index, op_is_ptr: bool) I
61286205
6129 const err_offset: u32 = @intCast(errUnionErrorOffset(payload_ty, zcu));6206 const err_offset: u32 = @intCast(errUnionErrorOffset(payload_ty, zcu));
6130 if (op_is_ptr or isByRef(eu_ty, zcu, cg.target)) {6207 if (op_is_ptr or isByRef(eu_ty, zcu, cg.target)) {
6131 break :result try cg.load(operand, Type.anyerror, err_offset);6208 break :result try cg.load(operand, .anyerror, .{ .offset = err_offset });
6132 } else {6209 } else {
6133 assert(!payload_ty.hasRuntimeBits(zcu));6210 assert(!payload_ty.hasRuntimeBits(zcu));
6134 break :result cg.reuseOperand(ty_op.operand, operand);6211 break :result operand;
6135 }6212 }
6136 };6213 };
6137 return cg.finishAir(inst, result, &.{ty_op.operand});6214 return cg.finishAir(inst, result, &.{ty_op.operand});
...@@ -6147,20 +6224,19 @@ fn airWrapErrUnionPayload(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -6147,20 +6224,19 @@ fn airWrapErrUnionPayload(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6147 const pl_ty = cg.typeOf(ty_op.operand);6224 const pl_ty = cg.typeOf(ty_op.operand);
6148 const result = result: {6225 const result = result: {
6149 if (!pl_ty.hasRuntimeBits(zcu)) {6226 if (!pl_ty.hasRuntimeBits(zcu)) {
6150 break :result cg.reuseOperand(ty_op.operand, operand);6227 break :result operand;
6151 }6228 }
61526229
6153 const err_union = try cg.allocStack(err_ty);6230 const err_union = try cg.allocStack(err_ty);
6154 const payload_ptr = try cg.buildPointerOffset(err_union, @as(u32, @intCast(errUnionPayloadOffset(pl_ty, zcu))), .new);6231 try cg.store(err_union, operand, pl_ty, .{ .offset = errUnionPayloadOffset(pl_ty, zcu) });
6155 try cg.store(payload_ptr, operand, pl_ty, 0);
61566232
6157 // ensure we also write '0' to the error part, so any present stack value gets overwritten by it.6233 // ensure we also write '0' to the error part, so any present stack value gets overwritten by it.
6158 try cg.emitWValue(err_union);6234 try cg.emitMemBase(err_union);
6159 try cg.addImm32(0);6235 try cg.addImm32(0);
6160 const err_val_offset: u32 = @intCast(errUnionErrorOffset(pl_ty, zcu));6236 const err_val_offset = errUnionErrorOffset(pl_ty, zcu);
6161 try cg.addMemArg(.i32_store16, .{6237 try cg.addMemArg(.i32_store16, .{
6162 .offset = err_union.offset() + err_val_offset,6238 .offset = err_union.memOffset() + err_val_offset,
6163 .alignment = 2,6239 .alignment = .@"2",
6164 });6240 });
6165 break :result err_union;6241 break :result err_union;
6166 };6242 };
...@@ -6177,15 +6253,15 @@ fn airWrapErrUnionErr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -6177,15 +6253,15 @@ fn airWrapErrUnionErr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
61776253
6178 const result = result: {6254 const result = result: {
6179 if (!pl_ty.hasRuntimeBits(zcu)) {6255 if (!pl_ty.hasRuntimeBits(zcu)) {
6180 break :result cg.reuseOperand(ty_op.operand, operand);6256 break :result operand;
6181 }6257 }
61826258
6183 const err_union = try cg.allocStack(err_ty);6259 const err_union = try cg.allocStack(err_ty);
6184 // store error value6260 // store error value
6185 try cg.store(err_union, operand, Type.anyerror, @intCast(errUnionErrorOffset(pl_ty, zcu)));6261 try cg.store(err_union, operand, .anyerror, .{ .offset = errUnionErrorOffset(pl_ty, zcu) });
61866262
6187 // write 'undefined' to the payload6263 // write 'undefined' to the payload
6188 const payload_ptr = try cg.buildPointerOffset(err_union, @as(u32, @intCast(errUnionPayloadOffset(pl_ty, zcu))), .new);6264 const payload_ptr = try cg.offsetPointer(err_union, errUnionPayloadOffset(pl_ty, zcu));
6189 const len = @as(u32, @intCast(err_ty.errorUnionPayload(zcu).abiSize(zcu)));6265 const len = @as(u32, @intCast(err_ty.errorUnionPayload(zcu).abiSize(zcu)));
6190 try cg.memset(Type.u8, payload_ptr, .{ .imm32 = len }, .{ .imm32 = 0xaa });6266 try cg.memset(Type.u8, payload_ptr, .{ .imm32 = len }, .{ .imm32 = 0xaa });
61916267
...@@ -6210,7 +6286,6 @@ fn airIsNull(cg: *CodeGen, inst: Air.Inst.Index, opcode: std.wasm.Opcode, op_kin...@@ -6210,7 +6286,6 @@ fn airIsNull(cg: *CodeGen, inst: Air.Inst.Index, opcode: std.wasm.Opcode, op_kin
6210fn isNull(cg: *CodeGen, operand: WValue, op_ty: Type, opcode: std.wasm.Opcode, op_kind: OpKind) InnerError!WValue {6286fn isNull(cg: *CodeGen, operand: WValue, op_ty: Type, opcode: std.wasm.Opcode, op_kind: OpKind) InnerError!WValue {
6211 const pt = cg.pt;6287 const pt = cg.pt;
6212 const zcu = pt.zcu;6288 const zcu = pt.zcu;
6213 try cg.emitWValue(operand);
6214 const optional_ty = switch (op_kind) {6289 const optional_ty = switch (op_kind) {
6215 .value => op_ty,6290 .value => op_ty,
6216 .ptr => op_ty.childType(zcu),6291 .ptr => op_ty.childType(zcu),
...@@ -6223,19 +6298,27 @@ fn isNull(cg: *CodeGen, operand: WValue, op_ty: Type, opcode: std.wasm.Opcode, o...@@ -6223,19 +6298,27 @@ fn isNull(cg: *CodeGen, operand: WValue, op_ty: Type, opcode: std.wasm.Opcode, o
6223 const offset = std.math.cast(u32, payload_ty.abiSize(zcu)) orelse {6298 const offset = std.math.cast(u32, payload_ty.abiSize(zcu)) orelse {
6224 return cg.fail("Optional type {f} too big to fit into stack frame", .{optional_ty.fmt(pt)});6299 return cg.fail("Optional type {f} too big to fit into stack frame", .{optional_ty.fmt(pt)});
6225 };6300 };
6226 try cg.addMemArg(.i32_load8_u, .{ .offset = operand.offset() + offset, .alignment = 1 });6301 try cg.emitMemBase(operand);
6302 try cg.addMemArg(.i32_load8_u, .{ .offset = operand.memOffset() + offset, .alignment = .@"1" });
6303 } else {
6304 try cg.emitWValue(operand);
6227 }6305 }
6228 } else if (payload_ty.isSlice(zcu)) {6306 } else if (payload_ty.isSlice(zcu)) {
6307 const base_offset = operand.memOffset();
6308 try cg.emitMemBase(operand);
6229 switch (cg.ptr_size) {6309 switch (cg.ptr_size) {
6230 .wasm32 => try cg.addMemArg(.i32_load, .{ .offset = operand.offset(), .alignment = 4 }),6310 .wasm32 => try cg.addMemArg(.i32_load, .{ .offset = base_offset, .alignment = .@"4" }),
6231 .wasm64 => try cg.addMemArg(.i64_load, .{ .offset = operand.offset(), .alignment = 8 }),6311 .wasm64 => try cg.addMemArg(.i64_load, .{ .offset = base_offset, .alignment = .@"8" }),
6232 }6312 }
6233 } else {6313 } else {
6234 if (op_kind == .ptr) {6314 if (op_kind == .ptr) {
6315 try cg.emitMemBase(operand);
6235 try cg.addMemArg(.i32_load, .{6316 try cg.addMemArg(.i32_load, .{
6236 .offset = operand.offset(),6317 .offset = operand.memOffset(),
6237 .alignment = 4,6318 .alignment = .@"4",
6238 });6319 });
6320 } else {
6321 try cg.emitWValue(operand);
6239 }6322 }
6240 }6323 }
62416324
...@@ -6257,13 +6340,13 @@ fn airOptionalPayload(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -6257,13 +6340,13 @@ fn airOptionalPayload(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
62576340
6258 const result = result: {6341 const result = result: {
6259 const operand = try cg.resolveInst(ty_op.operand);6342 const operand = try cg.resolveInst(ty_op.operand);
6260 if (opt_ty.optionalReprIsPayload(zcu)) break :result cg.reuseOperand(ty_op.operand, operand);6343 if (opt_ty.optionalReprIsPayload(zcu)) break :result operand;
62616344
6262 if (isByRef(payload_ty, zcu, cg.target)) {6345 if (isByRef(payload_ty, zcu, cg.target)) {
6263 break :result try cg.buildPointerOffset(operand, 0, .new);6346 break :result operand;
6264 }6347 }
62656348
6266 break :result try cg.load(operand, payload_ty, 0);6349 break :result try cg.load(operand, payload_ty, .{});
6267 };6350 };
6268 return cg.finishAir(inst, result, &.{ty_op.operand});6351 return cg.finishAir(inst, result, &.{ty_op.operand});
6269}6352}
...@@ -6277,10 +6360,10 @@ fn airOptionalPayloadPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -6277,10 +6360,10 @@ fn airOptionalPayloadPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6277 const result = result: {6360 const result = result: {
6278 const payload_ty = opt_ty.optionalChild(zcu);6361 const payload_ty = opt_ty.optionalChild(zcu);
6279 if (!payload_ty.hasRuntimeBits(zcu) or opt_ty.optionalReprIsPayload(zcu)) {6362 if (!payload_ty.hasRuntimeBits(zcu) or opt_ty.optionalReprIsPayload(zcu)) {
6280 break :result cg.reuseOperand(ty_op.operand, operand);6363 break :result operand;
6281 }6364 }
62826365
6283 break :result try cg.buildPointerOffset(operand, 0, .new);6366 break :result operand;
6284 };6367 };
6285 return cg.finishAir(inst, result, &.{ty_op.operand});6368 return cg.finishAir(inst, result, &.{ty_op.operand});
6286}6369}
...@@ -6301,12 +6384,11 @@ fn airOptionalPayloadPtrSet(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void...@@ -6301,12 +6384,11 @@ fn airOptionalPayloadPtrSet(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void
6301 return cg.fail("Optional type {f} too big to fit into stack frame", .{opt_ty.fmt(pt)});6384 return cg.fail("Optional type {f} too big to fit into stack frame", .{opt_ty.fmt(pt)});
6302 };6385 };
63036386
6304 try cg.emitWValue(operand);6387 try cg.emitMemBase(operand);
6305 try cg.addImm32(1);6388 try cg.addImm32(1);
6306 try cg.addMemArg(.i32_store8, .{ .offset = operand.offset() + offset, .alignment = 1 });6389 try cg.addMemArg(.i32_store8, .{ .offset = operand.memOffset() + offset, .alignment = .@"1" });
63076390
6308 const result = try cg.buildPointerOffset(operand, 0, .new);6391 return cg.finishAir(inst, operand, &.{ty_op.operand});
6309 return cg.finishAir(inst, result, &.{ty_op.operand});
6310}6392}
63116393
6312fn airWrapOptional(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {6394fn airWrapOptional(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
...@@ -6318,16 +6400,16 @@ fn airWrapOptional(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -6318,16 +6400,16 @@ fn airWrapOptional(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6318 const result = result: {6400 const result = result: {
6319 if (!payload_ty.hasRuntimeBits(zcu)) {6401 if (!payload_ty.hasRuntimeBits(zcu)) {
6320 const non_null_bit = try cg.allocStack(Type.u1);6402 const non_null_bit = try cg.allocStack(Type.u1);
6321 try cg.emitWValue(non_null_bit);6403 try cg.emitMemBase(non_null_bit);
6322 try cg.addImm32(1);6404 try cg.addImm32(1);
6323 try cg.addMemArg(.i32_store8, .{ .offset = non_null_bit.offset(), .alignment = 1 });6405 try cg.addMemArg(.i32_store8, .{ .offset = non_null_bit.memOffset(), .alignment = .@"1" });
6324 break :result non_null_bit;6406 break :result non_null_bit;
6325 }6407 }
63266408
6327 const operand = try cg.resolveInst(ty_op.operand);6409 const operand = try cg.resolveInst(ty_op.operand);
6328 const op_ty = cg.typeOfIndex(inst);6410 const op_ty = cg.typeOfIndex(inst);
6329 if (op_ty.optionalReprIsPayload(zcu)) {6411 if (op_ty.optionalReprIsPayload(zcu)) {
6330 break :result cg.reuseOperand(ty_op.operand, operand);6412 break :result operand;
6331 }6413 }
6332 const offset = std.math.cast(u32, payload_ty.abiSize(zcu)) orelse {6414 const offset = std.math.cast(u32, payload_ty.abiSize(zcu)) orelse {
6333 return cg.fail("Optional type {f} too big to fit into stack frame", .{op_ty.fmt(pt)});6415 return cg.fail("Optional type {f} too big to fit into stack frame", .{op_ty.fmt(pt)});
...@@ -6335,12 +6417,11 @@ fn airWrapOptional(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -6335,12 +6417,11 @@ fn airWrapOptional(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
63356417
6336 // Create optional type, set the non-null bit, and store the operand inside the optional type6418 // Create optional type, set the non-null bit, and store the operand inside the optional type
6337 const result_ptr = try cg.allocStack(op_ty);6419 const result_ptr = try cg.allocStack(op_ty);
6338 try cg.emitWValue(result_ptr);6420 try cg.emitMemBase(result_ptr);
6339 try cg.addImm32(1);6421 try cg.addImm32(1);
6340 try cg.addMemArg(.i32_store8, .{ .offset = result_ptr.offset() + offset, .alignment = 1 });6422 try cg.addMemArg(.i32_store8, .{ .offset = result_ptr.memOffset() + offset, .alignment = .@"1" });
63416423
6342 const payload_ptr = try cg.buildPointerOffset(result_ptr, 0, .new);6424 try cg.store(result_ptr, operand, payload_ty, .{});
6343 try cg.store(payload_ptr, operand, payload_ty, 0);
6344 break :result result_ptr;6425 break :result result_ptr;
6345 };6426 };
63466427
...@@ -6356,8 +6437,8 @@ fn airSlice(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -6356,8 +6437,8 @@ fn airSlice(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6356 const slice_ty = cg.typeOfIndex(inst);6437 const slice_ty = cg.typeOfIndex(inst);
63576438
6358 const slice = try cg.allocStack(slice_ty);6439 const slice = try cg.allocStack(slice_ty);
6359 try cg.store(slice, lhs, Type.usize, 0);6440 try cg.store(slice, lhs, .usize, .{});
6360 try cg.store(slice, rhs, Type.usize, cg.ptrSize());6441 try cg.store(slice, rhs, .usize, .{ .offset = cg.ptrSize() });
63616442
6362 return cg.finishAir(inst, slice, &.{ bin_op.lhs, bin_op.rhs });6443 return cg.finishAir(inst, slice, &.{ bin_op.lhs, bin_op.rhs });
6363}6444}
...@@ -6380,15 +6461,12 @@ fn airSliceElemVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -6380,15 +6461,12 @@ fn airSliceElemVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6380 const elem_size = elem_ty.abiSize(zcu);6461 const elem_size = elem_ty.abiSize(zcu);
63816462
6382 // load pointer onto stack6463 // load pointer onto stack
6383 _ = try cg.load(slice, Type.usize, 0);6464 _ = try cg.load(slice, .usize, .{});
63846465
6385 // calculate index into slice6466 // calculate index into slice
6386 try cg.emitWValue(index);6467 try cg.emitScaledPointerOffset(index, elem_size, .add);
6387 try cg.addImm32(@intCast(elem_size));
6388 try cg.addTag(.i32_mul);
6389 try cg.addTag(.i32_add);
63906468
6391 const elem_result = try cg.load(.stack, elem_ty, 0);6469 const elem_result = try cg.load(.stack, elem_ty, .{});
63926470
6393 return cg.finishAir(inst, elem_result, &.{ bin_op.lhs, bin_op.rhs });6471 return cg.finishAir(inst, elem_result, &.{ bin_op.lhs, bin_op.rhs });
6394}6472}
...@@ -6404,13 +6482,10 @@ fn airSliceElemPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -6404,13 +6482,10 @@ fn airSliceElemPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6404 const slice = try cg.resolveInst(bin_op.lhs);6482 const slice = try cg.resolveInst(bin_op.lhs);
6405 const index = try cg.resolveInst(bin_op.rhs);6483 const index = try cg.resolveInst(bin_op.rhs);
64066484
6407 _ = try cg.load(slice, Type.usize, 0);6485 _ = try cg.load(slice, .usize, .{});
64086486
6409 // calculate index into slice6487 // calculate index into slice
6410 try cg.emitWValue(index);6488 try cg.emitScaledPointerOffset(index, elem_size, .add);
6411 try cg.addImm32(@intCast(elem_size));
6412 try cg.addTag(.i32_mul);
6413 try cg.addTag(.i32_add);
64146489
6415 return cg.finishAir(inst, .stack, &.{ bin_op.lhs, bin_op.rhs });6490 return cg.finishAir(inst, .stack, &.{ bin_op.lhs, bin_op.rhs });
6416}6491}
...@@ -6422,12 +6497,12 @@ fn airSlicePtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -6422,12 +6497,12 @@ fn airSlicePtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6422}6497}
64236498
6424fn slicePtr(cg: *CodeGen, operand: WValue) InnerError!WValue {6499fn slicePtr(cg: *CodeGen, operand: WValue) InnerError!WValue {
6425 const ptr = try cg.load(operand, Type.usize, 0);6500 const ptr = try cg.load(operand, .usize, .{});
6426 return ptr.toLocal(cg, Type.usize);6501 return ptr.toLocal(cg, Type.usize);
6427}6502}
64286503
6429fn sliceLen(cg: *CodeGen, operand: WValue) InnerError!WValue {6504fn sliceLen(cg: *CodeGen, operand: WValue) InnerError!WValue {
6430 const len = try cg.load(operand, Type.usize, cg.ptrSize());6505 const len = try cg.load(operand, .usize, .{ .offset = cg.ptrSize() });
6431 return len.toLocal(cg, Type.usize);6506 return len.toLocal(cg, Type.usize);
6432}6507}
64336508
...@@ -6442,11 +6517,11 @@ fn airArrayToSlice(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -6442,11 +6517,11 @@ fn airArrayToSlice(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6442 // create a slice on the stack6517 // create a slice on the stack
6443 const slice_local = try cg.allocStack(slice_ty);6518 const slice_local = try cg.allocStack(slice_ty);
64446519
6445 try cg.store(slice_local, operand, Type.usize, 0);6520 try cg.store(slice_local, operand, .usize, .{});
64466521
6447 // store the length of the array in the slice6522 // store the length of the array in the slice
6448 const array_len: u32 = @intCast(array_ty.arrayLen(zcu));6523 const array_len: u32 = @intCast(array_ty.arrayLen(zcu));
6449 try cg.store(slice_local, .{ .imm32 = array_len }, Type.usize, cg.ptrSize());6524 try cg.store(slice_local, .{ .imm32 = array_len }, .usize, .{ .offset = cg.ptrSize() });
64506525
6451 return cg.finishAir(inst, slice_local, &.{ty_op.operand});6526 return cg.finishAir(inst, slice_local, &.{ty_op.operand});
6452}6527}
...@@ -6463,18 +6538,15 @@ fn airPtrElemVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -6463,18 +6538,15 @@ fn airPtrElemVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
64636538
6464 // load pointer onto the stack6539 // load pointer onto the stack
6465 if (ptr_ty.isSlice(zcu)) {6540 if (ptr_ty.isSlice(zcu)) {
6466 _ = try cg.load(ptr, Type.usize, 0);6541 _ = try cg.load(ptr, .usize, .{});
6467 } else {6542 } else {
6468 try cg.lowerToStack(ptr);6543 try cg.emitWValue(ptr);
6469 }6544 }
64706545
6471 // calculate index into slice6546 // calculate index into slice
6472 try cg.emitWValue(index);6547 try cg.emitScaledPointerOffset(index, elem_size, .add);
6473 try cg.addImm32(@intCast(elem_size));
6474 try cg.addTag(.i32_mul);
6475 try cg.addTag(.i32_add);
64766548
6477 const elem_result = try cg.load(.stack, elem_ty, 0);6549 const elem_result = try cg.load(.stack, elem_ty, .{});
64786550
6479 return cg.finishAir(inst, elem_result, &.{ bin_op.lhs, bin_op.rhs });6551 return cg.finishAir(inst, elem_result, &.{ bin_op.lhs, bin_op.rhs });
6480}6552}
...@@ -6493,21 +6565,18 @@ fn airPtrElemPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -6493,21 +6565,18 @@ fn airPtrElemPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
64936565
6494 // load pointer onto the stack6566 // load pointer onto the stack
6495 if (ptr_ty.isSlice(zcu)) {6567 if (ptr_ty.isSlice(zcu)) {
6496 _ = try cg.load(ptr, Type.usize, 0);6568 _ = try cg.load(ptr, .usize, .{});
6497 } else {6569 } else {
6498 try cg.lowerToStack(ptr);6570 try cg.emitWValue(ptr);
6499 }6571 }
65006572
6501 // calculate index into ptr6573 // calculate index into ptr
6502 try cg.emitWValue(index);6574 try cg.emitScaledPointerOffset(index, elem_size, .add);
6503 try cg.addImm32(@intCast(elem_size));
6504 try cg.addTag(.i32_mul);
6505 try cg.addTag(.i32_add);
65066575
6507 return cg.finishAir(inst, .stack, &.{ bin_op.lhs, bin_op.rhs });6576 return cg.finishAir(inst, .stack, &.{ bin_op.lhs, bin_op.rhs });
6508}6577}
65096578
6510fn airPtrBinOp(cg: *CodeGen, inst: Air.Inst.Index, op: enum { add, sub }) InnerError!void {6579fn airPtrBinOp(cg: *CodeGen, inst: Air.Inst.Index, op: PointerOffsetOp) InnerError!void {
6511 const zcu = cg.pt.zcu;6580 const zcu = cg.pt.zcu;
6512 const ty_pl = cg.air.instructions.items(.data)[@backingInt(inst)].ty_pl;6581 const ty_pl = cg.air.instructions.items(.data)[@backingInt(inst)].ty_pl;
6513 const bin_op = cg.air.extraData(Air.Bin, ty_pl.payload).data;6582 const bin_op = cg.air.extraData(Air.Bin, ty_pl.payload).data;
...@@ -6520,27 +6589,8 @@ fn airPtrBinOp(cg: *CodeGen, inst: Air.Inst.Index, op: enum { add, sub }) InnerE...@@ -6520,27 +6589,8 @@ fn airPtrBinOp(cg: *CodeGen, inst: Air.Inst.Index, op: enum { add, sub }) InnerE
6520 else => ptr_ty.childType(zcu),6589 else => ptr_ty.childType(zcu),
6521 };6590 };
65226591
6523 try cg.lowerToStack(ptr);6592 try cg.emitWValue(ptr);
6524 try cg.emitWValue(offset);6593 try cg.emitScaledPointerOffset(offset, pointee_ty.abiSize(zcu), op);
6525
6526 switch (cg.ptr_size) {
6527 .wasm32 => {
6528 try cg.addImm32(@intCast(pointee_ty.abiSize(zcu)));
6529 try cg.addTag(.i32_mul);
6530 try cg.addTag(switch (op) {
6531 .add => .i32_add,
6532 .sub => .i32_sub,
6533 });
6534 },
6535 .wasm64 => {
6536 try cg.addImm64(pointee_ty.abiSize(zcu));
6537 try cg.addTag(.i64_mul);
6538 try cg.addTag(switch (op) {
6539 .add => .i64_add,
6540 .sub => .i64_sub,
6541 });
6542 },
6543 }
65446594
6545 return cg.finishAir(inst, .stack, &.{ bin_op.lhs, bin_op.rhs });6595 return cg.finishAir(inst, .stack, &.{ bin_op.lhs, bin_op.rhs });
6546}6596}
...@@ -6552,11 +6602,6 @@ fn airMemset(cg: *CodeGen, inst: Air.Inst.Index, safety: bool) InnerError!void {...@@ -6552,11 +6602,6 @@ fn airMemset(cg: *CodeGen, inst: Air.Inst.Index, safety: bool) InnerError!void {
6552 const ptr = try cg.resolveInst(bin_op.lhs);6602 const ptr = try cg.resolveInst(bin_op.lhs);
6553 const ptr_ty = cg.typeOf(bin_op.lhs);6603 const ptr_ty = cg.typeOf(bin_op.lhs);
6554 const value = try cg.resolveInst(bin_op.rhs);6604 const value = try cg.resolveInst(bin_op.rhs);
6555 const len = switch (ptr_ty.ptrSize(zcu)) {
6556 .slice => try cg.sliceLen(ptr),
6557 .one => @as(WValue, .{ .imm32 = @as(u32, @intCast(ptr_ty.childType(zcu).arrayLen(zcu))) }),
6558 .c, .many => unreachable,
6559 };
65606605
6561 const elem_ty = if (ptr_ty.ptrSize(zcu) == .one)6606 const elem_ty = if (ptr_ty.ptrSize(zcu) == .one)
6562 ptr_ty.childType(zcu).childType(zcu)6607 ptr_ty.childType(zcu).childType(zcu)
...@@ -6567,26 +6612,38 @@ fn airMemset(cg: *CodeGen, inst: Air.Inst.Index, safety: bool) InnerError!void {...@@ -6567,26 +6612,38 @@ fn airMemset(cg: *CodeGen, inst: Air.Inst.Index, safety: bool) InnerError!void {
6567 return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });6612 return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });
6568 }6613 }
65696614
6570 const dst_ptr = try cg.sliceOrArrayPtr(ptr, ptr_ty);6615 var dst_ptr = try cg.sliceOrArrayPtr(ptr, ptr_ty);
6616 defer if (ptr_ty.isSlice(zcu)) dst_ptr.free(cg);
6617 var len = switch (ptr_ty.ptrSize(zcu)) {
6618 .slice => try cg.sliceLen(ptr),
6619 .one => @as(WValue, .{ .imm32 = @as(u32, @intCast(ptr_ty.childType(zcu).arrayLen(zcu))) }),
6620 .c, .many => unreachable,
6621 };
6622 defer len.free(cg);
6571 try cg.memset(elem_ty, dst_ptr, len, value);6623 try cg.memset(elem_ty, dst_ptr, len, value);
65726624
6573 return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });6625 return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });
6574}6626}
65756627
6576/// Sets a region of memory at `ptr` to the value of `value`6628/// Sets a region of memory at `ptr` to the value of `value`
6577/// When the user has enabled the bulk_memory feature, we lower6629/// When the user has enabled the bulk_memory_opt feature, we lower
6578/// this to wasm's memset instruction. When the feature is not present,6630/// this to wasm's memset instruction. When the feature is not present,
6579/// we implement it manually.6631/// we implement it manually.
6580fn memset(cg: *CodeGen, elem_ty: Type, ptr: WValue, len: WValue, value: WValue) InnerError!void {6632fn memset(cg: *CodeGen, elem_ty: Type, ptr: WValue, len: WValue, value: WValue) InnerError!void {
6581 const zcu = cg.pt.zcu;6633 const zcu = cg.pt.zcu;
6582 const abi_size = @as(u32, @intCast(elem_ty.abiSize(zcu)));6634 const abi_size = @as(u32, @intCast(elem_ty.abiSize(zcu)));
65836635 const len_known_neq_0 = switch (len) {
6584 // When bulk_memory is enabled, we lower it to wasm's memset instruction.6636 .imm32 => |imm| if (imm != 0) true else return,
6637 .imm64 => |imm| if (imm != 0) true else return,
6638 else => false,
6639 };
6640 // When bulk_memory_opt is enabled, we lower it to wasm's memset instruction.
6585 // If not, we lower it ourselves.6641 // If not, we lower it ourselves.
6586 if (cg.target.cpu.has(.wasm, .bulk_memory) and abi_size == 1) {6642 if (cg.target.cpu.has(.wasm, .bulk_memory_opt) and abi_size == 1) {
6587 const len0_ok = cg.target.cpu.has(.wasm, .nontrapping_bulk_memory_len0);6643 const len0_ok = cg.target.cpu.has(.wasm, .nontrapping_bulk_memory_len0);
6644 const emit_check = !(len0_ok or len_known_neq_0);
65886645
6589 if (!len0_ok) {6646 if (emit_check) {
6590 try cg.startBlock(.block, .empty);6647 try cg.startBlock(.block, .empty);
65916648
6592 // Even if `len` is zero, the spec requires an implementation to trap if `ptr + len` is6649 // Even if `len` is zero, the spec requires an implementation to trap if `ptr + len` is
...@@ -6602,23 +6659,25 @@ fn memset(cg: *CodeGen, elem_ty: Type, ptr: WValue, len: WValue, value: WValue)...@@ -6602,23 +6659,25 @@ fn memset(cg: *CodeGen, elem_ty: Type, ptr: WValue, len: WValue, value: WValue)
6602 try cg.addLabel(.br_if, 0);6659 try cg.addLabel(.br_if, 0);
6603 }6660 }
66046661
6605 try cg.lowerToStack(ptr);6662 try cg.emitWValue(ptr);
6606 try cg.emitWValue(value);6663 try cg.emitWValue(value);
6607 try cg.emitWValue(len);6664 try cg.emitWValue(len);
6608 try cg.addExtended(.memory_fill);6665 try cg.addExtended(.memory_fill);
66096666
6610 if (!len0_ok) {6667 if (emit_check) {
6611 try cg.endBlock();6668 try cg.endBlock();
6612 }6669 }
66136670
6614 return;6671 return;
6615 }6672 }
66166673
6674 var owned_final_len: WValue = .none;
6675 defer owned_final_len.free(cg);
6617 const final_len: WValue = switch (len) {6676 const final_len: WValue = switch (len) {
6618 .imm32 => |val| .{ .imm32 = val * abi_size },6677 .imm32 => |val| .{ .imm32 = val * abi_size },
6619 .imm64 => |val| .{ .imm64 = val * abi_size },6678 .imm64 => |val| .{ .imm64 = val * abi_size },
6620 else => if (abi_size != 1) blk: {6679 else => if (abi_size != 1) blk: {
6621 const new_len = try cg.ensureAllocLocal(Type.usize);6680 owned_final_len = try cg.allocLocal(Type.usize);
6622 try cg.emitWValue(len);6681 try cg.emitWValue(len);
6623 switch (cg.ptr_size) {6682 switch (cg.ptr_size) {
6624 .wasm32 => {6683 .wasm32 => {
...@@ -6630,24 +6689,26 @@ fn memset(cg: *CodeGen, elem_ty: Type, ptr: WValue, len: WValue, value: WValue)...@@ -6630,24 +6689,26 @@ fn memset(cg: *CodeGen, elem_ty: Type, ptr: WValue, len: WValue, value: WValue)
6630 try cg.addTag(.i64_mul);6689 try cg.addTag(.i64_mul);
6631 },6690 },
6632 }6691 }
6633 try cg.addLocal(.local_set, new_len.local.value);6692 try cg.addLocal(.local_set, owned_final_len.local);
6634 break :blk new_len;6693 break :blk owned_final_len;
6635 } else len,6694 } else len,
6636 };6695 };
66376696
6638 var end_ptr = try cg.allocLocal(Type.usize);6697 var end_ptr = try cg.allocLocal(Type.usize);
6639 defer end_ptr.free(cg);6698 defer end_ptr.free(cg);
6640 var new_ptr = try cg.buildPointerOffset(ptr, 0, .new);6699 var new_ptr = try cg.allocLocal(Type.usize);
6641 defer new_ptr.free(cg);6700 defer new_ptr.free(cg);
6701 try cg.emitWValue(ptr);
6702 try cg.addLocal(.local_set, new_ptr.local);
66426703
6643 // get the loop conditional: if current pointer address equals final pointer's address6704 // get the loop conditional: if current pointer address equals final pointer's address
6644 try cg.lowerToStack(ptr);6705 try cg.emitWValue(ptr);
6645 try cg.emitWValue(final_len);6706 try cg.emitWValue(final_len);
6646 switch (cg.ptr_size) {6707 switch (cg.ptr_size) {
6647 .wasm32 => try cg.addTag(.i32_add),6708 .wasm32 => try cg.addTag(.i32_add),
6648 .wasm64 => try cg.addTag(.i64_add),6709 .wasm64 => try cg.addTag(.i64_add),
6649 }6710 }
6650 try cg.addLocal(.local_set, end_ptr.local.value);6711 try cg.addLocal(.local_set, end_ptr.local);
66516712
6652 // outer block to jump to when loop is done6713 // outer block to jump to when loop is done
6653 try cg.startBlock(.block, .empty);6714 try cg.startBlock(.block, .empty);
...@@ -6663,7 +6724,7 @@ fn memset(cg: *CodeGen, elem_ty: Type, ptr: WValue, len: WValue, value: WValue)...@@ -6663,7 +6724,7 @@ fn memset(cg: *CodeGen, elem_ty: Type, ptr: WValue, len: WValue, value: WValue)
6663 try cg.addLabel(.br_if, 1); // jump out of loop into outer block (finished)6724 try cg.addLabel(.br_if, 1); // jump out of loop into outer block (finished)
66646725
6665 // store the value at the current position of the pointer6726 // store the value at the current position of the pointer
6666 try cg.store(new_ptr, value, elem_ty, 0);6727 try cg.store(new_ptr, value, elem_ty, .{});
66676728
6668 // move the pointer to the next element6729 // move the pointer to the next element
6669 try cg.emitWValue(new_ptr);6730 try cg.emitWValue(new_ptr);
...@@ -6677,7 +6738,7 @@ fn memset(cg: *CodeGen, elem_ty: Type, ptr: WValue, len: WValue, value: WValue)...@@ -6677,7 +6738,7 @@ fn memset(cg: *CodeGen, elem_ty: Type, ptr: WValue, len: WValue, value: WValue)
6677 try cg.addTag(.i64_add);6738 try cg.addTag(.i64_add);
6678 },6739 },
6679 }6740 }
6680 try cg.addLocal(.local_set, new_ptr.local.value);6741 try cg.addLocal(.local_set, new_ptr.local);
66816742
6682 // end of loop6743 // end of loop
6683 try cg.addLabel(.br, 0); // jump to start of loop6744 try cg.addLabel(.br, 0); // jump to start of loop
...@@ -6696,11 +6757,8 @@ fn airArrayElemVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -6696,11 +6757,8 @@ fn airArrayElemVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6696 const elem_size = elem_ty.abiSize(zcu);6757 const elem_size = elem_ty.abiSize(zcu);
66976758
6698 if (isByRef(array_ty, zcu, cg.target)) {6759 if (isByRef(array_ty, zcu, cg.target)) {
6699 try cg.lowerToStack(array);6760 try cg.emitWValue(array);
6700 try cg.emitWValue(index);6761 try cg.emitScaledPointerOffset(index, elem_size, .add);
6701 try cg.addImm32(@intCast(elem_size));
6702 try cg.addTag(.i32_mul);
6703 try cg.addTag(.i32_add);
6704 } else {6762 } else {
6705 assert(array_ty.zigTypeTag(zcu) == .vector);6763 assert(array_ty.zigTypeTag(zcu) == .vector);
67066764
...@@ -6726,14 +6784,11 @@ fn airArrayElemVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -6726,14 +6784,11 @@ fn airArrayElemVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6726 },6784 },
6727 else => {6785 else => {
6728 const stack_vec = try cg.allocStack(array_ty);6786 const stack_vec = try cg.allocStack(array_ty);
6729 try cg.store(stack_vec, array, array_ty, 0);6787 try cg.store(stack_vec, array, array_ty, .{});
67306788
6731 // Is a non-unrolled vector (v128)6789 // Is a non-unrolled vector (v128)
6732 try cg.lowerToStack(stack_vec);6790 try cg.emitWValue(stack_vec);
6733 try cg.emitWValue(index);6791 try cg.emitScaledPointerOffset(index, elem_size, .add);
6734 try cg.addImm32(@intCast(elem_size));
6735 try cg.addTag(.i32_mul);
6736 try cg.addTag(.i32_add);
6737 },6792 },
6738 }6793 }
6739 }6794 }
...@@ -6741,7 +6796,7 @@ fn airArrayElemVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -6741,7 +6796,7 @@ fn airArrayElemVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6741 const result = if (isByRef(elem_ty, zcu, cg.target))6796 const result = if (isByRef(elem_ty, zcu, cg.target))
6742 .stack6797 .stack
6743 else6798 else
6744 try cg.load(.stack, elem_ty, 0);6799 try cg.load(.stack, elem_ty, .{});
6745 return cg.finishAir(inst, result, &.{ bin_op.lhs, bin_op.rhs });6800 return cg.finishAir(inst, result, &.{ bin_op.lhs, bin_op.rhs });
6746}6801}
67476802
...@@ -6765,13 +6820,14 @@ fn airSplat(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -6765,13 +6820,14 @@ fn airSplat(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6765 64 => @backingInt(std.wasm.SimdOpcode.v128_load64_splat),6820 64 => @backingInt(std.wasm.SimdOpcode.v128_load64_splat),
6766 else => break :blk, // Cannot make use of simd-instructions6821 else => break :blk, // Cannot make use of simd-instructions
6767 };6822 };
6768 try cg.emitWValue(operand);6823 try cg.emitMemBase(operand);
6769 const extra_index: u32 = @intCast(cg.mir_extra.items.len);6824 const extra_index: u32 = @intCast(cg.mir_extra.items.len);
6770 // stores as := opcode, offset, alignment (opcode::memarg)6825 // stores as := opcode, offset, alignment (opcode::memarg)
6771 try cg.mir_extra.appendSlice(cg.gpa, &[_]u32{6826 try cg.mir_extra.ensureUnusedCapacity(cg.gpa, 4);
6772 opcode,6827 cg.mir_extra.appendAssumeCapacity(opcode);
6773 operand.offset(),6828 _ = try cg.addExtraAssumeCapacity(Mir.MemArg{
6774 @intCast(elem_ty.abiAlignment(zcu).toByteUnits().?),6829 .offset = operand.memOffset(),
6830 .alignment = elem_ty.abiAlignment(zcu),
6775 });6831 });
6776 try cg.addInst(.{ .tag = .simd_prefix, .data = .{ .payload = extra_index } });6832 try cg.addInst(.{ .tag = .simd_prefix, .data = .{ .payload = extra_index } });
6777 return cg.finishAir(inst, .stack, &.{ty_op.operand});6833 return cg.finishAir(inst, .stack, &.{ty_op.operand});
...@@ -6800,7 +6856,7 @@ fn airSplat(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -6800,7 +6856,7 @@ fn airSplat(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6800 var index: usize = 0;6856 var index: usize = 0;
6801 var offset: u32 = 0;6857 var offset: u32 = 0;
6802 while (index < vector_len) : (index += 1) {6858 while (index < vector_len) : (index += 1) {
6803 try cg.store(result, operand, elem_ty, offset);6859 try cg.store(result, operand, elem_ty, .{ .offset = offset });
6804 offset += elem_byte_size;6860 offset += elem_byte_size;
6805 }6861 }
68066862
...@@ -6837,12 +6893,12 @@ fn airShuffleOne(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -6837,12 +6893,12 @@ fn airShuffleOne(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
68376893
6838 const dest_alloc = try cg.allocStack(result_ty);6894 const dest_alloc = try cg.allocStack(result_ty);
6839 for (mask, 0..) |mask_elem, out_idx| {6895 for (mask, 0..) |mask_elem, out_idx| {
6840 try cg.emitWValue(dest_alloc);6896 try cg.emitMemBase(dest_alloc);
6841 const elem_val = switch (mask_elem.unwrap()) {6897 const elem_val = switch (mask_elem.unwrap()) {
6842 .elem => |idx| try cg.load(operand, elem_ty, @intCast(elem_size * idx)),6898 .elem => |idx| try cg.load(operand, elem_ty, .{ .offset = elem_size * idx }),
6843 .value => |val| try cg.lowerConstant(.fromInterned(val)),6899 .value => |val| try cg.lowerConstant(.fromInterned(val)),
6844 };6900 };
6845 try cg.store(.stack, elem_val, elem_ty, @intCast(dest_alloc.offset() + elem_size * out_idx));6901 try cg.store(.stack, elem_val, elem_ty, .{ .offset = dest_alloc.memOffset() + elem_size * out_idx });
6846 }6902 }
6847 return cg.finishAir(inst, dest_alloc, &.{unwrapped.operand});6903 return cg.finishAir(inst, dest_alloc, &.{unwrapped.operand});
6848}6904}
...@@ -6905,13 +6961,13 @@ fn airShuffleTwo(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -6905,13 +6961,13 @@ fn airShuffleTwo(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
69056961
6906 const dest_alloc = try cg.allocStack(result_ty);6962 const dest_alloc = try cg.allocStack(result_ty);
6907 for (mask, 0..) |mask_elem, out_idx| {6963 for (mask, 0..) |mask_elem, out_idx| {
6908 try cg.emitWValue(dest_alloc);6964 try cg.emitMemBase(dest_alloc);
6909 const elem_val = switch (mask_elem.unwrap()) {6965 const elem_val = switch (mask_elem.unwrap()) {
6910 .a_elem => |idx| try cg.load(operand_a, elem_ty, @intCast(elem_size * idx)),6966 .a_elem => |idx| try cg.load(operand_a, elem_ty, .{ .offset = elem_size * idx }),
6911 .b_elem => |idx| try cg.load(operand_b, elem_ty, @intCast(elem_size * idx)),6967 .b_elem => |idx| try cg.load(operand_b, elem_ty, .{ .offset = elem_size * idx }),
6912 .undef => try cg.emitUndefined(elem_ty),6968 .undef => try cg.emitUndefined(elem_ty),
6913 };6969 };
6914 try cg.store(.stack, elem_val, elem_ty, @intCast(dest_alloc.offset() + elem_size * out_idx));6970 try cg.store(.stack, elem_val, elem_ty, .{ .offset = dest_alloc.memOffset() + elem_size * out_idx });
6915 }6971 }
6916 return cg.finishAir(inst, dest_alloc, &.{ unwrapped.operand_a, unwrapped.operand_b });6972 return cg.finishAir(inst, dest_alloc, &.{ unwrapped.operand_a, unwrapped.operand_b });
6917}6973}
...@@ -6937,39 +6993,18 @@ fn airAggregateInit(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -6937,39 +6993,18 @@ fn airAggregateInit(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6937 .array, .vector => {6993 .array, .vector => {
6938 const result = try cg.allocStack(result_ty);6994 const result = try cg.allocStack(result_ty);
6939 const elem_ty = result_ty.childType(zcu);6995 const elem_ty = result_ty.childType(zcu);
6940 const elem_size = @as(u32, @intCast(elem_ty.abiSize(zcu)));6996 const elem_size = elem_ty.abiSize(zcu);
6941 const sentinel = result_ty.sentinel(zcu);6997 const sentinel = result_ty.sentinel(zcu);
69426998
6943 // When the element type is by reference, we must copy the entire6999 var offset: u64 = 0;
6944 // value. It is therefore safer to move the offset pointer and store7000 for (elements) |elem| {
6945 // each value individually, instead of using store offsets.7001 const elem_val = try cg.resolveInst(elem);
6946 if (isByRef(elem_ty, zcu, cg.target)) {7002 try cg.store(result, elem_val, elem_ty, .{ .offset = offset });
6947 // copy stack pointer into a temporary local, which is7003 offset += elem_size;
6948 // moved for each element to store each value in the right position.7004 }
6949 const offset = try cg.buildPointerOffset(result, 0, .new);7005 if (sentinel) |s| {
6950 for (elements, 0..) |elem, elem_index| {7006 const val = try cg.resolveValue(s);
6951 const elem_val = try cg.resolveInst(elem);7007 try cg.store(result, val, elem_ty, .{ .offset = offset });
6952 try cg.store(offset, elem_val, elem_ty, 0);
6953
6954 if (elem_index < elements.len - 1 or sentinel != null) {
6955 _ = try cg.buildPointerOffset(offset, elem_size, .modify);
6956 }
6957 }
6958 if (sentinel) |s| {
6959 const val = try cg.resolveValue(s);
6960 try cg.store(offset, val, elem_ty, 0);
6961 }
6962 } else {
6963 var offset: u32 = 0;
6964 for (elements) |elem| {
6965 const elem_val = try cg.resolveInst(elem);
6966 try cg.store(result, elem_val, elem_ty, offset);
6967 offset += elem_size;
6968 }
6969 if (sentinel) |s| {
6970 const val = try cg.resolveValue(s);
6971 try cg.store(result, val, elem_ty, offset);
6972 }
6973 }7008 }
6974 break :result_value result;7009 break :result_value result;
6975 },7010 },
...@@ -6982,10 +7017,9 @@ fn airAggregateInit(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -6982,10 +7017,9 @@ fn airAggregateInit(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
69827017
6983 const elem_ty = result_ty.fieldType(elem_index, zcu);7018 const elem_ty = result_ty.fieldType(elem_index, zcu);
6984 const field_offset = result_ty.structFieldOffset(elem_index, zcu);7019 const field_offset = result_ty.structFieldOffset(elem_index, zcu);
6985 const offset = try cg.buildPointerOffset(result, field_offset, .new);
69867020
6987 const value = try cg.resolveInst(elem);7021 const value = try cg.resolveInst(elem);
6988 try cg.store(offset, value, elem_ty, 0);7022 try cg.store(result, value, elem_ty, .{ .offset = field_offset });
6989 }7023 }
69907024
6991 break :result_value result;7025 break :result_value result;
...@@ -7032,24 +7066,19 @@ fn airUnionInit(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -7032,24 +7066,19 @@ fn airUnionInit(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7032 const result_ptr = try cg.allocStack(union_ty);7066 const result_ptr = try cg.allocStack(union_ty);
7033 const payload = try cg.resolveInst(extra.init);7067 const payload = try cg.resolveInst(extra.init);
7034 if (layout.tag_align.compare(.gte, layout.payload_align)) {7068 if (layout.tag_align.compare(.gte, layout.payload_align)) {
7035 if (isByRef(field_ty, zcu, cg.target)) {7069 try cg.store(result_ptr, payload, field_ty, .{ .offset = layout.tag_size });
7036 const payload_ptr = try cg.buildPointerOffset(result_ptr, layout.tag_size, .new);
7037 try cg.store(payload_ptr, payload, field_ty, 0);
7038 } else {
7039 try cg.store(result_ptr, payload, field_ty, @intCast(layout.tag_size));
7040 }
70417070
7042 if (layout.tag_size > 0) {7071 if (layout.tag_size > 0) {
7043 try cg.store(result_ptr, tag_int, .fromInterned(union_obj.enum_tag_type), 0);7072 try cg.store(result_ptr, tag_int, .fromInterned(union_obj.enum_tag_type), .{});
7044 }7073 }
7045 } else {7074 } else {
7046 try cg.store(result_ptr, payload, field_ty, 0);7075 try cg.store(result_ptr, payload, field_ty, .{});
7047 if (layout.tag_size > 0) {7076 if (layout.tag_size > 0) {
7048 try cg.store(7077 try cg.store(
7049 result_ptr,7078 result_ptr,
7050 tag_int,7079 tag_int,
7051 .fromInterned(union_obj.enum_tag_type),7080 .fromInterned(union_obj.enum_tag_type),
7052 @intCast(layout.payload_size),7081 .{ .offset = layout.payload_size },
7053 );7082 );
7054 }7083 }
7055 }7084 }
...@@ -7095,7 +7124,7 @@ fn airSetUnionTag(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -7095,7 +7124,7 @@ fn airSetUnionTag(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7095 const union_ptr = try cg.resolveInst(bin_op.lhs);7124 const union_ptr = try cg.resolveInst(bin_op.lhs);
7096 const new_tag = try cg.resolveInst(bin_op.rhs);7125 const new_tag = try cg.resolveInst(bin_op.rhs);
7097 if (layout.payload_size == 0) {7126 if (layout.payload_size == 0) {
7098 try cg.store(union_ptr, new_tag, tag_ty, 0);7127 try cg.store(union_ptr, new_tag, tag_ty, .{});
7099 return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });7128 return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });
7100 }7129 }
71017130
...@@ -7104,7 +7133,7 @@ fn airSetUnionTag(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -7104,7 +7133,7 @@ fn airSetUnionTag(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7104 const offset: u32 = if (layout.tag_align.compare(.lt, layout.payload_align)) blk: {7133 const offset: u32 = if (layout.tag_align.compare(.lt, layout.payload_align)) blk: {
7105 break :blk @intCast(layout.payload_size);7134 break :blk @intCast(layout.payload_size);
7106 } else 0;7135 } else 0;
7107 try cg.store(union_ptr, new_tag, tag_ty, offset);7136 try cg.store(union_ptr, new_tag, tag_ty, .{ .offset = offset });
7108 return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });7137 return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });
7109}7138}
71107139
...@@ -7124,7 +7153,7 @@ fn airGetUnionTag(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -7124,7 +7153,7 @@ fn airGetUnionTag(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7124 @intCast(layout.payload_size)7153 @intCast(layout.payload_size)
7125 else7154 else
7126 0;7155 0;
7127 const result = try cg.load(operand, tag_ty, offset);7156 const result = try cg.load(operand, tag_ty, .{ .offset = offset });
7128 return cg.finishAir(inst, result, &.{ty_op.operand});7157 return cg.finishAir(inst, result, &.{ty_op.operand});
7129}7158}
71307159
...@@ -7140,17 +7169,15 @@ fn airErrUnionPayloadPtrSet(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void...@@ -7140,17 +7169,15 @@ fn airErrUnionPayloadPtrSet(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void
7140 try cg.store(7169 try cg.store(
7141 operand,7170 operand,
7142 .{ .imm32 = 0 },7171 .{ .imm32 = 0 },
7143 Type.anyerror,7172 .anyerror,
7144 @intCast(errUnionErrorOffset(payload_ty, zcu)),7173 .{ .offset = errUnionErrorOffset(payload_ty, zcu) },
7145 );7174 );
71467175
7147 const result = result: {7176 if (!payload_ty.hasRuntimeBits(zcu)) {
7148 if (!payload_ty.hasRuntimeBits(zcu)) {7177 return cg.finishAir(inst, operand, &.{ty_op.operand});
7149 break :result cg.reuseOperand(ty_op.operand, operand);7178 }
7150 }
71517179
7152 break :result try cg.buildPointerOffset(operand, @as(u32, @intCast(errUnionPayloadOffset(payload_ty, zcu))), .new);7180 const result = try cg.offsetPointer(operand, errUnionPayloadOffset(payload_ty, zcu));
7153 };
7154 return cg.finishAir(inst, result, &.{ty_op.operand});7181 return cg.finishAir(inst, result, &.{ty_op.operand});
7155}7182}
71567183
...@@ -7175,16 +7202,23 @@ fn airFieldParentPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -7175,16 +7202,23 @@ fn airFieldParentPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7175 },7202 },
7176 };7203 };
71777204
7178 const result = if (field_offset != 0) result: {7205 if (field_offset == 0) {
7179 const base = try cg.buildPointerOffset(field_ptr, 0, .new);7206 return cg.finishAir(inst, field_ptr, &.{extra.field_ptr});
7180 try cg.addLocal(.local_get, base.local.value);7207 }
7181 try cg.addImm32(@intCast(field_offset));
7182 try cg.addTag(.i32_sub);
7183 try cg.addLocal(.local_set, base.local.value);
7184 break :result base;
7185 } else cg.reuseOperand(extra.field_ptr, field_ptr);
71867208
7187 return cg.finishAir(inst, result, &.{extra.field_ptr});7209 // maybe use offsetPointer? sub looks better in disasm
7210 try cg.emitWValue(field_ptr);
7211 switch (cg.ptr_size) {
7212 .wasm32 => {
7213 try cg.addImm32(@intCast(field_offset));
7214 try cg.addTag(.i32_sub);
7215 },
7216 .wasm64 => {
7217 try cg.addImm64(field_offset);
7218 try cg.addTag(.i64_sub);
7219 },
7220 }
7221 return cg.finishAir(inst, .stack, &.{extra.field_ptr});
7188}7222}
71897223
7190fn sliceOrArrayPtr(cg: *CodeGen, ptr: WValue, ptr_ty: Type) InnerError!WValue {7224fn sliceOrArrayPtr(cg: *CodeGen, ptr: WValue, ptr_ty: Type) InnerError!WValue {
...@@ -7204,14 +7238,14 @@ fn airMemcpy(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -7204,14 +7238,14 @@ fn airMemcpy(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7204 const ptr_elem_ty = dst_ty.childType(zcu);7238 const ptr_elem_ty = dst_ty.childType(zcu);
7205 const src = try cg.resolveInst(bin_op.rhs);7239 const src = try cg.resolveInst(bin_op.rhs);
7206 const src_ty = cg.typeOf(bin_op.rhs);7240 const src_ty = cg.typeOf(bin_op.rhs);
7207 const len = switch (dst_ty.ptrSize(zcu)) {7241 var len = switch (dst_ty.ptrSize(zcu)) {
7208 .slice => blk: {7242 .slice => blk: {
7209 const slice_len = try cg.sliceLen(dst);7243 const slice_len = try cg.sliceLen(dst);
7210 if (ptr_elem_ty.abiSize(zcu) != 1) {7244 if (ptr_elem_ty.abiSize(zcu) != 1) {
7211 try cg.emitWValue(slice_len);7245 try cg.emitWValue(slice_len);
7212 try cg.emitWValue(.{ .imm32 = @as(u32, @intCast(ptr_elem_ty.abiSize(zcu))) });7246 try cg.emitWValue(.{ .imm32 = @as(u32, @intCast(ptr_elem_ty.abiSize(zcu))) });
7213 try cg.addTag(.i32_mul);7247 try cg.addTag(.i32_mul);
7214 try cg.addLocal(.local_set, slice_len.local.value);7248 try cg.addLocal(.local_set, slice_len.local);
7215 }7249 }
7216 break :blk slice_len;7250 break :blk slice_len;
7217 },7251 },
...@@ -7220,8 +7254,11 @@ fn airMemcpy(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -7220,8 +7254,11 @@ fn airMemcpy(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7220 }),7254 }),
7221 .c, .many => unreachable,7255 .c, .many => unreachable,
7222 };7256 };
7223 const dst_ptr = try cg.sliceOrArrayPtr(dst, dst_ty);7257 defer len.free(cg);
7224 const src_ptr = try cg.sliceOrArrayPtr(src, src_ty);7258 var dst_ptr = try cg.sliceOrArrayPtr(dst, dst_ty);
7259 defer if (dst_ty.isSlice(zcu)) dst_ptr.free(cg);
7260 var src_ptr = try cg.sliceOrArrayPtr(src, src_ty);
7261 defer if (src_ty.isSlice(zcu)) src_ptr.free(cg);
7225 try cg.memcpy(dst_ptr, src_ptr, len);7262 try cg.memcpy(dst_ptr, src_ptr, len);
72267263
7227 return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });7264 return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });
...@@ -7235,14 +7272,14 @@ fn airMemmove(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -7235,14 +7272,14 @@ fn airMemmove(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7235 const ptr_elem_ty = dst_ty.childType(zcu);7272 const ptr_elem_ty = dst_ty.childType(zcu);
7236 const src = try cg.resolveInst(bin_op.rhs);7273 const src = try cg.resolveInst(bin_op.rhs);
7237 const src_ty = cg.typeOf(bin_op.rhs);7274 const src_ty = cg.typeOf(bin_op.rhs);
7238 const len = switch (dst_ty.ptrSize(zcu)) {7275 var len = switch (dst_ty.ptrSize(zcu)) {
7239 .slice => blk: {7276 .slice => blk: {
7240 const slice_len = try cg.sliceLen(dst);7277 const slice_len = try cg.sliceLen(dst);
7241 if (ptr_elem_ty.abiSize(zcu) != 1) {7278 if (ptr_elem_ty.abiSize(zcu) != 1) {
7242 try cg.emitWValue(slice_len);7279 try cg.emitWValue(slice_len);
7243 try cg.emitWValue(.{ .imm32 = @as(u32, @intCast(ptr_elem_ty.abiSize(zcu))) });7280 try cg.emitWValue(.{ .imm32 = @as(u32, @intCast(ptr_elem_ty.abiSize(zcu))) });
7244 try cg.addTag(.i32_mul);7281 try cg.addTag(.i32_mul);
7245 try cg.addLocal(.local_set, slice_len.local.value);7282 try cg.addLocal(.local_set, slice_len.local);
7246 }7283 }
7247 break :blk slice_len;7284 break :blk slice_len;
7248 },7285 },
...@@ -7251,8 +7288,11 @@ fn airMemmove(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -7251,8 +7288,11 @@ fn airMemmove(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7251 }),7288 }),
7252 .c, .many => unreachable,7289 .c, .many => unreachable,
7253 };7290 };
7254 const dst_ptr = try cg.sliceOrArrayPtr(dst, dst_ty);7291 defer len.free(cg);
7255 const src_ptr = try cg.sliceOrArrayPtr(src, src_ty);7292 var dst_ptr = try cg.sliceOrArrayPtr(dst, dst_ty);
7293 defer if (dst_ty.isSlice(zcu)) dst_ptr.free(cg);
7294 var src_ptr = try cg.sliceOrArrayPtr(src, src_ty);
7295 defer if (src_ty.isSlice(zcu)) src_ptr.free(cg);
7256 try cg.memmove(dst_ptr, src_ptr, len);7296 try cg.memmove(dst_ptr, src_ptr, len);
72577297
7258 return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });7298 return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });
...@@ -7283,19 +7323,7 @@ fn airErrorName(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -7283,19 +7323,7 @@ fn airErrorName(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7283 // Lowers to a i32.const or i64.const with the error table memory address.7323 // Lowers to a i32.const or i64.const with the error table memory address.
7284 cg.error_name_table_ref_count += 1;7324 cg.error_name_table_ref_count += 1;
7285 try cg.addTag(.error_name_table_ref);7325 try cg.addTag(.error_name_table_ref);
7286 try cg.emitWValue(operand);7326 try cg.emitScaledPointerOffset(operand, abi_size, .add);
7287 switch (cg.ptr_size) {
7288 .wasm32 => {
7289 try cg.addImm32(@intCast(abi_size));
7290 try cg.addTag(.i32_mul);
7291 try cg.addTag(.i32_add);
7292 },
7293 .wasm64 => {
7294 try cg.addImm64(abi_size);
7295 try cg.addTag(.i64_mul);
7296 try cg.addTag(.i64_add);
7297 },
7298 }
72997327
7300 return cg.finishAir(inst, .stack, &.{un_op});7328 return cg.finishAir(inst, .stack, &.{un_op});
7301}7329}
...@@ -7303,7 +7331,7 @@ fn airErrorName(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -7303,7 +7331,7 @@ fn airErrorName(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7303fn airPtrSliceFieldPtr(cg: *CodeGen, inst: Air.Inst.Index, offset: u32) InnerError!void {7331fn airPtrSliceFieldPtr(cg: *CodeGen, inst: Air.Inst.Index, offset: u32) InnerError!void {
7304 const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;7332 const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
7305 const slice_ptr = try cg.resolveInst(ty_op.operand);7333 const slice_ptr = try cg.resolveInst(ty_op.operand);
7306 const result = try cg.buildPointerOffset(slice_ptr, offset, .new);7334 const result = try cg.offsetPointer(slice_ptr, offset);
7307 return cg.finishAir(inst, result, &.{ty_op.operand});7335 return cg.finishAir(inst, result, &.{ty_op.operand});
7308}7336}
73097337
...@@ -7332,7 +7360,8 @@ fn airDbgVar(...@@ -7332,7 +7360,8 @@ fn airDbgVar(
7332) InnerError!void {7360) InnerError!void {
7333 _ = is_ptr;7361 _ = is_ptr;
7334 _ = local_tag;7362 _ = local_tag;
7335 return cg.finishAir(inst, .none, &.{});7363 const pl_op = cg.air.instructions.items(.data)[@backingInt(inst)].pl_op;
7364 return cg.finishAir(inst, .none, &.{pl_op.operand});
7336}7365}
73377366
7338fn airTry(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {7367fn airTry(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
...@@ -7340,8 +7369,13 @@ fn airTry(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -7340,8 +7369,13 @@ fn airTry(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7340 const body = unwrapped_try.else_body;7369 const body = unwrapped_try.else_body;
7341 const err_union = try cg.resolveInst(unwrapped_try.error_union);7370 const err_union = try cg.resolveInst(unwrapped_try.error_union);
7342 const err_union_ty = cg.typeOf(unwrapped_try.error_union);7371 const err_union_ty = cg.typeOf(unwrapped_try.error_union);
7343 const result = try lowerTry(cg, inst, err_union, body, err_union_ty, false);7372 const result = try lowerTry(cg, inst, unwrapped_try.error_union, err_union, body, err_union_ty, false);
7344 return cg.finishAir(inst, result, &.{unwrapped_try.error_union});7373 return cg.finishAirWithBranchDeaths(
7374 inst,
7375 result,
7376 &.{unwrapped_try.error_union},
7377 cg.liveness.getCondBr(inst).then_deaths,
7378 );
7345}7379}
73467380
7347fn airTryPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {7381fn airTryPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
...@@ -7350,20 +7384,26 @@ fn airTryPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -7350,20 +7384,26 @@ fn airTryPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7350 const err_union_ptr = try cg.resolveInst(unwrapped_try.error_union_ptr);7384 const err_union_ptr = try cg.resolveInst(unwrapped_try.error_union_ptr);
7351 const body = unwrapped_try.else_body;7385 const body = unwrapped_try.else_body;
7352 const err_union_ty = cg.typeOf(unwrapped_try.error_union_ptr).childType(zcu);7386 const err_union_ty = cg.typeOf(unwrapped_try.error_union_ptr).childType(zcu);
7353 const result = try lowerTry(cg, inst, err_union_ptr, body, err_union_ty, true);7387 const result = try lowerTry(cg, inst, unwrapped_try.error_union_ptr, err_union_ptr, body, err_union_ty, true);
7354 return cg.finishAir(inst, result, &.{unwrapped_try.error_union_ptr});7388 return cg.finishAirWithBranchDeaths(
7389 inst,
7390 result,
7391 &.{unwrapped_try.error_union_ptr},
7392 cg.liveness.getCondBr(inst).then_deaths,
7393 );
7355}7394}
73567395
7357fn lowerTry(7396fn lowerTry(
7358 cg: *CodeGen,7397 cg: *CodeGen,
7359 inst: Air.Inst.Index,7398 inst: Air.Inst.Index,
7399 operand_ref: Air.Inst.Ref,
7360 err_union: WValue,7400 err_union: WValue,
7361 body: []const Air.Inst.Index,7401 body: []const Air.Inst.Index,
7362 err_union_ty: Type,7402 err_union_ty: Type,
7363 operand_is_ptr: bool,7403 operand_is_ptr: bool,
7364) InnerError!WValue {7404) InnerError!WValue {
7365 _ = inst;
7366 const zcu = cg.pt.zcu;7405 const zcu = cg.pt.zcu;
7406 const branch_liveness = cg.liveness.getCondBr(inst);
73677407
7368 const pl_ty = err_union_ty.errorUnionPayload(zcu);7408 const pl_ty = err_union_ty.errorUnionPayload(zcu);
7369 const pl_has_bits = pl_ty.hasRuntimeBits(zcu);7409 const pl_has_bits = pl_ty.hasRuntimeBits(zcu);
...@@ -7373,23 +7413,26 @@ fn lowerTry(...@@ -7373,23 +7413,26 @@ fn lowerTry(
7373 try cg.startBlock(.block, .empty);7413 try cg.startBlock(.block, .empty);
73747414
7375 // check if the error tag is set for the error union.7415 // check if the error tag is set for the error union.
7376 try cg.emitWValue(err_union);
7377 if (pl_has_bits or operand_is_ptr) {7416 if (pl_has_bits or operand_is_ptr) {
7378 const err_offset: u32 = @intCast(errUnionErrorOffset(pl_ty, zcu));7417 try cg.emitMemBase(err_union);
7418 const err_offset = errUnionErrorOffset(pl_ty, zcu);
7379 try cg.addMemArg(.i32_load16_u, .{7419 try cg.addMemArg(.i32_load16_u, .{
7380 .offset = err_union.offset() + err_offset,7420 .offset = err_union.memOffset() + err_offset,
7381 .alignment = @intCast(Type.anyerror.abiAlignment(zcu).toByteUnits().?),7421 .alignment = Type.anyerror.abiAlignment(zcu),
7382 });7422 });
7423 } else {
7424 try cg.emitWValue(err_union);
7383 }7425 }
7384 try cg.addTag(.i32_eqz);7426 try cg.addTag(.i32_eqz);
7385 try cg.addLabel(.br_if, 0); // jump out of block when error is '0'7427 try cg.addLabel(.br_if, 0); // jump out of block when error is '0'
73867428
7387 try cg.branches.append(cg.gpa, .{});7429 {
7388 defer {7430 try cg.pushBranch();
7389 var branch = cg.branches.pop().?;7431 defer cg.popBranch();
7390 branch.deinit(cg.gpa);7432 if (cg.liveness.operandDies(inst, 0)) cg.processDeath(operand_ref);
7433 for (branch_liveness.else_deaths) |death| cg.processDeath(death.toRef());
7434 try cg.genBody(body);
7391 }7435 }
7392 try cg.genBody(body);
7393 try cg.endBlock();7436 try cg.endBlock();
7394 }7437 }
73957438
...@@ -7398,12 +7441,11 @@ fn lowerTry(...@@ -7398,12 +7441,11 @@ fn lowerTry(
7398 return .none;7441 return .none;
7399 }7442 }
74007443
7401 const pl_offset: u32 = @intCast(errUnionPayloadOffset(pl_ty, zcu));7444 const pl_offset = errUnionPayloadOffset(pl_ty, zcu);
7402 if (operand_is_ptr or isByRef(pl_ty, zcu, cg.target)) {7445 if (operand_is_ptr or isByRef(pl_ty, zcu, cg.target)) {
7403 return buildPointerOffset(cg, err_union, pl_offset, .new);7446 return cg.offsetPointer(err_union, pl_offset);
7404 }7447 }
7405 const payload = try cg.load(err_union, pl_ty, pl_offset);7448 return cg.load(err_union, pl_ty, .{ .offset = pl_offset });
7406 return payload.toLocal(cg, pl_ty);
7407}7449}
74087450
7409/// Calls a compiler-rt intrinsic by creating an undefined symbol,7451/// Calls a compiler-rt intrinsic by creating an undefined symbol,
...@@ -7429,7 +7471,7 @@ fn callIntrinsic(...@@ -7429,7 +7471,7 @@ fn callIntrinsic(
7429 // and emit it as our first argument7471 // and emit it as our first argument
7430 const sret = if (want_sret_param) blk: {7472 const sret = if (want_sret_param) blk: {
7431 const sret_local = try cg.allocStack(return_type);7473 const sret_local = try cg.allocStack(return_type);
7432 try cg.lowerToStack(sret_local);7474 try cg.emitWValue(sret_local);
7433 break :blk sret_local;7475 break :blk sret_local;
7434 } else .none;7476 } else .none;
74357477
...@@ -7457,21 +7499,9 @@ fn airTagName(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -7457,21 +7499,9 @@ fn airTagName(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7457 const enum_ty = cg.typeOf(un_op);7499 const enum_ty = cg.typeOf(un_op);
74587500
7459 try cg.addInst(.{ .tag = .enum_tag_name_table_ref, .data = .{ .ip_index = enum_ty.toIntern() } });7501 try cg.addInst(.{ .tag = .enum_tag_name_table_ref, .data = .{ .ip_index = enum_ty.toIntern() } });
7460 try cg.lowerToStack(operand);7502 try cg.emitWValue(operand);
7461 try cg.addInst(.{ .tag = .call_tag_index, .data = .{ .ip_index = enum_ty.toIntern() } });7503 try cg.addInst(.{ .tag = .call_tag_index, .data = .{ .ip_index = enum_ty.toIntern() } });
74627504 try cg.emitScaledPointerOffset(.stack, 8, .add);
7463 switch (cg.ptr_size) {
7464 .wasm32 => {
7465 try cg.addImm32(@intCast(8));
7466 try cg.addTag(.i32_mul);
7467 try cg.addTag(.i32_add);
7468 },
7469 .wasm64 => {
7470 try cg.addImm64(8);
7471 try cg.addTag(.i64_mul);
7472 try cg.addTag(.i64_add);
7473 },
7474 }
74757505
7476 return cg.finishAir(inst, .stack, &.{un_op});7506 return cg.finishAir(inst, .stack, &.{un_op});
7477}7507}
...@@ -7481,7 +7511,7 @@ fn airIsNamedEnumValue(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -7481,7 +7511,7 @@ fn airIsNamedEnumValue(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7481 const operand = try cg.resolveInst(un_op);7511 const operand = try cg.resolveInst(un_op);
7482 const enum_ty = cg.typeOf(un_op);7512 const enum_ty = cg.typeOf(un_op);
74837513
7484 try cg.lowerToStack(operand);7514 try cg.emitWValue(operand);
7485 try cg.addInst(.{ .tag = .call_tag_index, .data = .{ .ip_index = enum_ty.toIntern() } });7515 try cg.addInst(.{ .tag = .call_tag_index, .data = .{ .ip_index = enum_ty.toIntern() } });
7486 try cg.addImm32(~@as(u32, 0));7516 try cg.addImm32(~@as(u32, 0));
7487 try cg.addTag(.i32_ne);7517 try cg.addTag(.i32_ne);
...@@ -7559,13 +7589,13 @@ fn airErrorSetHasValue(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -7559,13 +7589,13 @@ fn airErrorSetHasValue(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7559 // 'false' branch (i.e. error set does not have value7589 // 'false' branch (i.e. error set does not have value
7560 // ensure we set local to 0 in case the local was re-used.7590 // ensure we set local to 0 in case the local was re-used.
7561 try cg.addImm32(0);7591 try cg.addImm32(0);
7562 try cg.addLocal(.local_set, result.local.value);7592 try cg.addLocal(.local_set, result.local);
7563 try cg.addLabel(.br, 1);7593 try cg.addLabel(.br, 1);
7564 try cg.endBlock();7594 try cg.endBlock();
75657595
7566 // 'true' branch7596 // 'true' branch
7567 try cg.addImm32(1);7597 try cg.addImm32(1);
7568 try cg.addLocal(.local_set, result.local.value);7598 try cg.addLocal(.local_set, result.local);
7569 try cg.addLabel(.br, 0);7599 try cg.addLabel(.br, 0);
7570 try cg.endBlock();7600 try cg.endBlock();
75717601
...@@ -7591,13 +7621,14 @@ fn airCmpxchg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -7591,13 +7621,14 @@ fn airCmpxchg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7591 const expected_val = try cg.resolveInst(extra.expected_value);7621 const expected_val = try cg.resolveInst(extra.expected_value);
7592 const new_val = try cg.resolveInst(extra.new_value);7622 const new_val = try cg.resolveInst(extra.new_value);
75937623
7594 const cmp_result = try cg.allocLocal(Type.bool);7624 var cmp_result = try cg.allocLocal(Type.bool);
7625 defer cmp_result.free(cg);
75957626
7596 const ptr_val = if (cg.useAtomicFeature()) val: {7627 var ptr_val = if (cg.useAtomicFeature()) val: {
7597 const val_local = try cg.allocLocal(ty);7628 const val_local = try cg.allocLocal(ty);
7598 try cg.emitWValue(ptr_operand);7629 try cg.emitMemBase(ptr_operand);
7599 try cg.lowerToStack(expected_val);7630 try cg.emitWValue(expected_val);
7600 try cg.lowerToStack(new_val);7631 try cg.emitWValue(new_val);
7601 try cg.addAtomicMemArg(switch (ty.abiSize(zcu)) {7632 try cg.addAtomicMemArg(switch (ty.abiSize(zcu)) {
7602 1 => .i32_atomic_rmw8_cmpxchg_u,7633 1 => .i32_atomic_rmw8_cmpxchg_u,
7603 2 => .i32_atomic_rmw16_cmpxchg_u,7634 2 => .i32_atomic_rmw16_cmpxchg_u,
...@@ -7605,26 +7636,26 @@ fn airCmpxchg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -7605,26 +7636,26 @@ fn airCmpxchg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7605 8 => .i32_atomic_rmw_cmpxchg,7636 8 => .i32_atomic_rmw_cmpxchg,
7606 else => |size| return cg.fail("TODO: implement `@cmpxchg` for types with abi size '{d}'", .{size}),7637 else => |size| return cg.fail("TODO: implement `@cmpxchg` for types with abi size '{d}'", .{size}),
7607 }, .{7638 }, .{
7608 .offset = ptr_operand.offset(),7639 .offset = ptr_operand.memOffset(),
7609 .alignment = @intCast(ty.abiAlignment(zcu).toByteUnits().?),7640 .alignment = ty.abiAlignment(zcu),
7610 });7641 });
7611 try cg.addLocal(.local_tee, val_local.local.value);7642 try cg.addLocal(.local_tee, val_local.local);
7612 _ = try cg.intCmp(int_ty, .eq, .stack, expected_val);7643 _ = try cg.intCmp(int_ty, .eq, .stack, expected_val);
7613 try cg.addLocal(.local_set, cmp_result.local.value);7644 try cg.addLocal(.local_set, cmp_result.local);
7614 break :val val_local;7645 break :val val_local;
7615 } else val: {7646 } else val: {
7616 if (ty.abiSize(zcu) > 8) {7647 if (ty.abiSize(zcu) > 8) {
7617 return cg.fail("TODO: Implement `@cmpxchg` for types larger than abi size of 8 bytes", .{});7648 return cg.fail("TODO: Implement `@cmpxchg` for types larger than abi size of 8 bytes", .{});
7618 }7649 }
7619 const ptr_val = try WValue.toLocal(try cg.load(ptr_operand, ty, 0), cg, ty);7650 const ptr_val = try WValue.toLocal(try cg.load(ptr_operand, ty, .{}), cg, ty);
76207651
7621 try cg.lowerToStack(ptr_operand);7652 try cg.emitMemBase(ptr_operand);
7622 try cg.lowerToStack(new_val);7653 try cg.emitWValue(new_val);
7623 try cg.emitWValue(ptr_val);7654 try cg.emitWValue(ptr_val);
7624 _ = try cg.intCmp(int_ty, .eq, ptr_val, expected_val);7655 _ = try cg.intCmp(int_ty, .eq, ptr_val, expected_val);
7625 try cg.addLocal(.local_tee, cmp_result.local.value);7656 try cg.addLocal(.local_tee, cmp_result.local);
7626 try cg.addTag(.select);7657 try cg.addTag(.select);
7627 try cg.store(.stack, .stack, ty, 0);7658 try cg.store(.stack, .stack, ty, .{ .offset = ptr_operand.memOffset() });
76287659
7629 break :val ptr_val;7660 break :val ptr_val;
7630 };7661 };
...@@ -7635,10 +7666,11 @@ fn airCmpxchg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -7635,10 +7666,11 @@ fn airCmpxchg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7635 try cg.addTag(.i32_xor);7666 try cg.addTag(.i32_xor);
7636 try cg.addImm32(1);7667 try cg.addImm32(1);
7637 try cg.addTag(.i32_and);7668 try cg.addTag(.i32_and);
7638 const and_result = try WValue.toLocal(.stack, cg, Type.bool);7669 var and_result = try WValue.toLocal(.stack, cg, Type.bool);
7670 defer and_result.free(cg);
7639 const result_ptr = try cg.allocStack(result_ty);7671 const result_ptr = try cg.allocStack(result_ty);
7640 try cg.store(result_ptr, and_result, Type.bool, @as(u32, @intCast(ty.abiSize(zcu))));7672 try cg.store(result_ptr, and_result, .bool, .{ .offset = ty.abiSize(zcu) });
7641 try cg.store(result_ptr, ptr_val, ty, 0);7673 try cg.store(result_ptr, ptr_val, ty, .{});
7642 break :val result_ptr;7674 break :val result_ptr;
7643 } else val: {7675 } else val: {
7644 try cg.addImm32(0);7676 try cg.addImm32(0);
...@@ -7647,6 +7679,7 @@ fn airCmpxchg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -7647,6 +7679,7 @@ fn airCmpxchg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7647 try cg.addTag(.select);7679 try cg.addTag(.select);
7648 break :val .stack;7680 break :val .stack;
7649 };7681 };
7682 defer ptr_val.free(cg);
76507683
7651 return cg.finishAir(inst, result, &.{ extra.ptr, extra.expected_value, extra.new_value });7684 return cg.finishAir(inst, result, &.{ extra.ptr, extra.expected_value, extra.new_value });
7652}7685}
...@@ -7665,13 +7698,13 @@ fn airAtomicLoad(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -7665,13 +7698,13 @@ fn airAtomicLoad(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7665 8 => .i64_atomic_load,7698 8 => .i64_atomic_load,
7666 else => |size| return cg.fail("TODO: @atomicLoad for types with abi size {d}", .{size}),7699 else => |size| return cg.fail("TODO: @atomicLoad for types with abi size {d}", .{size}),
7667 };7700 };
7668 try cg.emitWValue(ptr);7701 try cg.emitMemBase(ptr);
7669 try cg.addAtomicMemArg(tag, .{7702 try cg.addAtomicMemArg(tag, .{
7670 .offset = ptr.offset(),7703 .offset = ptr.memOffset(),
7671 .alignment = @intCast(ty.abiAlignment(zcu).toByteUnits().?),7704 .alignment = ty.abiAlignment(zcu),
7672 });7705 });
7673 } else {7706 } else {
7674 _ = try cg.load(ptr, ty, 0);7707 _ = try cg.load(ptr, ty, .{});
7675 }7708 }
76767709
7677 return cg.finishAir(inst, .stack, &.{atomic_load.ptr});7710 return cg.finishAir(inst, .stack, &.{atomic_load.ptr});
...@@ -7694,13 +7727,13 @@ fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -7694,13 +7727,13 @@ fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7694 .Min,7727 .Min,
7695 .Nand,7728 .Nand,
7696 => {7729 => {
7697 const tmp = try cg.load(ptr, ty, 0);7730 const tmp = try cg.load(ptr, ty, .{});
7698 const value = try tmp.toLocal(cg, ty);7731 const value = try tmp.toLocal(cg, ty);
76997732
7700 // create a loop to cmpxchg the new value7733 // create a loop to cmpxchg the new value
7701 try cg.startBlock(.loop, .empty);7734 try cg.startBlock(.loop, .empty);
77027735
7703 try cg.emitWValue(ptr);7736 try cg.emitMemBase(ptr);
7704 try cg.emitWValue(value);7737 try cg.emitWValue(value);
7705 if (op == .Nand) {7738 if (op == .Nand) {
7706 const and_res = try cg.intAnd(int_ty, value, operand);7739 const and_res = try cg.intAnd(int_ty, value, operand);
...@@ -7727,16 +7760,17 @@ fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -7727,16 +7760,17 @@ fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7727 else => return cg.fail("TODO: implement `@atomicRmw` with operation `{s}` for types larger than 64 bits", .{@tagName(op)}),7760 else => return cg.fail("TODO: implement `@atomicRmw` with operation `{s}` for types larger than 64 bits", .{@tagName(op)}),
7728 },7761 },
7729 .{7762 .{
7730 .offset = ptr.offset(),7763 .offset = ptr.memOffset(),
7731 .alignment = @intCast(ty.abiAlignment(zcu).toByteUnits().?),7764 .alignment = ty.abiAlignment(zcu),
7732 },7765 },
7733 );7766 );
7734 const select_res = try cg.allocLocal(ty);7767 var select_res = try cg.allocLocal(ty);
7735 try cg.addLocal(.local_tee, select_res.local.value);7768 defer select_res.free(cg);
7769 try cg.addLocal(.local_tee, select_res.local);
7736 _ = try cg.intCmp(int_ty, .neq, .stack, value); // leave on stack so we can use it for br_if7770 _ = try cg.intCmp(int_ty, .neq, .stack, value); // leave on stack so we can use it for br_if
77377771
7738 try cg.emitWValue(select_res);7772 try cg.emitWValue(select_res);
7739 try cg.addLocal(.local_set, value.local.value);7773 try cg.addLocal(.local_set, value.local);
77407774
7741 try cg.addLabel(.br_if, 0);7775 try cg.addLabel(.br_if, 0);
7742 try cg.endBlock();7776 try cg.endBlock();
...@@ -7745,7 +7779,7 @@ fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -7745,7 +7779,7 @@ fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
77457779
7746 // the other operations have their own instructions for Wasm.7780 // the other operations have their own instructions for Wasm.
7747 else => {7781 else => {
7748 try cg.emitWValue(ptr);7782 try cg.emitMemBase(ptr);
7749 try cg.emitWValue(operand);7783 try cg.emitWValue(operand);
7750 const tag: std.wasm.AtomicsOpcode = switch (ty.abiSize(zcu)) {7784 const tag: std.wasm.AtomicsOpcode = switch (ty.abiSize(zcu)) {
7751 1 => switch (op) {7785 1 => switch (op) {
...@@ -7787,42 +7821,42 @@ fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -7787,42 +7821,42 @@ fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7787 else => |size| return cg.fail("TODO: Implement `@atomicRmw` for types with abi size {d}", .{size}),7821 else => |size| return cg.fail("TODO: Implement `@atomicRmw` for types with abi size {d}", .{size}),
7788 };7822 };
7789 try cg.addAtomicMemArg(tag, .{7823 try cg.addAtomicMemArg(tag, .{
7790 .offset = ptr.offset(),7824 .offset = ptr.memOffset(),
7791 .alignment = @intCast(ty.abiAlignment(zcu).toByteUnits().?),7825 .alignment = ty.abiAlignment(zcu),
7792 });7826 });
7793 return cg.finishAir(inst, .stack, &.{ pl_op.operand, extra.operand });7827 return cg.finishAir(inst, .stack, &.{ pl_op.operand, extra.operand });
7794 },7828 },
7795 }7829 }
7796 } else {7830 } else {
7797 const loaded = try cg.load(ptr, ty, 0);7831 const loaded = try cg.load(ptr, ty, .{});
7798 const result = try loaded.toLocal(cg, ty);7832 const result = try loaded.toLocal(cg, ty);
77997833
7800 switch (op) {7834 switch (op) {
7801 .Xchg => {7835 .Xchg => {
7802 try cg.store(ptr, operand, ty, 0);7836 try cg.store(ptr, operand, ty, .{});
7803 },7837 },
7804 .Add,7838 .Add,
7805 .Sub,7839 .Sub,
7806 => {7840 => {
7807 if (ty.isAnyFloat()) {7841 if (ty.isAnyFloat()) {
7808 const float_ty: FloatType = .fromType(cg, ty);7842 const float_ty: FloatType = .fromType(cg, ty);
7809 try cg.emitWValue(ptr);7843 try cg.emitMemBase(ptr);
7810 _ = switch (op) {7844 _ = switch (op) {
7811 .Add => try cg.floatAdd(float_ty, result, operand),7845 .Add => try cg.floatAdd(float_ty, result, operand),
7812 .Sub => try cg.floatSub(float_ty, result, operand),7846 .Sub => try cg.floatSub(float_ty, result, operand),
7813 else => unreachable,7847 else => unreachable,
7814 };7848 };
7815 try cg.store(.stack, .stack, ty, ptr.offset());7849 try cg.store(.stack, .stack, ty, .{ .offset = ptr.memOffset() });
7816 } else {7850 } else {
7817 const int_ty: IntType = .fromType(cg, ty);7851 const int_ty: IntType = .fromType(cg, ty);
7818 try cg.emitWValue(ptr);7852 try cg.emitMemBase(ptr);
7819 _ = switch (op) {7853 _ = switch (op) {
7820 .Add => try cg.intAdd(int_ty, result, operand),7854 .Add => try cg.intAdd(int_ty, result, operand),
7821 .Sub => try cg.intSub(int_ty, result, operand),7855 .Sub => try cg.intSub(int_ty, result, operand),
7822 else => unreachable,7856 else => unreachable,
7823 };7857 };
7824 _ = try cg.intWrap(int_ty, .stack);7858 _ = try cg.intWrap(int_ty, .stack);
7825 try cg.store(.stack, .stack, ty, ptr.offset());7859 try cg.store(.stack, .stack, ty, .{ .offset = ptr.memOffset() });
7826 }7860 }
7827 },7861 },
7828 .And,7862 .And,
...@@ -7830,39 +7864,39 @@ fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -7830,39 +7864,39 @@ fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7830 .Xor,7864 .Xor,
7831 => {7865 => {
7832 const int_ty: IntType = .fromType(cg, ty);7866 const int_ty: IntType = .fromType(cg, ty);
7833 try cg.emitWValue(ptr);7867 try cg.emitMemBase(ptr);
7834 _ = switch (op) {7868 _ = switch (op) {
7835 .And => try cg.intAnd(int_ty, result, operand),7869 .And => try cg.intAnd(int_ty, result, operand),
7836 .Or => try cg.intOr(int_ty, result, operand),7870 .Or => try cg.intOr(int_ty, result, operand),
7837 .Xor => try cg.intXor(int_ty, result, operand),7871 .Xor => try cg.intXor(int_ty, result, operand),
7838 else => unreachable,7872 else => unreachable,
7839 };7873 };
7840 try cg.store(.stack, .stack, ty, ptr.offset());7874 try cg.store(.stack, .stack, ty, .{ .offset = ptr.memOffset() });
7841 },7875 },
7842 .Max,7876 .Max,
7843 .Min,7877 .Min,
7844 => {7878 => {
7845 if (ty.isAnyFloat()) {7879 if (ty.isAnyFloat()) {
7846 const float_ty: FloatType = .fromType(cg, ty);7880 const float_ty: FloatType = .fromType(cg, ty);
7847 try cg.emitWValue(ptr);7881 try cg.emitMemBase(ptr);
7848 try cg.emitWValue(result);7882 try cg.emitWValue(result);
7849 try cg.emitWValue(operand);7883 try cg.emitWValue(operand);
7850 _ = try cg.floatCmp(float_ty, if (op == .Max) .gt else .lt, result, operand);7884 _ = try cg.floatCmp(float_ty, if (op == .Max) .gt else .lt, result, operand);
7851 try cg.addTag(.select);7885 try cg.addTag(.select);
7852 try cg.store(.stack, .stack, ty, ptr.offset());7886 try cg.store(.stack, .stack, ty, .{ .offset = ptr.memOffset() });
7853 } else {7887 } else {
7854 const int_ty: IntType = .fromType(cg, ty);7888 const int_ty: IntType = .fromType(cg, ty);
7855 try cg.emitWValue(ptr);7889 try cg.emitMemBase(ptr);
7856 try cg.emitWValue(result);7890 try cg.emitWValue(result);
7857 try cg.emitWValue(operand);7891 try cg.emitWValue(operand);
7858 _ = try cg.intCmp(int_ty, if (op == .Max) .gt else .lt, result, operand);7892 _ = try cg.intCmp(int_ty, if (op == .Max) .gt else .lt, result, operand);
7859 try cg.addTag(.select);7893 try cg.addTag(.select);
7860 try cg.store(.stack, .stack, ty, ptr.offset());7894 try cg.store(.stack, .stack, ty, .{ .offset = ptr.memOffset() });
7861 }7895 }
7862 },7896 },
7863 .Nand => {7897 .Nand => {
7864 const int_ty: IntType = .fromType(cg, ty);7898 const int_ty: IntType = .fromType(cg, ty);
7865 try cg.emitWValue(ptr);7899 try cg.emitMemBase(ptr);
7866 const and_res = try cg.intAnd(int_ty, result, operand);7900 const and_res = try cg.intAnd(int_ty, result, operand);
7867 if (int_ty.bits <= 32) {7901 if (int_ty.bits <= 32) {
7868 try cg.addImm32(~@as(u32, 0));7902 try cg.addImm32(~@as(u32, 0));
...@@ -7872,7 +7906,7 @@ fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -7872,7 +7906,7 @@ fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7872 return cg.fail("TODO: `@atomicRmw` with operator `Nand` for types larger than 64 bits", .{});7906 return cg.fail("TODO: `@atomicRmw` with operator `Nand` for types larger than 64 bits", .{});
7873 }7907 }
7874 _ = try cg.intXor(int_ty, and_res, .stack);7908 _ = try cg.intXor(int_ty, and_res, .stack);
7875 try cg.store(.stack, .stack, ty, ptr.offset());7909 try cg.store(.stack, .stack, ty, .{ .offset = ptr.memOffset() });
7876 },7910 },
7877 }7911 }
78787912
...@@ -7897,23 +7931,20 @@ fn airAtomicStore(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -7897,23 +7931,20 @@ fn airAtomicStore(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7897 8 => .i64_atomic_store,7931 8 => .i64_atomic_store,
7898 else => |size| return cg.fail("TODO: @atomicLoad for types with abi size {d}", .{size}),7932 else => |size| return cg.fail("TODO: @atomicLoad for types with abi size {d}", .{size}),
7899 };7933 };
7900 try cg.emitWValue(ptr);7934 try cg.emitMemBase(ptr);
7901 try cg.lowerToStack(operand);7935 try cg.emitWValue(operand);
7902 try cg.addAtomicMemArg(tag, .{7936 try cg.addAtomicMemArg(tag, .{
7903 .offset = ptr.offset(),7937 .offset = ptr.memOffset(),
7904 .alignment = @intCast(ty.abiAlignment(zcu).toByteUnits().?),7938 .alignment = ty.abiAlignment(zcu),
7905 });7939 });
7906 } else {7940 } else {
7907 try cg.store(ptr, operand, ty, 0);7941 try cg.store(ptr, operand, ty, .{});
7908 }7942 }
79097943
7910 return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });7944 return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });
7911}7945}
79127946
7913fn airFrameAddress(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {7947fn airFrameAddress(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7914 if (cg.initial_stack_value == .none) {
7915 try cg.initializeStack();
7916 }
7917 try cg.emitWValue(cg.bottom_stack_value);7948 try cg.emitWValue(cg.bottom_stack_value);
7918 return cg.finishAir(inst, .stack, &.{});7949 return cg.finishAir(inst, .stack, &.{});
7919}7950}
...@@ -7947,6 +7978,11 @@ fn airAsm(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -7947,6 +7978,11 @@ fn airAsm(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7947 if (unwrapped_asm.source.len != 0) {7978 if (unwrapped_asm.source.len != 0) {
7948 var local_map: assembly.LocalMap = .empty;7979 var local_map: assembly.LocalMap = .empty;
7949 defer local_map.deinit(cg.gpa);7980 defer local_map.deinit(cg.gpa);
7981 var input_locals: std.ArrayList(WValue) = try .initCapacity(cg.gpa, inputs.len);
7982 defer {
7983 for (input_locals.items) |*input_local| input_local.free(cg);
7984 input_locals.deinit(cg.gpa);
7985 }
79507986
7951 {7987 {
7952 var it = unwrapped_asm.iterateOutputs();7988 var it = unwrapped_asm.iterateOutputs();
...@@ -7959,7 +7995,7 @@ fn airAsm(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -7959,7 +7995,7 @@ fn airAsm(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7959 return cg.fail("Self-hosted wasm backend requires output constraint to be equal \"=r\"", .{});7995 return cg.fail("Self-hosted wasm backend requires output constraint to be equal \"=r\"", .{});
7960 }7996 }
79617997
7962 const gop = try local_map.getOrPutValue(cg.gpa, name, result.local.value);7998 const gop = try local_map.getOrPutValue(cg.gpa, name, result.local);
7963 assert(!gop.found_existing); // first value7999 assert(!gop.found_existing); // first value
79648000
7965 assert(it.next() == null);8001 assert(it.next() == null);
...@@ -7977,10 +8013,11 @@ fn airAsm(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -7977,10 +8013,11 @@ fn airAsm(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7977 return cg.fail("Self-hosted wasm backend requires input constraint to be equal \"r\"", .{});8013 return cg.fail("Self-hosted wasm backend requires input constraint to be equal \"r\"", .{});
7978 }8014 }
79798015
7980 try cg.lowerToStack(operand);8016 try cg.emitWValue(operand);
7981 const op_local = try WValue.toLocal(.stack, cg, cg.typeOf(input.operand));8017 const op_local = try WValue.toLocal(.stack, cg, cg.typeOf(input.operand));
8018 input_locals.appendAssumeCapacity(op_local);
79828019
7983 const gop = try local_map.getOrPutValue(cg.gpa, name, op_local.local.value);8020 const gop = try local_map.getOrPutValue(cg.gpa, name, op_local.local);
7984 if (gop.found_existing) {8021 if (gop.found_existing) {
7985 return cg.fail("Duplicate asm variable name \"{s}\"", .{name});8022 return cg.fail("Duplicate asm variable name \"{s}\"", .{name});
7986 }8023 }
src/codegen/wasm/Emit.zig+10-10
...@@ -56,13 +56,15 @@ pub fn lower(emit: *Emit) Error!void {...@@ -56,13 +56,15 @@ pub fn lower(emit: *Emit) Error!void {
56 writeSleb128(code, aligned_stack);56 writeSleb128(code, aligned_stack);
57 // subtract it from the current stack pointer57 // subtract it from the current stack pointer
58 code.appendAssumeCapacity(@backingInt(std.wasm.Opcode.i32_sub));58 code.appendAssumeCapacity(@backingInt(std.wasm.Opcode.i32_sub));
59 // Get negative stack alignment59 if (align_bytes != 16) {
60 const neg_stack_align = @as(i32, @intCast(align_bytes)) * -1;60 // Get negative stack alignment
61 code.appendAssumeCapacity(@backingInt(std.wasm.Opcode.i32_const));61 const neg_stack_align = @as(i32, @intCast(align_bytes)) * -1;
62 writeSleb128(code, neg_stack_align);62 code.appendAssumeCapacity(@backingInt(std.wasm.Opcode.i32_const));
63 // Bitwise-and the value to get the new stack pointer to ensure the63 writeSleb128(code, neg_stack_align);
64 // pointers are aligned with the abi alignment.64 // Bitwise-and the value to get the new stack pointer to ensure the
65 code.appendAssumeCapacity(@backingInt(std.wasm.Opcode.i32_and));65 // pointers are aligned with the abi alignment.
66 code.appendAssumeCapacity(@backingInt(std.wasm.Opcode.i32_and));
67 }
66 // The bottom will be used to calculate all stack pointer offsets.68 // The bottom will be used to calculate all stack pointer offsets.
67 code.appendAssumeCapacity(@backingInt(std.wasm.Opcode.local_tee));69 code.appendAssumeCapacity(@backingInt(std.wasm.Opcode.local_tee));
68 writeUleb128(code, mir.prologue.bottom_stack_local);70 writeUleb128(code, mir.prologue.bottom_stack_local);
...@@ -988,9 +990,7 @@ pub fn lower(emit: *Emit) Error!void {...@@ -988,9 +990,7 @@ pub fn lower(emit: *Emit) Error!void {
988/// Asserts 20 unused capacity.990/// Asserts 20 unused capacity.
989fn encodeMemArg(code: *ArrayList(u8), mem_arg: Mir.MemArg) void {991fn encodeMemArg(code: *ArrayList(u8), mem_arg: Mir.MemArg) void {
990 assert(code.unusedCapacitySlice().len >= 20);992 assert(code.unusedCapacitySlice().len >= 20);
991 // Wasm encodes alignment as power of 2, rather than natural alignment.993 writeUleb128(code, mem_arg.alignment.toLog2Units());
992 const encoded_alignment = @ctz(mem_arg.alignment);
993 writeUleb128(code, encoded_alignment);
994 writeUleb128(code, mem_arg.offset);994 writeUleb128(code, mem_arg.offset);
995}995}
996996
src/codegen/wasm/Mir.zig+9-2
...@@ -694,11 +694,18 @@ pub fn extraData(self: *const Mir, comptime T: type, index: usize) struct { data...@@ -694,11 +694,18 @@ pub fn extraData(self: *const Mir, comptime T: type, index: usize) struct { data
694 inline for (info.field_names, info.field_types) |field_name, field_type| {694 inline for (info.field_names, info.field_types) |field_name, field_type| {
695 @field(result, field_name) = switch (field_type) {695 @field(result, field_name) = switch (field_type) {
696 u32 => self.extra[i],696 u32 => self.extra[i],
697 u64 => value: {
698 const lo = @as(u64, self.extra[i]);
699 const hi = @as(u64, self.extra[i + 1]) << 32;
700 i += 1;
701 break :value hi | lo;
702 },
697 i32 => @bitCast(self.extra[i]),703 i32 => @bitCast(self.extra[i]),
698 Wasm.UavsObjIndex,704 Wasm.UavsObjIndex,
699 Wasm.UavsExeIndex,705 Wasm.UavsExeIndex,
700 InternPool.Nav.Index,706 InternPool.Nav.Index,
701 InternPool.Index,707 InternPool.Index,
708 Alignment,
702 => @fromBackingInt(@intCast(self.extra[i])),709 => @fromBackingInt(@intCast(self.extra[i])),
703 else => @compileError("Unsupported field type " ++ @typeName(field_type)),710 else => @compileError("Unsupported field type " ++ @typeName(field_type)),
704 };711 };
...@@ -747,8 +754,8 @@ pub const Float64 = struct {...@@ -747,8 +754,8 @@ pub const Float64 = struct {
747};754};
748755
749pub const MemArg = struct {756pub const MemArg = struct {
750 offset: u32,757 offset: u64,
751 alignment: u32,758 alignment: Alignment,
752};759};
753760
754pub const UavRefOff = struct {761pub const UavRefOff = struct {
src/codegen/wasm/assembly.zig+4-4
...@@ -679,20 +679,20 @@ fn parseMemArg(cg: *CodeGen, inst: []const u8, word_it: *mem.TokenIterator(u8, ....@@ -679,20 +679,20 @@ fn parseMemArg(cg: *CodeGen, inst: []const u8, word_it: *mem.TokenIterator(u8, .
679 const offset_str = oa[0];679 const offset_str = oa[0];
680 const alignment_str = oa[1];680 const alignment_str = oa[1];
681681
682 const offset = std.fmt.parseInt(i32, offset_str, 0) catch {682 const offset = std.fmt.parseInt(i64, offset_str, 0) catch {
683 return cg.fail("Malformed assembly, malformed offset \"{s}\"", .{offset_str});683 return cg.fail("Malformed assembly, malformed offset \"{s}\"", .{offset_str});
684 };684 };
685 const alignment_p2 = std.fmt.parseInt(u5, alignment_str, 0) catch {685 const alignment_p2 = std.fmt.parseInt(u5, alignment_str, 0) catch {
686 return cg.fail("Malformed assembly, malformed offset \"{s}\"", .{alignment_str});686 return cg.fail("Malformed assembly, malformed offset \"{s}\"", .{alignment_str});
687 };687 };
688688
689 return .{ .offset = @bitCast(offset), .alignment = @as(u32, 1) << alignment_p2 };689 return .{ .offset = @bitCast(offset), .alignment = .fromLog2Units(alignment_p2) };
690 } else {690 } else {
691 const offset_str = mem_arg_str;691 const offset_str = mem_arg_str;
692 const offset = std.fmt.parseInt(i32, offset_str, 0) catch {692 const offset = std.fmt.parseInt(i64, offset_str, 0) catch {
693 return cg.fail("Malformed assembly, malformed offset \"{s}\"", .{offset_str});693 return cg.fail("Malformed assembly, malformed offset \"{s}\"", .{offset_str});
694 };694 };
695 return .{ .offset = @bitCast(offset), .alignment = 1 };695 return .{ .offset = @bitCast(offset), .alignment = .@"1" };
696 }696 }
697}697}
698698