authorgravatar for luuk@degram.devLuuk de Gram <luuk@degram.dev> 2022-08-12 08:07:09+02:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2022-08-12 08:07:09+02:00
log645c396d02b7570d10d4d6b80653e7f9db42234e
treee057b4683d59b910a4cfd3c862e9cc8496facf80
parente67a43a673269edf477245931c70806f4b1abae1
parentb42ba7c3d411cde31ede290b3915150c3e8acfbb
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #12394 from Luukdegram/wasm-reuse-locals

stage2: wasm - reuse (temporary) locals

1 files changed, 418 insertions(+), 325 deletions(-)

src/arch/wasm/CodeGen.zig+418-325
......@@ -29,6 +29,8 @@ const errUnionErrorOffset = codegen.errUnionErrorOffset;
2929const WValue = union(enum) {
3030 /// May be referenced but is unused
3131 none: void,
32 /// The value lives on top of the stack
33 stack: void,
3234 /// Index of the local variable
3335 local: u32,
3436 /// An immediate 32bit value
......@@ -55,7 +57,7 @@ const WValue = union(enum) {
5557 /// In wasm function pointers are indexes into a function table,
5658 /// rather than an address in the data section.
5759 function_index: u32,
58 /// Offset from the bottom of the stack, with the offset
60 /// Offset from the bottom of the virtual stack, with the offset
5961 /// pointing to where the value lives.
6062 stack_offset: u32,
6163
......@@ -71,6 +73,38 @@ const WValue = union(enum) {
7173 else => return 0,
7274 }
7375 }
76
77 /// Promotes a `WValue` to a local when given value is on top of the stack.
78 /// When encountering a `local` or `stack_offset` this is essentially a no-op.
79 /// All other tags are illegal.
80 fn toLocal(value: WValue, gen: *Self, ty: Type) InnerError!WValue {
81 switch (value) {
82 .stack => {
83 const local = try gen.allocLocal(ty);
84 try gen.addLabel(.local_set, local.local);
85 return local;
86 },
87 .local, .stack_offset => return value,
88 else => unreachable,
89 }
90 }
91
92 /// Marks a local as no longer being referenced and essentially allows
93 /// us to re-use it somewhere else within the function.
94 /// The valtype of the local is deducted by using the index of the given.
95 fn free(value: *WValue, gen: *Self) void {
96 if (value.* != .local) return;
97 const local_value = value.local;
98 const index = local_value - gen.args.len - @boolToInt(gen.return_value != .none);
99 const valtype = @intToEnum(wasm.Valtype, gen.locals.items[index]);
100 switch (valtype) {
101 .i32 => gen.free_locals_i32.append(gen.gpa, local_value) catch return, // It's ok to fail any of those, a new local can be allocated instead
102 .i64 => gen.free_locals_i64.append(gen.gpa, local_value) catch return,
103 .f32 => gen.free_locals_f32.append(gen.gpa, local_value) catch return,
104 .f64 => gen.free_locals_f64.append(gen.gpa, local_value) catch return,
105 }
106 value.* = WValue{ .none = {} };
107 }
74108};
75109
76110/// Wasm ops, but without input/output/signedness information
......@@ -601,6 +635,21 @@ stack_size: u32 = 0,
601635/// However, local variables or the usage of `@setAlignStack` can overwrite this default.
602636stack_alignment: u32 = 16,
603637
638// For each individual Wasm valtype we store a seperate free list which
639// allows us to re-use locals that are no longer used. e.g. a temporary local.
640/// A list of indexes which represents a local of valtype `i32`.
641/// It is illegal to store a non-i32 valtype in this list.
642free_locals_i32: std.ArrayListUnmanaged(u32) = .{},
643/// A list of indexes which represents a local of valtype `i64`.
644/// It is illegal to store a non-i32 valtype in this list.
645free_locals_i64: std.ArrayListUnmanaged(u32) = .{},
646/// A list of indexes which represents a local of valtype `f32`.
647/// It is illegal to store a non-i32 valtype in this list.
648free_locals_f32: std.ArrayListUnmanaged(u32) = .{},
649/// A list of indexes which represents a local of valtype `f64`.
650/// It is illegal to store a non-i32 valtype in this list.
651free_locals_f64: std.ArrayListUnmanaged(u32) = .{},
652
604653const InnerError = error{
605654 OutOfMemory,
606655 /// An error occurred when trying to lower AIR to MIR.
......@@ -759,7 +808,7 @@ fn genBlockType(ty: Type, target: std.Target) u8 {
759808/// Writes the bytecode depending on the given `WValue` in `val`
760809fn emitWValue(self: *Self, value: WValue) InnerError!void {
761810 switch (value) {
762 .none => {}, // no-op
811 .none, .stack => {}, // no-op
763812 .local => |idx| try self.addLabel(.local_get, idx),
764813 .imm32 => |val| try self.addImm32(@bitCast(i32, val)),
765814 .imm64 => |val| try self.addImm64(val),
......@@ -781,9 +830,30 @@ fn emitWValue(self: *Self, value: WValue) InnerError!void {
781830/// Creates one locals for a given `Type`.
782831/// Returns a corresponding `Wvalue` with `local` as active tag
783832fn allocLocal(self: *Self, ty: Type) InnerError!WValue {
833 const valtype = typeToValtype(ty, self.target);
834 switch (valtype) {
835 .i32 => if (self.free_locals_i32.popOrNull()) |index| {
836 return WValue{ .local = index };
837 },
838 .i64 => if (self.free_locals_i64.popOrNull()) |index| {
839 return WValue{ .local = index };
840 },
841 .f32 => if (self.free_locals_f32.popOrNull()) |index| {
842 return WValue{ .local = index };
843 },
844 .f64 => if (self.free_locals_f64.popOrNull()) |index| {
845 return WValue{ .local = index };
846 },
847 }
848 // no local was free to be re-used, so allocate a new local instead
849 return self.ensureAllocLocal(ty);
850}
851
852/// Ensures a new local will be created. This is useful when it's useful
853/// to use a zero-initialized local.
854fn ensureAllocLocal(self: *Self, ty: Type) InnerError!WValue {
855 try self.locals.append(self.gpa, genValtype(ty, self.target));
784856 const initial_index = self.local_index;
785 const valtype = genValtype(ty, self.target);
786 try self.locals.append(self.gpa, valtype);
787857 self.local_index += 1;
788858 return WValue{ .local = initial_index };
789859}
......@@ -1135,9 +1205,9 @@ fn initializeStack(self: *Self) !void {
11351205 // Reserve a local to store the current stack pointer
11361206 // We can later use this local to set the stack pointer back to the value
11371207 // we have stored here.
1138 self.initial_stack_value = try self.allocLocal(Type.usize);
1208 self.initial_stack_value = try self.ensureAllocLocal(Type.usize);
11391209 // Also reserve a local to store the bottom stack value
1140 self.bottom_stack_value = try self.allocLocal(Type.usize);
1210 self.bottom_stack_value = try self.ensureAllocLocal(Type.usize);
11411211}
11421212
11431213/// Reads the stack pointer from `Context.initial_stack_value` and writes it
......@@ -1268,7 +1338,9 @@ fn memcpy(self: *Self, dst: WValue, src: WValue, len: WValue) !void {
12681338 else => {
12691339 // TODO: We should probably lower this to a call to compiler_rt
12701340 // But for now, we implement it manually
1271 const offset = try self.allocLocal(Type.usize); // local for counter
1341 var offset = try self.ensureAllocLocal(Type.usize); // local for counter
1342 defer offset.free(self);
1343
12721344 // outer block to jump to when loop is done
12731345 try self.startBlock(.block, wasm.block_empty);
12741346 try self.startBlock(.loop, wasm.block_empty);
......@@ -1405,7 +1477,7 @@ fn buildPointerOffset(self: *Self, ptr_value: WValue, offset: u64, action: enum
14051477 // do not perform arithmetic when offset is 0.
14061478 if (offset == 0 and ptr_value.offset() == 0 and action == .modify) return ptr_value;
14071479 const result_ptr: WValue = switch (action) {
1408 .new => try self.allocLocal(Type.usize),
1480 .new => try self.ensureAllocLocal(Type.usize),
14091481 .modify => ptr_value,
14101482 };
14111483 try self.emitWValue(ptr_value);
......@@ -1653,7 +1725,10 @@ fn genInst(self: *Self, inst: Air.Inst.Index) !WValue {
16531725fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
16541726 for (body) |inst| {
16551727 const result = try self.genInst(inst);
1656 try self.values.putNoClobber(self.gpa, Air.indexToRef(inst), result);
1728 if (result != .none) {
1729 assert(result != .stack); // not allowed to store stack values as we cannot keep track of where they are on the stack
1730 try self.values.putNoClobber(self.gpa, Air.indexToRef(inst), result);
1731 }
16571732 }
16581733}
16591734
......@@ -1727,8 +1802,8 @@ fn airRetLoad(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
17271802
17281803 const fn_info = self.decl.ty.fnInfo();
17291804 if (!firstParamSRet(fn_info.cc, fn_info.return_type, self.target)) {
1730 const result = try self.load(operand, ret_ty, 0);
1731 try self.emitWValue(result);
1805 // leave on the stack
1806 _ = try self.load(operand, ret_ty, 0);
17321807 }
17331808
17341809 try self.restoreStackPointer();
......@@ -1847,6 +1922,7 @@ fn airStore(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
18471922}
18481923
18491924fn store(self: *Self, lhs: WValue, rhs: WValue, ty: Type, offset: u32) InnerError!void {
1925 assert(!(lhs != .stack and rhs == .stack));
18501926 switch (ty.zigTypeTag()) {
18511927 .ErrorUnion => {
18521928 const pl_ty = ty.errorUnionPayload();
......@@ -1880,20 +1956,26 @@ fn store(self: *Self, lhs: WValue, rhs: WValue, ty: Type, offset: u32) InnerErro
18801956 .Pointer => {
18811957 if (ty.isSlice()) {
18821958 // store pointer first
1959 // lower it to the stack so we do not have to store rhs into a local first
1960 try self.emitWValue(lhs);
18831961 const ptr_local = try self.load(rhs, Type.usize, 0);
1884 try self.store(lhs, ptr_local, Type.usize, 0);
1962 try self.store(.{ .stack = {} }, ptr_local, Type.usize, 0 + lhs.offset());
18851963
18861964 // retrieve length from rhs, and store that alongside lhs as well
1965 try self.emitWValue(lhs);
18871966 const len_local = try self.load(rhs, Type.usize, self.ptrSize());
1888 try self.store(lhs, len_local, Type.usize, self.ptrSize());
1967 try self.store(.{ .stack = {} }, len_local, Type.usize, self.ptrSize() + lhs.offset());
18891968 return;
18901969 }
18911970 },
18921971 .Int => if (ty.intInfo(self.target).bits > 64) {
1972 try self.emitWValue(lhs);
18931973 const lsb = try self.load(rhs, Type.u64, 0);
1974 try self.store(.{ .stack = {} }, lsb, Type.u64, 0 + lhs.offset());
1975
1976 try self.emitWValue(lhs);
18941977 const msb = try self.load(rhs, Type.u64, 8);
1895 try self.store(lhs, lsb, Type.u64, 0);
1896 try self.store(lhs, msb, Type.u64, 8);
1978 try self.store(.{ .stack = {} }, msb, Type.u64, 8 + lhs.offset());
18971979 return;
18981980 },
18991981 else => {},
......@@ -1932,9 +2014,12 @@ fn airLoad(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
19322014 return new_local;
19332015 }
19342016
1935 return self.load(operand, ty, 0);
2017 const stack_loaded = try self.load(operand, ty, 0);
2018 return stack_loaded.toLocal(self, ty);
19362019}
19372020
2021/// Loads an operand from the linear memory section.
2022/// NOTE: Leaves the value on the stack.
19382023fn load(self: *Self, operand: WValue, ty: Type, offset: u32) InnerError!WValue {
19392024 // load local's value from memory by its stack position
19402025 try self.emitWValue(operand);
......@@ -1952,10 +2037,7 @@ fn load(self: *Self, operand: WValue, ty: Type, offset: u32) InnerError!WValue {
19522037 .{ .offset = offset + operand.offset(), .alignment = ty.abiAlignment(self.target) },
19532038 );
19542039
1955 // store the result in a local
1956 const result = try self.allocLocal(ty);
1957 try self.addLabel(.local_set, result.local);
1958 return result;
2040 return WValue{ .stack = {} };
19592041}
19602042
19612043fn airArg(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
......@@ -2025,10 +2107,14 @@ fn airBinOp(self: *Self, inst: Air.Inst.Index, op: Op) InnerError!WValue {
20252107 const rhs = try self.resolveInst(bin_op.rhs);
20262108 const ty = self.air.typeOf(bin_op.lhs);
20272109
2028 return self.binOp(lhs, rhs, ty, op);
2110 const stack_value = try self.binOp(lhs, rhs, ty, op);
2111 return stack_value.toLocal(self, ty);
20292112}
20302113
2114/// Performs a binary operation on the given `WValue`'s
2115/// NOTE: THis leaves the value on top of the stack.
20312116fn binOp(self: *Self, lhs: WValue, rhs: WValue, ty: Type, op: Op) InnerError!WValue {
2117 assert(!(lhs != .stack and rhs == .stack));
20322118 if (isByRef(ty, self.target)) {
20332119 if (ty.zigTypeTag() == .Int) {
20342120 return self.binOpBigInt(lhs, rhs, ty, op);
......@@ -2054,24 +2140,18 @@ fn binOp(self: *Self, lhs: WValue, rhs: WValue, ty: Type, op: Op) InnerError!WVa
20542140
20552141 try self.addTag(Mir.Inst.Tag.fromOpcode(opcode));
20562142
2057 // save the result in a temporary
2058 const bin_local = try self.allocLocal(ty);
2059 try self.addLabel(.local_set, bin_local.local);
2060 return bin_local;
2143 return WValue{ .stack = {} };
20612144}
20622145
2146/// Performs a binary operation for 16-bit floats.
2147/// NOTE: Leaves the result value on the stack
20632148fn binOpFloat16(self: *Self, lhs: WValue, rhs: WValue, op: Op) InnerError!WValue {
2064 const ext_lhs = try self.fpext(lhs, Type.f16, Type.f32);
2065 const ext_rhs = try self.fpext(rhs, Type.f16, Type.f32);
2066
20672149 const opcode: wasm.Opcode = buildOpcode(.{ .op = op, .valtype1 = .f32, .signedness = .unsigned });
2068 try self.emitWValue(ext_lhs);
2069 try self.emitWValue(ext_rhs);
2150 _ = try self.fpext(lhs, Type.f16, Type.f32);
2151 _ = try self.fpext(rhs, Type.f16, Type.f32);
20702152 try self.addTag(Mir.Inst.Tag.fromOpcode(opcode));
20712153
2072 // re-use temporary local
2073 try self.addLabel(.local_set, ext_lhs.local);
2074 return self.fptrunc(ext_lhs, Type.f32, Type.f16);
2154 return self.fptrunc(.{ .stack = {} }, Type.f32, Type.f16);
20752155}
20762156
20772157fn binOpBigInt(self: *Self, lhs: WValue, rhs: WValue, ty: Type, op: Op) InnerError!WValue {
......@@ -2084,13 +2164,16 @@ fn binOpBigInt(self: *Self, lhs: WValue, rhs: WValue, ty: Type, op: Op) InnerErr
20842164 }
20852165
20862166 const result = try self.allocStack(ty);
2087 const lhs_high_bit = try self.load(lhs, Type.u64, 0);
2167 var lhs_high_bit = try (try self.load(lhs, Type.u64, 0)).toLocal(self, Type.u64);
2168 defer lhs_high_bit.free(self);
2169 var rhs_high_bit = try (try self.load(rhs, Type.u64, 0)).toLocal(self, Type.u64);
2170 defer rhs_high_bit.free(self);
2171 var high_op_res = try (try self.binOp(lhs_high_bit, rhs_high_bit, Type.u64, op)).toLocal(self, Type.u64);
2172 defer high_op_res.free(self);
2173
20882174 const lhs_low_bit = try self.load(lhs, Type.u64, 8);
2089 const rhs_high_bit = try self.load(rhs, Type.u64, 0);
20902175 const rhs_low_bit = try self.load(rhs, Type.u64, 8);
2091
20922176 const low_op_res = try self.binOp(lhs_low_bit, rhs_low_bit, Type.u64, op);
2093 const high_op_res = try self.binOp(lhs_high_bit, rhs_high_bit, Type.u64, op);
20942177
20952178 const lt = if (op == .add) blk: {
20962179 break :blk try self.cmp(high_op_res, rhs_high_bit, Type.u64, .lt);
......@@ -2098,7 +2181,8 @@ fn binOpBigInt(self: *Self, lhs: WValue, rhs: WValue, ty: Type, op: Op) InnerErr
20982181 break :blk try self.cmp(lhs_high_bit, rhs_high_bit, Type.u64, .lt);
20992182 } else unreachable;
21002183 const tmp = try self.intcast(lt, Type.u32, Type.u64);
2101 const tmp_op = try self.binOp(low_op_res, tmp, Type.u64, op);
2184 var tmp_op = try (try self.binOp(low_op_res, tmp, Type.u64, op)).toLocal(self, Type.u64);
2185 defer tmp_op.free(self);
21022186
21032187 try self.store(result, high_op_res, Type.u64, 0);
21042188 try self.store(result, tmp_op, Type.u64, 8);
......@@ -2115,40 +2199,22 @@ fn airWrapBinOp(self: *Self, inst: Air.Inst.Index, op: Op) InnerError!WValue {
21152199 return self.fail("TODO: Implement wrapping arithmetic for vectors", .{});
21162200 }
21172201
2118 return self.wrapBinOp(lhs, rhs, ty, op);
2202 return (try self.wrapBinOp(lhs, rhs, ty, op)).toLocal(self, ty);
21192203}
21202204
2205/// Performs a wrapping binary operation.
2206/// Asserts rhs is not a stack value when lhs also isn't.
2207/// NOTE: Leaves the result on the stack when its Type is <= 64 bits
21212208fn wrapBinOp(self: *Self, lhs: WValue, rhs: WValue, ty: Type, op: Op) InnerError!WValue {
2122 const bit_size = ty.intInfo(self.target).bits;
2123 var wasm_bits = toWasmBits(bit_size) orelse {
2124 return self.fail("TODO: Implement wrapping arithmetic for integers with bitsize: {d}\n", .{bit_size});
2125 };
2126
2127 if (wasm_bits == 128) {
2128 const bin_op = try self.binOpBigInt(lhs, rhs, ty, op);
2129 return self.wrapOperand(bin_op, ty);
2130 }
2131
2132 const opcode: wasm.Opcode = buildOpcode(.{
2133 .op = op,
2134 .valtype1 = typeToValtype(ty, self.target),
2135 .signedness = if (ty.isSignedInt()) .signed else .unsigned,
2136 });
2137
2138 try self.emitWValue(lhs);
2139 try self.emitWValue(rhs);
2140 try self.addTag(Mir.Inst.Tag.fromOpcode(opcode));
2141 const bin_local = try self.allocLocal(ty);
2142 try self.addLabel(.local_set, bin_local.local);
2143
2209 const bin_local = try self.binOp(lhs, rhs, ty, op);
21442210 return self.wrapOperand(bin_local, ty);
21452211}
21462212
21472213/// Wraps an operand based on a given type's bitsize.
21482214/// Asserts `Type` is <= 128 bits.
2215/// NOTE: When the Type is <= 64 bits, leaves the value on top of the stack.
21492216fn wrapOperand(self: *Self, operand: WValue, ty: Type) InnerError!WValue {
21502217 assert(ty.abiSize(self.target) <= 16);
2151 const result_local = try self.allocLocal(ty);
21522218 const bitsize = ty.intInfo(self.target).bits;
21532219 const wasm_bits = toWasmBits(bitsize) orelse {
21542220 return self.fail("TODO: Implement wrapOperand for bitsize '{d}'", .{bitsize});
......@@ -2157,14 +2223,15 @@ fn wrapOperand(self: *Self, operand: WValue, ty: Type) InnerError!WValue {
21572223 if (wasm_bits == bitsize) return operand;
21582224
21592225 if (wasm_bits == 128) {
2160 const msb = try self.load(operand, Type.u64, 0);
2226 assert(operand != .stack);
21612227 const lsb = try self.load(operand, Type.u64, 8);
21622228
21632229 const result_ptr = try self.allocStack(ty);
2164 try self.store(result_ptr, lsb, Type.u64, 8);
2230 try self.emitWValue(result_ptr);
2231 try self.store(.{ .stack = {} }, lsb, Type.u64, 8 + result_ptr.offset());
21652232 const result = (@as(u64, 1) << @intCast(u6, 64 - (wasm_bits - bitsize))) - 1;
21662233 try self.emitWValue(result_ptr);
2167 try self.emitWValue(msb);
2234 _ = try self.load(operand, Type.u64, 0);
21682235 try self.addImm64(result);
21692236 try self.addTag(.i64_and);
21702237 try self.addMemArg(.i64_store, .{ .offset = result_ptr.offset(), .alignment = 8 });
......@@ -2181,8 +2248,7 @@ fn wrapOperand(self: *Self, operand: WValue, ty: Type) InnerError!WValue {
21812248 try self.addTag(.i64_and);
21822249 } else unreachable;
21832250
2184 try self.addLabel(.local_set, result_local.local);
2185 return result_local;
2251 return WValue{ .stack = {} };
21862252}
21872253
21882254fn lowerParentPtr(self: *Self, ptr_val: Value, ptr_child_ty: Type) InnerError!WValue {
......@@ -2594,10 +2660,14 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: std.math.CompareOperator) Inner
25942660 const lhs = try self.resolveInst(bin_op.lhs);
25952661 const rhs = try self.resolveInst(bin_op.rhs);
25962662 const operand_ty = self.air.typeOf(bin_op.lhs);
2597 return self.cmp(lhs, rhs, operand_ty, op);
2663 return (try self.cmp(lhs, rhs, operand_ty, op)).toLocal(self, Type.u32); // comparison result is always 32 bits
25982664}
25992665
2666/// Compares two operands.
2667/// Asserts rhs is not a stack value when the lhs isn't a stack value either
2668/// NOTE: This leaves the result on top of the stack, rather than a new local.
26002669fn cmp(self: *Self, lhs: WValue, rhs: WValue, ty: Type, op: std.math.CompareOperator) InnerError!WValue {
2670 assert(!(lhs != .stack and rhs == .stack));
26012671 if (ty.zigTypeTag() == .Optional and !ty.optionalReprIsPayload()) {
26022672 var buf: Type.Payload.ElemType = undefined;
26032673 const payload_ty = ty.optionalChild(&buf);
......@@ -2639,15 +2709,12 @@ fn cmp(self: *Self, lhs: WValue, rhs: WValue, ty: Type, op: std.math.CompareOper
26392709 });
26402710 try self.addTag(Mir.Inst.Tag.fromOpcode(opcode));
26412711
2642 const cmp_tmp = try self.allocLocal(Type.initTag(.i32)); // bool is always i32
2643 try self.addLabel(.local_set, cmp_tmp.local);
2644 return cmp_tmp;
2712 return WValue{ .stack = {} };
26452713}
26462714
2715/// Compares 16-bit floats
2716/// NOTE: The result value remains on top of the stack.
26472717fn cmpFloat16(self: *Self, lhs: WValue, rhs: WValue, op: std.math.CompareOperator) InnerError!WValue {
2648 const ext_lhs = try self.fpext(lhs, Type.f16, Type.f32);
2649 const ext_rhs = try self.fpext(rhs, Type.f16, Type.f32);
2650
26512718 const opcode: wasm.Opcode = buildOpcode(.{
26522719 .op = switch (op) {
26532720 .lt => .lt,
......@@ -2660,13 +2727,11 @@ fn cmpFloat16(self: *Self, lhs: WValue, rhs: WValue, op: std.math.CompareOperato
26602727 .valtype1 = .f32,
26612728 .signedness = .unsigned,
26622729 });
2663 try self.emitWValue(ext_lhs);
2664 try self.emitWValue(ext_rhs);
2730 _ = try self.fpext(lhs, Type.f16, Type.f32);
2731 _ = try self.fpext(rhs, Type.f16, Type.f32);
26652732 try self.addTag(Mir.Inst.Tag.fromOpcode(opcode));
26662733
2667 const result = try self.allocLocal(Type.initTag(.i32)); // bool is always i32
2668 try self.addLabel(.local_set, result.local);
2669 return result;
2734 return WValue{ .stack = {} };
26702735}
26712736
26722737fn airCmpVector(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
......@@ -2727,21 +2792,23 @@ fn airNot(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
27272792 switch (wasm_bits) {
27282793 32 => {
27292794 const bin_op = try self.binOp(operand, .{ .imm32 = ~@as(u32, 0) }, operand_ty, .xor);
2730 return self.wrapOperand(bin_op, operand_ty);
2795 return (try self.wrapOperand(bin_op, operand_ty)).toLocal(self, operand_ty);
27312796 },
27322797 64 => {
27332798 const bin_op = try self.binOp(operand, .{ .imm64 = ~@as(u64, 0) }, operand_ty, .xor);
2734 return self.wrapOperand(bin_op, operand_ty);
2799 return (try self.wrapOperand(bin_op, operand_ty)).toLocal(self, operand_ty);
27352800 },
27362801 128 => {
27372802 const result_ptr = try self.allocStack(operand_ty);
2803 try self.emitWValue(result_ptr);
27382804 const msb = try self.load(operand, Type.u64, 0);
2739 const lsb = try self.load(operand, Type.u64, 8);
2740
27412805 const msb_xor = try self.binOp(msb, .{ .imm64 = ~@as(u64, 0) }, Type.u64, .xor);
2806 try self.store(.{ .stack = {} }, msb_xor, Type.u64, 0 + result_ptr.offset());
2807
2808 try self.emitWValue(result_ptr);
2809 const lsb = try self.load(operand, Type.u64, 8);
27422810 const lsb_xor = try self.binOp(lsb, .{ .imm64 = ~@as(u64, 0) }, Type.u64, .xor);
2743 try self.store(result_ptr, msb_xor, Type.u64, 0);
2744 try self.store(result_ptr, lsb_xor, Type.u64, 8);
2811 try self.store(result_ptr, lsb_xor, Type.u64, 8 + result_ptr.offset());
27452812 return result_ptr;
27462813 },
27472814 else => unreachable,
......@@ -2829,7 +2896,8 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
28292896 }
28302897 }
28312898
2832 return self.load(operand, field_ty, offset);
2899 const field = try self.load(operand, field_ty, offset);
2900 return field.toLocal(self, field_ty);
28332901}
28342902
28352903fn airSwitchBr(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
......@@ -3039,7 +3107,9 @@ fn airUnwrapErrUnionPayload(self: *Self, inst: Air.Inst.Index, op_is_ptr: bool)
30393107 if (op_is_ptr or isByRef(payload_ty, self.target)) {
30403108 return self.buildPointerOffset(operand, pl_offset, .new);
30413109 }
3042 return self.load(operand, payload_ty, pl_offset);
3110
3111 const payload = try self.load(operand, payload_ty, pl_offset);
3112 return payload.toLocal(self, payload_ty);
30433113}
30443114
30453115fn airUnwrapErrUnionError(self: *Self, inst: Air.Inst.Index, op_is_ptr: bool) InnerError!WValue {
......@@ -3059,7 +3129,8 @@ fn airUnwrapErrUnionError(self: *Self, inst: Air.Inst.Index, op_is_ptr: bool) In
30593129 return operand;
30603130 }
30613131
3062 return self.load(operand, Type.anyerror, @intCast(u32, errUnionErrorOffset(payload_ty, self.target)));
3132 const error_val = try self.load(operand, Type.anyerror, @intCast(u32, errUnionErrorOffset(payload_ty, self.target)));
3133 return error_val.toLocal(self, Type.anyerror);
30633134}
30643135
30653136fn airWrapErrUnionPayload(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
......@@ -3125,12 +3196,13 @@ fn airIntcast(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
31253196 return self.fail("todo Wasm intcast for bitsize > 128", .{});
31263197 }
31273198
3128 return self.intcast(operand, operand_ty, ty);
3199 return (try self.intcast(operand, operand_ty, ty)).toLocal(self, ty);
31293200}
31303201
31313202/// Upcasts or downcasts an integer based on the given and wanted types,
31323203/// and stores the result in a new operand.
31333204/// Asserts type's bitsize <= 128
3205/// NOTE: May leave the result on the top of the stack.
31343206fn intcast(self: *Self, operand: WValue, given: Type, wanted: Type) InnerError!WValue {
31353207 const given_info = given.intInfo(self.target);
31363208 const wanted_info = wanted.intInfo(self.target);
......@@ -3153,25 +3225,22 @@ fn intcast(self: *Self, operand: WValue, given: Type, wanted: Type) InnerError!W
31533225 } else if (wanted_bits == 128) {
31543226 // for 128bit integers we store the integer in the virtual stack, rather than a local
31553227 const stack_ptr = try self.allocStack(wanted);
3228 try self.emitWValue(stack_ptr);
31563229
31573230 // for 32 bit integers, we first coerce the value into a 64 bit integer before storing it
31583231 // meaning less store operations are required.
31593232 const lhs = if (op_bits == 32) blk: {
3160 const tmp = try self.intcast(
3161 operand,
3162 given,
3163 if (wanted.isSignedInt()) Type.i64 else Type.u64,
3164 );
3165 break :blk tmp;
3233 break :blk try self.intcast(operand, given, if (wanted.isSignedInt()) Type.i64 else Type.u64);
31663234 } else operand;
31673235
31683236 // store msb first
3169 try self.store(stack_ptr, lhs, Type.u64, 0);
3237 try self.store(.{ .stack = {} }, lhs, Type.u64, 0 + stack_ptr.offset());
31703238
31713239 // For signed integers we shift msb by 63 (64bit integer - 1 sign bit) and store remaining value
31723240 if (wanted.isSignedInt()) {
3241 try self.emitWValue(stack_ptr);
31733242 const shr = try self.binOp(lhs, .{ .imm64 = 63 }, Type.i64, .shr);
3174 try self.store(stack_ptr, shr, Type.u64, 8);
3243 try self.store(.{ .stack = {} }, shr, Type.u64, 8 + stack_ptr.offset());
31753244 } else {
31763245 // Ensure memory of lsb is zero'd
31773246 try self.store(stack_ptr, .{ .imm64 = 0 }, Type.u64, 8);
......@@ -3179,9 +3248,7 @@ fn intcast(self: *Self, operand: WValue, given: Type, wanted: Type) InnerError!W
31793248 return stack_ptr;
31803249 } else return self.load(operand, wanted, 0);
31813250
3182 const result = try self.allocLocal(wanted);
3183 try self.addLabel(.local_set, result.local);
3184 return result;
3251 return WValue{ .stack = {} };
31853252}
31863253
31873254fn airIsNull(self: *Self, inst: Air.Inst.Index, opcode: wasm.Opcode, op_kind: enum { value, ptr }) InnerError!WValue {
......@@ -3190,9 +3257,12 @@ fn airIsNull(self: *Self, inst: Air.Inst.Index, opcode: wasm.Opcode, op_kind: en
31903257
31913258 const op_ty = self.air.typeOf(un_op);
31923259 const optional_ty = if (op_kind == .ptr) op_ty.childType() else op_ty;
3193 return self.isNull(operand, optional_ty, opcode);
3260 const is_null = try self.isNull(operand, optional_ty, opcode);
3261 return is_null.toLocal(self, optional_ty);
31943262}
31953263
3264/// For a given type and operand, checks if it's considered `null`.
3265/// NOTE: Leaves the result on the stack
31963266fn isNull(self: *Self, operand: WValue, optional_ty: Type, opcode: wasm.Opcode) InnerError!WValue {
31973267 try self.emitWValue(operand);
31983268 if (!optional_ty.optionalReprIsPayload()) {
......@@ -3209,9 +3279,7 @@ fn isNull(self: *Self, operand: WValue, optional_ty: Type, opcode: wasm.Opcode)
32093279 try self.addImm32(0);
32103280 try self.addTag(Mir.Inst.Tag.fromOpcode(opcode));
32113281
3212 const is_null_tmp = try self.allocLocal(Type.initTag(.i32));
3213 try self.addLabel(.local_set, is_null_tmp.local);
3214 return is_null_tmp;
3282 return WValue{ .stack = {} };
32153283}
32163284
32173285fn airOptionalPayload(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
......@@ -3229,7 +3297,8 @@ fn airOptionalPayload(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
32293297 return self.buildPointerOffset(operand, offset, .new);
32303298 }
32313299
3232 return self.load(operand, payload_ty, @intCast(u32, offset));
3300 const payload = try self.load(operand, payload_ty, @intCast(u32, offset));
3301 return payload.toLocal(self, payload_ty);
32333302}
32343303
32353304fn airOptionalPayloadPtr(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
......@@ -3332,7 +3401,8 @@ fn airSliceLen(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
33323401 const ty_op = self.air.instructions.items(.data)[inst].ty_op;
33333402 const operand = try self.resolveInst(ty_op.operand);
33343403
3335 return self.load(operand, Type.usize, self.ptrSize());
3404 const len = try self.load(operand, Type.usize, self.ptrSize());
3405 return len.toLocal(self, Type.usize);
33363406}
33373407
33383408fn airSliceElemVal(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
......@@ -3346,8 +3416,7 @@ fn airSliceElemVal(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
33463416 const elem_size = elem_ty.abiSize(self.target);
33473417
33483418 // load pointer onto stack
3349 const slice_ptr = try self.load(slice, Type.usize, 0);
3350 try self.addLabel(.local_get, slice_ptr.local);
3419 _ = try self.load(slice, Type.usize, 0);
33513420
33523421 // calculate index into slice
33533422 try self.emitWValue(index);
......@@ -3361,7 +3430,9 @@ fn airSliceElemVal(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
33613430 if (isByRef(elem_ty, self.target)) {
33623431 return result;
33633432 }
3364 return self.load(result, elem_ty, 0);
3433
3434 const elem_val = try self.load(result, elem_ty, 0);
3435 return elem_val.toLocal(self, elem_ty);
33653436}
33663437
33673438fn airSliceElemPtr(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
......@@ -3374,8 +3445,7 @@ fn airSliceElemPtr(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
33743445 const slice = try self.resolveInst(bin_op.lhs);
33753446 const index = try self.resolveInst(bin_op.rhs);
33763447
3377 const slice_ptr = try self.load(slice, Type.usize, 0);
3378 try self.addLabel(.local_get, slice_ptr.local);
3448 _ = try self.load(slice, Type.usize, 0);
33793449
33803450 // calculate index into slice
33813451 try self.emitWValue(index);
......@@ -3383,7 +3453,7 @@ fn airSliceElemPtr(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
33833453 try self.addTag(.i32_mul);
33843454 try self.addTag(.i32_add);
33853455
3386 const result = try self.allocLocal(Type.initTag(.i32));
3456 const result = try self.allocLocal(Type.i32);
33873457 try self.addLabel(.local_set, result.local);
33883458 return result;
33893459}
......@@ -3392,7 +3462,8 @@ fn airSlicePtr(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
33923462 if (self.liveness.isUnused(inst)) return WValue{ .none = {} };
33933463 const ty_op = self.air.instructions.items(.data)[inst].ty_op;
33943464 const operand = try self.resolveInst(ty_op.operand);
3395 return self.load(operand, Type.usize, 0);
3465 const ptr = try self.load(operand, Type.usize, 0);
3466 return ptr.toLocal(self, Type.usize);
33963467}
33973468
33983469fn airTrunc(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
......@@ -3407,13 +3478,13 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
34073478 return self.fail("TODO: Implement wasm integer truncation for integer bitsize: {d}", .{int_info.bits});
34083479 }
34093480
3410 const result = try self.intcast(operand, op_ty, wanted_ty);
3481 var result = try self.intcast(operand, op_ty, wanted_ty);
34113482 const wanted_bits = wanted_ty.intInfo(self.target).bits;
34123483 const wasm_bits = toWasmBits(wanted_bits).?;
34133484 if (wasm_bits != wanted_bits) {
3414 return self.wrapOperand(result, wanted_ty);
3485 result = try self.wrapOperand(result, wanted_ty);
34153486 }
3416 return result;
3487 return result.toLocal(self, wanted_ty);
34173488}
34183489
34193490fn airBoolToInt(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
......@@ -3466,8 +3537,7 @@ fn airPtrElemVal(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
34663537
34673538 // load pointer onto the stack
34683539 if (ptr_ty.isSlice()) {
3469 const ptr_local = try self.load(ptr, Type.usize, 0);
3470 try self.addLabel(.local_get, ptr_local.local);
3540 _ = try self.load(ptr, Type.usize, 0);
34713541 } else {
34723542 try self.lowerToStack(ptr);
34733543 }
......@@ -3478,12 +3548,15 @@ fn airPtrElemVal(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
34783548 try self.addTag(.i32_mul);
34793549 try self.addTag(.i32_add);
34803550
3481 const result = try self.allocLocal(elem_ty);
3551 var result = try self.allocLocal(elem_ty);
34823552 try self.addLabel(.local_set, result.local);
34833553 if (isByRef(elem_ty, self.target)) {
34843554 return result;
34853555 }
3486 return self.load(result, elem_ty, 0);
3556 defer result.free(self); // only free if it's not returned like above
3557
3558 const elem_val = try self.load(result, elem_ty, 0);
3559 return elem_val.toLocal(self, elem_ty);
34873560}
34883561
34893562fn airPtrElemPtr(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
......@@ -3499,8 +3572,7 @@ fn airPtrElemPtr(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
34993572
35003573 // load pointer onto the stack
35013574 if (ptr_ty.isSlice()) {
3502 const ptr_local = try self.load(ptr, Type.usize, 0);
3503 try self.addLabel(.local_get, ptr_local.local);
3575 _ = try self.load(ptr, Type.usize, 0);
35043576 } else {
35053577 try self.lowerToStack(ptr);
35063578 }
......@@ -3511,7 +3583,7 @@ fn airPtrElemPtr(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
35113583 try self.addTag(.i32_mul);
35123584 try self.addTag(.i32_add);
35133585
3514 const result = try self.allocLocal(Type.initTag(.i32));
3586 const result = try self.allocLocal(Type.i32);
35153587 try self.addLabel(.local_set, result.local);
35163588 return result;
35173589}
......@@ -3599,7 +3671,7 @@ fn memset(self: *Self, ptr: WValue, len: WValue, value: WValue) InnerError!void
35993671 else => {
36003672 // TODO: We should probably lower this to a call to compiler_rt
36013673 // But for now, we implement it manually
3602 const offset = try self.allocLocal(Type.usize); // local for counter
3674 const offset = try self.ensureAllocLocal(Type.usize); // local for counter
36033675 // outer block to jump to when loop is done
36043676 try self.startBlock(.block, wasm.block_empty);
36053677 try self.startBlock(.loop, wasm.block_empty);
......@@ -3656,13 +3728,16 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
36563728 try self.addTag(.i32_mul);
36573729 try self.addTag(.i32_add);
36583730
3659 const result = try self.allocLocal(Type.usize);
3731 var result = try self.allocLocal(Type.usize);
36603732 try self.addLabel(.local_set, result.local);
36613733
36623734 if (isByRef(elem_ty, self.target)) {
36633735 return result;
36643736 }
3665 return self.load(result, elem_ty, 0);
3737 defer result.free(self); // only free if no longer needed and not returned like above
3738
3739 const elem_val = try self.load(result, elem_ty, 0);
3740 return elem_val.toLocal(self, elem_ty);
36663741}
36673742
36683743fn airFloatToInt(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
......@@ -3685,11 +3760,8 @@ fn airFloatToInt(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
36853760 .signedness = if (dest_ty.isSignedInt()) .signed else .unsigned,
36863761 });
36873762 try self.addTag(Mir.Inst.Tag.fromOpcode(op));
3688
3689 const result = try self.allocLocal(dest_ty);
3690 try self.addLabel(.local_set, result.local);
3691
3692 return self.wrapOperand(result, dest_ty);
3763 const wrapped = try self.wrapOperand(.{ .stack = {} }, dest_ty);
3764 return wrapped.toLocal(self, dest_ty);
36933765}
36943766
36953767fn airIntToFloat(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
......@@ -3887,24 +3959,19 @@ fn cmpOptionals(self: *Self, lhs: WValue, rhs: WValue, operand_ty: Type, op: std
38873959 const payload_ty = operand_ty.optionalChild(&buf);
38883960 const offset = @intCast(u32, operand_ty.abiSize(self.target) - payload_ty.abiSize(self.target));
38893961
3890 const lhs_is_null = try self.isNull(lhs, operand_ty, .i32_eq);
3891 const rhs_is_null = try self.isNull(rhs, operand_ty, .i32_eq);
3892
38933962 // We store the final result in here that will be validated
38943963 // if the optional is truly equal.
3895 const result = try self.allocLocal(Type.initTag(.i32));
3964 var result = try self.ensureAllocLocal(Type.initTag(.i32));
3965 defer result.free(self);
38963966
38973967 try self.startBlock(.block, wasm.block_empty);
3898 try self.emitWValue(lhs_is_null);
3899 try self.emitWValue(rhs_is_null);
3968 _ = try self.isNull(lhs, operand_ty, .i32_eq);
3969 _ = try self.isNull(rhs, operand_ty, .i32_eq);
39003970 try self.addTag(.i32_ne); // inverse so we can exit early
39013971 try self.addLabel(.br_if, 0);
39023972
3903 const lhs_pl = try self.load(lhs, payload_ty, offset);
3904 const rhs_pl = try self.load(rhs, payload_ty, offset);
3905
3906 try self.emitWValue(lhs_pl);
3907 try self.emitWValue(rhs_pl);
3973 _ = try self.load(lhs, payload_ty, offset);
3974 _ = try self.load(rhs, payload_ty, offset);
39083975 const opcode = buildOpcode(.{ .op = .ne, .valtype1 = typeToValtype(payload_ty, self.target) });
39093976 try self.addTag(Mir.Inst.Tag.fromOpcode(opcode));
39103977 try self.addLabel(.br_if, 0);
......@@ -3916,26 +3983,29 @@ fn cmpOptionals(self: *Self, lhs: WValue, rhs: WValue, operand_ty: Type, op: std
39163983 try self.emitWValue(result);
39173984 try self.addImm32(0);
39183985 try self.addTag(if (op == .eq) .i32_ne else .i32_eq);
3919 try self.addLabel(.local_set, result.local);
3920 return result;
3986 return WValue{ .stack = {} };
39213987}
39223988
39233989/// Compares big integers by checking both its high bits and low bits.
3990/// NOTE: Leaves the result of the comparison on top of the stack.
39243991/// TODO: Lower this to compiler_rt call when bitsize > 128
39253992fn cmpBigInt(self: *Self, lhs: WValue, rhs: WValue, operand_ty: Type, op: std.math.CompareOperator) InnerError!WValue {
39263993 assert(operand_ty.abiSize(self.target) >= 16);
3994 assert(!(lhs != .stack and rhs == .stack));
39273995 if (operand_ty.intInfo(self.target).bits > 128) {
39283996 return self.fail("TODO: Support cmpBigInt for integer bitsize: '{d}'", .{operand_ty.intInfo(self.target).bits});
39293997 }
39303998
3931 const lhs_high_bit = try self.load(lhs, Type.u64, 0);
3932 const lhs_low_bit = try self.load(lhs, Type.u64, 8);
3933 const rhs_high_bit = try self.load(rhs, Type.u64, 0);
3934 const rhs_low_bit = try self.load(rhs, Type.u64, 8);
3999 var lhs_high_bit = try (try self.load(lhs, Type.u64, 0)).toLocal(self, Type.u64);
4000 defer lhs_high_bit.free(self);
4001 var rhs_high_bit = try (try self.load(rhs, Type.u64, 0)).toLocal(self, Type.u64);
4002 defer rhs_high_bit.free(self);
39354003
39364004 switch (op) {
39374005 .eq, .neq => {
39384006 const xor_high = try self.binOp(lhs_high_bit, rhs_high_bit, Type.u64, .xor);
4007 const lhs_low_bit = try self.load(lhs, Type.u64, 8);
4008 const rhs_low_bit = try self.load(rhs, Type.u64, 8);
39394009 const xor_low = try self.binOp(lhs_low_bit, rhs_low_bit, Type.u64, .xor);
39404010 const or_result = try self.binOp(xor_high, xor_low, Type.u64, .@"or");
39414011
......@@ -3947,20 +4017,17 @@ fn cmpBigInt(self: *Self, lhs: WValue, rhs: WValue, operand_ty: Type, op: std.ma
39474017 },
39484018 else => {
39494019 const ty = if (operand_ty.isSignedInt()) Type.i64 else Type.u64;
3950 const high_bit_eql = try self.cmp(lhs_high_bit, rhs_high_bit, ty, .eq);
3951 const high_bit_cmp = try self.cmp(lhs_high_bit, rhs_high_bit, ty, op);
3952 const low_bit_cmp = try self.cmp(lhs_low_bit, rhs_low_bit, ty, op);
3953
3954 try self.emitWValue(low_bit_cmp);
3955 try self.emitWValue(high_bit_cmp);
3956 try self.emitWValue(high_bit_eql);
4020 // leave those value on top of the stack for '.select'
4021 const lhs_low_bit = try self.load(lhs, Type.u64, 8);
4022 const rhs_low_bit = try self.load(rhs, Type.u64, 8);
4023 _ = try self.cmp(lhs_low_bit, rhs_low_bit, ty, op);
4024 _ = try self.cmp(lhs_high_bit, rhs_high_bit, ty, op);
4025 _ = try self.cmp(lhs_high_bit, rhs_high_bit, ty, .eq);
39574026 try self.addTag(.select);
39584027 },
39594028 }
39604029
3961 const result = try self.allocLocal(Type.initTag(.i32));
3962 try self.addLabel(.local_set, result.local);
3963 return result;
4030 return WValue{ .stack = {} };
39644031}
39654032
39664033fn airSetUnionTag(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
......@@ -4000,7 +4067,8 @@ fn airGetUnionTag(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
40004067 const offset = if (layout.tag_align < layout.payload_align) blk: {
40014068 break :blk @intCast(u32, layout.payload_size);
40024069 } else @as(u32, 0);
4003 return self.load(operand, tag_ty, offset);
4070 const tag = try self.load(operand, tag_ty, offset);
4071 return tag.toLocal(self, tag_ty);
40044072}
40054073
40064074fn airFpext(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
......@@ -4010,19 +4078,20 @@ fn airFpext(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
40104078 const dest_ty = self.air.typeOfIndex(inst);
40114079 const operand = try self.resolveInst(ty_op.operand);
40124080
4013 return self.fpext(operand, self.air.typeOf(ty_op.operand), dest_ty);
4081 const extended = try self.fpext(operand, self.air.typeOf(ty_op.operand), dest_ty);
4082 return extended.toLocal(self, dest_ty);
40144083}
40154084
4085/// Extends a float from a given `Type` to a larger wanted `Type`
4086/// NOTE: Leaves the result on the stack
40164087fn fpext(self: *Self, operand: WValue, given: Type, wanted: Type) InnerError!WValue {
40174088 const given_bits = given.floatBits(self.target);
40184089 const wanted_bits = wanted.floatBits(self.target);
40194090
40204091 if (wanted_bits == 64 and given_bits == 32) {
4021 const result = try self.allocLocal(wanted);
40224092 try self.emitWValue(operand);
40234093 try self.addTag(.f64_promote_f32);
4024 try self.addLabel(.local_set, result.local);
4025 return result;
4094 return WValue{ .stack = {} };
40264095 } else if (given_bits == 16) {
40274096 // call __extendhfsf2(f16) f32
40284097 const f32_result = try self.callIntrinsic(
......@@ -4036,11 +4105,9 @@ fn fpext(self: *Self, operand: WValue, given: Type, wanted: Type) InnerError!WVa
40364105 return f32_result;
40374106 }
40384107 if (wanted_bits == 64) {
4039 const result = try self.allocLocal(wanted);
40404108 try self.emitWValue(f32_result);
40414109 try self.addTag(.f64_promote_f32);
4042 try self.addLabel(.local_set, result.local);
4043 return result;
4110 return WValue{ .stack = {} };
40444111 }
40454112 return self.fail("TODO: Implement 'fpext' for floats with bitsize: {d}", .{wanted_bits});
40464113 } else {
......@@ -4055,26 +4122,25 @@ fn airFptrunc(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
40554122 const ty_op = self.air.instructions.items(.data)[inst].ty_op;
40564123 const dest_ty = self.air.typeOfIndex(inst);
40574124 const operand = try self.resolveInst(ty_op.operand);
4058 return self.fptrunc(operand, self.air.typeOf(ty_op.operand), dest_ty);
4125 const trunc = try self.fptrunc(operand, self.air.typeOf(ty_op.operand), dest_ty);
4126 return trunc.toLocal(self, dest_ty);
40594127}
40604128
4129/// Truncates a float from a given `Type` to its wanted `Type`
4130/// NOTE: The result value remains on the stack
40614131fn fptrunc(self: *Self, operand: WValue, given: Type, wanted: Type) InnerError!WValue {
40624132 const given_bits = given.floatBits(self.target);
40634133 const wanted_bits = wanted.floatBits(self.target);
40644134
40654135 if (wanted_bits == 32 and given_bits == 64) {
4066 const result = try self.allocLocal(wanted);
40674136 try self.emitWValue(operand);
40684137 try self.addTag(.f32_demote_f64);
4069 try self.addLabel(.local_set, result.local);
4070 return result;
4138 return WValue{ .stack = {} };
40714139 } else if (wanted_bits == 16) {
40724140 const op: WValue = if (given_bits == 64) blk: {
4073 const tmp = try self.allocLocal(Type.f32);
40744141 try self.emitWValue(operand);
40754142 try self.addTag(.f32_demote_f64);
4076 try self.addLabel(.local_set, tmp.local);
4077 break :blk tmp;
4143 break :blk WValue{ .stack = {} };
40784144 } else operand;
40794145
40804146 // call __truncsfhf2(f32) f16
......@@ -4159,12 +4225,9 @@ fn airPopcount(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
41594225
41604226 switch (wasm_bits) {
41614227 128 => {
4162 const msb = try self.load(operand, Type.u64, 0);
4163 const lsb = try self.load(operand, Type.u64, 8);
4164
4165 try self.emitWValue(msb);
4228 _ = try self.load(operand, Type.u64, 0);
41664229 try self.addTag(.i64_popcnt);
4167 try self.emitWValue(lsb);
4230 _ = try self.load(operand, Type.u64, 8);
41684231 try self.addTag(.i64_popcnt);
41694232 try self.addTag(.i64_add);
41704233 try self.addTag(.i32_wrap_i64);
......@@ -4268,24 +4331,26 @@ fn airAddSubWithOverflow(self: *Self, inst: Air.Inst.Index, op: Op) InnerError!W
42684331
42694332 // for signed integers, we first apply signed shifts by the difference in bits
42704333 // to get the signed value, as we store it internally as 2's complement.
4271 const lhs = if (wasm_bits != int_info.bits and is_signed) blk: {
4272 break :blk try self.signAbsValue(lhs_op, lhs_ty);
4334 var lhs = if (wasm_bits != int_info.bits and is_signed) blk: {
4335 break :blk try (try self.signAbsValue(lhs_op, lhs_ty)).toLocal(self, lhs_ty);
42734336 } else lhs_op;
4274 const rhs = if (wasm_bits != int_info.bits and is_signed) blk: {
4275 break :blk try self.signAbsValue(rhs_op, lhs_ty);
4337 var rhs = if (wasm_bits != int_info.bits and is_signed) blk: {
4338 break :blk try (try self.signAbsValue(rhs_op, lhs_ty)).toLocal(self, lhs_ty);
42764339 } else rhs_op;
42774340
4278 const bin_op = try self.binOp(lhs, rhs, lhs_ty, op);
4279 const result = if (wasm_bits != int_info.bits) blk: {
4280 break :blk try self.wrapOperand(bin_op, lhs_ty);
4341 var bin_op = try (try self.binOp(lhs, rhs, lhs_ty, op)).toLocal(self, lhs_ty);
4342 defer bin_op.free(self);
4343 var result = if (wasm_bits != int_info.bits) blk: {
4344 break :blk try (try self.wrapOperand(bin_op, lhs_ty)).toLocal(self, lhs_ty);
42814345 } else bin_op;
4346 defer result.free(self); // no-op when wasm_bits == int_info.bits
42824347
42834348 const cmp_op: std.math.CompareOperator = if (op == .sub) .gt else .lt;
42844349 const overflow_bit: WValue = if (is_signed) blk: {
42854350 if (wasm_bits == int_info.bits) {
42864351 const cmp_zero = try self.cmp(rhs, zero, lhs_ty, cmp_op);
42874352 const lt = try self.cmp(bin_op, lhs, lhs_ty, .lt);
4288 break :blk try self.binOp(cmp_zero, lt, Type.u32, .xor); // result of cmp_zero and lt is always 32bit
4353 break :blk try self.binOp(cmp_zero, lt, Type.u32, .xor);
42894354 }
42904355 const abs = try self.signAbsValue(bin_op, lhs_ty);
42914356 break :blk try self.cmp(abs, bin_op, lhs_ty, .neq);
......@@ -4293,11 +4358,22 @@ fn airAddSubWithOverflow(self: *Self, inst: Air.Inst.Index, op: Op) InnerError!W
42934358 try self.cmp(bin_op, lhs, lhs_ty, cmp_op)
42944359 else
42954360 try self.cmp(bin_op, result, lhs_ty, .neq);
4361 var overflow_local = try overflow_bit.toLocal(self, Type.u32);
4362 defer overflow_local.free(self);
42964363
42974364 const result_ptr = try self.allocStack(self.air.typeOfIndex(inst));
42984365 try self.store(result_ptr, result, lhs_ty, 0);
42994366 const offset = @intCast(u32, lhs_ty.abiSize(self.target));
4300 try self.store(result_ptr, overflow_bit, Type.initTag(.u1), offset);
4367 try self.store(result_ptr, overflow_local, Type.initTag(.u1), offset);
4368
4369 // in this case, we performed a signAbsValue which created a temporary local
4370 // so let's free this so it can be re-used instead.
4371 // In the other case we do not want to free it, because that would free the
4372 // resolved instructions which may be referenced by other instructions.
4373 if (wasm_bits != int_info.bits and is_signed) {
4374 lhs.free(self);
4375 rhs.free(self);
4376 }
43014377
43024378 return result_ptr;
43034379}
......@@ -4310,52 +4386,58 @@ fn airAddSubWithOverflowBigInt(self: *Self, lhs: WValue, rhs: WValue, ty: Type,
43104386 return self.fail("TODO: Implement @{{add/sub}}WithOverflow for integer bitsize '{d}'", .{int_info.bits});
43114387 }
43124388
4313 const lhs_high_bit = try self.load(lhs, Type.u64, 0);
4314 const lhs_low_bit = try self.load(lhs, Type.u64, 8);
4315 const rhs_high_bit = try self.load(rhs, Type.u64, 0);
4316 const rhs_low_bit = try self.load(rhs, Type.u64, 8);
4389 var lhs_high_bit = try (try self.load(lhs, Type.u64, 0)).toLocal(self, Type.u64);
4390 defer lhs_high_bit.free(self);
4391 var lhs_low_bit = try (try self.load(lhs, Type.u64, 8)).toLocal(self, Type.u64);
4392 defer lhs_low_bit.free(self);
4393 var rhs_high_bit = try (try self.load(rhs, Type.u64, 0)).toLocal(self, Type.u64);
4394 defer rhs_high_bit.free(self);
4395 var rhs_low_bit = try (try self.load(rhs, Type.u64, 8)).toLocal(self, Type.u64);
4396 defer rhs_low_bit.free(self);
43174397
4318 const low_op_res = try self.binOp(lhs_low_bit, rhs_low_bit, Type.u64, op);
4319 const high_op_res = try self.binOp(lhs_high_bit, rhs_high_bit, Type.u64, op);
4398 var low_op_res = try (try self.binOp(lhs_low_bit, rhs_low_bit, Type.u64, op)).toLocal(self, Type.u64);
4399 defer low_op_res.free(self);
4400 var high_op_res = try (try self.binOp(lhs_high_bit, rhs_high_bit, Type.u64, op)).toLocal(self, Type.u64);
4401 defer high_op_res.free(self);
43204402
4321 const lt = if (op == .add) blk: {
4322 break :blk try self.cmp(high_op_res, lhs_high_bit, Type.u64, .lt);
4403 var lt = if (op == .add) blk: {
4404 break :blk try (try self.cmp(high_op_res, lhs_high_bit, Type.u64, .lt)).toLocal(self, Type.u32);
43234405 } else if (op == .sub) blk: {
4324 break :blk try self.cmp(lhs_high_bit, rhs_high_bit, Type.u64, .lt);
4406 break :blk try (try self.cmp(lhs_high_bit, rhs_high_bit, Type.u64, .lt)).toLocal(self, Type.u32);
43254407 } else unreachable;
4326 const tmp = try self.intcast(lt, Type.u32, Type.u64);
4327 const tmp_op = try self.binOp(low_op_res, tmp, Type.u64, op);
4408 defer lt.free(self);
4409 var tmp = try (try self.intcast(lt, Type.u32, Type.u64)).toLocal(self, Type.u64);
4410 defer tmp.free(self);
4411 var tmp_op = try (try self.binOp(low_op_res, tmp, Type.u64, op)).toLocal(self, Type.u64);
4412 defer tmp_op.free(self);
43284413
43294414 const overflow_bit = if (is_signed) blk: {
4330 const xor_op = try self.binOp(lhs_low_bit, tmp_op, Type.u64, .xor);
43314415 const xor_low = try self.binOp(lhs_low_bit, rhs_low_bit, Type.u64, .xor);
43324416 const to_wrap = if (op == .add) wrap: {
43334417 break :wrap try self.binOp(xor_low, .{ .imm64 = ~@as(u64, 0) }, Type.u64, .xor);
43344418 } else xor_low;
4419 const xor_op = try self.binOp(lhs_low_bit, tmp_op, Type.u64, .xor);
43354420 const wrap = try self.binOp(to_wrap, xor_op, Type.u64, .@"and");
43364421 break :blk try self.cmp(wrap, .{ .imm64 = 0 }, Type.i64, .lt); // i64 because signed
43374422 } else blk: {
4338 const eq = try self.cmp(tmp_op, lhs_low_bit, Type.u64, .eq);
4339 const op_eq = try self.cmp(tmp_op, lhs_low_bit, Type.u64, if (op == .add) .lt else .gt);
4340
43414423 const first_arg = if (op == .sub) arg: {
43424424 break :arg try self.cmp(high_op_res, lhs_high_bit, Type.u64, .gt);
43434425 } else lt;
43444426
43454427 try self.emitWValue(first_arg);
4346 try self.emitWValue(op_eq);
4347 try self.emitWValue(eq);
4428 _ = try self.cmp(tmp_op, lhs_low_bit, Type.u64, if (op == .add) .lt else .gt);
4429 _ = try self.cmp(tmp_op, lhs_low_bit, Type.u64, .eq);
43484430 try self.addTag(.select);
43494431
4350 const overflow_bit = try self.allocLocal(Type.initTag(.u1));
4351 try self.addLabel(.local_set, overflow_bit.local);
4352 break :blk overflow_bit;
4432 break :blk WValue{ .stack = {} };
43534433 };
4434 var overflow_local = try overflow_bit.toLocal(self, Type.initTag(.u1));
4435 defer overflow_local.free(self);
43544436
43554437 const result_ptr = try self.allocStack(result_ty);
43564438 try self.store(result_ptr, high_op_res, Type.u64, 0);
43574439 try self.store(result_ptr, tmp_op, Type.u64, 8);
4358 try self.store(result_ptr, overflow_bit, Type.initTag(.u1), 16);
4440 try self.store(result_ptr, overflow_local, Type.initTag(.u1), 16);
43594441
43604442 return result_ptr;
43614443}
......@@ -4377,24 +4459,31 @@ fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
43774459 return self.fail("TODO: Implement shl_with_overflow for integer bitsize: {d}", .{int_info.bits});
43784460 };
43794461
4380 const shl = try self.binOp(lhs, rhs, lhs_ty, .shl);
4381 const result = if (wasm_bits != int_info.bits) blk: {
4382 break :blk try self.wrapOperand(shl, lhs_ty);
4462 var shl = try (try self.binOp(lhs, rhs, lhs_ty, .shl)).toLocal(self, lhs_ty);
4463 defer shl.free(self);
4464 var result = if (wasm_bits != int_info.bits) blk: {
4465 break :blk try (try self.wrapOperand(shl, lhs_ty)).toLocal(self, lhs_ty);
43834466 } else shl;
4467 defer result.free(self); // it's a no-op to free the same local twice (when wasm_bits == int_info.bits)
43844468
43854469 const overflow_bit = if (wasm_bits != int_info.bits and is_signed) blk: {
4470 // emit lhs to stack to we can keep 'wrapped' on the stack also
4471 try self.emitWValue(lhs);
43864472 const abs = try self.signAbsValue(shl, lhs_ty);
43874473 const wrapped = try self.wrapBinOp(abs, rhs, lhs_ty, .shr);
4388 break :blk try self.cmp(lhs, wrapped, lhs_ty, .neq);
4474 break :blk try self.cmp(.{ .stack = {} }, wrapped, lhs_ty, .neq);
43894475 } else blk: {
4476 try self.emitWValue(lhs);
43904477 const shr = try self.binOp(result, rhs, lhs_ty, .shr);
4391 break :blk try self.cmp(lhs, shr, lhs_ty, .neq);
4478 break :blk try self.cmp(.{ .stack = {} }, shr, lhs_ty, .neq);
43924479 };
4480 var overflow_local = try overflow_bit.toLocal(self, Type.initTag(.u1));
4481 defer overflow_local.free(self);
43934482
43944483 const result_ptr = try self.allocStack(self.air.typeOfIndex(inst));
43954484 try self.store(result_ptr, result, lhs_ty, 0);
43964485 const offset = @intCast(u32, lhs_ty.abiSize(self.target));
4397 try self.store(result_ptr, overflow_bit, Type.initTag(.u1), offset);
4486 try self.store(result_ptr, overflow_local, Type.initTag(.u1), offset);
43984487
43994488 return result_ptr;
44004489}
......@@ -4412,7 +4501,9 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
44124501
44134502 // We store the bit if it's overflowed or not in this. As it's zero-initialized
44144503 // we only need to update it if an overflow (or underflow) occurred.
4415 const overflow_bit = try self.allocLocal(Type.initTag(.u1));
4504 var overflow_bit = try self.ensureAllocLocal(Type.initTag(.u1));
4505 defer overflow_bit.free(self);
4506
44164507 const int_info = lhs_ty.intInfo(self.target);
44174508 const wasm_bits = toWasmBits(int_info.bits) orelse {
44184509 return self.fail("TODO: Implement overflow arithmetic for integer bitsize: {d}", .{int_info.bits});
......@@ -4433,49 +4524,49 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
44334524 const new_ty = if (int_info.signedness == .signed) Type.i64 else Type.u64;
44344525 const lhs_upcast = try self.intcast(lhs, lhs_ty, new_ty);
44354526 const rhs_upcast = try self.intcast(rhs, lhs_ty, new_ty);
4436 const bin_op = try self.binOp(lhs_upcast, rhs_upcast, new_ty, .mul);
4527 const bin_op = try (try self.binOp(lhs_upcast, rhs_upcast, new_ty, .mul)).toLocal(self, new_ty);
44374528 if (int_info.signedness == .unsigned) {
44384529 const shr = try self.binOp(bin_op, .{ .imm64 = int_info.bits }, new_ty, .shr);
44394530 const wrap = try self.intcast(shr, new_ty, lhs_ty);
4440 const cmp_res = try self.cmp(wrap, zero, lhs_ty, .neq);
4441 try self.emitWValue(cmp_res);
4531 _ = try self.cmp(wrap, zero, lhs_ty, .neq);
44424532 try self.addLabel(.local_set, overflow_bit.local);
44434533 break :blk try self.intcast(bin_op, new_ty, lhs_ty);
44444534 } else {
4445 const down_cast = try self.intcast(bin_op, new_ty, lhs_ty);
4446 const shr = try self.binOp(down_cast, .{ .imm32 = int_info.bits - 1 }, lhs_ty, .shr);
4535 const down_cast = try (try self.intcast(bin_op, new_ty, lhs_ty)).toLocal(self, lhs_ty);
4536 var shr = try (try self.binOp(down_cast, .{ .imm32 = int_info.bits - 1 }, lhs_ty, .shr)).toLocal(self, lhs_ty);
4537 defer shr.free(self);
44474538
44484539 const shr_res = try self.binOp(bin_op, .{ .imm64 = int_info.bits }, new_ty, .shr);
44494540 const down_shr_res = try self.intcast(shr_res, new_ty, lhs_ty);
4450 const cmp_res = try self.cmp(down_shr_res, shr, lhs_ty, .neq);
4451 try self.emitWValue(cmp_res);
4541 _ = try self.cmp(down_shr_res, shr, lhs_ty, .neq);
44524542 try self.addLabel(.local_set, overflow_bit.local);
44534543 break :blk down_cast;
44544544 }
44554545 } else if (int_info.signedness == .signed) blk: {
44564546 const lhs_abs = try self.signAbsValue(lhs, lhs_ty);
44574547 const rhs_abs = try self.signAbsValue(rhs, lhs_ty);
4458 const bin_op = try self.binOp(lhs_abs, rhs_abs, lhs_ty, .mul);
4548 const bin_op = try (try self.binOp(lhs_abs, rhs_abs, lhs_ty, .mul)).toLocal(self, lhs_ty);
44594549 const mul_abs = try self.signAbsValue(bin_op, lhs_ty);
4460 const cmp_op = try self.cmp(mul_abs, bin_op, lhs_ty, .neq);
4461 try self.emitWValue(cmp_op);
4550 _ = try self.cmp(mul_abs, bin_op, lhs_ty, .neq);
44624551 try self.addLabel(.local_set, overflow_bit.local);
44634552 break :blk try self.wrapOperand(bin_op, lhs_ty);
44644553 } else blk: {
4465 const bin_op = try self.binOp(lhs, rhs, lhs_ty, .mul);
4554 var bin_op = try (try self.binOp(lhs, rhs, lhs_ty, .mul)).toLocal(self, lhs_ty);
4555 defer bin_op.free(self);
44664556 const shift_imm = if (wasm_bits == 32)
44674557 WValue{ .imm32 = int_info.bits }
44684558 else
44694559 WValue{ .imm64 = int_info.bits };
44704560 const shr = try self.binOp(bin_op, shift_imm, lhs_ty, .shr);
4471 const cmp_op = try self.cmp(shr, zero, lhs_ty, .neq);
4472 try self.emitWValue(cmp_op);
4561 _ = try self.cmp(shr, zero, lhs_ty, .neq);
44734562 try self.addLabel(.local_set, overflow_bit.local);
44744563 break :blk try self.wrapOperand(bin_op, lhs_ty);
44754564 };
4565 var bin_op_local = try bin_op.toLocal(self, lhs_ty);
4566 defer bin_op_local.free(self);
44764567
44774568 const result_ptr = try self.allocStack(self.air.typeOfIndex(inst));
4478 try self.store(result_ptr, bin_op, lhs_ty, 0);
4569 try self.store(result_ptr, bin_op_local, lhs_ty, 0);
44794570 const offset = @intCast(u32, lhs_ty.abiSize(self.target));
44804571 try self.store(result_ptr, overflow_bit, Type.initTag(.u1), offset);
44814572
......@@ -4497,12 +4588,10 @@ fn airMaxMin(self: *Self, inst: Air.Inst.Index, op: enum { max, min }) InnerErro
44974588 const lhs = try self.resolveInst(bin_op.lhs);
44984589 const rhs = try self.resolveInst(bin_op.rhs);
44994590
4500 const cmp_result = try self.cmp(lhs, rhs, ty, if (op == .max) .gt else .lt);
4501
45024591 // operands to select from
45034592 try self.lowerToStack(lhs);
45044593 try self.lowerToStack(rhs);
4505 try self.emitWValue(cmp_result);
4594 _ = try self.cmp(lhs, rhs, ty, if (op == .max) .gt else .lt);
45064595
45074596 // based on the result from comparison, return operand 0 or 1.
45084597 try self.addTag(.select);
......@@ -4528,21 +4617,22 @@ fn airMulAdd(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
45284617 const rhs = try self.resolveInst(bin_op.rhs);
45294618
45304619 if (ty.floatBits(self.target) == 16) {
4531 const addend_ext = try self.fpext(addend, ty, Type.f32);
4532 const lhs_ext = try self.fpext(lhs, ty, Type.f32);
45334620 const rhs_ext = try self.fpext(rhs, ty, Type.f32);
4621 const lhs_ext = try self.fpext(lhs, ty, Type.f32);
4622 const addend_ext = try self.fpext(addend, ty, Type.f32);
45344623 // call to compiler-rt `fn fmaf(f32, f32, f32) f32`
4535 const result = try self.callIntrinsic(
4624 var result = try self.callIntrinsic(
45364625 "fmaf",
45374626 &.{ Type.f32, Type.f32, Type.f32 },
45384627 Type.f32,
45394628 &.{ rhs_ext, lhs_ext, addend_ext },
45404629 );
4541 return try self.fptrunc(result, Type.f32, ty);
4630 defer result.free(self);
4631 return try (try self.fptrunc(result, Type.f32, ty)).toLocal(self, ty);
45424632 }
45434633
45444634 const mul_result = try self.binOp(lhs, rhs, ty, .mul);
4545 return self.binOp(mul_result, addend, ty, .add);
4635 return (try self.binOp(mul_result, addend, ty, .add)).toLocal(self, ty);
45464636}
45474637
45484638fn airClz(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
......@@ -4571,17 +4661,16 @@ fn airClz(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
45714661 try self.addTag(.i32_wrap_i64);
45724662 },
45734663 128 => {
4574 const msb = try self.load(operand, Type.u64, 0);
4575 const lsb = try self.load(operand, Type.u64, 8);
4576 const neq = try self.cmp(lsb, .{ .imm64 = 0 }, Type.u64, .neq);
4664 var lsb = try (try self.load(operand, Type.u64, 8)).toLocal(self, Type.u64);
4665 defer lsb.free(self);
45774666
45784667 try self.emitWValue(lsb);
45794668 try self.addTag(.i64_clz);
4580 try self.emitWValue(msb);
4669 _ = try self.load(operand, Type.u64, 0);
45814670 try self.addTag(.i64_clz);
45824671 try self.emitWValue(.{ .imm64 = 64 });
45834672 try self.addTag(.i64_add);
4584 try self.emitWValue(neq);
4673 _ = try self.cmp(lsb, .{ .imm64 = 0 }, Type.u64, .neq);
45854674 try self.addTag(.select);
45864675 try self.addTag(.i32_wrap_i64);
45874676 },
......@@ -4618,28 +4707,27 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
46184707 32 => {
46194708 if (wasm_bits != int_info.bits) {
46204709 const val: u32 = @as(u32, 1) << @intCast(u5, int_info.bits);
4621 const bin_op = try self.binOp(operand, .{ .imm32 = val }, ty, .@"or");
4622 try self.emitWValue(bin_op);
4710 // leave value on the stack
4711 _ = try self.binOp(operand, .{ .imm32 = val }, ty, .@"or");
46234712 } else try self.emitWValue(operand);
46244713 try self.addTag(.i32_ctz);
46254714 },
46264715 64 => {
46274716 if (wasm_bits != int_info.bits) {
46284717 const val: u64 = @as(u64, 1) << @intCast(u6, int_info.bits);
4629 const bin_op = try self.binOp(operand, .{ .imm64 = val }, ty, .@"or");
4630 try self.emitWValue(bin_op);
4718 // leave value on the stack
4719 _ = try self.binOp(operand, .{ .imm64 = val }, ty, .@"or");
46314720 } else try self.emitWValue(operand);
46324721 try self.addTag(.i64_ctz);
46334722 try self.addTag(.i32_wrap_i64);
46344723 },
46354724 128 => {
4636 const msb = try self.load(operand, Type.u64, 0);
4637 const lsb = try self.load(operand, Type.u64, 8);
4638 const neq = try self.cmp(msb, .{ .imm64 = 0 }, Type.u64, .neq);
4725 var msb = try (try self.load(operand, Type.u64, 0)).toLocal(self, Type.u64);
4726 defer msb.free(self);
46394727
46404728 try self.emitWValue(msb);
46414729 try self.addTag(.i64_ctz);
4642 try self.emitWValue(lsb);
4730 _ = try self.load(operand, Type.u64, 8);
46434731 if (wasm_bits != int_info.bits) {
46444732 try self.addImm64(@as(u64, 1) << @intCast(u6, int_info.bits - 64));
46454733 try self.addTag(.i64_or);
......@@ -4651,7 +4739,7 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
46514739 } else {
46524740 try self.addTag(.i64_add);
46534741 }
4654 try self.emitWValue(neq);
4742 _ = try self.cmp(msb, .{ .imm64 = 0 }, Type.u64, .neq);
46554743 try self.addTag(.select);
46564744 try self.addTag(.i32_wrap_i64);
46574745 },
......@@ -4777,7 +4865,8 @@ fn lowerTry(
47774865 if (isByRef(pl_ty, self.target)) {
47784866 return buildPointerOffset(self, err_union, pl_offset, .new);
47794867 }
4780 return self.load(err_union, pl_ty, pl_offset);
4868 const payload = try self.load(err_union, pl_ty, pl_offset);
4869 return payload.toLocal(self, pl_ty);
47814870}
47824871
47834872fn airByteSwap(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
......@@ -4807,11 +4896,11 @@ fn airByteSwap(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
48074896 const res = if (int_info.signedness == .signed) blk: {
48084897 break :blk try self.wrapOperand(shr_res, Type.u8);
48094898 } else shr_res;
4810 return self.binOp(lhs, res, ty, .@"or");
4899 return (try self.binOp(lhs, res, ty, .@"or")).toLocal(self, ty);
48114900 },
48124901 24 => {
4813 const msb = try self.wrapOperand(operand, Type.u16);
4814 const lsb = try self.wrapBinOp(operand, .{ .imm32 = 16 }, Type.u8, .shr);
4902 var msb = try (try self.wrapOperand(operand, Type.u16)).toLocal(self, Type.u16);
4903 defer msb.free(self);
48154904
48164905 const shl_res = try self.binOp(msb, .{ .imm32 = 8 }, Type.u16, .shl);
48174906 const lhs = try self.binOp(shl_res, .{ .imm32 = 0xFF0000 }, Type.u16, .@"and");
......@@ -4825,22 +4914,26 @@ fn airByteSwap(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
48254914 const rhs_wrap = try self.wrapOperand(msb, Type.u8);
48264915 const rhs_result = try self.binOp(rhs_wrap, .{ .imm32 = 16 }, ty, .shl);
48274916
4917 const lsb = try self.wrapBinOp(operand, .{ .imm32 = 16 }, Type.u8, .shr);
48284918 const tmp = try self.binOp(lhs_result, rhs_result, ty, .@"or");
4829 return self.binOp(tmp, lsb, ty, .@"or");
4919 return (try self.binOp(tmp, lsb, ty, .@"or")).toLocal(self, ty);
48304920 },
48314921 32 => {
48324922 const shl_tmp = try self.binOp(operand, .{ .imm32 = 8 }, ty, .shl);
4833 const lhs = try self.binOp(shl_tmp, .{ .imm32 = 0xFF00FF00 }, ty, .@"and");
4923 var lhs = try (try self.binOp(shl_tmp, .{ .imm32 = 0xFF00FF00 }, ty, .@"and")).toLocal(self, ty);
4924 defer lhs.free(self);
48344925 const shr_tmp = try self.binOp(operand, .{ .imm32 = 8 }, ty, .shr);
4835 const rhs = try self.binOp(shr_tmp, .{ .imm32 = 0xFF00FF }, ty, .@"and");
4836 const tmp_or = try self.binOp(lhs, rhs, ty, .@"or");
4926 var rhs = try (try self.binOp(shr_tmp, .{ .imm32 = 0xFF00FF }, ty, .@"and")).toLocal(self, ty);
4927 defer rhs.free(self);
4928 var tmp_or = try (try self.binOp(lhs, rhs, ty, .@"or")).toLocal(self, ty);
4929 defer tmp_or.free(self);
48374930
48384931 const shl = try self.binOp(tmp_or, .{ .imm32 = 16 }, ty, .shl);
48394932 const shr = try self.binOp(tmp_or, .{ .imm32 = 16 }, ty, .shr);
48404933 const res = if (int_info.signedness == .signed) blk: {
48414934 break :blk try self.wrapOperand(shr, Type.u16);
48424935 } else shr;
4843 return self.binOp(shl, res, ty, .@"or");
4936 return (try self.binOp(shl, res, ty, .@"or")).toLocal(self, ty);
48444937 },
48454938 else => return self.fail("TODO: @byteSwap for integers with bitsize {d}", .{int_info.bits}),
48464939 }
......@@ -4857,7 +4950,7 @@ fn airDiv(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
48574950 if (ty.isSignedInt()) {
48584951 return self.divSigned(lhs, rhs, ty);
48594952 }
4860 return self.binOp(lhs, rhs, ty, .div);
4953 return (try self.binOp(lhs, rhs, ty, .div)).toLocal(self, ty);
48614954}
48624955
48634956fn airDivFloor(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
......@@ -4869,33 +4962,31 @@ fn airDivFloor(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
48694962 const rhs = try self.resolveInst(bin_op.rhs);
48704963
48714964 if (ty.isUnsignedInt()) {
4872 return self.binOp(lhs, rhs, ty, .div);
4965 return (try self.binOp(lhs, rhs, ty, .div)).toLocal(self, ty);
48734966 } else if (ty.isSignedInt()) {
48744967 const int_bits = ty.intInfo(self.target).bits;
48754968 const wasm_bits = toWasmBits(int_bits) orelse {
48764969 return self.fail("TODO: `@divFloor` for signed integers larger than '{d}' bits", .{int_bits});
48774970 };
48784971 const lhs_res = if (wasm_bits != int_bits) blk: {
4879 break :blk try self.signAbsValue(lhs, ty);
4972 break :blk try (try self.signAbsValue(lhs, ty)).toLocal(self, ty);
48804973 } else lhs;
48814974 const rhs_res = if (wasm_bits != int_bits) blk: {
4882 break :blk try self.signAbsValue(rhs, ty);
4975 break :blk try (try self.signAbsValue(rhs, ty)).toLocal(self, ty);
48834976 } else rhs;
48844977
4885 const div_result = try self.binOp(lhs_res, rhs_res, ty, .div);
4886 const rem_result = try self.binOp(lhs_res, rhs_res, ty, .rem);
4887
48884978 const zero = switch (wasm_bits) {
48894979 32 => WValue{ .imm32 = 0 },
48904980 64 => WValue{ .imm64 = 0 },
48914981 else => unreachable,
48924982 };
4893 const lhs_less_than_zero = try self.cmp(lhs_res, zero, ty, .lt);
4894 const rhs_less_than_zero = try self.cmp(rhs_res, zero, ty, .lt);
48954983
4896 try self.emitWValue(div_result);
4897 try self.emitWValue(lhs_less_than_zero);
4898 try self.emitWValue(rhs_less_than_zero);
4984 const div_result = try self.allocLocal(ty);
4985 // leave on stack
4986 _ = try self.binOp(lhs_res, rhs_res, ty, .div);
4987 try self.addLabel(.local_tee, div_result.local);
4988 _ = try self.cmp(lhs_res, zero, ty, .lt);
4989 _ = try self.cmp(rhs_res, zero, ty, .lt);
48994990 switch (wasm_bits) {
49004991 32 => {
49014992 try self.addTag(.i32_xor);
......@@ -4908,7 +4999,8 @@ fn airDivFloor(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
49084999 else => unreachable,
49095000 }
49105001 try self.emitWValue(div_result);
4911 try self.emitWValue(rem_result);
5002 // leave value on the stack
5003 _ = try self.binOp(lhs_res, rhs_res, ty, .rem);
49125004 try self.addTag(.select);
49135005 } else {
49145006 const float_bits = ty.floatBits(self.target);
......@@ -4940,9 +5032,7 @@ fn airDivFloor(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
49405032 }
49415033
49425034 if (is_f16) {
4943 // we can re-use temporary local
4944 try self.addLabel(.local_set, lhs_operand.local);
4945 return self.fptrunc(lhs_operand, Type.f32, Type.f16);
5035 _ = try self.fptrunc(.{ .stack = {} }, Type.f32, Type.f16);
49465036 }
49475037 }
49485038
......@@ -4962,10 +5052,9 @@ fn divSigned(self: *Self, lhs: WValue, rhs: WValue, ty: Type) InnerError!WValue
49625052 }
49635053
49645054 if (wasm_bits != int_bits) {
4965 const lhs_abs = try self.signAbsValue(lhs, ty);
4966 const rhs_abs = try self.signAbsValue(rhs, ty);
4967 try self.emitWValue(lhs_abs);
4968 try self.emitWValue(rhs_abs);
5055 // Leave both values on the stack
5056 _ = try self.signAbsValue(lhs, ty);
5057 _ = try self.signAbsValue(rhs, ty);
49695058 } else {
49705059 try self.emitWValue(lhs);
49715060 try self.emitWValue(rhs);
......@@ -4977,6 +5066,8 @@ fn divSigned(self: *Self, lhs: WValue, rhs: WValue, ty: Type) InnerError!WValue
49775066 return result;
49785067}
49795068
5069/// Retrieves the absolute value of a signed integer
5070/// NOTE: Leaves the result value on the stack.
49805071fn signAbsValue(self: *Self, operand: WValue, ty: Type) InnerError!WValue {
49815072 const int_bits = ty.intInfo(self.target).bits;
49825073 const wasm_bits = toWasmBits(int_bits) orelse {
......@@ -5005,9 +5096,8 @@ fn signAbsValue(self: *Self, operand: WValue, ty: Type) InnerError!WValue {
50055096 },
50065097 else => unreachable,
50075098 }
5008 const result = try self.allocLocal(ty);
5009 try self.addLabel(.local_set, result.local);
5010 return result;
5099
5100 return WValue{ .stack = {} };
50115101}
50125102
50135103fn airCeilFloorTrunc(self: *Self, inst: Air.Inst.Index, op: Op) InnerError!WValue {
......@@ -5034,9 +5124,7 @@ fn airCeilFloorTrunc(self: *Self, inst: Air.Inst.Index, op: Op) InnerError!WValu
50345124 try self.addTag(Mir.Inst.Tag.fromOpcode(opcode));
50355125
50365126 if (is_f16) {
5037 // re-use temporary to save locals
5038 try self.addLabel(.local_set, op_to_lower.local);
5039 return self.fptrunc(op_to_lower, Type.f32, Type.f16);
5127 _ = try self.fptrunc(.{ .stack = {} }, Type.f32, Type.f16);
50405128 }
50415129
50425130 const result = try self.allocLocal(ty);
......@@ -5065,7 +5153,8 @@ fn airSatBinOp(self: *Self, inst: Air.Inst.Index, op: Op) InnerError!WValue {
50655153 }
50665154
50675155 const wasm_bits = toWasmBits(int_info.bits).?;
5068 const bin_result = try self.binOp(lhs, rhs, ty, op);
5156 var bin_result = try (try self.binOp(lhs, rhs, ty, op)).toLocal(self, ty);
5157 defer bin_result.free(self);
50695158 if (wasm_bits != int_info.bits and op == .add) {
50705159 const val: u64 = @intCast(u64, (@as(u65, 1) << @intCast(u7, int_info.bits)) - 1);
50715160 const imm_val = switch (wasm_bits) {
......@@ -5074,19 +5163,17 @@ fn airSatBinOp(self: *Self, inst: Air.Inst.Index, op: Op) InnerError!WValue {
50745163 else => unreachable,
50755164 };
50765165
5077 const cmp_result = try self.cmp(bin_result, imm_val, ty, .lt);
50785166 try self.emitWValue(bin_result);
50795167 try self.emitWValue(imm_val);
5080 try self.emitWValue(cmp_result);
5168 _ = try self.cmp(bin_result, imm_val, ty, .lt);
50815169 } else {
5082 const cmp_result = try self.cmp(bin_result, lhs, ty, if (op == .add) .lt else .gt);
50835170 switch (wasm_bits) {
50845171 32 => try self.addImm32(if (op == .add) @as(i32, -1) else 0),
50855172 64 => try self.addImm64(if (op == .add) @bitCast(u64, @as(i64, -1)) else 0),
50865173 else => unreachable,
50875174 }
50885175 try self.emitWValue(bin_result);
5089 try self.emitWValue(cmp_result);
5176 _ = try self.cmp(bin_result, lhs, ty, if (op == .add) .lt else .gt);
50905177 }
50915178
50925179 try self.addTag(.select);
......@@ -5100,8 +5187,12 @@ fn signedSat(self: *Self, lhs_operand: WValue, rhs_operand: WValue, ty: Type, op
51005187 const wasm_bits = toWasmBits(int_info.bits).?;
51015188 const is_wasm_bits = wasm_bits == int_info.bits;
51025189
5103 const lhs = if (!is_wasm_bits) try self.signAbsValue(lhs_operand, ty) else lhs_operand;
5104 const rhs = if (!is_wasm_bits) try self.signAbsValue(rhs_operand, ty) else rhs_operand;
5190 var lhs = if (!is_wasm_bits) lhs: {
5191 break :lhs try (try self.signAbsValue(lhs_operand, ty)).toLocal(self, ty);
5192 } else lhs_operand;
5193 var rhs = if (!is_wasm_bits) rhs: {
5194 break :rhs try (try self.signAbsValue(rhs_operand, ty)).toLocal(self, ty);
5195 } else rhs_operand;
51055196
51065197 const max_val: u64 = @intCast(u64, (@as(u65, 1) << @intCast(u7, int_info.bits - 1)) - 1);
51075198 const min_val: i64 = (-@intCast(i64, @intCast(u63, max_val))) - 1;
......@@ -5116,38 +5207,38 @@ fn signedSat(self: *Self, lhs_operand: WValue, rhs_operand: WValue, ty: Type, op
51165207 else => unreachable,
51175208 };
51185209
5119 const bin_result = try self.binOp(lhs, rhs, ty, op);
5210 var bin_result = try (try self.binOp(lhs, rhs, ty, op)).toLocal(self, ty);
51205211 if (!is_wasm_bits) {
5121 const cmp_result_lt = try self.cmp(bin_result, max_wvalue, ty, .lt);
5212 defer bin_result.free(self); // not returned in this branch
5213 defer lhs.free(self); // uses temporary local for absvalue
5214 defer rhs.free(self); // uses temporary local for absvalue
51225215 try self.emitWValue(bin_result);
51235216 try self.emitWValue(max_wvalue);
5124 try self.emitWValue(cmp_result_lt);
5217 _ = try self.cmp(bin_result, max_wvalue, ty, .lt);
51255218 try self.addTag(.select);
51265219 try self.addLabel(.local_set, bin_result.local); // re-use local
51275220
5128 const cmp_result_gt = try self.cmp(bin_result, min_wvalue, ty, .gt);
51295221 try self.emitWValue(bin_result);
51305222 try self.emitWValue(min_wvalue);
5131 try self.emitWValue(cmp_result_gt);
5223 _ = try self.cmp(bin_result, min_wvalue, ty, .gt);
51325224 try self.addTag(.select);
51335225 try self.addLabel(.local_set, bin_result.local); // re-use local
5134 return self.wrapOperand(bin_result, ty);
5226 return (try self.wrapOperand(bin_result, ty)).toLocal(self, ty);
51355227 } else {
51365228 const zero = switch (wasm_bits) {
51375229 32 => WValue{ .imm32 = 0 },
51385230 64 => WValue{ .imm64 = 0 },
51395231 else => unreachable,
51405232 };
5141 const cmp_bin_result = try self.cmp(bin_result, lhs, ty, .lt);
5142 const cmp_zero_result = try self.cmp(rhs, zero, ty, if (op == .add) .lt else .gt);
5143 const xor = try self.binOp(cmp_zero_result, cmp_bin_result, Type.u32, .xor); // comparisons always return i32, so provide u32 as type to xor.
5144 const cmp_bin_zero_result = try self.cmp(bin_result, zero, ty, .lt);
51455233 try self.emitWValue(max_wvalue);
51465234 try self.emitWValue(min_wvalue);
5147 try self.emitWValue(cmp_bin_zero_result);
5235 _ = try self.cmp(bin_result, zero, ty, .lt);
51485236 try self.addTag(.select);
51495237 try self.emitWValue(bin_result);
5150 try self.emitWValue(xor);
5238 // leave on stack
5239 const cmp_zero_result = try self.cmp(rhs, zero, ty, if (op == .add) .lt else .gt);
5240 const cmp_bin_result = try self.cmp(bin_result, lhs, ty, .lt);
5241 _ = try self.binOp(cmp_zero_result, cmp_bin_result, Type.u32, .xor); // comparisons always return i32, so provide u32 as type to xor.
51515242 try self.addTag(.select);
51525243 try self.addLabel(.local_set, bin_result.local); // re-use local
51535244 return bin_result;
......@@ -5171,9 +5262,10 @@ fn airShlSat(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
51715262 const result = try self.allocLocal(ty);
51725263
51735264 if (wasm_bits == int_info.bits) {
5174 const shl = try self.binOp(lhs, rhs, ty, .shl);
5175 const shr = try self.binOp(shl, rhs, ty, .shr);
5176 const cmp_result = try self.cmp(lhs, shr, ty, .neq);
5265 var shl = try (try self.binOp(lhs, rhs, ty, .shl)).toLocal(self, ty);
5266 defer shl.free(self);
5267 var shr = try (try self.binOp(shl, rhs, ty, .shr)).toLocal(self, ty);
5268 defer shr.free(self);
51775269
51785270 switch (wasm_bits) {
51795271 32 => blk: {
......@@ -5181,10 +5273,9 @@ fn airShlSat(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
51815273 try self.addImm32(-1);
51825274 break :blk;
51835275 }
5184 const less_than_zero = try self.cmp(lhs, .{ .imm32 = 0 }, ty, .lt);
51855276 try self.addImm32(std.math.minInt(i32));
51865277 try self.addImm32(std.math.maxInt(i32));
5187 try self.emitWValue(less_than_zero);
5278 _ = try self.cmp(lhs, .{ .imm32 = 0 }, ty, .lt);
51885279 try self.addTag(.select);
51895280 },
51905281 64 => blk: {
......@@ -5192,16 +5283,15 @@ fn airShlSat(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
51925283 try self.addImm64(@bitCast(u64, @as(i64, -1)));
51935284 break :blk;
51945285 }
5195 const less_than_zero = try self.cmp(lhs, .{ .imm64 = 0 }, ty, .lt);
51965286 try self.addImm64(@bitCast(u64, @as(i64, std.math.minInt(i64))));
51975287 try self.addImm64(@bitCast(u64, @as(i64, std.math.maxInt(i64))));
5198 try self.emitWValue(less_than_zero);
5288 _ = try self.cmp(lhs, .{ .imm64 = 0 }, ty, .lt);
51995289 try self.addTag(.select);
52005290 },
52015291 else => unreachable,
52025292 }
52035293 try self.emitWValue(shl);
5204 try self.emitWValue(cmp_result);
5294 _ = try self.cmp(lhs, shr, ty, .neq);
52055295 try self.addTag(.select);
52065296 try self.addLabel(.local_set, result.local);
52075297 return result;
......@@ -5213,10 +5303,12 @@ fn airShlSat(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
52135303 else => unreachable,
52145304 };
52155305
5216 const shl_res = try self.binOp(lhs, shift_value, ty, .shl);
5217 const shl = try self.binOp(shl_res, rhs, ty, .shl);
5218 const shr = try self.binOp(shl, rhs, ty, .shr);
5219 const cmp_result = try self.cmp(shl_res, shr, ty, .neq);
5306 var shl_res = try (try self.binOp(lhs, shift_value, ty, .shl)).toLocal(self, ty);
5307 defer shl_res.free(self);
5308 var shl = try (try self.binOp(shl_res, rhs, ty, .shl)).toLocal(self, ty);
5309 defer shl.free(self);
5310 var shr = try (try self.binOp(shl, rhs, ty, .shr)).toLocal(self, ty);
5311 defer shr.free(self);
52205312
52215313 switch (wasm_bits) {
52225314 32 => blk: {
......@@ -5225,10 +5317,9 @@ fn airShlSat(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
52255317 break :blk;
52265318 }
52275319
5228 const less_than_zero = try self.cmp(shl_res, .{ .imm32 = 0 }, ty, .lt);
52295320 try self.addImm32(std.math.minInt(i32));
52305321 try self.addImm32(std.math.maxInt(i32));
5231 try self.emitWValue(less_than_zero);
5322 _ = try self.cmp(shl_res, .{ .imm32 = 0 }, ty, .lt);
52325323 try self.addTag(.select);
52335324 },
52345325 64 => blk: {
......@@ -5237,29 +5328,30 @@ fn airShlSat(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
52375328 break :blk;
52385329 }
52395330
5240 const less_than_zero = try self.cmp(shl_res, .{ .imm64 = 0 }, ty, .lt);
52415331 try self.addImm64(@bitCast(u64, @as(i64, std.math.minInt(i64))));
52425332 try self.addImm64(@bitCast(u64, @as(i64, std.math.maxInt(i64))));
5243 try self.emitWValue(less_than_zero);
5333 _ = try self.cmp(shl_res, .{ .imm64 = 0 }, ty, .lt);
52445334 try self.addTag(.select);
52455335 },
52465336 else => unreachable,
52475337 }
52485338 try self.emitWValue(shl);
5249 try self.emitWValue(cmp_result);
5339 _ = try self.cmp(shl_res, shr, ty, .neq);
52505340 try self.addTag(.select);
52515341 try self.addLabel(.local_set, result.local);
5252 const shift_result = try self.binOp(result, shift_value, ty, .shr);
5342 var shift_result = try self.binOp(result, shift_value, ty, .shr);
52535343 if (is_signed) {
5254 return self.wrapOperand(shift_result, ty);
5344 shift_result = try self.wrapOperand(shift_result, ty);
52555345 }
5256 return shift_result;
5346 return shift_result.toLocal(self, ty);
52575347 }
52585348}
52595349
52605350/// Calls a compiler-rt intrinsic by creating an undefined symbol,
52615351/// then lowering the arguments and calling the symbol as a function call.
52625352/// This function call assumes the C-ABI.
5353/// Asserts arguments are not stack values when the return value is
5354/// passed as the first parameter.
52635355fn callIntrinsic(
52645356 self: *Self,
52655357 name: []const u8,
......@@ -5289,6 +5381,7 @@ fn callIntrinsic(
52895381
52905382 // Lower all arguments to the stack before we call our function
52915383 for (args) |arg, arg_i| {
5384 assert(!(want_sret_param and arg == .stack));
52925385 assert(param_types[arg_i].hasRuntimeBitsIgnoreComptime());
52935386 try self.lowerArg(.C, param_types[arg_i], arg);
52945387 }