authorgravatar for luuk@degram.devLuuk de Gram <luuk@degram.dev> 2022-08-12 08:07:09+02:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2022-08-12 08:07:09+02:00
log645c396d02b7570d10d4d6b80653e7f9db42234e
treee057b4683d59b910a4cfd3c862e9cc8496facf80
parente67a43a673269edf477245931c70806f4b1abae1
parentb42ba7c3d411cde31ede290b3915150c3e8acfbb
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #12394 from Luukdegram/wasm-reuse-locals

stage2: wasm - reuse (temporary) locals

1 files changed, 418 insertions(+), 325 deletions(-)

src/arch/wasm/CodeGen.zig+418-325
...@@ -29,6 +29,8 @@ const errUnionErrorOffset = codegen.errUnionErrorOffset;...@@ -29,6 +29,8 @@ const errUnionErrorOffset = codegen.errUnionErrorOffset;
29const WValue = union(enum) {29const WValue = union(enum) {
30 /// May be referenced but is unused30 /// May be referenced but is unused
31 none: void,31 none: void,
32 /// The value lives on top of the stack
33 stack: void,
32 /// Index of the local variable34 /// Index of the local variable
33 local: u32,35 local: u32,
34 /// An immediate 32bit value36 /// An immediate 32bit value
...@@ -55,7 +57,7 @@ const WValue = union(enum) {...@@ -55,7 +57,7 @@ const WValue = union(enum) {
55 /// In wasm function pointers are indexes into a function table,57 /// In wasm function pointers are indexes into a function table,
56 /// rather than an address in the data section.58 /// rather than an address in the data section.
57 function_index: u32,59 function_index: u32,
58 /// Offset from the bottom of the stack, with the offset60 /// Offset from the bottom of the virtual stack, with the offset
59 /// pointing to where the value lives.61 /// pointing to where the value lives.
60 stack_offset: u32,62 stack_offset: u32,
6163
...@@ -71,6 +73,38 @@ const WValue = union(enum) {...@@ -71,6 +73,38 @@ const WValue = union(enum) {
71 else => return 0,73 else => return 0,
72 }74 }
73 }75 }
76
77 /// Promotes a `WValue` to a local when given value is on top of the stack.
78 /// When encountering a `local` or `stack_offset` this is essentially a no-op.
79 /// All other tags are illegal.
80 fn toLocal(value: WValue, gen: *Self, ty: Type) InnerError!WValue {
81 switch (value) {
82 .stack => {
83 const local = try gen.allocLocal(ty);
84 try gen.addLabel(.local_set, local.local);
85 return local;
86 },
87 .local, .stack_offset => return value,
88 else => unreachable,
89 }
90 }
91
92 /// Marks a local as no longer being referenced and essentially allows
93 /// us to re-use it somewhere else within the function.
94 /// The valtype of the local is deducted by using the index of the given.
95 fn free(value: *WValue, gen: *Self) void {
96 if (value.* != .local) return;
97 const local_value = value.local;
98 const index = local_value - gen.args.len - @boolToInt(gen.return_value != .none);
99 const valtype = @intToEnum(wasm.Valtype, gen.locals.items[index]);
100 switch (valtype) {
101 .i32 => gen.free_locals_i32.append(gen.gpa, local_value) catch return, // It's ok to fail any of those, a new local can be allocated instead
102 .i64 => gen.free_locals_i64.append(gen.gpa, local_value) catch return,
103 .f32 => gen.free_locals_f32.append(gen.gpa, local_value) catch return,
104 .f64 => gen.free_locals_f64.append(gen.gpa, local_value) catch return,
105 }
106 value.* = WValue{ .none = {} };
107 }
74};108};
75109
76/// Wasm ops, but without input/output/signedness information110/// Wasm ops, but without input/output/signedness information
...@@ -601,6 +635,21 @@ stack_size: u32 = 0,...@@ -601,6 +635,21 @@ stack_size: u32 = 0,
601/// However, local variables or the usage of `@setAlignStack` can overwrite this default.635/// However, local variables or the usage of `@setAlignStack` can overwrite this default.
602stack_alignment: u32 = 16,636stack_alignment: u32 = 16,
603637
638// For each individual Wasm valtype we store a seperate free list which
639// allows us to re-use locals that are no longer used. e.g. a temporary local.
640/// A list of indexes which represents a local of valtype `i32`.
641/// It is illegal to store a non-i32 valtype in this list.
642free_locals_i32: std.ArrayListUnmanaged(u32) = .{},
643/// A list of indexes which represents a local of valtype `i64`.
644/// It is illegal to store a non-i32 valtype in this list.
645free_locals_i64: std.ArrayListUnmanaged(u32) = .{},
646/// A list of indexes which represents a local of valtype `f32`.
647/// It is illegal to store a non-i32 valtype in this list.
648free_locals_f32: std.ArrayListUnmanaged(u32) = .{},
649/// A list of indexes which represents a local of valtype `f64`.
650/// It is illegal to store a non-i32 valtype in this list.
651free_locals_f64: std.ArrayListUnmanaged(u32) = .{},
652
604const InnerError = error{653const InnerError = error{
605 OutOfMemory,654 OutOfMemory,
606 /// An error occurred when trying to lower AIR to MIR.655 /// An error occurred when trying to lower AIR to MIR.
...@@ -759,7 +808,7 @@ fn genBlockType(ty: Type, target: std.Target) u8 {...@@ -759,7 +808,7 @@ fn genBlockType(ty: Type, target: std.Target) u8 {
759/// Writes the bytecode depending on the given `WValue` in `val`808/// Writes the bytecode depending on the given `WValue` in `val`
760fn emitWValue(self: *Self, value: WValue) InnerError!void {809fn emitWValue(self: *Self, value: WValue) InnerError!void {
761 switch (value) {810 switch (value) {
762 .none => {}, // no-op811 .none, .stack => {}, // no-op
763 .local => |idx| try self.addLabel(.local_get, idx),812 .local => |idx| try self.addLabel(.local_get, idx),
764 .imm32 => |val| try self.addImm32(@bitCast(i32, val)),813 .imm32 => |val| try self.addImm32(@bitCast(i32, val)),
765 .imm64 => |val| try self.addImm64(val),814 .imm64 => |val| try self.addImm64(val),
...@@ -781,9 +830,30 @@ fn emitWValue(self: *Self, value: WValue) InnerError!void {...@@ -781,9 +830,30 @@ fn emitWValue(self: *Self, value: WValue) InnerError!void {
781/// Creates one locals for a given `Type`.830/// Creates one locals for a given `Type`.
782/// Returns a corresponding `Wvalue` with `local` as active tag831/// Returns a corresponding `Wvalue` with `local` as active tag
783fn allocLocal(self: *Self, ty: Type) InnerError!WValue {832fn allocLocal(self: *Self, ty: Type) InnerError!WValue {
833 const valtype = typeToValtype(ty, self.target);
834 switch (valtype) {
835 .i32 => if (self.free_locals_i32.popOrNull()) |index| {
836 return WValue{ .local = index };
837 },
838 .i64 => if (self.free_locals_i64.popOrNull()) |index| {
839 return WValue{ .local = index };
840 },
841 .f32 => if (self.free_locals_f32.popOrNull()) |index| {
842 return WValue{ .local = index };
843 },
844 .f64 => if (self.free_locals_f64.popOrNull()) |index| {
845 return WValue{ .local = index };
846 },
847 }
848 // no local was free to be re-used, so allocate a new local instead
849 return self.ensureAllocLocal(ty);
850}
851
852/// Ensures a new local will be created. This is useful when it's useful
853/// to use a zero-initialized local.
854fn ensureAllocLocal(self: *Self, ty: Type) InnerError!WValue {
855 try self.locals.append(self.gpa, genValtype(ty, self.target));
784 const initial_index = self.local_index;856 const initial_index = self.local_index;
785 const valtype = genValtype(ty, self.target);
786 try self.locals.append(self.gpa, valtype);
787 self.local_index += 1;857 self.local_index += 1;
788 return WValue{ .local = initial_index };858 return WValue{ .local = initial_index };
789}859}
...@@ -1135,9 +1205,9 @@ fn initializeStack(self: *Self) !void {...@@ -1135,9 +1205,9 @@ fn initializeStack(self: *Self) !void {
1135 // Reserve a local to store the current stack pointer1205 // Reserve a local to store the current stack pointer
1136 // We can later use this local to set the stack pointer back to the value1206 // We can later use this local to set the stack pointer back to the value
1137 // we have stored here.1207 // we have stored here.
1138 self.initial_stack_value = try self.allocLocal(Type.usize);1208 self.initial_stack_value = try self.ensureAllocLocal(Type.usize);
1139 // Also reserve a local to store the bottom stack value1209 // Also reserve a local to store the bottom stack value
1140 self.bottom_stack_value = try self.allocLocal(Type.usize);1210 self.bottom_stack_value = try self.ensureAllocLocal(Type.usize);
1141}1211}
11421212
1143/// Reads the stack pointer from `Context.initial_stack_value` and writes it1213/// Reads the stack pointer from `Context.initial_stack_value` and writes it
...@@ -1268,7 +1338,9 @@ fn memcpy(self: *Self, dst: WValue, src: WValue, len: WValue) !void {...@@ -1268,7 +1338,9 @@ fn memcpy(self: *Self, dst: WValue, src: WValue, len: WValue) !void {
1268 else => {1338 else => {
1269 // TODO: We should probably lower this to a call to compiler_rt1339 // TODO: We should probably lower this to a call to compiler_rt
1270 // But for now, we implement it manually1340 // But for now, we implement it manually
1271 const offset = try self.allocLocal(Type.usize); // local for counter1341 var offset = try self.ensureAllocLocal(Type.usize); // local for counter
1342 defer offset.free(self);
1343
1272 // outer block to jump to when loop is done1344 // outer block to jump to when loop is done
1273 try self.startBlock(.block, wasm.block_empty);1345 try self.startBlock(.block, wasm.block_empty);
1274 try self.startBlock(.loop, wasm.block_empty);1346 try self.startBlock(.loop, wasm.block_empty);
...@@ -1405,7 +1477,7 @@ fn buildPointerOffset(self: *Self, ptr_value: WValue, offset: u64, action: enum...@@ -1405,7 +1477,7 @@ fn buildPointerOffset(self: *Self, ptr_value: WValue, offset: u64, action: enum
1405 // do not perform arithmetic when offset is 0.1477 // do not perform arithmetic when offset is 0.
1406 if (offset == 0 and ptr_value.offset() == 0 and action == .modify) return ptr_value;1478 if (offset == 0 and ptr_value.offset() == 0 and action == .modify) return ptr_value;
1407 const result_ptr: WValue = switch (action) {1479 const result_ptr: WValue = switch (action) {
1408 .new => try self.allocLocal(Type.usize),1480 .new => try self.ensureAllocLocal(Type.usize),
1409 .modify => ptr_value,1481 .modify => ptr_value,
1410 };1482 };
1411 try self.emitWValue(ptr_value);1483 try self.emitWValue(ptr_value);
...@@ -1653,7 +1725,10 @@ fn genInst(self: *Self, inst: Air.Inst.Index) !WValue {...@@ -1653,7 +1725,10 @@ fn genInst(self: *Self, inst: Air.Inst.Index) !WValue {
1653fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {1725fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
1654 for (body) |inst| {1726 for (body) |inst| {
1655 const result = try self.genInst(inst);1727 const result = try self.genInst(inst);
1656 try self.values.putNoClobber(self.gpa, Air.indexToRef(inst), result);1728 if (result != .none) {
1729 assert(result != .stack); // not allowed to store stack values as we cannot keep track of where they are on the stack
1730 try self.values.putNoClobber(self.gpa, Air.indexToRef(inst), result);
1731 }
1657 }1732 }
1658}1733}
16591734
...@@ -1727,8 +1802,8 @@ fn airRetLoad(self: *Self, inst: Air.Inst.Index) InnerError!WValue {...@@ -1727,8 +1802,8 @@ fn airRetLoad(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
17271802
1728 const fn_info = self.decl.ty.fnInfo();1803 const fn_info = self.decl.ty.fnInfo();
1729 if (!firstParamSRet(fn_info.cc, fn_info.return_type, self.target)) {1804 if (!firstParamSRet(fn_info.cc, fn_info.return_type, self.target)) {
1730 const result = try self.load(operand, ret_ty, 0);1805 // leave on the stack
1731 try self.emitWValue(result);1806 _ = try self.load(operand, ret_ty, 0);
1732 }1807 }
17331808
1734 try self.restoreStackPointer();1809 try self.restoreStackPointer();
...@@ -1847,6 +1922,7 @@ fn airStore(self: *Self, inst: Air.Inst.Index) InnerError!WValue {...@@ -1847,6 +1922,7 @@ fn airStore(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
1847}1922}
18481923
1849fn store(self: *Self, lhs: WValue, rhs: WValue, ty: Type, offset: u32) InnerError!void {1924fn store(self: *Self, lhs: WValue, rhs: WValue, ty: Type, offset: u32) InnerError!void {
1925 assert(!(lhs != .stack and rhs == .stack));
1850 switch (ty.zigTypeTag()) {1926 switch (ty.zigTypeTag()) {
1851 .ErrorUnion => {1927 .ErrorUnion => {
1852 const pl_ty = ty.errorUnionPayload();1928 const pl_ty = ty.errorUnionPayload();
...@@ -1880,20 +1956,26 @@ fn store(self: *Self, lhs: WValue, rhs: WValue, ty: Type, offset: u32) InnerErro...@@ -1880,20 +1956,26 @@ fn store(self: *Self, lhs: WValue, rhs: WValue, ty: Type, offset: u32) InnerErro
1880 .Pointer => {1956 .Pointer => {
1881 if (ty.isSlice()) {1957 if (ty.isSlice()) {
1882 // store pointer first1958 // store pointer first
1959 // lower it to the stack so we do not have to store rhs into a local first
1960 try self.emitWValue(lhs);
1883 const ptr_local = try self.load(rhs, Type.usize, 0);1961 const ptr_local = try self.load(rhs, Type.usize, 0);
1884 try self.store(lhs, ptr_local, Type.usize, 0);1962 try self.store(.{ .stack = {} }, ptr_local, Type.usize, 0 + lhs.offset());
18851963
1886 // retrieve length from rhs, and store that alongside lhs as well1964 // retrieve length from rhs, and store that alongside lhs as well
1965 try self.emitWValue(lhs);
1887 const len_local = try self.load(rhs, Type.usize, self.ptrSize());1966 const len_local = try self.load(rhs, Type.usize, self.ptrSize());
1888 try self.store(lhs, len_local, Type.usize, self.ptrSize());1967 try self.store(.{ .stack = {} }, len_local, Type.usize, self.ptrSize() + lhs.offset());
1889 return;1968 return;
1890 }1969 }
1891 },1970 },
1892 .Int => if (ty.intInfo(self.target).bits > 64) {1971 .Int => if (ty.intInfo(self.target).bits > 64) {
1972 try self.emitWValue(lhs);
1893 const lsb = try self.load(rhs, Type.u64, 0);1973 const lsb = try self.load(rhs, Type.u64, 0);
1974 try self.store(.{ .stack = {} }, lsb, Type.u64, 0 + lhs.offset());
1975
1976 try self.emitWValue(lhs);
1894 const msb = try self.load(rhs, Type.u64, 8);1977 const msb = try self.load(rhs, Type.u64, 8);
1895 try self.store(lhs, lsb, Type.u64, 0);1978 try self.store(.{ .stack = {} }, msb, Type.u64, 8 + lhs.offset());
1896 try self.store(lhs, msb, Type.u64, 8);
1897 return;1979 return;
1898 },1980 },
1899 else => {},1981 else => {},
...@@ -1932,9 +2014,12 @@ fn airLoad(self: *Self, inst: Air.Inst.Index) InnerError!WValue {...@@ -1932,9 +2014,12 @@ fn airLoad(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
1932 return new_local;2014 return new_local;
1933 }2015 }
19342016
1935 return self.load(operand, ty, 0);2017 const stack_loaded = try self.load(operand, ty, 0);
2018 return stack_loaded.toLocal(self, ty);
1936}2019}
19372020
2021/// Loads an operand from the linear memory section.
2022/// NOTE: Leaves the value on the stack.
1938fn load(self: *Self, operand: WValue, ty: Type, offset: u32) InnerError!WValue {2023fn load(self: *Self, operand: WValue, ty: Type, offset: u32) InnerError!WValue {
1939 // load local's value from memory by its stack position2024 // load local's value from memory by its stack position
1940 try self.emitWValue(operand);2025 try self.emitWValue(operand);
...@@ -1952,10 +2037,7 @@ fn load(self: *Self, operand: WValue, ty: Type, offset: u32) InnerError!WValue {...@@ -1952,10 +2037,7 @@ fn load(self: *Self, operand: WValue, ty: Type, offset: u32) InnerError!WValue {
1952 .{ .offset = offset + operand.offset(), .alignment = ty.abiAlignment(self.target) },2037 .{ .offset = offset + operand.offset(), .alignment = ty.abiAlignment(self.target) },
1953 );2038 );
19542039
1955 // store the result in a local2040 return WValue{ .stack = {} };
1956 const result = try self.allocLocal(ty);
1957 try self.addLabel(.local_set, result.local);
1958 return result;
1959}2041}
19602042
1961fn airArg(self: *Self, inst: Air.Inst.Index) InnerError!WValue {2043fn airArg(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
...@@ -2025,10 +2107,14 @@ fn airBinOp(self: *Self, inst: Air.Inst.Index, op: Op) InnerError!WValue {...@@ -2025,10 +2107,14 @@ fn airBinOp(self: *Self, inst: Air.Inst.Index, op: Op) InnerError!WValue {
2025 const rhs = try self.resolveInst(bin_op.rhs);2107 const rhs = try self.resolveInst(bin_op.rhs);
2026 const ty = self.air.typeOf(bin_op.lhs);2108 const ty = self.air.typeOf(bin_op.lhs);
20272109
2028 return self.binOp(lhs, rhs, ty, op);2110 const stack_value = try self.binOp(lhs, rhs, ty, op);
2111 return stack_value.toLocal(self, ty);
2029}2112}
20302113
2114/// Performs a binary operation on the given `WValue`'s
2115/// NOTE: THis leaves the value on top of the stack.
2031fn binOp(self: *Self, lhs: WValue, rhs: WValue, ty: Type, op: Op) InnerError!WValue {2116fn binOp(self: *Self, lhs: WValue, rhs: WValue, ty: Type, op: Op) InnerError!WValue {
2117 assert(!(lhs != .stack and rhs == .stack));
2032 if (isByRef(ty, self.target)) {2118 if (isByRef(ty, self.target)) {
2033 if (ty.zigTypeTag() == .Int) {2119 if (ty.zigTypeTag() == .Int) {
2034 return self.binOpBigInt(lhs, rhs, ty, op);2120 return self.binOpBigInt(lhs, rhs, ty, op);
...@@ -2054,24 +2140,18 @@ fn binOp(self: *Self, lhs: WValue, rhs: WValue, ty: Type, op: Op) InnerError!WVa...@@ -2054,24 +2140,18 @@ fn binOp(self: *Self, lhs: WValue, rhs: WValue, ty: Type, op: Op) InnerError!WVa
20542140
2055 try self.addTag(Mir.Inst.Tag.fromOpcode(opcode));2141 try self.addTag(Mir.Inst.Tag.fromOpcode(opcode));
20562142
2057 // save the result in a temporary2143 return WValue{ .stack = {} };
2058 const bin_local = try self.allocLocal(ty);
2059 try self.addLabel(.local_set, bin_local.local);
2060 return bin_local;
2061}2144}
20622145
2146/// Performs a binary operation for 16-bit floats.
2147/// NOTE: Leaves the result value on the stack
2063fn binOpFloat16(self: *Self, lhs: WValue, rhs: WValue, op: Op) InnerError!WValue {2148fn binOpFloat16(self: *Self, lhs: WValue, rhs: WValue, op: Op) InnerError!WValue {
2064 const ext_lhs = try self.fpext(lhs, Type.f16, Type.f32);
2065 const ext_rhs = try self.fpext(rhs, Type.f16, Type.f32);
2066
2067 const opcode: wasm.Opcode = buildOpcode(.{ .op = op, .valtype1 = .f32, .signedness = .unsigned });2149 const opcode: wasm.Opcode = buildOpcode(.{ .op = op, .valtype1 = .f32, .signedness = .unsigned });
2068 try self.emitWValue(ext_lhs);2150 _ = try self.fpext(lhs, Type.f16, Type.f32);
2069 try self.emitWValue(ext_rhs);2151 _ = try self.fpext(rhs, Type.f16, Type.f32);
2070 try self.addTag(Mir.Inst.Tag.fromOpcode(opcode));2152 try self.addTag(Mir.Inst.Tag.fromOpcode(opcode));
20712153
2072 // re-use temporary local2154 return self.fptrunc(.{ .stack = {} }, Type.f32, Type.f16);
2073 try self.addLabel(.local_set, ext_lhs.local);
2074 return self.fptrunc(ext_lhs, Type.f32, Type.f16);
2075}2155}
20762156
2077fn binOpBigInt(self: *Self, lhs: WValue, rhs: WValue, ty: Type, op: Op) InnerError!WValue {2157fn binOpBigInt(self: *Self, lhs: WValue, rhs: WValue, ty: Type, op: Op) InnerError!WValue {
...@@ -2084,13 +2164,16 @@ fn binOpBigInt(self: *Self, lhs: WValue, rhs: WValue, ty: Type, op: Op) InnerErr...@@ -2084,13 +2164,16 @@ fn binOpBigInt(self: *Self, lhs: WValue, rhs: WValue, ty: Type, op: Op) InnerErr
2084 }2164 }
20852165
2086 const result = try self.allocStack(ty);2166 const result = try self.allocStack(ty);
2087 const lhs_high_bit = try self.load(lhs, Type.u64, 0);2167 var lhs_high_bit = try (try self.load(lhs, Type.u64, 0)).toLocal(self, Type.u64);
2168 defer lhs_high_bit.free(self);
2169 var rhs_high_bit = try (try self.load(rhs, Type.u64, 0)).toLocal(self, Type.u64);
2170 defer rhs_high_bit.free(self);
2171 var high_op_res = try (try self.binOp(lhs_high_bit, rhs_high_bit, Type.u64, op)).toLocal(self, Type.u64);
2172 defer high_op_res.free(self);
2173
2088 const lhs_low_bit = try self.load(lhs, Type.u64, 8);2174 const lhs_low_bit = try self.load(lhs, Type.u64, 8);
2089 const rhs_high_bit = try self.load(rhs, Type.u64, 0);
2090 const rhs_low_bit = try self.load(rhs, Type.u64, 8);2175 const rhs_low_bit = try self.load(rhs, Type.u64, 8);
2091
2092 const low_op_res = try self.binOp(lhs_low_bit, rhs_low_bit, Type.u64, op);2176 const low_op_res = try self.binOp(lhs_low_bit, rhs_low_bit, Type.u64, op);
2093 const high_op_res = try self.binOp(lhs_high_bit, rhs_high_bit, Type.u64, op);
20942177
2095 const lt = if (op == .add) blk: {2178 const lt = if (op == .add) blk: {
2096 break :blk try self.cmp(high_op_res, rhs_high_bit, Type.u64, .lt);2179 break :blk try self.cmp(high_op_res, rhs_high_bit, Type.u64, .lt);
...@@ -2098,7 +2181,8 @@ fn binOpBigInt(self: *Self, lhs: WValue, rhs: WValue, ty: Type, op: Op) InnerErr...@@ -2098,7 +2181,8 @@ fn binOpBigInt(self: *Self, lhs: WValue, rhs: WValue, ty: Type, op: Op) InnerErr
2098 break :blk try self.cmp(lhs_high_bit, rhs_high_bit, Type.u64, .lt);2181 break :blk try self.cmp(lhs_high_bit, rhs_high_bit, Type.u64, .lt);
2099 } else unreachable;2182 } else unreachable;
2100 const tmp = try self.intcast(lt, Type.u32, Type.u64);2183 const tmp = try self.intcast(lt, Type.u32, Type.u64);
2101 const tmp_op = try self.binOp(low_op_res, tmp, Type.u64, op);2184 var tmp_op = try (try self.binOp(low_op_res, tmp, Type.u64, op)).toLocal(self, Type.u64);
2185 defer tmp_op.free(self);
21022186
2103 try self.store(result, high_op_res, Type.u64, 0);2187 try self.store(result, high_op_res, Type.u64, 0);
2104 try self.store(result, tmp_op, Type.u64, 8);2188 try self.store(result, tmp_op, Type.u64, 8);
...@@ -2115,40 +2199,22 @@ fn airWrapBinOp(self: *Self, inst: Air.Inst.Index, op: Op) InnerError!WValue {...@@ -2115,40 +2199,22 @@ fn airWrapBinOp(self: *Self, inst: Air.Inst.Index, op: Op) InnerError!WValue {
2115 return self.fail("TODO: Implement wrapping arithmetic for vectors", .{});2199 return self.fail("TODO: Implement wrapping arithmetic for vectors", .{});
2116 }2200 }
21172201
2118 return self.wrapBinOp(lhs, rhs, ty, op);2202 return (try self.wrapBinOp(lhs, rhs, ty, op)).toLocal(self, ty);
2119}2203}
21202204
2205/// Performs a wrapping binary operation.
2206/// Asserts rhs is not a stack value when lhs also isn't.
2207/// NOTE: Leaves the result on the stack when its Type is <= 64 bits
2121fn wrapBinOp(self: *Self, lhs: WValue, rhs: WValue, ty: Type, op: Op) InnerError!WValue {2208fn wrapBinOp(self: *Self, lhs: WValue, rhs: WValue, ty: Type, op: Op) InnerError!WValue {
2122 const bit_size = ty.intInfo(self.target).bits;2209 const bin_local = try self.binOp(lhs, rhs, ty, op);
2123 var wasm_bits = toWasmBits(bit_size) orelse {
2124 return self.fail("TODO: Implement wrapping arithmetic for integers with bitsize: {d}\n", .{bit_size});
2125 };
2126
2127 if (wasm_bits == 128) {
2128 const bin_op = try self.binOpBigInt(lhs, rhs, ty, op);
2129 return self.wrapOperand(bin_op, ty);
2130 }
2131
2132 const opcode: wasm.Opcode = buildOpcode(.{
2133 .op = op,
2134 .valtype1 = typeToValtype(ty, self.target),
2135 .signedness = if (ty.isSignedInt()) .signed else .unsigned,
2136 });
2137
2138 try self.emitWValue(lhs);
2139 try self.emitWValue(rhs);
2140 try self.addTag(Mir.Inst.Tag.fromOpcode(opcode));
2141 const bin_local = try self.allocLocal(ty);
2142 try self.addLabel(.local_set, bin_local.local);
2143
2144 return self.wrapOperand(bin_local, ty);2210 return self.wrapOperand(bin_local, ty);
2145}2211}
21462212
2147/// Wraps an operand based on a given type's bitsize.2213/// Wraps an operand based on a given type's bitsize.
2148/// Asserts `Type` is <= 128 bits.2214/// Asserts `Type` is <= 128 bits.
2215/// NOTE: When the Type is <= 64 bits, leaves the value on top of the stack.
2149fn wrapOperand(self: *Self, operand: WValue, ty: Type) InnerError!WValue {2216fn wrapOperand(self: *Self, operand: WValue, ty: Type) InnerError!WValue {
2150 assert(ty.abiSize(self.target) <= 16);2217 assert(ty.abiSize(self.target) <= 16);
2151 const result_local = try self.allocLocal(ty);
2152 const bitsize = ty.intInfo(self.target).bits;2218 const bitsize = ty.intInfo(self.target).bits;
2153 const wasm_bits = toWasmBits(bitsize) orelse {2219 const wasm_bits = toWasmBits(bitsize) orelse {
2154 return self.fail("TODO: Implement wrapOperand for bitsize '{d}'", .{bitsize});2220 return self.fail("TODO: Implement wrapOperand for bitsize '{d}'", .{bitsize});
...@@ -2157,14 +2223,15 @@ fn wrapOperand(self: *Self, operand: WValue, ty: Type) InnerError!WValue {...@@ -2157,14 +2223,15 @@ fn wrapOperand(self: *Self, operand: WValue, ty: Type) InnerError!WValue {
2157 if (wasm_bits == bitsize) return operand;2223 if (wasm_bits == bitsize) return operand;
21582224
2159 if (wasm_bits == 128) {2225 if (wasm_bits == 128) {
2160 const msb = try self.load(operand, Type.u64, 0);2226 assert(operand != .stack);
2161 const lsb = try self.load(operand, Type.u64, 8);2227 const lsb = try self.load(operand, Type.u64, 8);
21622228
2163 const result_ptr = try self.allocStack(ty);2229 const result_ptr = try self.allocStack(ty);
2164 try self.store(result_ptr, lsb, Type.u64, 8);2230 try self.emitWValue(result_ptr);
2231 try self.store(.{ .stack = {} }, lsb, Type.u64, 8 + result_ptr.offset());
2165 const result = (@as(u64, 1) << @intCast(u6, 64 - (wasm_bits - bitsize))) - 1;2232 const result = (@as(u64, 1) << @intCast(u6, 64 - (wasm_bits - bitsize))) - 1;
2166 try self.emitWValue(result_ptr);2233 try self.emitWValue(result_ptr);
2167 try self.emitWValue(msb);2234 _ = try self.load(operand, Type.u64, 0);
2168 try self.addImm64(result);2235 try self.addImm64(result);
2169 try self.addTag(.i64_and);2236 try self.addTag(.i64_and);
2170 try self.addMemArg(.i64_store, .{ .offset = result_ptr.offset(), .alignment = 8 });2237 try self.addMemArg(.i64_store, .{ .offset = result_ptr.offset(), .alignment = 8 });
...@@ -2181,8 +2248,7 @@ fn wrapOperand(self: *Self, operand: WValue, ty: Type) InnerError!WValue {...@@ -2181,8 +2248,7 @@ fn wrapOperand(self: *Self, operand: WValue, ty: Type) InnerError!WValue {
2181 try self.addTag(.i64_and);2248 try self.addTag(.i64_and);
2182 } else unreachable;2249 } else unreachable;
21832250
2184 try self.addLabel(.local_set, result_local.local);2251 return WValue{ .stack = {} };
2185 return result_local;
2186}2252}
21872253
2188fn lowerParentPtr(self: *Self, ptr_val: Value, ptr_child_ty: Type) InnerError!WValue {2254fn lowerParentPtr(self: *Self, ptr_val: Value, ptr_child_ty: Type) InnerError!WValue {
...@@ -2594,10 +2660,14 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: std.math.CompareOperator) Inner...@@ -2594,10 +2660,14 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: std.math.CompareOperator) Inner
2594 const lhs = try self.resolveInst(bin_op.lhs);2660 const lhs = try self.resolveInst(bin_op.lhs);
2595 const rhs = try self.resolveInst(bin_op.rhs);2661 const rhs = try self.resolveInst(bin_op.rhs);
2596 const operand_ty = self.air.typeOf(bin_op.lhs);2662 const operand_ty = self.air.typeOf(bin_op.lhs);
2597 return self.cmp(lhs, rhs, operand_ty, op);2663 return (try self.cmp(lhs, rhs, operand_ty, op)).toLocal(self, Type.u32); // comparison result is always 32 bits
2598}2664}
25992665
2666/// Compares two operands.
2667/// Asserts rhs is not a stack value when the lhs isn't a stack value either
2668/// NOTE: This leaves the result on top of the stack, rather than a new local.
2600fn cmp(self: *Self, lhs: WValue, rhs: WValue, ty: Type, op: std.math.CompareOperator) InnerError!WValue {2669fn cmp(self: *Self, lhs: WValue, rhs: WValue, ty: Type, op: std.math.CompareOperator) InnerError!WValue {
2670 assert(!(lhs != .stack and rhs == .stack));
2601 if (ty.zigTypeTag() == .Optional and !ty.optionalReprIsPayload()) {2671 if (ty.zigTypeTag() == .Optional and !ty.optionalReprIsPayload()) {
2602 var buf: Type.Payload.ElemType = undefined;2672 var buf: Type.Payload.ElemType = undefined;
2603 const payload_ty = ty.optionalChild(&buf);2673 const payload_ty = ty.optionalChild(&buf);
...@@ -2639,15 +2709,12 @@ fn cmp(self: *Self, lhs: WValue, rhs: WValue, ty: Type, op: std.math.CompareOper...@@ -2639,15 +2709,12 @@ fn cmp(self: *Self, lhs: WValue, rhs: WValue, ty: Type, op: std.math.CompareOper
2639 });2709 });
2640 try self.addTag(Mir.Inst.Tag.fromOpcode(opcode));2710 try self.addTag(Mir.Inst.Tag.fromOpcode(opcode));
26412711
2642 const cmp_tmp = try self.allocLocal(Type.initTag(.i32)); // bool is always i322712 return WValue{ .stack = {} };
2643 try self.addLabel(.local_set, cmp_tmp.local);
2644 return cmp_tmp;
2645}2713}
26462714
2715/// Compares 16-bit floats
2716/// NOTE: The result value remains on top of the stack.
2647fn cmpFloat16(self: *Self, lhs: WValue, rhs: WValue, op: std.math.CompareOperator) InnerError!WValue {2717fn cmpFloat16(self: *Self, lhs: WValue, rhs: WValue, op: std.math.CompareOperator) InnerError!WValue {
2648 const ext_lhs = try self.fpext(lhs, Type.f16, Type.f32);
2649 const ext_rhs = try self.fpext(rhs, Type.f16, Type.f32);
2650
2651 const opcode: wasm.Opcode = buildOpcode(.{2718 const opcode: wasm.Opcode = buildOpcode(.{
2652 .op = switch (op) {2719 .op = switch (op) {
2653 .lt => .lt,2720 .lt => .lt,
...@@ -2660,13 +2727,11 @@ fn cmpFloat16(self: *Self, lhs: WValue, rhs: WValue, op: std.math.CompareOperato...@@ -2660,13 +2727,11 @@ fn cmpFloat16(self: *Self, lhs: WValue, rhs: WValue, op: std.math.CompareOperato
2660 .valtype1 = .f32,2727 .valtype1 = .f32,
2661 .signedness = .unsigned,2728 .signedness = .unsigned,
2662 });2729 });
2663 try self.emitWValue(ext_lhs);2730 _ = try self.fpext(lhs, Type.f16, Type.f32);
2664 try self.emitWValue(ext_rhs);2731 _ = try self.fpext(rhs, Type.f16, Type.f32);
2665 try self.addTag(Mir.Inst.Tag.fromOpcode(opcode));2732 try self.addTag(Mir.Inst.Tag.fromOpcode(opcode));
26662733
2667 const result = try self.allocLocal(Type.initTag(.i32)); // bool is always i322734 return WValue{ .stack = {} };
2668 try self.addLabel(.local_set, result.local);
2669 return result;
2670}2735}
26712736
2672fn airCmpVector(self: *Self, inst: Air.Inst.Index) InnerError!WValue {2737fn airCmpVector(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
...@@ -2727,21 +2792,23 @@ fn airNot(self: *Self, inst: Air.Inst.Index) InnerError!WValue {...@@ -2727,21 +2792,23 @@ fn airNot(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
2727 switch (wasm_bits) {2792 switch (wasm_bits) {
2728 32 => {2793 32 => {
2729 const bin_op = try self.binOp(operand, .{ .imm32 = ~@as(u32, 0) }, operand_ty, .xor);2794 const bin_op = try self.binOp(operand, .{ .imm32 = ~@as(u32, 0) }, operand_ty, .xor);
2730 return self.wrapOperand(bin_op, operand_ty);2795 return (try self.wrapOperand(bin_op, operand_ty)).toLocal(self, operand_ty);
2731 },2796 },
2732 64 => {2797 64 => {
2733 const bin_op = try self.binOp(operand, .{ .imm64 = ~@as(u64, 0) }, operand_ty, .xor);2798 const bin_op = try self.binOp(operand, .{ .imm64 = ~@as(u64, 0) }, operand_ty, .xor);
2734 return self.wrapOperand(bin_op, operand_ty);2799 return (try self.wrapOperand(bin_op, operand_ty)).toLocal(self, operand_ty);
2735 },2800 },
2736 128 => {2801 128 => {
2737 const result_ptr = try self.allocStack(operand_ty);2802 const result_ptr = try self.allocStack(operand_ty);
2803 try self.emitWValue(result_ptr);
2738 const msb = try self.load(operand, Type.u64, 0);2804 const msb = try self.load(operand, Type.u64, 0);
2739 const lsb = try self.load(operand, Type.u64, 8);
2740
2741 const msb_xor = try self.binOp(msb, .{ .imm64 = ~@as(u64, 0) }, Type.u64, .xor);2805 const msb_xor = try self.binOp(msb, .{ .imm64 = ~@as(u64, 0) }, Type.u64, .xor);
2806 try self.store(.{ .stack = {} }, msb_xor, Type.u64, 0 + result_ptr.offset());
2807
2808 try self.emitWValue(result_ptr);
2809 const lsb = try self.load(operand, Type.u64, 8);
2742 const lsb_xor = try self.binOp(lsb, .{ .imm64 = ~@as(u64, 0) }, Type.u64, .xor);2810 const lsb_xor = try self.binOp(lsb, .{ .imm64 = ~@as(u64, 0) }, Type.u64, .xor);
2743 try self.store(result_ptr, msb_xor, Type.u64, 0);2811 try self.store(result_ptr, lsb_xor, Type.u64, 8 + result_ptr.offset());
2744 try self.store(result_ptr, lsb_xor, Type.u64, 8);
2745 return result_ptr;2812 return result_ptr;
2746 },2813 },
2747 else => unreachable,2814 else => unreachable,
...@@ -2829,7 +2896,8 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) InnerError!WValue {...@@ -2829,7 +2896,8 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
2829 }2896 }
2830 }2897 }
28312898
2832 return self.load(operand, field_ty, offset);2899 const field = try self.load(operand, field_ty, offset);
2900 return field.toLocal(self, field_ty);
2833}2901}
28342902
2835fn airSwitchBr(self: *Self, inst: Air.Inst.Index) InnerError!WValue {2903fn airSwitchBr(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
...@@ -3039,7 +3107,9 @@ fn airUnwrapErrUnionPayload(self: *Self, inst: Air.Inst.Index, op_is_ptr: bool)...@@ -3039,7 +3107,9 @@ fn airUnwrapErrUnionPayload(self: *Self, inst: Air.Inst.Index, op_is_ptr: bool)
3039 if (op_is_ptr or isByRef(payload_ty, self.target)) {3107 if (op_is_ptr or isByRef(payload_ty, self.target)) {
3040 return self.buildPointerOffset(operand, pl_offset, .new);3108 return self.buildPointerOffset(operand, pl_offset, .new);
3041 }3109 }
3042 return self.load(operand, payload_ty, pl_offset);3110
3111 const payload = try self.load(operand, payload_ty, pl_offset);
3112 return payload.toLocal(self, payload_ty);
3043}3113}
30443114
3045fn airUnwrapErrUnionError(self: *Self, inst: Air.Inst.Index, op_is_ptr: bool) InnerError!WValue {3115fn airUnwrapErrUnionError(self: *Self, inst: Air.Inst.Index, op_is_ptr: bool) InnerError!WValue {
...@@ -3059,7 +3129,8 @@ fn airUnwrapErrUnionError(self: *Self, inst: Air.Inst.Index, op_is_ptr: bool) In...@@ -3059,7 +3129,8 @@ fn airUnwrapErrUnionError(self: *Self, inst: Air.Inst.Index, op_is_ptr: bool) In
3059 return operand;3129 return operand;
3060 }3130 }
30613131
3062 return self.load(operand, Type.anyerror, @intCast(u32, errUnionErrorOffset(payload_ty, self.target)));3132 const error_val = try self.load(operand, Type.anyerror, @intCast(u32, errUnionErrorOffset(payload_ty, self.target)));
3133 return error_val.toLocal(self, Type.anyerror);
3063}3134}
30643135
3065fn airWrapErrUnionPayload(self: *Self, inst: Air.Inst.Index) InnerError!WValue {3136fn airWrapErrUnionPayload(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
...@@ -3125,12 +3196,13 @@ fn airIntcast(self: *Self, inst: Air.Inst.Index) InnerError!WValue {...@@ -3125,12 +3196,13 @@ fn airIntcast(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
3125 return self.fail("todo Wasm intcast for bitsize > 128", .{});3196 return self.fail("todo Wasm intcast for bitsize > 128", .{});
3126 }3197 }
31273198
3128 return self.intcast(operand, operand_ty, ty);3199 return (try self.intcast(operand, operand_ty, ty)).toLocal(self, ty);
3129}3200}
31303201
3131/// Upcasts or downcasts an integer based on the given and wanted types,3202/// Upcasts or downcasts an integer based on the given and wanted types,
3132/// and stores the result in a new operand.3203/// and stores the result in a new operand.
3133/// Asserts type's bitsize <= 1283204/// Asserts type's bitsize <= 128
3205/// NOTE: May leave the result on the top of the stack.
3134fn intcast(self: *Self, operand: WValue, given: Type, wanted: Type) InnerError!WValue {3206fn intcast(self: *Self, operand: WValue, given: Type, wanted: Type) InnerError!WValue {
3135 const given_info = given.intInfo(self.target);3207 const given_info = given.intInfo(self.target);
3136 const wanted_info = wanted.intInfo(self.target);3208 const wanted_info = wanted.intInfo(self.target);
...@@ -3153,25 +3225,22 @@ fn intcast(self: *Self, operand: WValue, given: Type, wanted: Type) InnerError!W...@@ -3153,25 +3225,22 @@ fn intcast(self: *Self, operand: WValue, given: Type, wanted: Type) InnerError!W
3153 } else if (wanted_bits == 128) {3225 } else if (wanted_bits == 128) {
3154 // for 128bit integers we store the integer in the virtual stack, rather than a local3226 // for 128bit integers we store the integer in the virtual stack, rather than a local
3155 const stack_ptr = try self.allocStack(wanted);3227 const stack_ptr = try self.allocStack(wanted);
3228 try self.emitWValue(stack_ptr);
31563229
3157 // for 32 bit integers, we first coerce the value into a 64 bit integer before storing it3230 // for 32 bit integers, we first coerce the value into a 64 bit integer before storing it
3158 // meaning less store operations are required.3231 // meaning less store operations are required.
3159 const lhs = if (op_bits == 32) blk: {3232 const lhs = if (op_bits == 32) blk: {
3160 const tmp = try self.intcast(3233 break :blk try self.intcast(operand, given, if (wanted.isSignedInt()) Type.i64 else Type.u64);
3161 operand,
3162 given,
3163 if (wanted.isSignedInt()) Type.i64 else Type.u64,
3164 );
3165 break :blk tmp;
3166 } else operand;3234 } else operand;
31673235
3168 // store msb first3236 // store msb first
3169 try self.store(stack_ptr, lhs, Type.u64, 0);3237 try self.store(.{ .stack = {} }, lhs, Type.u64, 0 + stack_ptr.offset());
31703238
3171 // For signed integers we shift msb by 63 (64bit integer - 1 sign bit) and store remaining value3239 // For signed integers we shift msb by 63 (64bit integer - 1 sign bit) and store remaining value
3172 if (wanted.isSignedInt()) {3240 if (wanted.isSignedInt()) {
3241 try self.emitWValue(stack_ptr);
3173 const shr = try self.binOp(lhs, .{ .imm64 = 63 }, Type.i64, .shr);3242 const shr = try self.binOp(lhs, .{ .imm64 = 63 }, Type.i64, .shr);
3174 try self.store(stack_ptr, shr, Type.u64, 8);3243 try self.store(.{ .stack = {} }, shr, Type.u64, 8 + stack_ptr.offset());
3175 } else {3244 } else {
3176 // Ensure memory of lsb is zero'd3245 // Ensure memory of lsb is zero'd
3177 try self.store(stack_ptr, .{ .imm64 = 0 }, Type.u64, 8);3246 try self.store(stack_ptr, .{ .imm64 = 0 }, Type.u64, 8);
...@@ -3179,9 +3248,7 @@ fn intcast(self: *Self, operand: WValue, given: Type, wanted: Type) InnerError!W...@@ -3179,9 +3248,7 @@ fn intcast(self: *Self, operand: WValue, given: Type, wanted: Type) InnerError!W
3179 return stack_ptr;3248 return stack_ptr;
3180 } else return self.load(operand, wanted, 0);3249 } else return self.load(operand, wanted, 0);
31813250
3182 const result = try self.allocLocal(wanted);3251 return WValue{ .stack = {} };
3183 try self.addLabel(.local_set, result.local);
3184 return result;
3185}3252}
31863253
3187fn airIsNull(self: *Self, inst: Air.Inst.Index, opcode: wasm.Opcode, op_kind: enum { value, ptr }) InnerError!WValue {3254fn airIsNull(self: *Self, inst: Air.Inst.Index, opcode: wasm.Opcode, op_kind: enum { value, ptr }) InnerError!WValue {
...@@ -3190,9 +3257,12 @@ fn airIsNull(self: *Self, inst: Air.Inst.Index, opcode: wasm.Opcode, op_kind: en...@@ -3190,9 +3257,12 @@ fn airIsNull(self: *Self, inst: Air.Inst.Index, opcode: wasm.Opcode, op_kind: en
31903257
3191 const op_ty = self.air.typeOf(un_op);3258 const op_ty = self.air.typeOf(un_op);
3192 const optional_ty = if (op_kind == .ptr) op_ty.childType() else op_ty;3259 const optional_ty = if (op_kind == .ptr) op_ty.childType() else op_ty;
3193 return self.isNull(operand, optional_ty, opcode);3260 const is_null = try self.isNull(operand, optional_ty, opcode);
3261 return is_null.toLocal(self, optional_ty);
3194}3262}
31953263
3264/// For a given type and operand, checks if it's considered `null`.
3265/// NOTE: Leaves the result on the stack
3196fn isNull(self: *Self, operand: WValue, optional_ty: Type, opcode: wasm.Opcode) InnerError!WValue {3266fn isNull(self: *Self, operand: WValue, optional_ty: Type, opcode: wasm.Opcode) InnerError!WValue {
3197 try self.emitWValue(operand);3267 try self.emitWValue(operand);
3198 if (!optional_ty.optionalReprIsPayload()) {3268 if (!optional_ty.optionalReprIsPayload()) {
...@@ -3209,9 +3279,7 @@ fn isNull(self: *Self, operand: WValue, optional_ty: Type, opcode: wasm.Opcode)...@@ -3209,9 +3279,7 @@ fn isNull(self: *Self, operand: WValue, optional_ty: Type, opcode: wasm.Opcode)
3209 try self.addImm32(0);3279 try self.addImm32(0);
3210 try self.addTag(Mir.Inst.Tag.fromOpcode(opcode));3280 try self.addTag(Mir.Inst.Tag.fromOpcode(opcode));
32113281
3212 const is_null_tmp = try self.allocLocal(Type.initTag(.i32));3282 return WValue{ .stack = {} };
3213 try self.addLabel(.local_set, is_null_tmp.local);
3214 return is_null_tmp;
3215}3283}
32163284
3217fn airOptionalPayload(self: *Self, inst: Air.Inst.Index) InnerError!WValue {3285fn airOptionalPayload(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
...@@ -3229,7 +3297,8 @@ fn airOptionalPayload(self: *Self, inst: Air.Inst.Index) InnerError!WValue {...@@ -3229,7 +3297,8 @@ fn airOptionalPayload(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
3229 return self.buildPointerOffset(operand, offset, .new);3297 return self.buildPointerOffset(operand, offset, .new);
3230 }3298 }
32313299
3232 return self.load(operand, payload_ty, @intCast(u32, offset));3300 const payload = try self.load(operand, payload_ty, @intCast(u32, offset));
3301 return payload.toLocal(self, payload_ty);
3233}3302}
32343303
3235fn airOptionalPayloadPtr(self: *Self, inst: Air.Inst.Index) InnerError!WValue {3304fn airOptionalPayloadPtr(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
...@@ -3332,7 +3401,8 @@ fn airSliceLen(self: *Self, inst: Air.Inst.Index) InnerError!WValue {...@@ -3332,7 +3401,8 @@ fn airSliceLen(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
3332 const ty_op = self.air.instructions.items(.data)[inst].ty_op;3401 const ty_op = self.air.instructions.items(.data)[inst].ty_op;
3333 const operand = try self.resolveInst(ty_op.operand);3402 const operand = try self.resolveInst(ty_op.operand);
33343403
3335 return self.load(operand, Type.usize, self.ptrSize());3404 const len = try self.load(operand, Type.usize, self.ptrSize());
3405 return len.toLocal(self, Type.usize);
3336}3406}
33373407
3338fn airSliceElemVal(self: *Self, inst: Air.Inst.Index) InnerError!WValue {3408fn airSliceElemVal(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
...@@ -3346,8 +3416,7 @@ fn airSliceElemVal(self: *Self, inst: Air.Inst.Index) InnerError!WValue {...@@ -3346,8 +3416,7 @@ fn airSliceElemVal(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
3346 const elem_size = elem_ty.abiSize(self.target);3416 const elem_size = elem_ty.abiSize(self.target);
33473417
3348 // load pointer onto stack3418 // load pointer onto stack
3349 const slice_ptr = try self.load(slice, Type.usize, 0);3419 _ = try self.load(slice, Type.usize, 0);
3350 try self.addLabel(.local_get, slice_ptr.local);
33513420
3352 // calculate index into slice3421 // calculate index into slice
3353 try self.emitWValue(index);3422 try self.emitWValue(index);
...@@ -3361,7 +3430,9 @@ fn airSliceElemVal(self: *Self, inst: Air.Inst.Index) InnerError!WValue {...@@ -3361,7 +3430,9 @@ fn airSliceElemVal(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
3361 if (isByRef(elem_ty, self.target)) {3430 if (isByRef(elem_ty, self.target)) {
3362 return result;3431 return result;
3363 }3432 }
3364 return self.load(result, elem_ty, 0);3433
3434 const elem_val = try self.load(result, elem_ty, 0);
3435 return elem_val.toLocal(self, elem_ty);
3365}3436}
33663437
3367fn airSliceElemPtr(self: *Self, inst: Air.Inst.Index) InnerError!WValue {3438fn airSliceElemPtr(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
...@@ -3374,8 +3445,7 @@ fn airSliceElemPtr(self: *Self, inst: Air.Inst.Index) InnerError!WValue {...@@ -3374,8 +3445,7 @@ fn airSliceElemPtr(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
3374 const slice = try self.resolveInst(bin_op.lhs);3445 const slice = try self.resolveInst(bin_op.lhs);
3375 const index = try self.resolveInst(bin_op.rhs);3446 const index = try self.resolveInst(bin_op.rhs);
33763447
3377 const slice_ptr = try self.load(slice, Type.usize, 0);3448 _ = try self.load(slice, Type.usize, 0);
3378 try self.addLabel(.local_get, slice_ptr.local);
33793449
3380 // calculate index into slice3450 // calculate index into slice
3381 try self.emitWValue(index);3451 try self.emitWValue(index);
...@@ -3383,7 +3453,7 @@ fn airSliceElemPtr(self: *Self, inst: Air.Inst.Index) InnerError!WValue {...@@ -3383,7 +3453,7 @@ fn airSliceElemPtr(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
3383 try self.addTag(.i32_mul);3453 try self.addTag(.i32_mul);
3384 try self.addTag(.i32_add);3454 try self.addTag(.i32_add);
33853455
3386 const result = try self.allocLocal(Type.initTag(.i32));3456 const result = try self.allocLocal(Type.i32);
3387 try self.addLabel(.local_set, result.local);3457 try self.addLabel(.local_set, result.local);
3388 return result;3458 return result;
3389}3459}
...@@ -3392,7 +3462,8 @@ fn airSlicePtr(self: *Self, inst: Air.Inst.Index) InnerError!WValue {...@@ -3392,7 +3462,8 @@ fn airSlicePtr(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
3392 if (self.liveness.isUnused(inst)) return WValue{ .none = {} };3462 if (self.liveness.isUnused(inst)) return WValue{ .none = {} };
3393 const ty_op = self.air.instructions.items(.data)[inst].ty_op;3463 const ty_op = self.air.instructions.items(.data)[inst].ty_op;
3394 const operand = try self.resolveInst(ty_op.operand);3464 const operand = try self.resolveInst(ty_op.operand);
3395 return self.load(operand, Type.usize, 0);3465 const ptr = try self.load(operand, Type.usize, 0);
3466 return ptr.toLocal(self, Type.usize);
3396}3467}
33973468
3398fn airTrunc(self: *Self, inst: Air.Inst.Index) InnerError!WValue {3469fn airTrunc(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
...@@ -3407,13 +3478,13 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) InnerError!WValue {...@@ -3407,13 +3478,13 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
3407 return self.fail("TODO: Implement wasm integer truncation for integer bitsize: {d}", .{int_info.bits});3478 return self.fail("TODO: Implement wasm integer truncation for integer bitsize: {d}", .{int_info.bits});
3408 }3479 }
34093480
3410 const result = try self.intcast(operand, op_ty, wanted_ty);3481 var result = try self.intcast(operand, op_ty, wanted_ty);
3411 const wanted_bits = wanted_ty.intInfo(self.target).bits;3482 const wanted_bits = wanted_ty.intInfo(self.target).bits;
3412 const wasm_bits = toWasmBits(wanted_bits).?;3483 const wasm_bits = toWasmBits(wanted_bits).?;
3413 if (wasm_bits != wanted_bits) {3484 if (wasm_bits != wanted_bits) {
3414 return self.wrapOperand(result, wanted_ty);3485 result = try self.wrapOperand(result, wanted_ty);
3415 }3486 }
3416 return result;3487 return result.toLocal(self, wanted_ty);
3417}3488}
34183489
3419fn airBoolToInt(self: *Self, inst: Air.Inst.Index) InnerError!WValue {3490fn airBoolToInt(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
...@@ -3466,8 +3537,7 @@ fn airPtrElemVal(self: *Self, inst: Air.Inst.Index) InnerError!WValue {...@@ -3466,8 +3537,7 @@ fn airPtrElemVal(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
34663537
3467 // load pointer onto the stack3538 // load pointer onto the stack
3468 if (ptr_ty.isSlice()) {3539 if (ptr_ty.isSlice()) {
3469 const ptr_local = try self.load(ptr, Type.usize, 0);3540 _ = try self.load(ptr, Type.usize, 0);
3470 try self.addLabel(.local_get, ptr_local.local);
3471 } else {3541 } else {
3472 try self.lowerToStack(ptr);3542 try self.lowerToStack(ptr);
3473 }3543 }
...@@ -3478,12 +3548,15 @@ fn airPtrElemVal(self: *Self, inst: Air.Inst.Index) InnerError!WValue {...@@ -3478,12 +3548,15 @@ fn airPtrElemVal(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
3478 try self.addTag(.i32_mul);3548 try self.addTag(.i32_mul);
3479 try self.addTag(.i32_add);3549 try self.addTag(.i32_add);
34803550
3481 const result = try self.allocLocal(elem_ty);3551 var result = try self.allocLocal(elem_ty);
3482 try self.addLabel(.local_set, result.local);3552 try self.addLabel(.local_set, result.local);
3483 if (isByRef(elem_ty, self.target)) {3553 if (isByRef(elem_ty, self.target)) {
3484 return result;3554 return result;
3485 }3555 }
3486 return self.load(result, elem_ty, 0);3556 defer result.free(self); // only free if it's not returned like above
3557
3558 const elem_val = try self.load(result, elem_ty, 0);
3559 return elem_val.toLocal(self, elem_ty);
3487}3560}
34883561
3489fn airPtrElemPtr(self: *Self, inst: Air.Inst.Index) InnerError!WValue {3562fn airPtrElemPtr(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
...@@ -3499,8 +3572,7 @@ fn airPtrElemPtr(self: *Self, inst: Air.Inst.Index) InnerError!WValue {...@@ -3499,8 +3572,7 @@ fn airPtrElemPtr(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
34993572
3500 // load pointer onto the stack3573 // load pointer onto the stack
3501 if (ptr_ty.isSlice()) {3574 if (ptr_ty.isSlice()) {
3502 const ptr_local = try self.load(ptr, Type.usize, 0);3575 _ = try self.load(ptr, Type.usize, 0);
3503 try self.addLabel(.local_get, ptr_local.local);
3504 } else {3576 } else {
3505 try self.lowerToStack(ptr);3577 try self.lowerToStack(ptr);
3506 }3578 }
...@@ -3511,7 +3583,7 @@ fn airPtrElemPtr(self: *Self, inst: Air.Inst.Index) InnerError!WValue {...@@ -3511,7 +3583,7 @@ fn airPtrElemPtr(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
3511 try self.addTag(.i32_mul);3583 try self.addTag(.i32_mul);
3512 try self.addTag(.i32_add);3584 try self.addTag(.i32_add);
35133585
3514 const result = try self.allocLocal(Type.initTag(.i32));3586 const result = try self.allocLocal(Type.i32);
3515 try self.addLabel(.local_set, result.local);3587 try self.addLabel(.local_set, result.local);
3516 return result;3588 return result;
3517}3589}
...@@ -3599,7 +3671,7 @@ fn memset(self: *Self, ptr: WValue, len: WValue, value: WValue) InnerError!void...@@ -3599,7 +3671,7 @@ fn memset(self: *Self, ptr: WValue, len: WValue, value: WValue) InnerError!void
3599 else => {3671 else => {
3600 // TODO: We should probably lower this to a call to compiler_rt3672 // TODO: We should probably lower this to a call to compiler_rt
3601 // But for now, we implement it manually3673 // But for now, we implement it manually
3602 const offset = try self.allocLocal(Type.usize); // local for counter3674 const offset = try self.ensureAllocLocal(Type.usize); // local for counter
3603 // outer block to jump to when loop is done3675 // outer block to jump to when loop is done
3604 try self.startBlock(.block, wasm.block_empty);3676 try self.startBlock(.block, wasm.block_empty);
3605 try self.startBlock(.loop, wasm.block_empty);3677 try self.startBlock(.loop, wasm.block_empty);
...@@ -3656,13 +3728,16 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) InnerError!WValue {...@@ -3656,13 +3728,16 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
3656 try self.addTag(.i32_mul);3728 try self.addTag(.i32_mul);
3657 try self.addTag(.i32_add);3729 try self.addTag(.i32_add);
36583730
3659 const result = try self.allocLocal(Type.usize);3731 var result = try self.allocLocal(Type.usize);
3660 try self.addLabel(.local_set, result.local);3732 try self.addLabel(.local_set, result.local);
36613733
3662 if (isByRef(elem_ty, self.target)) {3734 if (isByRef(elem_ty, self.target)) {
3663 return result;3735 return result;
3664 }3736 }
3665 return self.load(result, elem_ty, 0);3737 defer result.free(self); // only free if no longer needed and not returned like above
3738
3739 const elem_val = try self.load(result, elem_ty, 0);
3740 return elem_val.toLocal(self, elem_ty);
3666}3741}
36673742
3668fn airFloatToInt(self: *Self, inst: Air.Inst.Index) InnerError!WValue {3743fn airFloatToInt(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
...@@ -3685,11 +3760,8 @@ fn airFloatToInt(self: *Self, inst: Air.Inst.Index) InnerError!WValue {...@@ -3685,11 +3760,8 @@ fn airFloatToInt(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
3685 .signedness = if (dest_ty.isSignedInt()) .signed else .unsigned,3760 .signedness = if (dest_ty.isSignedInt()) .signed else .unsigned,
3686 });3761 });
3687 try self.addTag(Mir.Inst.Tag.fromOpcode(op));3762 try self.addTag(Mir.Inst.Tag.fromOpcode(op));
36883763 const wrapped = try self.wrapOperand(.{ .stack = {} }, dest_ty);
3689 const result = try self.allocLocal(dest_ty);3764 return wrapped.toLocal(self, dest_ty);
3690 try self.addLabel(.local_set, result.local);
3691
3692 return self.wrapOperand(result, dest_ty);
3693}3765}
36943766
3695fn airIntToFloat(self: *Self, inst: Air.Inst.Index) InnerError!WValue {3767fn airIntToFloat(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
...@@ -3887,24 +3959,19 @@ fn cmpOptionals(self: *Self, lhs: WValue, rhs: WValue, operand_ty: Type, op: std...@@ -3887,24 +3959,19 @@ fn cmpOptionals(self: *Self, lhs: WValue, rhs: WValue, operand_ty: Type, op: std
3887 const payload_ty = operand_ty.optionalChild(&buf);3959 const payload_ty = operand_ty.optionalChild(&buf);
3888 const offset = @intCast(u32, operand_ty.abiSize(self.target) - payload_ty.abiSize(self.target));3960 const offset = @intCast(u32, operand_ty.abiSize(self.target) - payload_ty.abiSize(self.target));
38893961
3890 const lhs_is_null = try self.isNull(lhs, operand_ty, .i32_eq);
3891 const rhs_is_null = try self.isNull(rhs, operand_ty, .i32_eq);
3892
3893 // We store the final result in here that will be validated3962 // We store the final result in here that will be validated
3894 // if the optional is truly equal.3963 // if the optional is truly equal.
3895 const result = try self.allocLocal(Type.initTag(.i32));3964 var result = try self.ensureAllocLocal(Type.initTag(.i32));
3965 defer result.free(self);
38963966
3897 try self.startBlock(.block, wasm.block_empty);3967 try self.startBlock(.block, wasm.block_empty);
3898 try self.emitWValue(lhs_is_null);3968 _ = try self.isNull(lhs, operand_ty, .i32_eq);
3899 try self.emitWValue(rhs_is_null);3969 _ = try self.isNull(rhs, operand_ty, .i32_eq);
3900 try self.addTag(.i32_ne); // inverse so we can exit early3970 try self.addTag(.i32_ne); // inverse so we can exit early
3901 try self.addLabel(.br_if, 0);3971 try self.addLabel(.br_if, 0);
39023972
3903 const lhs_pl = try self.load(lhs, payload_ty, offset);3973 _ = try self.load(lhs, payload_ty, offset);
3904 const rhs_pl = try self.load(rhs, payload_ty, offset);3974 _ = try self.load(rhs, payload_ty, offset);
3905
3906 try self.emitWValue(lhs_pl);
3907 try self.emitWValue(rhs_pl);
3908 const opcode = buildOpcode(.{ .op = .ne, .valtype1 = typeToValtype(payload_ty, self.target) });3975 const opcode = buildOpcode(.{ .op = .ne, .valtype1 = typeToValtype(payload_ty, self.target) });
3909 try self.addTag(Mir.Inst.Tag.fromOpcode(opcode));3976 try self.addTag(Mir.Inst.Tag.fromOpcode(opcode));
3910 try self.addLabel(.br_if, 0);3977 try self.addLabel(.br_if, 0);
...@@ -3916,26 +3983,29 @@ fn cmpOptionals(self: *Self, lhs: WValue, rhs: WValue, operand_ty: Type, op: std...@@ -3916,26 +3983,29 @@ fn cmpOptionals(self: *Self, lhs: WValue, rhs: WValue, operand_ty: Type, op: std
3916 try self.emitWValue(result);3983 try self.emitWValue(result);
3917 try self.addImm32(0);3984 try self.addImm32(0);
3918 try self.addTag(if (op == .eq) .i32_ne else .i32_eq);3985 try self.addTag(if (op == .eq) .i32_ne else .i32_eq);
3919 try self.addLabel(.local_set, result.local);3986 return WValue{ .stack = {} };
3920 return result;
3921}3987}
39223988
3923/// Compares big integers by checking both its high bits and low bits.3989/// Compares big integers by checking both its high bits and low bits.
3990/// NOTE: Leaves the result of the comparison on top of the stack.
3924/// TODO: Lower this to compiler_rt call when bitsize > 1283991/// TODO: Lower this to compiler_rt call when bitsize > 128
3925fn cmpBigInt(self: *Self, lhs: WValue, rhs: WValue, operand_ty: Type, op: std.math.CompareOperator) InnerError!WValue {3992fn cmpBigInt(self: *Self, lhs: WValue, rhs: WValue, operand_ty: Type, op: std.math.CompareOperator) InnerError!WValue {
3926 assert(operand_ty.abiSize(self.target) >= 16);3993 assert(operand_ty.abiSize(self.target) >= 16);
3994 assert(!(lhs != .stack and rhs == .stack));
3927 if (operand_ty.intInfo(self.target).bits > 128) {3995 if (operand_ty.intInfo(self.target).bits > 128) {
3928 return self.fail("TODO: Support cmpBigInt for integer bitsize: '{d}'", .{operand_ty.intInfo(self.target).bits});3996 return self.fail("TODO: Support cmpBigInt for integer bitsize: '{d}'", .{operand_ty.intInfo(self.target).bits});
3929 }3997 }
39303998
3931 const lhs_high_bit = try self.load(lhs, Type.u64, 0);3999 var lhs_high_bit = try (try self.load(lhs, Type.u64, 0)).toLocal(self, Type.u64);
3932 const lhs_low_bit = try self.load(lhs, Type.u64, 8);4000 defer lhs_high_bit.free(self);
3933 const rhs_high_bit = try self.load(rhs, Type.u64, 0);4001 var rhs_high_bit = try (try self.load(rhs, Type.u64, 0)).toLocal(self, Type.u64);
3934 const rhs_low_bit = try self.load(rhs, Type.u64, 8);4002 defer rhs_high_bit.free(self);
39354003
3936 switch (op) {4004 switch (op) {
3937 .eq, .neq => {4005 .eq, .neq => {
3938 const xor_high = try self.binOp(lhs_high_bit, rhs_high_bit, Type.u64, .xor);4006 const xor_high = try self.binOp(lhs_high_bit, rhs_high_bit, Type.u64, .xor);
4007 const lhs_low_bit = try self.load(lhs, Type.u64, 8);
4008 const rhs_low_bit = try self.load(rhs, Type.u64, 8);
3939 const xor_low = try self.binOp(lhs_low_bit, rhs_low_bit, Type.u64, .xor);4009 const xor_low = try self.binOp(lhs_low_bit, rhs_low_bit, Type.u64, .xor);
3940 const or_result = try self.binOp(xor_high, xor_low, Type.u64, .@"or");4010 const or_result = try self.binOp(xor_high, xor_low, Type.u64, .@"or");
39414011
...@@ -3947,20 +4017,17 @@ fn cmpBigInt(self: *Self, lhs: WValue, rhs: WValue, operand_ty: Type, op: std.ma...@@ -3947,20 +4017,17 @@ fn cmpBigInt(self: *Self, lhs: WValue, rhs: WValue, operand_ty: Type, op: std.ma
3947 },4017 },
3948 else => {4018 else => {
3949 const ty = if (operand_ty.isSignedInt()) Type.i64 else Type.u64;4019 const ty = if (operand_ty.isSignedInt()) Type.i64 else Type.u64;
3950 const high_bit_eql = try self.cmp(lhs_high_bit, rhs_high_bit, ty, .eq);4020 // leave those value on top of the stack for '.select'
3951 const high_bit_cmp = try self.cmp(lhs_high_bit, rhs_high_bit, ty, op);4021 const lhs_low_bit = try self.load(lhs, Type.u64, 8);
3952 const low_bit_cmp = try self.cmp(lhs_low_bit, rhs_low_bit, ty, op);4022 const rhs_low_bit = try self.load(rhs, Type.u64, 8);
39534023 _ = try self.cmp(lhs_low_bit, rhs_low_bit, ty, op);
3954 try self.emitWValue(low_bit_cmp);4024 _ = try self.cmp(lhs_high_bit, rhs_high_bit, ty, op);
3955 try self.emitWValue(high_bit_cmp);4025 _ = try self.cmp(lhs_high_bit, rhs_high_bit, ty, .eq);
3956 try self.emitWValue(high_bit_eql);
3957 try self.addTag(.select);4026 try self.addTag(.select);
3958 },4027 },
3959 }4028 }
39604029
3961 const result = try self.allocLocal(Type.initTag(.i32));4030 return WValue{ .stack = {} };
3962 try self.addLabel(.local_set, result.local);
3963 return result;
3964}4031}
39654032
3966fn airSetUnionTag(self: *Self, inst: Air.Inst.Index) InnerError!WValue {4033fn airSetUnionTag(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
...@@ -4000,7 +4067,8 @@ fn airGetUnionTag(self: *Self, inst: Air.Inst.Index) InnerError!WValue {...@@ -4000,7 +4067,8 @@ fn airGetUnionTag(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
4000 const offset = if (layout.tag_align < layout.payload_align) blk: {4067 const offset = if (layout.tag_align < layout.payload_align) blk: {
4001 break :blk @intCast(u32, layout.payload_size);4068 break :blk @intCast(u32, layout.payload_size);
4002 } else @as(u32, 0);4069 } else @as(u32, 0);
4003 return self.load(operand, tag_ty, offset);4070 const tag = try self.load(operand, tag_ty, offset);
4071 return tag.toLocal(self, tag_ty);
4004}4072}
40054073
4006fn airFpext(self: *Self, inst: Air.Inst.Index) InnerError!WValue {4074fn airFpext(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
...@@ -4010,19 +4078,20 @@ fn airFpext(self: *Self, inst: Air.Inst.Index) InnerError!WValue {...@@ -4010,19 +4078,20 @@ fn airFpext(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
4010 const dest_ty = self.air.typeOfIndex(inst);4078 const dest_ty = self.air.typeOfIndex(inst);
4011 const operand = try self.resolveInst(ty_op.operand);4079 const operand = try self.resolveInst(ty_op.operand);
40124080
4013 return self.fpext(operand, self.air.typeOf(ty_op.operand), dest_ty);4081 const extended = try self.fpext(operand, self.air.typeOf(ty_op.operand), dest_ty);
4082 return extended.toLocal(self, dest_ty);
4014}4083}
40154084
4085/// Extends a float from a given `Type` to a larger wanted `Type`
4086/// NOTE: Leaves the result on the stack
4016fn fpext(self: *Self, operand: WValue, given: Type, wanted: Type) InnerError!WValue {4087fn fpext(self: *Self, operand: WValue, given: Type, wanted: Type) InnerError!WValue {
4017 const given_bits = given.floatBits(self.target);4088 const given_bits = given.floatBits(self.target);
4018 const wanted_bits = wanted.floatBits(self.target);4089 const wanted_bits = wanted.floatBits(self.target);
40194090
4020 if (wanted_bits == 64 and given_bits == 32) {4091 if (wanted_bits == 64 and given_bits == 32) {
4021 const result = try self.allocLocal(wanted);
4022 try self.emitWValue(operand);4092 try self.emitWValue(operand);
4023 try self.addTag(.f64_promote_f32);4093 try self.addTag(.f64_promote_f32);
4024 try self.addLabel(.local_set, result.local);4094 return WValue{ .stack = {} };
4025 return result;
4026 } else if (given_bits == 16) {4095 } else if (given_bits == 16) {
4027 // call __extendhfsf2(f16) f324096 // call __extendhfsf2(f16) f32
4028 const f32_result = try self.callIntrinsic(4097 const f32_result = try self.callIntrinsic(
...@@ -4036,11 +4105,9 @@ fn fpext(self: *Self, operand: WValue, given: Type, wanted: Type) InnerError!WVa...@@ -4036,11 +4105,9 @@ fn fpext(self: *Self, operand: WValue, given: Type, wanted: Type) InnerError!WVa
4036 return f32_result;4105 return f32_result;
4037 }4106 }
4038 if (wanted_bits == 64) {4107 if (wanted_bits == 64) {
4039 const result = try self.allocLocal(wanted);
4040 try self.emitWValue(f32_result);4108 try self.emitWValue(f32_result);
4041 try self.addTag(.f64_promote_f32);4109 try self.addTag(.f64_promote_f32);
4042 try self.addLabel(.local_set, result.local);4110 return WValue{ .stack = {} };
4043 return result;
4044 }4111 }
4045 return self.fail("TODO: Implement 'fpext' for floats with bitsize: {d}", .{wanted_bits});4112 return self.fail("TODO: Implement 'fpext' for floats with bitsize: {d}", .{wanted_bits});
4046 } else {4113 } else {
...@@ -4055,26 +4122,25 @@ fn airFptrunc(self: *Self, inst: Air.Inst.Index) InnerError!WValue {...@@ -4055,26 +4122,25 @@ fn airFptrunc(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
4055 const ty_op = self.air.instructions.items(.data)[inst].ty_op;4122 const ty_op = self.air.instructions.items(.data)[inst].ty_op;
4056 const dest_ty = self.air.typeOfIndex(inst);4123 const dest_ty = self.air.typeOfIndex(inst);
4057 const operand = try self.resolveInst(ty_op.operand);4124 const operand = try self.resolveInst(ty_op.operand);
4058 return self.fptrunc(operand, self.air.typeOf(ty_op.operand), dest_ty);4125 const trunc = try self.fptrunc(operand, self.air.typeOf(ty_op.operand), dest_ty);
4126 return trunc.toLocal(self, dest_ty);
4059}4127}
40604128
4129/// Truncates a float from a given `Type` to its wanted `Type`
4130/// NOTE: The result value remains on the stack
4061fn fptrunc(self: *Self, operand: WValue, given: Type, wanted: Type) InnerError!WValue {4131fn fptrunc(self: *Self, operand: WValue, given: Type, wanted: Type) InnerError!WValue {
4062 const given_bits = given.floatBits(self.target);4132 const given_bits = given.floatBits(self.target);
4063 const wanted_bits = wanted.floatBits(self.target);4133 const wanted_bits = wanted.floatBits(self.target);
40644134
4065 if (wanted_bits == 32 and given_bits == 64) {4135 if (wanted_bits == 32 and given_bits == 64) {
4066 const result = try self.allocLocal(wanted);
4067 try self.emitWValue(operand);4136 try self.emitWValue(operand);
4068 try self.addTag(.f32_demote_f64);4137 try self.addTag(.f32_demote_f64);
4069 try self.addLabel(.local_set, result.local);4138 return WValue{ .stack = {} };
4070 return result;
4071 } else if (wanted_bits == 16) {4139 } else if (wanted_bits == 16) {
4072 const op: WValue = if (given_bits == 64) blk: {4140 const op: WValue = if (given_bits == 64) blk: {
4073 const tmp = try self.allocLocal(Type.f32);
4074 try self.emitWValue(operand);4141 try self.emitWValue(operand);
4075 try self.addTag(.f32_demote_f64);4142 try self.addTag(.f32_demote_f64);
4076 try self.addLabel(.local_set, tmp.local);4143 break :blk WValue{ .stack = {} };
4077 break :blk tmp;
4078 } else operand;4144 } else operand;
40794145
4080 // call __truncsfhf2(f32) f164146 // call __truncsfhf2(f32) f16
...@@ -4159,12 +4225,9 @@ fn airPopcount(self: *Self, inst: Air.Inst.Index) InnerError!WValue {...@@ -4159,12 +4225,9 @@ fn airPopcount(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
41594225
4160 switch (wasm_bits) {4226 switch (wasm_bits) {
4161 128 => {4227 128 => {
4162 const msb = try self.load(operand, Type.u64, 0);4228 _ = try self.load(operand, Type.u64, 0);
4163 const lsb = try self.load(operand, Type.u64, 8);
4164
4165 try self.emitWValue(msb);
4166 try self.addTag(.i64_popcnt);4229 try self.addTag(.i64_popcnt);
4167 try self.emitWValue(lsb);4230 _ = try self.load(operand, Type.u64, 8);
4168 try self.addTag(.i64_popcnt);4231 try self.addTag(.i64_popcnt);
4169 try self.addTag(.i64_add);4232 try self.addTag(.i64_add);
4170 try self.addTag(.i32_wrap_i64);4233 try self.addTag(.i32_wrap_i64);
...@@ -4268,24 +4331,26 @@ fn airAddSubWithOverflow(self: *Self, inst: Air.Inst.Index, op: Op) InnerError!W...@@ -4268,24 +4331,26 @@ fn airAddSubWithOverflow(self: *Self, inst: Air.Inst.Index, op: Op) InnerError!W
42684331
4269 // for signed integers, we first apply signed shifts by the difference in bits4332 // for signed integers, we first apply signed shifts by the difference in bits
4270 // to get the signed value, as we store it internally as 2's complement.4333 // to get the signed value, as we store it internally as 2's complement.
4271 const lhs = if (wasm_bits != int_info.bits and is_signed) blk: {4334 var lhs = if (wasm_bits != int_info.bits and is_signed) blk: {
4272 break :blk try self.signAbsValue(lhs_op, lhs_ty);4335 break :blk try (try self.signAbsValue(lhs_op, lhs_ty)).toLocal(self, lhs_ty);
4273 } else lhs_op;4336 } else lhs_op;
4274 const rhs = if (wasm_bits != int_info.bits and is_signed) blk: {4337 var rhs = if (wasm_bits != int_info.bits and is_signed) blk: {
4275 break :blk try self.signAbsValue(rhs_op, lhs_ty);4338 break :blk try (try self.signAbsValue(rhs_op, lhs_ty)).toLocal(self, lhs_ty);
4276 } else rhs_op;4339 } else rhs_op;
42774340
4278 const bin_op = try self.binOp(lhs, rhs, lhs_ty, op);4341 var bin_op = try (try self.binOp(lhs, rhs, lhs_ty, op)).toLocal(self, lhs_ty);
4279 const result = if (wasm_bits != int_info.bits) blk: {4342 defer bin_op.free(self);
4280 break :blk try self.wrapOperand(bin_op, lhs_ty);4343 var result = if (wasm_bits != int_info.bits) blk: {
4344 break :blk try (try self.wrapOperand(bin_op, lhs_ty)).toLocal(self, lhs_ty);
4281 } else bin_op;4345 } else bin_op;
4346 defer result.free(self); // no-op when wasm_bits == int_info.bits
42824347
4283 const cmp_op: std.math.CompareOperator = if (op == .sub) .gt else .lt;4348 const cmp_op: std.math.CompareOperator = if (op == .sub) .gt else .lt;
4284 const overflow_bit: WValue = if (is_signed) blk: {4349 const overflow_bit: WValue = if (is_signed) blk: {
4285 if (wasm_bits == int_info.bits) {4350 if (wasm_bits == int_info.bits) {
4286 const cmp_zero = try self.cmp(rhs, zero, lhs_ty, cmp_op);4351 const cmp_zero = try self.cmp(rhs, zero, lhs_ty, cmp_op);
4287 const lt = try self.cmp(bin_op, lhs, lhs_ty, .lt);4352 const lt = try self.cmp(bin_op, lhs, lhs_ty, .lt);
4288 break :blk try self.binOp(cmp_zero, lt, Type.u32, .xor); // result of cmp_zero and lt is always 32bit4353 break :blk try self.binOp(cmp_zero, lt, Type.u32, .xor);
4289 }4354 }
4290 const abs = try self.signAbsValue(bin_op, lhs_ty);4355 const abs = try self.signAbsValue(bin_op, lhs_ty);
4291 break :blk try self.cmp(abs, bin_op, lhs_ty, .neq);4356 break :blk try self.cmp(abs, bin_op, lhs_ty, .neq);
...@@ -4293,11 +4358,22 @@ fn airAddSubWithOverflow(self: *Self, inst: Air.Inst.Index, op: Op) InnerError!W...@@ -4293,11 +4358,22 @@ fn airAddSubWithOverflow(self: *Self, inst: Air.Inst.Index, op: Op) InnerError!W
4293 try self.cmp(bin_op, lhs, lhs_ty, cmp_op)4358 try self.cmp(bin_op, lhs, lhs_ty, cmp_op)
4294 else4359 else
4295 try self.cmp(bin_op, result, lhs_ty, .neq);4360 try self.cmp(bin_op, result, lhs_ty, .neq);
4361 var overflow_local = try overflow_bit.toLocal(self, Type.u32);
4362 defer overflow_local.free(self);
42964363
4297 const result_ptr = try self.allocStack(self.air.typeOfIndex(inst));4364 const result_ptr = try self.allocStack(self.air.typeOfIndex(inst));
4298 try self.store(result_ptr, result, lhs_ty, 0);4365 try self.store(result_ptr, result, lhs_ty, 0);
4299 const offset = @intCast(u32, lhs_ty.abiSize(self.target));4366 const offset = @intCast(u32, lhs_ty.abiSize(self.target));
4300 try self.store(result_ptr, overflow_bit, Type.initTag(.u1), offset);4367 try self.store(result_ptr, overflow_local, Type.initTag(.u1), offset);
4368
4369 // in this case, we performed a signAbsValue which created a temporary local
4370 // so let's free this so it can be re-used instead.
4371 // In the other case we do not want to free it, because that would free the
4372 // resolved instructions which may be referenced by other instructions.
4373 if (wasm_bits != int_info.bits and is_signed) {
4374 lhs.free(self);
4375 rhs.free(self);
4376 }
43014377
4302 return result_ptr;4378 return result_ptr;
4303}4379}
...@@ -4310,52 +4386,58 @@ fn airAddSubWithOverflowBigInt(self: *Self, lhs: WValue, rhs: WValue, ty: Type,...@@ -4310,52 +4386,58 @@ fn airAddSubWithOverflowBigInt(self: *Self, lhs: WValue, rhs: WValue, ty: Type,
4310 return self.fail("TODO: Implement @{{add/sub}}WithOverflow for integer bitsize '{d}'", .{int_info.bits});4386 return self.fail("TODO: Implement @{{add/sub}}WithOverflow for integer bitsize '{d}'", .{int_info.bits});
4311 }4387 }
43124388
4313 const lhs_high_bit = try self.load(lhs, Type.u64, 0);4389 var lhs_high_bit = try (try self.load(lhs, Type.u64, 0)).toLocal(self, Type.u64);
4314 const lhs_low_bit = try self.load(lhs, Type.u64, 8);4390 defer lhs_high_bit.free(self);
4315 const rhs_high_bit = try self.load(rhs, Type.u64, 0);4391 var lhs_low_bit = try (try self.load(lhs, Type.u64, 8)).toLocal(self, Type.u64);
4316 const rhs_low_bit = try self.load(rhs, Type.u64, 8);4392 defer lhs_low_bit.free(self);
4393 var rhs_high_bit = try (try self.load(rhs, Type.u64, 0)).toLocal(self, Type.u64);
4394 defer rhs_high_bit.free(self);
4395 var rhs_low_bit = try (try self.load(rhs, Type.u64, 8)).toLocal(self, Type.u64);
4396 defer rhs_low_bit.free(self);
43174397
4318 const low_op_res = try self.binOp(lhs_low_bit, rhs_low_bit, Type.u64, op);4398 var low_op_res = try (try self.binOp(lhs_low_bit, rhs_low_bit, Type.u64, op)).toLocal(self, Type.u64);
4319 const high_op_res = try self.binOp(lhs_high_bit, rhs_high_bit, Type.u64, op);4399 defer low_op_res.free(self);
4400 var high_op_res = try (try self.binOp(lhs_high_bit, rhs_high_bit, Type.u64, op)).toLocal(self, Type.u64);
4401 defer high_op_res.free(self);
43204402
4321 const lt = if (op == .add) blk: {4403 var lt = if (op == .add) blk: {
4322 break :blk try self.cmp(high_op_res, lhs_high_bit, Type.u64, .lt);4404 break :blk try (try self.cmp(high_op_res, lhs_high_bit, Type.u64, .lt)).toLocal(self, Type.u32);
4323 } else if (op == .sub) blk: {4405 } else if (op == .sub) blk: {
4324 break :blk try self.cmp(lhs_high_bit, rhs_high_bit, Type.u64, .lt);4406 break :blk try (try self.cmp(lhs_high_bit, rhs_high_bit, Type.u64, .lt)).toLocal(self, Type.u32);
4325 } else unreachable;4407 } else unreachable;
4326 const tmp = try self.intcast(lt, Type.u32, Type.u64);4408 defer lt.free(self);
4327 const tmp_op = try self.binOp(low_op_res, tmp, Type.u64, op);4409 var tmp = try (try self.intcast(lt, Type.u32, Type.u64)).toLocal(self, Type.u64);
4410 defer tmp.free(self);
4411 var tmp_op = try (try self.binOp(low_op_res, tmp, Type.u64, op)).toLocal(self, Type.u64);
4412 defer tmp_op.free(self);
43284413
4329 const overflow_bit = if (is_signed) blk: {4414 const overflow_bit = if (is_signed) blk: {
4330 const xor_op = try self.binOp(lhs_low_bit, tmp_op, Type.u64, .xor);
4331 const xor_low = try self.binOp(lhs_low_bit, rhs_low_bit, Type.u64, .xor);4415 const xor_low = try self.binOp(lhs_low_bit, rhs_low_bit, Type.u64, .xor);
4332 const to_wrap = if (op == .add) wrap: {4416 const to_wrap = if (op == .add) wrap: {
4333 break :wrap try self.binOp(xor_low, .{ .imm64 = ~@as(u64, 0) }, Type.u64, .xor);4417 break :wrap try self.binOp(xor_low, .{ .imm64 = ~@as(u64, 0) }, Type.u64, .xor);
4334 } else xor_low;4418 } else xor_low;
4419 const xor_op = try self.binOp(lhs_low_bit, tmp_op, Type.u64, .xor);
4335 const wrap = try self.binOp(to_wrap, xor_op, Type.u64, .@"and");4420 const wrap = try self.binOp(to_wrap, xor_op, Type.u64, .@"and");
4336 break :blk try self.cmp(wrap, .{ .imm64 = 0 }, Type.i64, .lt); // i64 because signed4421 break :blk try self.cmp(wrap, .{ .imm64 = 0 }, Type.i64, .lt); // i64 because signed
4337 } else blk: {4422 } else blk: {
4338 const eq = try self.cmp(tmp_op, lhs_low_bit, Type.u64, .eq);
4339 const op_eq = try self.cmp(tmp_op, lhs_low_bit, Type.u64, if (op == .add) .lt else .gt);
4340
4341 const first_arg = if (op == .sub) arg: {4423 const first_arg = if (op == .sub) arg: {
4342 break :arg try self.cmp(high_op_res, lhs_high_bit, Type.u64, .gt);4424 break :arg try self.cmp(high_op_res, lhs_high_bit, Type.u64, .gt);
4343 } else lt;4425 } else lt;
43444426
4345 try self.emitWValue(first_arg);4427 try self.emitWValue(first_arg);
4346 try self.emitWValue(op_eq);4428 _ = try self.cmp(tmp_op, lhs_low_bit, Type.u64, if (op == .add) .lt else .gt);
4347 try self.emitWValue(eq);4429 _ = try self.cmp(tmp_op, lhs_low_bit, Type.u64, .eq);
4348 try self.addTag(.select);4430 try self.addTag(.select);
43494431
4350 const overflow_bit = try self.allocLocal(Type.initTag(.u1));4432 break :blk WValue{ .stack = {} };
4351 try self.addLabel(.local_set, overflow_bit.local);
4352 break :blk overflow_bit;
4353 };4433 };
4434 var overflow_local = try overflow_bit.toLocal(self, Type.initTag(.u1));
4435 defer overflow_local.free(self);
43544436
4355 const result_ptr = try self.allocStack(result_ty);4437 const result_ptr = try self.allocStack(result_ty);
4356 try self.store(result_ptr, high_op_res, Type.u64, 0);4438 try self.store(result_ptr, high_op_res, Type.u64, 0);
4357 try self.store(result_ptr, tmp_op, Type.u64, 8);4439 try self.store(result_ptr, tmp_op, Type.u64, 8);
4358 try self.store(result_ptr, overflow_bit, Type.initTag(.u1), 16);4440 try self.store(result_ptr, overflow_local, Type.initTag(.u1), 16);
43594441
4360 return result_ptr;4442 return result_ptr;
4361}4443}
...@@ -4377,24 +4459,31 @@ fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) InnerError!WValue {...@@ -4377,24 +4459,31 @@ fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
4377 return self.fail("TODO: Implement shl_with_overflow for integer bitsize: {d}", .{int_info.bits});4459 return self.fail("TODO: Implement shl_with_overflow for integer bitsize: {d}", .{int_info.bits});
4378 };4460 };
43794461
4380 const shl = try self.binOp(lhs, rhs, lhs_ty, .shl);4462 var shl = try (try self.binOp(lhs, rhs, lhs_ty, .shl)).toLocal(self, lhs_ty);
4381 const result = if (wasm_bits != int_info.bits) blk: {4463 defer shl.free(self);
4382 break :blk try self.wrapOperand(shl, lhs_ty);4464 var result = if (wasm_bits != int_info.bits) blk: {
4465 break :blk try (try self.wrapOperand(shl, lhs_ty)).toLocal(self, lhs_ty);
4383 } else shl;4466 } else shl;
4467 defer result.free(self); // it's a no-op to free the same local twice (when wasm_bits == int_info.bits)
43844468
4385 const overflow_bit = if (wasm_bits != int_info.bits and is_signed) blk: {4469 const overflow_bit = if (wasm_bits != int_info.bits and is_signed) blk: {
4470 // emit lhs to stack to we can keep 'wrapped' on the stack also
4471 try self.emitWValue(lhs);
4386 const abs = try self.signAbsValue(shl, lhs_ty);4472 const abs = try self.signAbsValue(shl, lhs_ty);
4387 const wrapped = try self.wrapBinOp(abs, rhs, lhs_ty, .shr);4473 const wrapped = try self.wrapBinOp(abs, rhs, lhs_ty, .shr);
4388 break :blk try self.cmp(lhs, wrapped, lhs_ty, .neq);4474 break :blk try self.cmp(.{ .stack = {} }, wrapped, lhs_ty, .neq);
4389 } else blk: {4475 } else blk: {
4476 try self.emitWValue(lhs);
4390 const shr = try self.binOp(result, rhs, lhs_ty, .shr);4477 const shr = try self.binOp(result, rhs, lhs_ty, .shr);
4391 break :blk try self.cmp(lhs, shr, lhs_ty, .neq);4478 break :blk try self.cmp(.{ .stack = {} }, shr, lhs_ty, .neq);
4392 };4479 };
4480 var overflow_local = try overflow_bit.toLocal(self, Type.initTag(.u1));
4481 defer overflow_local.free(self);
43934482
4394 const result_ptr = try self.allocStack(self.air.typeOfIndex(inst));4483 const result_ptr = try self.allocStack(self.air.typeOfIndex(inst));
4395 try self.store(result_ptr, result, lhs_ty, 0);4484 try self.store(result_ptr, result, lhs_ty, 0);
4396 const offset = @intCast(u32, lhs_ty.abiSize(self.target));4485 const offset = @intCast(u32, lhs_ty.abiSize(self.target));
4397 try self.store(result_ptr, overflow_bit, Type.initTag(.u1), offset);4486 try self.store(result_ptr, overflow_local, Type.initTag(.u1), offset);
43984487
4399 return result_ptr;4488 return result_ptr;
4400}4489}
...@@ -4412,7 +4501,9 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) InnerError!WValue {...@@ -4412,7 +4501,9 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
44124501
4413 // We store the bit if it's overflowed or not in this. As it's zero-initialized4502 // We store the bit if it's overflowed or not in this. As it's zero-initialized
4414 // we only need to update it if an overflow (or underflow) occurred.4503 // we only need to update it if an overflow (or underflow) occurred.
4415 const overflow_bit = try self.allocLocal(Type.initTag(.u1));4504 var overflow_bit = try self.ensureAllocLocal(Type.initTag(.u1));
4505 defer overflow_bit.free(self);
4506
4416 const int_info = lhs_ty.intInfo(self.target);4507 const int_info = lhs_ty.intInfo(self.target);
4417 const wasm_bits = toWasmBits(int_info.bits) orelse {4508 const wasm_bits = toWasmBits(int_info.bits) orelse {
4418 return self.fail("TODO: Implement overflow arithmetic for integer bitsize: {d}", .{int_info.bits});4509 return self.fail("TODO: Implement overflow arithmetic for integer bitsize: {d}", .{int_info.bits});
...@@ -4433,49 +4524,49 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) InnerError!WValue {...@@ -4433,49 +4524,49 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
4433 const new_ty = if (int_info.signedness == .signed) Type.i64 else Type.u64;4524 const new_ty = if (int_info.signedness == .signed) Type.i64 else Type.u64;
4434 const lhs_upcast = try self.intcast(lhs, lhs_ty, new_ty);4525 const lhs_upcast = try self.intcast(lhs, lhs_ty, new_ty);
4435 const rhs_upcast = try self.intcast(rhs, lhs_ty, new_ty);4526 const rhs_upcast = try self.intcast(rhs, lhs_ty, new_ty);
4436 const bin_op = try self.binOp(lhs_upcast, rhs_upcast, new_ty, .mul);4527 const bin_op = try (try self.binOp(lhs_upcast, rhs_upcast, new_ty, .mul)).toLocal(self, new_ty);
4437 if (int_info.signedness == .unsigned) {4528 if (int_info.signedness == .unsigned) {
4438 const shr = try self.binOp(bin_op, .{ .imm64 = int_info.bits }, new_ty, .shr);4529 const shr = try self.binOp(bin_op, .{ .imm64 = int_info.bits }, new_ty, .shr);
4439 const wrap = try self.intcast(shr, new_ty, lhs_ty);4530 const wrap = try self.intcast(shr, new_ty, lhs_ty);
4440 const cmp_res = try self.cmp(wrap, zero, lhs_ty, .neq);4531 _ = try self.cmp(wrap, zero, lhs_ty, .neq);
4441 try self.emitWValue(cmp_res);
4442 try self.addLabel(.local_set, overflow_bit.local);4532 try self.addLabel(.local_set, overflow_bit.local);
4443 break :blk try self.intcast(bin_op, new_ty, lhs_ty);4533 break :blk try self.intcast(bin_op, new_ty, lhs_ty);
4444 } else {4534 } else {
4445 const down_cast = try self.intcast(bin_op, new_ty, lhs_ty);4535 const down_cast = try (try self.intcast(bin_op, new_ty, lhs_ty)).toLocal(self, lhs_ty);
4446 const shr = try self.binOp(down_cast, .{ .imm32 = int_info.bits - 1 }, lhs_ty, .shr);4536 var shr = try (try self.binOp(down_cast, .{ .imm32 = int_info.bits - 1 }, lhs_ty, .shr)).toLocal(self, lhs_ty);
4537 defer shr.free(self);
44474538
4448 const shr_res = try self.binOp(bin_op, .{ .imm64 = int_info.bits }, new_ty, .shr);4539 const shr_res = try self.binOp(bin_op, .{ .imm64 = int_info.bits }, new_ty, .shr);
4449 const down_shr_res = try self.intcast(shr_res, new_ty, lhs_ty);4540 const down_shr_res = try self.intcast(shr_res, new_ty, lhs_ty);
4450 const cmp_res = try self.cmp(down_shr_res, shr, lhs_ty, .neq);4541 _ = try self.cmp(down_shr_res, shr, lhs_ty, .neq);
4451 try self.emitWValue(cmp_res);
4452 try self.addLabel(.local_set, overflow_bit.local);4542 try self.addLabel(.local_set, overflow_bit.local);
4453 break :blk down_cast;4543 break :blk down_cast;
4454 }4544 }
4455 } else if (int_info.signedness == .signed) blk: {4545 } else if (int_info.signedness == .signed) blk: {
4456 const lhs_abs = try self.signAbsValue(lhs, lhs_ty);4546 const lhs_abs = try self.signAbsValue(lhs, lhs_ty);
4457 const rhs_abs = try self.signAbsValue(rhs, lhs_ty);4547 const rhs_abs = try self.signAbsValue(rhs, lhs_ty);
4458 const bin_op = try self.binOp(lhs_abs, rhs_abs, lhs_ty, .mul);4548 const bin_op = try (try self.binOp(lhs_abs, rhs_abs, lhs_ty, .mul)).toLocal(self, lhs_ty);
4459 const mul_abs = try self.signAbsValue(bin_op, lhs_ty);4549 const mul_abs = try self.signAbsValue(bin_op, lhs_ty);
4460 const cmp_op = try self.cmp(mul_abs, bin_op, lhs_ty, .neq);4550 _ = try self.cmp(mul_abs, bin_op, lhs_ty, .neq);
4461 try self.emitWValue(cmp_op);
4462 try self.addLabel(.local_set, overflow_bit.local);4551 try self.addLabel(.local_set, overflow_bit.local);
4463 break :blk try self.wrapOperand(bin_op, lhs_ty);4552 break :blk try self.wrapOperand(bin_op, lhs_ty);
4464 } else blk: {4553 } else blk: {
4465 const bin_op = try self.binOp(lhs, rhs, lhs_ty, .mul);4554 var bin_op = try (try self.binOp(lhs, rhs, lhs_ty, .mul)).toLocal(self, lhs_ty);
4555 defer bin_op.free(self);
4466 const shift_imm = if (wasm_bits == 32)4556 const shift_imm = if (wasm_bits == 32)
4467 WValue{ .imm32 = int_info.bits }4557 WValue{ .imm32 = int_info.bits }
4468 else4558 else
4469 WValue{ .imm64 = int_info.bits };4559 WValue{ .imm64 = int_info.bits };
4470 const shr = try self.binOp(bin_op, shift_imm, lhs_ty, .shr);4560 const shr = try self.binOp(bin_op, shift_imm, lhs_ty, .shr);
4471 const cmp_op = try self.cmp(shr, zero, lhs_ty, .neq);4561 _ = try self.cmp(shr, zero, lhs_ty, .neq);
4472 try self.emitWValue(cmp_op);
4473 try self.addLabel(.local_set, overflow_bit.local);4562 try self.addLabel(.local_set, overflow_bit.local);
4474 break :blk try self.wrapOperand(bin_op, lhs_ty);4563 break :blk try self.wrapOperand(bin_op, lhs_ty);
4475 };4564 };
4565 var bin_op_local = try bin_op.toLocal(self, lhs_ty);
4566 defer bin_op_local.free(self);
44764567
4477 const result_ptr = try self.allocStack(self.air.typeOfIndex(inst));4568 const result_ptr = try self.allocStack(self.air.typeOfIndex(inst));
4478 try self.store(result_ptr, bin_op, lhs_ty, 0);4569 try self.store(result_ptr, bin_op_local, lhs_ty, 0);
4479 const offset = @intCast(u32, lhs_ty.abiSize(self.target));4570 const offset = @intCast(u32, lhs_ty.abiSize(self.target));
4480 try self.store(result_ptr, overflow_bit, Type.initTag(.u1), offset);4571 try self.store(result_ptr, overflow_bit, Type.initTag(.u1), offset);
44814572
...@@ -4497,12 +4588,10 @@ fn airMaxMin(self: *Self, inst: Air.Inst.Index, op: enum { max, min }) InnerErro...@@ -4497,12 +4588,10 @@ fn airMaxMin(self: *Self, inst: Air.Inst.Index, op: enum { max, min }) InnerErro
4497 const lhs = try self.resolveInst(bin_op.lhs);4588 const lhs = try self.resolveInst(bin_op.lhs);
4498 const rhs = try self.resolveInst(bin_op.rhs);4589 const rhs = try self.resolveInst(bin_op.rhs);
44994590
4500 const cmp_result = try self.cmp(lhs, rhs, ty, if (op == .max) .gt else .lt);
4501
4502 // operands to select from4591 // operands to select from
4503 try self.lowerToStack(lhs);4592 try self.lowerToStack(lhs);
4504 try self.lowerToStack(rhs);4593 try self.lowerToStack(rhs);
4505 try self.emitWValue(cmp_result);4594 _ = try self.cmp(lhs, rhs, ty, if (op == .max) .gt else .lt);
45064595
4507 // based on the result from comparison, return operand 0 or 1.4596 // based on the result from comparison, return operand 0 or 1.
4508 try self.addTag(.select);4597 try self.addTag(.select);
...@@ -4528,21 +4617,22 @@ fn airMulAdd(self: *Self, inst: Air.Inst.Index) InnerError!WValue {...@@ -4528,21 +4617,22 @@ fn airMulAdd(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
4528 const rhs = try self.resolveInst(bin_op.rhs);4617 const rhs = try self.resolveInst(bin_op.rhs);
45294618
4530 if (ty.floatBits(self.target) == 16) {4619 if (ty.floatBits(self.target) == 16) {
4531 const addend_ext = try self.fpext(addend, ty, Type.f32);
4532 const lhs_ext = try self.fpext(lhs, ty, Type.f32);
4533 const rhs_ext = try self.fpext(rhs, ty, Type.f32);4620 const rhs_ext = try self.fpext(rhs, ty, Type.f32);
4621 const lhs_ext = try self.fpext(lhs, ty, Type.f32);
4622 const addend_ext = try self.fpext(addend, ty, Type.f32);
4534 // call to compiler-rt `fn fmaf(f32, f32, f32) f32`4623 // call to compiler-rt `fn fmaf(f32, f32, f32) f32`
4535 const result = try self.callIntrinsic(4624 var result = try self.callIntrinsic(
4536 "fmaf",4625 "fmaf",
4537 &.{ Type.f32, Type.f32, Type.f32 },4626 &.{ Type.f32, Type.f32, Type.f32 },
4538 Type.f32,4627 Type.f32,
4539 &.{ rhs_ext, lhs_ext, addend_ext },4628 &.{ rhs_ext, lhs_ext, addend_ext },
4540 );4629 );
4541 return try self.fptrunc(result, Type.f32, ty);4630 defer result.free(self);
4631 return try (try self.fptrunc(result, Type.f32, ty)).toLocal(self, ty);
4542 }4632 }
45434633
4544 const mul_result = try self.binOp(lhs, rhs, ty, .mul);4634 const mul_result = try self.binOp(lhs, rhs, ty, .mul);
4545 return self.binOp(mul_result, addend, ty, .add);4635 return (try self.binOp(mul_result, addend, ty, .add)).toLocal(self, ty);
4546}4636}
45474637
4548fn airClz(self: *Self, inst: Air.Inst.Index) InnerError!WValue {4638fn airClz(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
...@@ -4571,17 +4661,16 @@ fn airClz(self: *Self, inst: Air.Inst.Index) InnerError!WValue {...@@ -4571,17 +4661,16 @@ fn airClz(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
4571 try self.addTag(.i32_wrap_i64);4661 try self.addTag(.i32_wrap_i64);
4572 },4662 },
4573 128 => {4663 128 => {
4574 const msb = try self.load(operand, Type.u64, 0);4664 var lsb = try (try self.load(operand, Type.u64, 8)).toLocal(self, Type.u64);
4575 const lsb = try self.load(operand, Type.u64, 8);4665 defer lsb.free(self);
4576 const neq = try self.cmp(lsb, .{ .imm64 = 0 }, Type.u64, .neq);
45774666
4578 try self.emitWValue(lsb);4667 try self.emitWValue(lsb);
4579 try self.addTag(.i64_clz);4668 try self.addTag(.i64_clz);
4580 try self.emitWValue(msb);4669 _ = try self.load(operand, Type.u64, 0);
4581 try self.addTag(.i64_clz);4670 try self.addTag(.i64_clz);
4582 try self.emitWValue(.{ .imm64 = 64 });4671 try self.emitWValue(.{ .imm64 = 64 });
4583 try self.addTag(.i64_add);4672 try self.addTag(.i64_add);
4584 try self.emitWValue(neq);4673 _ = try self.cmp(lsb, .{ .imm64 = 0 }, Type.u64, .neq);
4585 try self.addTag(.select);4674 try self.addTag(.select);
4586 try self.addTag(.i32_wrap_i64);4675 try self.addTag(.i32_wrap_i64);
4587 },4676 },
...@@ -4618,28 +4707,27 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) InnerError!WValue {...@@ -4618,28 +4707,27 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
4618 32 => {4707 32 => {
4619 if (wasm_bits != int_info.bits) {4708 if (wasm_bits != int_info.bits) {
4620 const val: u32 = @as(u32, 1) << @intCast(u5, int_info.bits);4709 const val: u32 = @as(u32, 1) << @intCast(u5, int_info.bits);
4621 const bin_op = try self.binOp(operand, .{ .imm32 = val }, ty, .@"or");4710 // leave value on the stack
4622 try self.emitWValue(bin_op);4711 _ = try self.binOp(operand, .{ .imm32 = val }, ty, .@"or");
4623 } else try self.emitWValue(operand);4712 } else try self.emitWValue(operand);
4624 try self.addTag(.i32_ctz);4713 try self.addTag(.i32_ctz);
4625 },4714 },
4626 64 => {4715 64 => {
4627 if (wasm_bits != int_info.bits) {4716 if (wasm_bits != int_info.bits) {
4628 const val: u64 = @as(u64, 1) << @intCast(u6, int_info.bits);4717 const val: u64 = @as(u64, 1) << @intCast(u6, int_info.bits);
4629 const bin_op = try self.binOp(operand, .{ .imm64 = val }, ty, .@"or");4718 // leave value on the stack
4630 try self.emitWValue(bin_op);4719 _ = try self.binOp(operand, .{ .imm64 = val }, ty, .@"or");
4631 } else try self.emitWValue(operand);4720 } else try self.emitWValue(operand);
4632 try self.addTag(.i64_ctz);4721 try self.addTag(.i64_ctz);
4633 try self.addTag(.i32_wrap_i64);4722 try self.addTag(.i32_wrap_i64);
4634 },4723 },
4635 128 => {4724 128 => {
4636 const msb = try self.load(operand, Type.u64, 0);4725 var msb = try (try self.load(operand, Type.u64, 0)).toLocal(self, Type.u64);
4637 const lsb = try self.load(operand, Type.u64, 8);4726 defer msb.free(self);
4638 const neq = try self.cmp(msb, .{ .imm64 = 0 }, Type.u64, .neq);
46394727
4640 try self.emitWValue(msb);4728 try self.emitWValue(msb);
4641 try self.addTag(.i64_ctz);4729 try self.addTag(.i64_ctz);
4642 try self.emitWValue(lsb);4730 _ = try self.load(operand, Type.u64, 8);
4643 if (wasm_bits != int_info.bits) {4731 if (wasm_bits != int_info.bits) {
4644 try self.addImm64(@as(u64, 1) << @intCast(u6, int_info.bits - 64));4732 try self.addImm64(@as(u64, 1) << @intCast(u6, int_info.bits - 64));
4645 try self.addTag(.i64_or);4733 try self.addTag(.i64_or);
...@@ -4651,7 +4739,7 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) InnerError!WValue {...@@ -4651,7 +4739,7 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
4651 } else {4739 } else {
4652 try self.addTag(.i64_add);4740 try self.addTag(.i64_add);
4653 }4741 }
4654 try self.emitWValue(neq);4742 _ = try self.cmp(msb, .{ .imm64 = 0 }, Type.u64, .neq);
4655 try self.addTag(.select);4743 try self.addTag(.select);
4656 try self.addTag(.i32_wrap_i64);4744 try self.addTag(.i32_wrap_i64);
4657 },4745 },
...@@ -4777,7 +4865,8 @@ fn lowerTry(...@@ -4777,7 +4865,8 @@ fn lowerTry(
4777 if (isByRef(pl_ty, self.target)) {4865 if (isByRef(pl_ty, self.target)) {
4778 return buildPointerOffset(self, err_union, pl_offset, .new);4866 return buildPointerOffset(self, err_union, pl_offset, .new);
4779 }4867 }
4780 return self.load(err_union, pl_ty, pl_offset);4868 const payload = try self.load(err_union, pl_ty, pl_offset);
4869 return payload.toLocal(self, pl_ty);
4781}4870}
47824871
4783fn airByteSwap(self: *Self, inst: Air.Inst.Index) InnerError!WValue {4872fn airByteSwap(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
...@@ -4807,11 +4896,11 @@ fn airByteSwap(self: *Self, inst: Air.Inst.Index) InnerError!WValue {...@@ -4807,11 +4896,11 @@ fn airByteSwap(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
4807 const res = if (int_info.signedness == .signed) blk: {4896 const res = if (int_info.signedness == .signed) blk: {
4808 break :blk try self.wrapOperand(shr_res, Type.u8);4897 break :blk try self.wrapOperand(shr_res, Type.u8);
4809 } else shr_res;4898 } else shr_res;
4810 return self.binOp(lhs, res, ty, .@"or");4899 return (try self.binOp(lhs, res, ty, .@"or")).toLocal(self, ty);
4811 },4900 },
4812 24 => {4901 24 => {
4813 const msb = try self.wrapOperand(operand, Type.u16);4902 var msb = try (try self.wrapOperand(operand, Type.u16)).toLocal(self, Type.u16);
4814 const lsb = try self.wrapBinOp(operand, .{ .imm32 = 16 }, Type.u8, .shr);4903 defer msb.free(self);
48154904
4816 const shl_res = try self.binOp(msb, .{ .imm32 = 8 }, Type.u16, .shl);4905 const shl_res = try self.binOp(msb, .{ .imm32 = 8 }, Type.u16, .shl);
4817 const lhs = try self.binOp(shl_res, .{ .imm32 = 0xFF0000 }, Type.u16, .@"and");4906 const lhs = try self.binOp(shl_res, .{ .imm32 = 0xFF0000 }, Type.u16, .@"and");
...@@ -4825,22 +4914,26 @@ fn airByteSwap(self: *Self, inst: Air.Inst.Index) InnerError!WValue {...@@ -4825,22 +4914,26 @@ fn airByteSwap(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
4825 const rhs_wrap = try self.wrapOperand(msb, Type.u8);4914 const rhs_wrap = try self.wrapOperand(msb, Type.u8);
4826 const rhs_result = try self.binOp(rhs_wrap, .{ .imm32 = 16 }, ty, .shl);4915 const rhs_result = try self.binOp(rhs_wrap, .{ .imm32 = 16 }, ty, .shl);
48274916
4917 const lsb = try self.wrapBinOp(operand, .{ .imm32 = 16 }, Type.u8, .shr);
4828 const tmp = try self.binOp(lhs_result, rhs_result, ty, .@"or");4918 const tmp = try self.binOp(lhs_result, rhs_result, ty, .@"or");
4829 return self.binOp(tmp, lsb, ty, .@"or");4919 return (try self.binOp(tmp, lsb, ty, .@"or")).toLocal(self, ty);
4830 },4920 },
4831 32 => {4921 32 => {
4832 const shl_tmp = try self.binOp(operand, .{ .imm32 = 8 }, ty, .shl);4922 const shl_tmp = try self.binOp(operand, .{ .imm32 = 8 }, ty, .shl);
4833 const lhs = try self.binOp(shl_tmp, .{ .imm32 = 0xFF00FF00 }, ty, .@"and");4923 var lhs = try (try self.binOp(shl_tmp, .{ .imm32 = 0xFF00FF00 }, ty, .@"and")).toLocal(self, ty);
4924 defer lhs.free(self);
4834 const shr_tmp = try self.binOp(operand, .{ .imm32 = 8 }, ty, .shr);4925 const shr_tmp = try self.binOp(operand, .{ .imm32 = 8 }, ty, .shr);
4835 const rhs = try self.binOp(shr_tmp, .{ .imm32 = 0xFF00FF }, ty, .@"and");4926 var rhs = try (try self.binOp(shr_tmp, .{ .imm32 = 0xFF00FF }, ty, .@"and")).toLocal(self, ty);
4836 const tmp_or = try self.binOp(lhs, rhs, ty, .@"or");4927 defer rhs.free(self);
4928 var tmp_or = try (try self.binOp(lhs, rhs, ty, .@"or")).toLocal(self, ty);
4929 defer tmp_or.free(self);
48374930
4838 const shl = try self.binOp(tmp_or, .{ .imm32 = 16 }, ty, .shl);4931 const shl = try self.binOp(tmp_or, .{ .imm32 = 16 }, ty, .shl);
4839 const shr = try self.binOp(tmp_or, .{ .imm32 = 16 }, ty, .shr);4932 const shr = try self.binOp(tmp_or, .{ .imm32 = 16 }, ty, .shr);
4840 const res = if (int_info.signedness == .signed) blk: {4933 const res = if (int_info.signedness == .signed) blk: {
4841 break :blk try self.wrapOperand(shr, Type.u16);4934 break :blk try self.wrapOperand(shr, Type.u16);
4842 } else shr;4935 } else shr;
4843 return self.binOp(shl, res, ty, .@"or");4936 return (try self.binOp(shl, res, ty, .@"or")).toLocal(self, ty);
4844 },4937 },
4845 else => return self.fail("TODO: @byteSwap for integers with bitsize {d}", .{int_info.bits}),4938 else => return self.fail("TODO: @byteSwap for integers with bitsize {d}", .{int_info.bits}),
4846 }4939 }
...@@ -4857,7 +4950,7 @@ fn airDiv(self: *Self, inst: Air.Inst.Index) InnerError!WValue {...@@ -4857,7 +4950,7 @@ fn airDiv(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
4857 if (ty.isSignedInt()) {4950 if (ty.isSignedInt()) {
4858 return self.divSigned(lhs, rhs, ty);4951 return self.divSigned(lhs, rhs, ty);
4859 }4952 }
4860 return self.binOp(lhs, rhs, ty, .div);4953 return (try self.binOp(lhs, rhs, ty, .div)).toLocal(self, ty);
4861}4954}
48624955
4863fn airDivFloor(self: *Self, inst: Air.Inst.Index) InnerError!WValue {4956fn airDivFloor(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
...@@ -4869,33 +4962,31 @@ fn airDivFloor(self: *Self, inst: Air.Inst.Index) InnerError!WValue {...@@ -4869,33 +4962,31 @@ fn airDivFloor(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
4869 const rhs = try self.resolveInst(bin_op.rhs);4962 const rhs = try self.resolveInst(bin_op.rhs);
48704963
4871 if (ty.isUnsignedInt()) {4964 if (ty.isUnsignedInt()) {
4872 return self.binOp(lhs, rhs, ty, .div);4965 return (try self.binOp(lhs, rhs, ty, .div)).toLocal(self, ty);
4873 } else if (ty.isSignedInt()) {4966 } else if (ty.isSignedInt()) {
4874 const int_bits = ty.intInfo(self.target).bits;4967 const int_bits = ty.intInfo(self.target).bits;
4875 const wasm_bits = toWasmBits(int_bits) orelse {4968 const wasm_bits = toWasmBits(int_bits) orelse {
4876 return self.fail("TODO: `@divFloor` for signed integers larger than '{d}' bits", .{int_bits});4969 return self.fail("TODO: `@divFloor` for signed integers larger than '{d}' bits", .{int_bits});
4877 };4970 };
4878 const lhs_res = if (wasm_bits != int_bits) blk: {4971 const lhs_res = if (wasm_bits != int_bits) blk: {
4879 break :blk try self.signAbsValue(lhs, ty);4972 break :blk try (try self.signAbsValue(lhs, ty)).toLocal(self, ty);
4880 } else lhs;4973 } else lhs;
4881 const rhs_res = if (wasm_bits != int_bits) blk: {4974 const rhs_res = if (wasm_bits != int_bits) blk: {
4882 break :blk try self.signAbsValue(rhs, ty);4975 break :blk try (try self.signAbsValue(rhs, ty)).toLocal(self, ty);
4883 } else rhs;4976 } else rhs;
48844977
4885 const div_result = try self.binOp(lhs_res, rhs_res, ty, .div);
4886 const rem_result = try self.binOp(lhs_res, rhs_res, ty, .rem);
4887
4888 const zero = switch (wasm_bits) {4978 const zero = switch (wasm_bits) {
4889 32 => WValue{ .imm32 = 0 },4979 32 => WValue{ .imm32 = 0 },
4890 64 => WValue{ .imm64 = 0 },4980 64 => WValue{ .imm64 = 0 },
4891 else => unreachable,4981 else => unreachable,
4892 };4982 };
4893 const lhs_less_than_zero = try self.cmp(lhs_res, zero, ty, .lt);
4894 const rhs_less_than_zero = try self.cmp(rhs_res, zero, ty, .lt);
48954983
4896 try self.emitWValue(div_result);4984 const div_result = try self.allocLocal(ty);
4897 try self.emitWValue(lhs_less_than_zero);4985 // leave on stack
4898 try self.emitWValue(rhs_less_than_zero);4986 _ = try self.binOp(lhs_res, rhs_res, ty, .div);
4987 try self.addLabel(.local_tee, div_result.local);
4988 _ = try self.cmp(lhs_res, zero, ty, .lt);
4989 _ = try self.cmp(rhs_res, zero, ty, .lt);
4899 switch (wasm_bits) {4990 switch (wasm_bits) {
4900 32 => {4991 32 => {
4901 try self.addTag(.i32_xor);4992 try self.addTag(.i32_xor);
...@@ -4908,7 +4999,8 @@ fn airDivFloor(self: *Self, inst: Air.Inst.Index) InnerError!WValue {...@@ -4908,7 +4999,8 @@ fn airDivFloor(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
4908 else => unreachable,4999 else => unreachable,
4909 }5000 }
4910 try self.emitWValue(div_result);5001 try self.emitWValue(div_result);
4911 try self.emitWValue(rem_result);5002 // leave value on the stack
5003 _ = try self.binOp(lhs_res, rhs_res, ty, .rem);
4912 try self.addTag(.select);5004 try self.addTag(.select);
4913 } else {5005 } else {
4914 const float_bits = ty.floatBits(self.target);5006 const float_bits = ty.floatBits(self.target);
...@@ -4940,9 +5032,7 @@ fn airDivFloor(self: *Self, inst: Air.Inst.Index) InnerError!WValue {...@@ -4940,9 +5032,7 @@ fn airDivFloor(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
4940 }5032 }
49415033
4942 if (is_f16) {5034 if (is_f16) {
4943 // we can re-use temporary local5035 _ = try self.fptrunc(.{ .stack = {} }, Type.f32, Type.f16);
4944 try self.addLabel(.local_set, lhs_operand.local);
4945 return self.fptrunc(lhs_operand, Type.f32, Type.f16);
4946 }5036 }
4947 }5037 }
49485038
...@@ -4962,10 +5052,9 @@ fn divSigned(self: *Self, lhs: WValue, rhs: WValue, ty: Type) InnerError!WValue...@@ -4962,10 +5052,9 @@ fn divSigned(self: *Self, lhs: WValue, rhs: WValue, ty: Type) InnerError!WValue
4962 }5052 }
49635053
4964 if (wasm_bits != int_bits) {5054 if (wasm_bits != int_bits) {
4965 const lhs_abs = try self.signAbsValue(lhs, ty);5055 // Leave both values on the stack
4966 const rhs_abs = try self.signAbsValue(rhs, ty);5056 _ = try self.signAbsValue(lhs, ty);
4967 try self.emitWValue(lhs_abs);5057 _ = try self.signAbsValue(rhs, ty);
4968 try self.emitWValue(rhs_abs);
4969 } else {5058 } else {
4970 try self.emitWValue(lhs);5059 try self.emitWValue(lhs);
4971 try self.emitWValue(rhs);5060 try self.emitWValue(rhs);
...@@ -4977,6 +5066,8 @@ fn divSigned(self: *Self, lhs: WValue, rhs: WValue, ty: Type) InnerError!WValue...@@ -4977,6 +5066,8 @@ fn divSigned(self: *Self, lhs: WValue, rhs: WValue, ty: Type) InnerError!WValue
4977 return result;5066 return result;
4978}5067}
49795068
5069/// Retrieves the absolute value of a signed integer
5070/// NOTE: Leaves the result value on the stack.
4980fn signAbsValue(self: *Self, operand: WValue, ty: Type) InnerError!WValue {5071fn signAbsValue(self: *Self, operand: WValue, ty: Type) InnerError!WValue {
4981 const int_bits = ty.intInfo(self.target).bits;5072 const int_bits = ty.intInfo(self.target).bits;
4982 const wasm_bits = toWasmBits(int_bits) orelse {5073 const wasm_bits = toWasmBits(int_bits) orelse {
...@@ -5005,9 +5096,8 @@ fn signAbsValue(self: *Self, operand: WValue, ty: Type) InnerError!WValue {...@@ -5005,9 +5096,8 @@ fn signAbsValue(self: *Self, operand: WValue, ty: Type) InnerError!WValue {
5005 },5096 },
5006 else => unreachable,5097 else => unreachable,
5007 }5098 }
5008 const result = try self.allocLocal(ty);5099
5009 try self.addLabel(.local_set, result.local);5100 return WValue{ .stack = {} };
5010 return result;
5011}5101}
50125102
5013fn airCeilFloorTrunc(self: *Self, inst: Air.Inst.Index, op: Op) InnerError!WValue {5103fn airCeilFloorTrunc(self: *Self, inst: Air.Inst.Index, op: Op) InnerError!WValue {
...@@ -5034,9 +5124,7 @@ fn airCeilFloorTrunc(self: *Self, inst: Air.Inst.Index, op: Op) InnerError!WValu...@@ -5034,9 +5124,7 @@ fn airCeilFloorTrunc(self: *Self, inst: Air.Inst.Index, op: Op) InnerError!WValu
5034 try self.addTag(Mir.Inst.Tag.fromOpcode(opcode));5124 try self.addTag(Mir.Inst.Tag.fromOpcode(opcode));
50355125
5036 if (is_f16) {5126 if (is_f16) {
5037 // re-use temporary to save locals5127 _ = try self.fptrunc(.{ .stack = {} }, Type.f32, Type.f16);
5038 try self.addLabel(.local_set, op_to_lower.local);
5039 return self.fptrunc(op_to_lower, Type.f32, Type.f16);
5040 }5128 }
50415129
5042 const result = try self.allocLocal(ty);5130 const result = try self.allocLocal(ty);
...@@ -5065,7 +5153,8 @@ fn airSatBinOp(self: *Self, inst: Air.Inst.Index, op: Op) InnerError!WValue {...@@ -5065,7 +5153,8 @@ fn airSatBinOp(self: *Self, inst: Air.Inst.Index, op: Op) InnerError!WValue {
5065 }5153 }
50665154
5067 const wasm_bits = toWasmBits(int_info.bits).?;5155 const wasm_bits = toWasmBits(int_info.bits).?;
5068 const bin_result = try self.binOp(lhs, rhs, ty, op);5156 var bin_result = try (try self.binOp(lhs, rhs, ty, op)).toLocal(self, ty);
5157 defer bin_result.free(self);
5069 if (wasm_bits != int_info.bits and op == .add) {5158 if (wasm_bits != int_info.bits and op == .add) {
5070 const val: u64 = @intCast(u64, (@as(u65, 1) << @intCast(u7, int_info.bits)) - 1);5159 const val: u64 = @intCast(u64, (@as(u65, 1) << @intCast(u7, int_info.bits)) - 1);
5071 const imm_val = switch (wasm_bits) {5160 const imm_val = switch (wasm_bits) {
...@@ -5074,19 +5163,17 @@ fn airSatBinOp(self: *Self, inst: Air.Inst.Index, op: Op) InnerError!WValue {...@@ -5074,19 +5163,17 @@ fn airSatBinOp(self: *Self, inst: Air.Inst.Index, op: Op) InnerError!WValue {
5074 else => unreachable,5163 else => unreachable,
5075 };5164 };
50765165
5077 const cmp_result = try self.cmp(bin_result, imm_val, ty, .lt);
5078 try self.emitWValue(bin_result);5166 try self.emitWValue(bin_result);
5079 try self.emitWValue(imm_val);5167 try self.emitWValue(imm_val);
5080 try self.emitWValue(cmp_result);5168 _ = try self.cmp(bin_result, imm_val, ty, .lt);
5081 } else {5169 } else {
5082 const cmp_result = try self.cmp(bin_result, lhs, ty, if (op == .add) .lt else .gt);
5083 switch (wasm_bits) {5170 switch (wasm_bits) {
5084 32 => try self.addImm32(if (op == .add) @as(i32, -1) else 0),5171 32 => try self.addImm32(if (op == .add) @as(i32, -1) else 0),
5085 64 => try self.addImm64(if (op == .add) @bitCast(u64, @as(i64, -1)) else 0),5172 64 => try self.addImm64(if (op == .add) @bitCast(u64, @as(i64, -1)) else 0),
5086 else => unreachable,5173 else => unreachable,
5087 }5174 }
5088 try self.emitWValue(bin_result);5175 try self.emitWValue(bin_result);
5089 try self.emitWValue(cmp_result);5176 _ = try self.cmp(bin_result, lhs, ty, if (op == .add) .lt else .gt);
5090 }5177 }
50915178
5092 try self.addTag(.select);5179 try self.addTag(.select);
...@@ -5100,8 +5187,12 @@ fn signedSat(self: *Self, lhs_operand: WValue, rhs_operand: WValue, ty: Type, op...@@ -5100,8 +5187,12 @@ fn signedSat(self: *Self, lhs_operand: WValue, rhs_operand: WValue, ty: Type, op
5100 const wasm_bits = toWasmBits(int_info.bits).?;5187 const wasm_bits = toWasmBits(int_info.bits).?;
5101 const is_wasm_bits = wasm_bits == int_info.bits;5188 const is_wasm_bits = wasm_bits == int_info.bits;
51025189
5103 const lhs = if (!is_wasm_bits) try self.signAbsValue(lhs_operand, ty) else lhs_operand;5190 var lhs = if (!is_wasm_bits) lhs: {
5104 const rhs = if (!is_wasm_bits) try self.signAbsValue(rhs_operand, ty) else rhs_operand;5191 break :lhs try (try self.signAbsValue(lhs_operand, ty)).toLocal(self, ty);
5192 } else lhs_operand;
5193 var rhs = if (!is_wasm_bits) rhs: {
5194 break :rhs try (try self.signAbsValue(rhs_operand, ty)).toLocal(self, ty);
5195 } else rhs_operand;
51055196
5106 const max_val: u64 = @intCast(u64, (@as(u65, 1) << @intCast(u7, int_info.bits - 1)) - 1);5197 const max_val: u64 = @intCast(u64, (@as(u65, 1) << @intCast(u7, int_info.bits - 1)) - 1);
5107 const min_val: i64 = (-@intCast(i64, @intCast(u63, max_val))) - 1;5198 const min_val: i64 = (-@intCast(i64, @intCast(u63, max_val))) - 1;
...@@ -5116,38 +5207,38 @@ fn signedSat(self: *Self, lhs_operand: WValue, rhs_operand: WValue, ty: Type, op...@@ -5116,38 +5207,38 @@ fn signedSat(self: *Self, lhs_operand: WValue, rhs_operand: WValue, ty: Type, op
5116 else => unreachable,5207 else => unreachable,
5117 };5208 };
51185209
5119 const bin_result = try self.binOp(lhs, rhs, ty, op);5210 var bin_result = try (try self.binOp(lhs, rhs, ty, op)).toLocal(self, ty);
5120 if (!is_wasm_bits) {5211 if (!is_wasm_bits) {
5121 const cmp_result_lt = try self.cmp(bin_result, max_wvalue, ty, .lt);5212 defer bin_result.free(self); // not returned in this branch
5213 defer lhs.free(self); // uses temporary local for absvalue
5214 defer rhs.free(self); // uses temporary local for absvalue
5122 try self.emitWValue(bin_result);5215 try self.emitWValue(bin_result);
5123 try self.emitWValue(max_wvalue);5216 try self.emitWValue(max_wvalue);
5124 try self.emitWValue(cmp_result_lt);5217 _ = try self.cmp(bin_result, max_wvalue, ty, .lt);
5125 try self.addTag(.select);5218 try self.addTag(.select);
5126 try self.addLabel(.local_set, bin_result.local); // re-use local5219 try self.addLabel(.local_set, bin_result.local); // re-use local
51275220
5128 const cmp_result_gt = try self.cmp(bin_result, min_wvalue, ty, .gt);
5129 try self.emitWValue(bin_result);5221 try self.emitWValue(bin_result);
5130 try self.emitWValue(min_wvalue);5222 try self.emitWValue(min_wvalue);
5131 try self.emitWValue(cmp_result_gt);5223 _ = try self.cmp(bin_result, min_wvalue, ty, .gt);
5132 try self.addTag(.select);5224 try self.addTag(.select);
5133 try self.addLabel(.local_set, bin_result.local); // re-use local5225 try self.addLabel(.local_set, bin_result.local); // re-use local
5134 return self.wrapOperand(bin_result, ty);5226 return (try self.wrapOperand(bin_result, ty)).toLocal(self, ty);
5135 } else {5227 } else {
5136 const zero = switch (wasm_bits) {5228 const zero = switch (wasm_bits) {
5137 32 => WValue{ .imm32 = 0 },5229 32 => WValue{ .imm32 = 0 },
5138 64 => WValue{ .imm64 = 0 },5230 64 => WValue{ .imm64 = 0 },
5139 else => unreachable,5231 else => unreachable,
5140 };5232 };
5141 const cmp_bin_result = try self.cmp(bin_result, lhs, ty, .lt);
5142 const cmp_zero_result = try self.cmp(rhs, zero, ty, if (op == .add) .lt else .gt);
5143 const xor = try self.binOp(cmp_zero_result, cmp_bin_result, Type.u32, .xor); // comparisons always return i32, so provide u32 as type to xor.
5144 const cmp_bin_zero_result = try self.cmp(bin_result, zero, ty, .lt);
5145 try self.emitWValue(max_wvalue);5233 try self.emitWValue(max_wvalue);
5146 try self.emitWValue(min_wvalue);5234 try self.emitWValue(min_wvalue);
5147 try self.emitWValue(cmp_bin_zero_result);5235 _ = try self.cmp(bin_result, zero, ty, .lt);
5148 try self.addTag(.select);5236 try self.addTag(.select);
5149 try self.emitWValue(bin_result);5237 try self.emitWValue(bin_result);
5150 try self.emitWValue(xor);5238 // leave on stack
5239 const cmp_zero_result = try self.cmp(rhs, zero, ty, if (op == .add) .lt else .gt);
5240 const cmp_bin_result = try self.cmp(bin_result, lhs, ty, .lt);
5241 _ = try self.binOp(cmp_zero_result, cmp_bin_result, Type.u32, .xor); // comparisons always return i32, so provide u32 as type to xor.
5151 try self.addTag(.select);5242 try self.addTag(.select);
5152 try self.addLabel(.local_set, bin_result.local); // re-use local5243 try self.addLabel(.local_set, bin_result.local); // re-use local
5153 return bin_result;5244 return bin_result;
...@@ -5171,9 +5262,10 @@ fn airShlSat(self: *Self, inst: Air.Inst.Index) InnerError!WValue {...@@ -5171,9 +5262,10 @@ fn airShlSat(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
5171 const result = try self.allocLocal(ty);5262 const result = try self.allocLocal(ty);
51725263
5173 if (wasm_bits == int_info.bits) {5264 if (wasm_bits == int_info.bits) {
5174 const shl = try self.binOp(lhs, rhs, ty, .shl);5265 var shl = try (try self.binOp(lhs, rhs, ty, .shl)).toLocal(self, ty);
5175 const shr = try self.binOp(shl, rhs, ty, .shr);5266 defer shl.free(self);
5176 const cmp_result = try self.cmp(lhs, shr, ty, .neq);5267 var shr = try (try self.binOp(shl, rhs, ty, .shr)).toLocal(self, ty);
5268 defer shr.free(self);
51775269
5178 switch (wasm_bits) {5270 switch (wasm_bits) {
5179 32 => blk: {5271 32 => blk: {
...@@ -5181,10 +5273,9 @@ fn airShlSat(self: *Self, inst: Air.Inst.Index) InnerError!WValue {...@@ -5181,10 +5273,9 @@ fn airShlSat(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
5181 try self.addImm32(-1);5273 try self.addImm32(-1);
5182 break :blk;5274 break :blk;
5183 }5275 }
5184 const less_than_zero = try self.cmp(lhs, .{ .imm32 = 0 }, ty, .lt);
5185 try self.addImm32(std.math.minInt(i32));5276 try self.addImm32(std.math.minInt(i32));
5186 try self.addImm32(std.math.maxInt(i32));5277 try self.addImm32(std.math.maxInt(i32));
5187 try self.emitWValue(less_than_zero);5278 _ = try self.cmp(lhs, .{ .imm32 = 0 }, ty, .lt);
5188 try self.addTag(.select);5279 try self.addTag(.select);
5189 },5280 },
5190 64 => blk: {5281 64 => blk: {
...@@ -5192,16 +5283,15 @@ fn airShlSat(self: *Self, inst: Air.Inst.Index) InnerError!WValue {...@@ -5192,16 +5283,15 @@ fn airShlSat(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
5192 try self.addImm64(@bitCast(u64, @as(i64, -1)));5283 try self.addImm64(@bitCast(u64, @as(i64, -1)));
5193 break :blk;5284 break :blk;
5194 }5285 }
5195 const less_than_zero = try self.cmp(lhs, .{ .imm64 = 0 }, ty, .lt);
5196 try self.addImm64(@bitCast(u64, @as(i64, std.math.minInt(i64))));5286 try self.addImm64(@bitCast(u64, @as(i64, std.math.minInt(i64))));
5197 try self.addImm64(@bitCast(u64, @as(i64, std.math.maxInt(i64))));5287 try self.addImm64(@bitCast(u64, @as(i64, std.math.maxInt(i64))));
5198 try self.emitWValue(less_than_zero);5288 _ = try self.cmp(lhs, .{ .imm64 = 0 }, ty, .lt);
5199 try self.addTag(.select);5289 try self.addTag(.select);
5200 },5290 },
5201 else => unreachable,5291 else => unreachable,
5202 }5292 }
5203 try self.emitWValue(shl);5293 try self.emitWValue(shl);
5204 try self.emitWValue(cmp_result);5294 _ = try self.cmp(lhs, shr, ty, .neq);
5205 try self.addTag(.select);5295 try self.addTag(.select);
5206 try self.addLabel(.local_set, result.local);5296 try self.addLabel(.local_set, result.local);
5207 return result;5297 return result;
...@@ -5213,10 +5303,12 @@ fn airShlSat(self: *Self, inst: Air.Inst.Index) InnerError!WValue {...@@ -5213,10 +5303,12 @@ fn airShlSat(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
5213 else => unreachable,5303 else => unreachable,
5214 };5304 };
52155305
5216 const shl_res = try self.binOp(lhs, shift_value, ty, .shl);5306 var shl_res = try (try self.binOp(lhs, shift_value, ty, .shl)).toLocal(self, ty);
5217 const shl = try self.binOp(shl_res, rhs, ty, .shl);5307 defer shl_res.free(self);
5218 const shr = try self.binOp(shl, rhs, ty, .shr);5308 var shl = try (try self.binOp(shl_res, rhs, ty, .shl)).toLocal(self, ty);
5219 const cmp_result = try self.cmp(shl_res, shr, ty, .neq);5309 defer shl.free(self);
5310 var shr = try (try self.binOp(shl, rhs, ty, .shr)).toLocal(self, ty);
5311 defer shr.free(self);
52205312
5221 switch (wasm_bits) {5313 switch (wasm_bits) {
5222 32 => blk: {5314 32 => blk: {
...@@ -5225,10 +5317,9 @@ fn airShlSat(self: *Self, inst: Air.Inst.Index) InnerError!WValue {...@@ -5225,10 +5317,9 @@ fn airShlSat(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
5225 break :blk;5317 break :blk;
5226 }5318 }
52275319
5228 const less_than_zero = try self.cmp(shl_res, .{ .imm32 = 0 }, ty, .lt);
5229 try self.addImm32(std.math.minInt(i32));5320 try self.addImm32(std.math.minInt(i32));
5230 try self.addImm32(std.math.maxInt(i32));5321 try self.addImm32(std.math.maxInt(i32));
5231 try self.emitWValue(less_than_zero);5322 _ = try self.cmp(shl_res, .{ .imm32 = 0 }, ty, .lt);
5232 try self.addTag(.select);5323 try self.addTag(.select);
5233 },5324 },
5234 64 => blk: {5325 64 => blk: {
...@@ -5237,29 +5328,30 @@ fn airShlSat(self: *Self, inst: Air.Inst.Index) InnerError!WValue {...@@ -5237,29 +5328,30 @@ fn airShlSat(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
5237 break :blk;5328 break :blk;
5238 }5329 }
52395330
5240 const less_than_zero = try self.cmp(shl_res, .{ .imm64 = 0 }, ty, .lt);
5241 try self.addImm64(@bitCast(u64, @as(i64, std.math.minInt(i64))));5331 try self.addImm64(@bitCast(u64, @as(i64, std.math.minInt(i64))));
5242 try self.addImm64(@bitCast(u64, @as(i64, std.math.maxInt(i64))));5332 try self.addImm64(@bitCast(u64, @as(i64, std.math.maxInt(i64))));
5243 try self.emitWValue(less_than_zero);5333 _ = try self.cmp(shl_res, .{ .imm64 = 0 }, ty, .lt);
5244 try self.addTag(.select);5334 try self.addTag(.select);
5245 },5335 },
5246 else => unreachable,5336 else => unreachable,
5247 }5337 }
5248 try self.emitWValue(shl);5338 try self.emitWValue(shl);
5249 try self.emitWValue(cmp_result);5339 _ = try self.cmp(shl_res, shr, ty, .neq);
5250 try self.addTag(.select);5340 try self.addTag(.select);
5251 try self.addLabel(.local_set, result.local);5341 try self.addLabel(.local_set, result.local);
5252 const shift_result = try self.binOp(result, shift_value, ty, .shr);5342 var shift_result = try self.binOp(result, shift_value, ty, .shr);
5253 if (is_signed) {5343 if (is_signed) {
5254 return self.wrapOperand(shift_result, ty);5344 shift_result = try self.wrapOperand(shift_result, ty);
5255 }5345 }
5256 return shift_result;5346 return shift_result.toLocal(self, ty);
5257 }5347 }
5258}5348}
52595349
5260/// Calls a compiler-rt intrinsic by creating an undefined symbol,5350/// Calls a compiler-rt intrinsic by creating an undefined symbol,
5261/// then lowering the arguments and calling the symbol as a function call.5351/// then lowering the arguments and calling the symbol as a function call.
5262/// This function call assumes the C-ABI.5352/// This function call assumes the C-ABI.
5353/// Asserts arguments are not stack values when the return value is
5354/// passed as the first parameter.
5263fn callIntrinsic(5355fn callIntrinsic(
5264 self: *Self,5356 self: *Self,
5265 name: []const u8,5357 name: []const u8,
...@@ -5289,6 +5381,7 @@ fn callIntrinsic(...@@ -5289,6 +5381,7 @@ fn callIntrinsic(
52895381
5290 // Lower all arguments to the stack before we call our function5382 // Lower all arguments to the stack before we call our function
5291 for (args) |arg, arg_i| {5383 for (args) |arg, arg_i| {
5384 assert(!(want_sret_param and arg == .stack));
5292 assert(param_types[arg_i].hasRuntimeBitsIgnoreComptime());5385 assert(param_types[arg_i].hasRuntimeBitsIgnoreComptime());
5293 try self.lowerArg(.C, param_types[arg_i], arg);5386 try self.lowerArg(.C, param_types[arg_i], arg);
5294 }5387 }