authorgravatar for luuk@degram.devLuuk de Gram <luuk@degram.dev> 2022-03-16 21:06:02+01:00
committergravatar for luuk@degram.devLuuk de Gram <luuk@degram.dev> 2022-03-17 20:41:23+01:00
logeafdc5562f5053ecc193041e83d3661ef0744ebb
tree2e12cfdc1b0348ba309b209ada085d5268a1862d
parent291f5055f4b169e53414913e5ac077cd99ace978
signaturelock-open Commit is signed but in an unrecognized format.

wasm: Implement 'memcpy' instruction

This implements the `memcpy` instruction and also updates the inline memcpy calls to make use of the same implementation. We use the fast-loop when the length is comptime known, and use a runtime loop when the length is runtime known. We also perform feature-dection to emit a simply wasm memory.copy instruction when the feature 'bulk-memory' is enabled. (off by default).

2 files changed, 138 insertions(+), 23 deletions(-)

src/arch/wasm/CodeGen.zig+138-22
......@@ -895,7 +895,7 @@ fn genFunc(self: *Self) InnerError!void {
895895 try prologue.append(.{ .tag = .i32_sub, .data = .{ .tag = {} } });
896896 // Get negative stack aligment
897897 try prologue.append(.{ .tag = .i32_const, .data = .{ .imm32 = @intCast(i32, self.stack_alignment) * -1 } });
898 // Bit and the value to get the new stack pointer to ensure the pointers are aligned with the abi alignment
898 // Bitwise-and the value to get the new stack pointer to ensure the pointers are aligned with the abi alignment
899899 try prologue.append(.{ .tag = .i32_and, .data = .{ .tag = {} } });
900900 // store the current stack pointer as the bottom, which will be used to calculate all stack pointer offsets
901901 try prologue.append(.{ .tag = .local_tee, .data = .{ .label = self.bottom_stack_value.local } });
......@@ -1074,22 +1074,123 @@ fn toWasmBits(bits: u16) ?u16 {
10741074
10751075/// Performs a copy of bytes for a given type. Copying all bytes
10761076/// from rhs to lhs.
1077///
1078/// TODO: Perform feature detection and when bulk_memory is available,
1079/// use wasm's mem.copy instruction.
1080fn memCopy(self: *Self, ty: Type, lhs: WValue, rhs: WValue) !void {
1081 const abi_size = ty.abiSize(self.target);
1082 var offset: u32 = 0;
1083 const lhs_base = lhs.offset();
1084 const rhs_base = rhs.offset();
1085 while (offset < abi_size) : (offset += 1) {
1086 // get lhs' address to store the result
1087 try self.emitWValue(lhs);
1088 // load byte from rhs' adress
1089 try self.emitWValue(rhs);
1090 try self.addMemArg(.i32_load8_u, .{ .offset = rhs_base + offset, .alignment = 1 });
1091 // store the result in lhs (we already have its address on the stack)
1092 try self.addMemArg(.i32_store8, .{ .offset = lhs_base + offset, .alignment = 1 });
1077fn memcpy(self: *Self, dst: WValue, src: WValue, len: WValue) !void {
1078 // When bulk_memory is enabled, we lower it to wasm's memcpy instruction.
1079 // If not, we lower it ourselves manually
1080 if (std.Target.wasm.featureSetHas(self.target.cpu.features, .bulk_memory)) {
1081 switch (dst) {
1082 .stack_offset => try self.emitWValue(try self.buildPointerOffset(dst, 0, .new)),
1083 else => try self.emitWValue(dst),
1084 }
1085 switch (src) {
1086 .stack_offset => try self.emitWValue(try self.buildPointerOffset(src, 0, .new)),
1087 else => try self.emitWValue(src),
1088 }
1089 try self.emitWValue(len);
1090 try self.addExtended(.memory_copy);
1091 return;
1092 }
1093
1094 // when the length is comptime-known, rather than a runtime value, we can optimize the generated code by having
1095 // the loop during codegen, rather than inserting a runtime loop into the binary.
1096 switch (len) {
1097 .imm32, .imm64 => {
1098 const length = switch (len) {
1099 .imm32 => |val| val,
1100 .imm64 => |val| val,
1101 else => unreachable,
1102 };
1103 var offset: u32 = 0;
1104 const lhs_base = dst.offset();
1105 const rhs_base = src.offset();
1106 while (offset < length) : (offset += 1) {
1107 // get dst's address to store the result
1108 try self.emitWValue(dst);
1109 // load byte from src's address
1110 try self.emitWValue(src);
1111 switch (self.arch()) {
1112 .wasm32 => {
1113 try self.addMemArg(.i32_load8_u, .{ .offset = rhs_base + offset, .alignment = 1 });
1114 try self.addMemArg(.i32_store8, .{ .offset = lhs_base + offset, .alignment = 1 });
1115 },
1116 .wasm64 => {
1117 try self.addMemArg(.i64_load8_u, .{ .offset = rhs_base + offset, .alignment = 1 });
1118 try self.addMemArg(.i64_store8, .{ .offset = lhs_base + offset, .alignment = 1 });
1119 },
1120 else => unreachable,
1121 }
1122 }
1123 },
1124 else => {
1125 // TODO: We should probably lower this to a call to compiler_rt
1126 // But for now, we implement it manually
1127 const offset = try self.allocLocal(Type.usize); // local for counter
1128 // outer block to jump to when loop is done
1129 try self.startBlock(.block, wasm.block_empty);
1130 try self.startBlock(.loop, wasm.block_empty);
1131
1132 // loop condition (offset == length -> break)
1133 {
1134 try self.emitWValue(offset);
1135 try self.emitWValue(len);
1136 switch (self.arch()) {
1137 .wasm32 => try self.addTag(.i32_eq),
1138 .wasm64 => try self.addTag(.i64_eq),
1139 else => unreachable,
1140 }
1141 try self.addLabel(.br_if, 1); // jump out of loop into outer block (finished)
1142 }
1143
1144 // get dst ptr
1145 {
1146 try self.emitWValue(dst);
1147 try self.emitWValue(offset);
1148 switch (self.arch()) {
1149 .wasm32 => try self.addTag(.i32_add),
1150 .wasm64 => try self.addTag(.i64_add),
1151 else => unreachable,
1152 }
1153 }
1154
1155 // get src value and also store in dst
1156 {
1157 try self.emitWValue(src);
1158 try self.emitWValue(offset);
1159 switch (self.arch()) {
1160 .wasm32 => {
1161 try self.addTag(.i32_add);
1162 try self.addMemArg(.i32_load8_u, .{ .offset = src.offset(), .alignment = 1 });
1163 try self.addMemArg(.i32_store8, .{ .offset = dst.offset(), .alignment = 1 });
1164 },
1165 .wasm64 => {
1166 try self.addTag(.i64_add);
1167 try self.addMemArg(.i64_load8_u, .{ .offset = src.offset(), .alignment = 1 });
1168 try self.addMemArg(.i64_store8, .{ .offset = dst.offset(), .alignment = 1 });
1169 },
1170 else => unreachable,
1171 }
1172 }
1173
1174 // increment loop counter
1175 {
1176 try self.emitWValue(offset);
1177 switch (self.arch()) {
1178 .wasm32 => {
1179 try self.addImm32(1);
1180 try self.addTag(.i32_add);
1181 },
1182 .wasm64 => {
1183 try self.addImm64(1);
1184 try self.addTag(.i64_add);
1185 },
1186 else => unreachable,
1187 }
1188 try self.addLabel(.local_set, offset.local);
1189 try self.addLabel(.br, 0); // jump to start of loop
1190 }
1191 try self.endBlock(); // close off loop block
1192 try self.endBlock(); // close off outer block
1193 },
10931194 }
10941195}
10951196
......@@ -1297,6 +1398,8 @@ fn genInst(self: *Self, inst: Air.Inst.Index) !WValue {
12971398 .wasm_memory_size => self.airWasmMemorySize(inst),
12981399 .wasm_memory_grow => self.airWasmMemoryGrow(inst),
12991400
1401 .memcpy => self.airMemcpy(inst),
1402
13001403 .add_sat,
13011404 .sub_sat,
13021405 .mul_sat,
......@@ -1337,7 +1440,6 @@ fn genInst(self: *Self, inst: Air.Inst.Index) !WValue {
13371440 .ptr_slice_len_ptr,
13381441 .ptr_slice_ptr_ptr,
13391442 .int_to_float,
1340 .memcpy,
13411443 .cmpxchg_weak,
13421444 .cmpxchg_strong,
13431445 .fence,
......@@ -1519,7 +1621,8 @@ fn store(self: *Self, lhs: WValue, rhs: WValue, ty: Type, offset: u32) InnerErro
15191621 return self.store(lhs, rhs, err_ty, 0);
15201622 }
15211623
1522 return self.memCopy(ty, lhs, rhs);
1624 const len = @intCast(u32, ty.abiSize(self.target));
1625 return self.memcpy(lhs, rhs, .{ .imm32 = len });
15231626 },
15241627 .Optional => {
15251628 if (ty.isPtrLikeOptional()) {
......@@ -1531,10 +1634,12 @@ fn store(self: *Self, lhs: WValue, rhs: WValue, ty: Type, offset: u32) InnerErro
15311634 return self.store(lhs, rhs, Type.u8, 0);
15321635 }
15331636
1534 return self.memCopy(ty, lhs, rhs);
1637 const len = @intCast(u32, ty.abiSize(self.target));
1638 return self.memcpy(lhs, rhs, .{ .imm32 = len });
15351639 },
15361640 .Struct, .Array, .Union, .Vector => {
1537 return self.memCopy(ty, lhs, rhs);
1641 const len = @intCast(u32, ty.abiSize(self.target));
1642 return self.memcpy(lhs, rhs, .{ .imm32 = len });
15381643 },
15391644 .Pointer => {
15401645 if (ty.isSlice()) {
......@@ -1549,7 +1654,8 @@ fn store(self: *Self, lhs: WValue, rhs: WValue, ty: Type, offset: u32) InnerErro
15491654 }
15501655 },
15511656 .Int => if (ty.intInfo(self.target).bits > 64) {
1552 return self.memCopy(ty, lhs, rhs);
1657 const len = @intCast(u32, ty.abiSize(self.target));
1658 return self.memcpy(lhs, rhs, .{ .imm32 = len });
15531659 },
15541660 else => {},
15551661 }
......@@ -3300,3 +3406,13 @@ fn airFieldParentPtr(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
33003406 try self.addLabel(.local_set, base.local);
33013407 return base;
33023408}
3409
3410fn airMemcpy(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
3411 const pl_op = self.air.instructions.items(.data)[inst].pl_op;
3412 const bin_op = self.air.extraData(Air.Bin, pl_op.payload).data;
3413 const dst = try self.resolveInst(pl_op.operand);
3414 const src = try self.resolveInst(bin_op.lhs);
3415 const len = try self.resolveInst(bin_op.rhs);
3416 try self.memcpy(dst, src, len);
3417 return WValue{ .none = {} };
3418}
test/behavior/basic.zig-1
......@@ -340,7 +340,6 @@ fn f2(x: bool) []const u8 {
340340test "memcpy and memset intrinsics" {
341341 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
342342 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
343 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
344343
345344 try testMemcpyMemset();
346345 // TODO add comptime test coverage