authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2022-03-17 18:25:15-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2022-03-17 18:25:15-07:00
log76b382072ae632d8f113bcf151976f140566e699
treebbbad6ab5372ec984a644fccf26b11b9b382d948
parent79679be50ddfab04d34cb3dc412d4e5088de2126
parent215a22541c2a5b9886173dca86e2990ae5c649d1
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #11200 from Luukdegram/wasm-memcpy

stage2: wasm - Implement memcpy instruction

2 files changed, 216 insertions(+), 64 deletions(-)

src/arch/wasm/CodeGen.zig+216-63
......@@ -895,7 +895,7 @@ fn genFunc(self: *Self) InnerError!void {
895895 try prologue.append(.{ .tag = .i32_sub, .data = .{ .tag = {} } });
896896 // Get negative stack aligment
897897 try prologue.append(.{ .tag = .i32_const, .data = .{ .imm32 = @intCast(i32, self.stack_alignment) * -1 } });
898 // Bit and the value to get the new stack pointer to ensure the pointers are aligned with the abi alignment
898 // Bitwise-and the value to get the new stack pointer to ensure the pointers are aligned with the abi alignment
899899 try prologue.append(.{ .tag = .i32_and, .data = .{ .tag = {} } });
900900 // store the current stack pointer as the bottom, which will be used to calculate all stack pointer offsets
901901 try prologue.append(.{ .tag = .local_tee, .data = .{ .label = self.bottom_stack_value.local } });
......@@ -1074,22 +1074,123 @@ fn toWasmBits(bits: u16) ?u16 {
10741074
10751075/// Performs a copy of bytes for a given type. Copying all bytes
10761076/// from rhs to lhs.
1077///
1078/// TODO: Perform feature detection and when bulk_memory is available,
1079/// use wasm's mem.copy instruction.
1080fn memCopy(self: *Self, ty: Type, lhs: WValue, rhs: WValue) !void {
1081 const abi_size = ty.abiSize(self.target);
1082 var offset: u32 = 0;
1083 const lhs_base = lhs.offset();
1084 const rhs_base = rhs.offset();
1085 while (offset < abi_size) : (offset += 1) {
1086 // get lhs' address to store the result
1087 try self.emitWValue(lhs);
1088 // load byte from rhs' adress
1089 try self.emitWValue(rhs);
1090 try self.addMemArg(.i32_load8_u, .{ .offset = rhs_base + offset, .alignment = 1 });
1091 // store the result in lhs (we already have its address on the stack)
1092 try self.addMemArg(.i32_store8, .{ .offset = lhs_base + offset, .alignment = 1 });
1077fn memcpy(self: *Self, dst: WValue, src: WValue, len: WValue) !void {
1078 // When bulk_memory is enabled, we lower it to wasm's memcpy instruction.
1079 // If not, we lower it ourselves manually
1080 if (std.Target.wasm.featureSetHas(self.target.cpu.features, .bulk_memory)) {
1081 switch (dst) {
1082 .stack_offset => try self.emitWValue(try self.buildPointerOffset(dst, 0, .new)),
1083 else => try self.emitWValue(dst),
1084 }
1085 switch (src) {
1086 .stack_offset => try self.emitWValue(try self.buildPointerOffset(src, 0, .new)),
1087 else => try self.emitWValue(src),
1088 }
1089 try self.emitWValue(len);
1090 try self.addExtended(.memory_copy);
1091 return;
1092 }
1093
1094 // when the length is comptime-known, rather than a runtime value, we can optimize the generated code by having
1095 // the loop during codegen, rather than inserting a runtime loop into the binary.
1096 switch (len) {
1097 .imm32, .imm64 => {
1098 const length = switch (len) {
1099 .imm32 => |val| val,
1100 .imm64 => |val| val,
1101 else => unreachable,
1102 };
1103 var offset: u32 = 0;
1104 const lhs_base = dst.offset();
1105 const rhs_base = src.offset();
1106 while (offset < length) : (offset += 1) {
1107 // get dst's address to store the result
1108 try self.emitWValue(dst);
1109 // load byte from src's address
1110 try self.emitWValue(src);
1111 switch (self.arch()) {
1112 .wasm32 => {
1113 try self.addMemArg(.i32_load8_u, .{ .offset = rhs_base + offset, .alignment = 1 });
1114 try self.addMemArg(.i32_store8, .{ .offset = lhs_base + offset, .alignment = 1 });
1115 },
1116 .wasm64 => {
1117 try self.addMemArg(.i64_load8_u, .{ .offset = rhs_base + offset, .alignment = 1 });
1118 try self.addMemArg(.i64_store8, .{ .offset = lhs_base + offset, .alignment = 1 });
1119 },
1120 else => unreachable,
1121 }
1122 }
1123 },
1124 else => {
1125 // TODO: We should probably lower this to a call to compiler_rt
1126 // But for now, we implement it manually
1127 const offset = try self.allocLocal(Type.usize); // local for counter
1128 // outer block to jump to when loop is done
1129 try self.startBlock(.block, wasm.block_empty);
1130 try self.startBlock(.loop, wasm.block_empty);
1131
1132 // loop condition (offset == length -> break)
1133 {
1134 try self.emitWValue(offset);
1135 try self.emitWValue(len);
1136 switch (self.arch()) {
1137 .wasm32 => try self.addTag(.i32_eq),
1138 .wasm64 => try self.addTag(.i64_eq),
1139 else => unreachable,
1140 }
1141 try self.addLabel(.br_if, 1); // jump out of loop into outer block (finished)
1142 }
1143
1144 // get dst ptr
1145 {
1146 try self.emitWValue(dst);
1147 try self.emitWValue(offset);
1148 switch (self.arch()) {
1149 .wasm32 => try self.addTag(.i32_add),
1150 .wasm64 => try self.addTag(.i64_add),
1151 else => unreachable,
1152 }
1153 }
1154
1155 // get src value and also store in dst
1156 {
1157 try self.emitWValue(src);
1158 try self.emitWValue(offset);
1159 switch (self.arch()) {
1160 .wasm32 => {
1161 try self.addTag(.i32_add);
1162 try self.addMemArg(.i32_load8_u, .{ .offset = src.offset(), .alignment = 1 });
1163 try self.addMemArg(.i32_store8, .{ .offset = dst.offset(), .alignment = 1 });
1164 },
1165 .wasm64 => {
1166 try self.addTag(.i64_add);
1167 try self.addMemArg(.i64_load8_u, .{ .offset = src.offset(), .alignment = 1 });
1168 try self.addMemArg(.i64_store8, .{ .offset = dst.offset(), .alignment = 1 });
1169 },
1170 else => unreachable,
1171 }
1172 }
1173
1174 // increment loop counter
1175 {
1176 try self.emitWValue(offset);
1177 switch (self.arch()) {
1178 .wasm32 => {
1179 try self.addImm32(1);
1180 try self.addTag(.i32_add);
1181 },
1182 .wasm64 => {
1183 try self.addImm64(1);
1184 try self.addTag(.i64_add);
1185 },
1186 else => unreachable,
1187 }
1188 try self.addLabel(.local_set, offset.local);
1189 try self.addLabel(.br, 0); // jump to start of loop
1190 }
1191 try self.endBlock(); // close off loop block
1192 try self.endBlock(); // close off outer block
1193 },
10931194 }
10941195}
10951196
......@@ -1298,6 +1399,8 @@ fn genInst(self: *Self, inst: Air.Inst.Index) !WValue {
12981399 .wasm_memory_size => self.airWasmMemorySize(inst),
12991400 .wasm_memory_grow => self.airWasmMemoryGrow(inst),
13001401
1402 .memcpy => self.airMemcpy(inst),
1403
13011404 .add_sat,
13021405 .sub_sat,
13031406 .mul_sat,
......@@ -1338,7 +1441,6 @@ fn genInst(self: *Self, inst: Air.Inst.Index) !WValue {
13381441 .ptr_slice_len_ptr,
13391442 .ptr_slice_ptr_ptr,
13401443 .int_to_float,
1341 .memcpy,
13421444 .cmpxchg_weak,
13431445 .cmpxchg_strong,
13441446 .fence,
......@@ -1520,7 +1622,8 @@ fn store(self: *Self, lhs: WValue, rhs: WValue, ty: Type, offset: u32) InnerErro
15201622 return self.store(lhs, rhs, err_ty, 0);
15211623 }
15221624
1523 return self.memCopy(ty, lhs, rhs);
1625 const len = @intCast(u32, ty.abiSize(self.target));
1626 return self.memcpy(lhs, rhs, .{ .imm32 = len });
15241627 },
15251628 .Optional => {
15261629 if (ty.isPtrLikeOptional()) {
......@@ -1532,10 +1635,12 @@ fn store(self: *Self, lhs: WValue, rhs: WValue, ty: Type, offset: u32) InnerErro
15321635 return self.store(lhs, rhs, Type.u8, 0);
15331636 }
15341637
1535 return self.memCopy(ty, lhs, rhs);
1638 const len = @intCast(u32, ty.abiSize(self.target));
1639 return self.memcpy(lhs, rhs, .{ .imm32 = len });
15361640 },
15371641 .Struct, .Array, .Union, .Vector => {
1538 return self.memCopy(ty, lhs, rhs);
1642 const len = @intCast(u32, ty.abiSize(self.target));
1643 return self.memcpy(lhs, rhs, .{ .imm32 = len });
15391644 },
15401645 .Pointer => {
15411646 if (ty.isSlice()) {
......@@ -1550,7 +1655,8 @@ fn store(self: *Self, lhs: WValue, rhs: WValue, ty: Type, offset: u32) InnerErro
15501655 }
15511656 },
15521657 .Int => if (ty.intInfo(self.target).bits > 64) {
1553 return self.memCopy(ty, lhs, rhs);
1658 const len = @intCast(u32, ty.abiSize(self.target));
1659 return self.memcpy(lhs, rhs, .{ .imm32 = len });
15541660 },
15551661 else => {},
15561662 }
......@@ -2414,8 +2520,16 @@ fn airWrapErrUnionErr(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
24142520 if (!err_ty.errorUnionPayload().hasRuntimeBits()) return operand;
24152521
24162522 const err_union = try self.allocStack(err_ty);
2417 // TODO: Also write 'undefined' to the payload
24182523 try self.store(err_union, operand, err_ty.errorUnionSet(), 0);
2524
2525 // write 'undefined' to the payload
2526 const err_align = err_ty.abiAlignment(self.target);
2527 const set_size = err_ty.errorUnionSet().abiSize(self.target);
2528 const offset = mem.alignForwardGeneric(u64, set_size, err_align);
2529 const payload_ptr = try self.buildPointerOffset(err_union, offset, .new);
2530 const len = @intCast(u32, err_ty.errorUnionPayload().abiSize(self.target));
2531 try self.memset(payload_ptr, .{ .imm32 = len }, .{ .imm32 = 0xaaaaaaaa });
2532
24192533 return err_union;
24202534}
24212535
......@@ -2867,7 +2981,7 @@ fn airMemset(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
28672981 const ptr = try self.resolveInst(pl_op.operand);
28682982 const value = try self.resolveInst(bin_op.lhs);
28692983 const len = try self.resolveInst(bin_op.rhs);
2870 try self.memSet(ptr, len, value);
2984 try self.memset(ptr, len, value);
28712985
28722986 return WValue{ .none = {} };
28732987}
......@@ -2876,7 +2990,7 @@ fn airMemset(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
28762990/// When the user has enabled the bulk_memory feature, we lower
28772991/// this to wasm's memset instruction. When the feature is not present,
28782992/// we implement it manually.
2879fn memSet(self: *Self, ptr: WValue, len: WValue, value: WValue) InnerError!void {
2993fn memset(self: *Self, ptr: WValue, len: WValue, value: WValue) InnerError!void {
28802994 // When bulk_memory is enabled, we lower it to wasm's memset instruction.
28812995 // If not, we lower it ourselves
28822996 if (std.Target.wasm.featureSetHas(self.target.cpu.features, .bulk_memory)) {
......@@ -2890,45 +3004,74 @@ fn memSet(self: *Self, ptr: WValue, len: WValue, value: WValue) InnerError!void
28903004 return;
28913005 }
28923006
2893 // TODO: We should probably lower this to a call to compiler_rt
2894 // But for now, we implement it manually
2895 const offset = try self.allocLocal(Type.usize); // local for counter
2896 // outer block to jump to when loop is done
2897 try self.startBlock(.block, wasm.block_empty);
2898 try self.startBlock(.loop, wasm.block_empty);
2899 try self.emitWValue(offset);
2900 try self.emitWValue(len);
2901 switch (self.ptrSize()) {
2902 4 => try self.addTag(.i32_eq),
2903 8 => try self.addTag(.i64_eq),
2904 else => unreachable,
2905 }
2906 try self.addLabel(.br_if, 1); // jump out of loop into outer block (finished)
2907 try self.emitWValue(ptr);
2908 try self.emitWValue(offset);
2909 switch (self.arch()) {
2910 .wasm32 => try self.addTag(.i32_add),
2911 .wasm64 => try self.addTag(.i64_add),
2912 else => unreachable,
2913 }
2914 try self.emitWValue(value);
2915 const mem_store_op: Mir.Inst.Tag = switch (self.arch()) {
2916 .wasm32 => .i32_store8,
2917 .wasm64 => .i64_store8,
2918 else => unreachable,
2919 };
2920 try self.addMemArg(mem_store_op, .{ .offset = ptr.offset(), .alignment = 1 });
2921 try self.emitWValue(offset);
2922 try self.addImm32(1);
2923 switch (self.ptrSize()) {
2924 4 => try self.addTag(.i32_add),
2925 8 => try self.addTag(.i64_add),
2926 else => unreachable,
3007 // When the length is comptime-known we do the loop at codegen, rather
3008 // than emitting a runtime loop into the binary
3009 switch (len) {
3010 .imm32, .imm64 => {
3011 const length = switch (len) {
3012 .imm32 => |val| val,
3013 .imm64 => |val| val,
3014 else => unreachable,
3015 };
3016
3017 var offset: u32 = 0;
3018 const base = ptr.offset();
3019 while (offset < length) : (offset += 1) {
3020 try self.emitWValue(ptr);
3021 try self.emitWValue(value);
3022 switch (self.arch()) {
3023 .wasm32 => {
3024 try self.addMemArg(.i32_store8, .{ .offset = base + offset, .alignment = 1 });
3025 },
3026 .wasm64 => {
3027 try self.addMemArg(.i64_store8, .{ .offset = base + offset, .alignment = 1 });
3028 },
3029 else => unreachable,
3030 }
3031 }
3032 },
3033 else => {
3034 // TODO: We should probably lower this to a call to compiler_rt
3035 // But for now, we implement it manually
3036 const offset = try self.allocLocal(Type.usize); // local for counter
3037 // outer block to jump to when loop is done
3038 try self.startBlock(.block, wasm.block_empty);
3039 try self.startBlock(.loop, wasm.block_empty);
3040 try self.emitWValue(offset);
3041 try self.emitWValue(len);
3042 switch (self.arch()) {
3043 .wasm32 => try self.addTag(.i32_eq),
3044 .wasm64 => try self.addTag(.i64_eq),
3045 else => unreachable,
3046 }
3047 try self.addLabel(.br_if, 1); // jump out of loop into outer block (finished)
3048 try self.emitWValue(ptr);
3049 try self.emitWValue(offset);
3050 switch (self.arch()) {
3051 .wasm32 => try self.addTag(.i32_add),
3052 .wasm64 => try self.addTag(.i64_add),
3053 else => unreachable,
3054 }
3055 try self.emitWValue(value);
3056 const mem_store_op: Mir.Inst.Tag = switch (self.arch()) {
3057 .wasm32 => .i32_store8,
3058 .wasm64 => .i64_store8,
3059 else => unreachable,
3060 };
3061 try self.addMemArg(mem_store_op, .{ .offset = ptr.offset(), .alignment = 1 });
3062 try self.emitWValue(offset);
3063 try self.addImm32(1);
3064 switch (self.arch()) {
3065 .wasm32 => try self.addTag(.i32_add),
3066 .wasm64 => try self.addTag(.i64_add),
3067 else => unreachable,
3068 }
3069 try self.addLabel(.local_set, offset.local);
3070 try self.addLabel(.br, 0); // jump to start of loop
3071 try self.endBlock();
3072 try self.endBlock();
3073 },
29273074 }
2928 try self.addLabel(.local_set, offset.local);
2929 try self.addLabel(.br, 0); // jump to start of loop
2930 try self.endBlock();
2931 try self.endBlock();
29323075}
29333076
29343077fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
......@@ -3309,3 +3452,13 @@ fn airFieldParentPtr(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
33093452 try self.addLabel(.local_set, base.local);
33103453 return base;
33113454}
3455
3456fn airMemcpy(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
3457 const pl_op = self.air.instructions.items(.data)[inst].pl_op;
3458 const bin_op = self.air.extraData(Air.Bin, pl_op.payload).data;
3459 const dst = try self.resolveInst(pl_op.operand);
3460 const src = try self.resolveInst(bin_op.lhs);
3461 const len = try self.resolveInst(bin_op.rhs);
3462 try self.memcpy(dst, src, len);
3463 return WValue{ .none = {} };
3464}
test/behavior/basic.zig-1
......@@ -340,7 +340,6 @@ fn f2(x: bool) []const u8 {
340340test "memcpy and memset intrinsics" {
341341 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
342342 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
343 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
344343
345344 try testMemcpyMemset();
346345 // TODO add comptime test coverage