authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2022-03-17 18:25:15-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2022-03-17 18:25:15-07:00
log76b382072ae632d8f113bcf151976f140566e699
treebbbad6ab5372ec984a644fccf26b11b9b382d948
parent79679be50ddfab04d34cb3dc412d4e5088de2126
parent215a22541c2a5b9886173dca86e2990ae5c649d1
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #11200 from Luukdegram/wasm-memcpy

stage2: wasm - Implement memcpy instruction

2 files changed, 216 insertions(+), 64 deletions(-)

src/arch/wasm/CodeGen.zig+216-63
...@@ -895,7 +895,7 @@ fn genFunc(self: *Self) InnerError!void {...@@ -895,7 +895,7 @@ fn genFunc(self: *Self) InnerError!void {
895 try prologue.append(.{ .tag = .i32_sub, .data = .{ .tag = {} } });895 try prologue.append(.{ .tag = .i32_sub, .data = .{ .tag = {} } });
896 // Get negative stack aligment896 // Get negative stack aligment
897 try prologue.append(.{ .tag = .i32_const, .data = .{ .imm32 = @intCast(i32, self.stack_alignment) * -1 } });897 try prologue.append(.{ .tag = .i32_const, .data = .{ .imm32 = @intCast(i32, self.stack_alignment) * -1 } });
898 // Bit and the value to get the new stack pointer to ensure the pointers are aligned with the abi alignment898 // Bitwise-and the value to get the new stack pointer to ensure the pointers are aligned with the abi alignment
899 try prologue.append(.{ .tag = .i32_and, .data = .{ .tag = {} } });899 try prologue.append(.{ .tag = .i32_and, .data = .{ .tag = {} } });
900 // store the current stack pointer as the bottom, which will be used to calculate all stack pointer offsets900 // store the current stack pointer as the bottom, which will be used to calculate all stack pointer offsets
901 try prologue.append(.{ .tag = .local_tee, .data = .{ .label = self.bottom_stack_value.local } });901 try prologue.append(.{ .tag = .local_tee, .data = .{ .label = self.bottom_stack_value.local } });
...@@ -1074,22 +1074,123 @@ fn toWasmBits(bits: u16) ?u16 {...@@ -1074,22 +1074,123 @@ fn toWasmBits(bits: u16) ?u16 {
10741074
1075/// Performs a copy of bytes for a given type. Copying all bytes1075/// Performs a copy of bytes for a given type. Copying all bytes
1076/// from rhs to lhs.1076/// from rhs to lhs.
1077///1077fn memcpy(self: *Self, dst: WValue, src: WValue, len: WValue) !void {
1078/// TODO: Perform feature detection and when bulk_memory is available,1078 // When bulk_memory is enabled, we lower it to wasm's memcpy instruction.
1079/// use wasm's mem.copy instruction.1079 // If not, we lower it ourselves manually
1080fn memCopy(self: *Self, ty: Type, lhs: WValue, rhs: WValue) !void {1080 if (std.Target.wasm.featureSetHas(self.target.cpu.features, .bulk_memory)) {
1081 const abi_size = ty.abiSize(self.target);1081 switch (dst) {
1082 var offset: u32 = 0;1082 .stack_offset => try self.emitWValue(try self.buildPointerOffset(dst, 0, .new)),
1083 const lhs_base = lhs.offset();1083 else => try self.emitWValue(dst),
1084 const rhs_base = rhs.offset();1084 }
1085 while (offset < abi_size) : (offset += 1) {1085 switch (src) {
1086 // get lhs' address to store the result1086 .stack_offset => try self.emitWValue(try self.buildPointerOffset(src, 0, .new)),
1087 try self.emitWValue(lhs);1087 else => try self.emitWValue(src),
1088 // load byte from rhs' adress1088 }
1089 try self.emitWValue(rhs);1089 try self.emitWValue(len);
1090 try self.addMemArg(.i32_load8_u, .{ .offset = rhs_base + offset, .alignment = 1 });1090 try self.addExtended(.memory_copy);
1091 // store the result in lhs (we already have its address on the stack)1091 return;
1092 try self.addMemArg(.i32_store8, .{ .offset = lhs_base + offset, .alignment = 1 });1092 }
1093
1094 // when the length is comptime-known, rather than a runtime value, we can optimize the generated code by having
1095 // the loop during codegen, rather than inserting a runtime loop into the binary.
1096 switch (len) {
1097 .imm32, .imm64 => {
1098 const length = switch (len) {
1099 .imm32 => |val| val,
1100 .imm64 => |val| val,
1101 else => unreachable,
1102 };
1103 var offset: u32 = 0;
1104 const lhs_base = dst.offset();
1105 const rhs_base = src.offset();
1106 while (offset < length) : (offset += 1) {
1107 // get dst's address to store the result
1108 try self.emitWValue(dst);
1109 // load byte from src's address
1110 try self.emitWValue(src);
1111 switch (self.arch()) {
1112 .wasm32 => {
1113 try self.addMemArg(.i32_load8_u, .{ .offset = rhs_base + offset, .alignment = 1 });
1114 try self.addMemArg(.i32_store8, .{ .offset = lhs_base + offset, .alignment = 1 });
1115 },
1116 .wasm64 => {
1117 try self.addMemArg(.i64_load8_u, .{ .offset = rhs_base + offset, .alignment = 1 });
1118 try self.addMemArg(.i64_store8, .{ .offset = lhs_base + offset, .alignment = 1 });
1119 },
1120 else => unreachable,
1121 }
1122 }
1123 },
1124 else => {
1125 // TODO: We should probably lower this to a call to compiler_rt
1126 // But for now, we implement it manually
1127 const offset = try self.allocLocal(Type.usize); // local for counter
1128 // outer block to jump to when loop is done
1129 try self.startBlock(.block, wasm.block_empty);
1130 try self.startBlock(.loop, wasm.block_empty);
1131
1132 // loop condition (offset == length -> break)
1133 {
1134 try self.emitWValue(offset);
1135 try self.emitWValue(len);
1136 switch (self.arch()) {
1137 .wasm32 => try self.addTag(.i32_eq),
1138 .wasm64 => try self.addTag(.i64_eq),
1139 else => unreachable,
1140 }
1141 try self.addLabel(.br_if, 1); // jump out of loop into outer block (finished)
1142 }
1143
1144 // get dst ptr
1145 {
1146 try self.emitWValue(dst);
1147 try self.emitWValue(offset);
1148 switch (self.arch()) {
1149 .wasm32 => try self.addTag(.i32_add),
1150 .wasm64 => try self.addTag(.i64_add),
1151 else => unreachable,
1152 }
1153 }
1154
1155 // get src value and also store in dst
1156 {
1157 try self.emitWValue(src);
1158 try self.emitWValue(offset);
1159 switch (self.arch()) {
1160 .wasm32 => {
1161 try self.addTag(.i32_add);
1162 try self.addMemArg(.i32_load8_u, .{ .offset = src.offset(), .alignment = 1 });
1163 try self.addMemArg(.i32_store8, .{ .offset = dst.offset(), .alignment = 1 });
1164 },
1165 .wasm64 => {
1166 try self.addTag(.i64_add);
1167 try self.addMemArg(.i64_load8_u, .{ .offset = src.offset(), .alignment = 1 });
1168 try self.addMemArg(.i64_store8, .{ .offset = dst.offset(), .alignment = 1 });
1169 },
1170 else => unreachable,
1171 }
1172 }
1173
1174 // increment loop counter
1175 {
1176 try self.emitWValue(offset);
1177 switch (self.arch()) {
1178 .wasm32 => {
1179 try self.addImm32(1);
1180 try self.addTag(.i32_add);
1181 },
1182 .wasm64 => {
1183 try self.addImm64(1);
1184 try self.addTag(.i64_add);
1185 },
1186 else => unreachable,
1187 }
1188 try self.addLabel(.local_set, offset.local);
1189 try self.addLabel(.br, 0); // jump to start of loop
1190 }
1191 try self.endBlock(); // close off loop block
1192 try self.endBlock(); // close off outer block
1193 },
1093 }1194 }
1094}1195}
10951196
...@@ -1298,6 +1399,8 @@ fn genInst(self: *Self, inst: Air.Inst.Index) !WValue {...@@ -1298,6 +1399,8 @@ fn genInst(self: *Self, inst: Air.Inst.Index) !WValue {
1298 .wasm_memory_size => self.airWasmMemorySize(inst),1399 .wasm_memory_size => self.airWasmMemorySize(inst),
1299 .wasm_memory_grow => self.airWasmMemoryGrow(inst),1400 .wasm_memory_grow => self.airWasmMemoryGrow(inst),
13001401
1402 .memcpy => self.airMemcpy(inst),
1403
1301 .add_sat,1404 .add_sat,
1302 .sub_sat,1405 .sub_sat,
1303 .mul_sat,1406 .mul_sat,
...@@ -1338,7 +1441,6 @@ fn genInst(self: *Self, inst: Air.Inst.Index) !WValue {...@@ -1338,7 +1441,6 @@ fn genInst(self: *Self, inst: Air.Inst.Index) !WValue {
1338 .ptr_slice_len_ptr,1441 .ptr_slice_len_ptr,
1339 .ptr_slice_ptr_ptr,1442 .ptr_slice_ptr_ptr,
1340 .int_to_float,1443 .int_to_float,
1341 .memcpy,
1342 .cmpxchg_weak,1444 .cmpxchg_weak,
1343 .cmpxchg_strong,1445 .cmpxchg_strong,
1344 .fence,1446 .fence,
...@@ -1520,7 +1622,8 @@ fn store(self: *Self, lhs: WValue, rhs: WValue, ty: Type, offset: u32) InnerErro...@@ -1520,7 +1622,8 @@ fn store(self: *Self, lhs: WValue, rhs: WValue, ty: Type, offset: u32) InnerErro
1520 return self.store(lhs, rhs, err_ty, 0);1622 return self.store(lhs, rhs, err_ty, 0);
1521 }1623 }
15221624
1523 return self.memCopy(ty, lhs, rhs);1625 const len = @intCast(u32, ty.abiSize(self.target));
1626 return self.memcpy(lhs, rhs, .{ .imm32 = len });
1524 },1627 },
1525 .Optional => {1628 .Optional => {
1526 if (ty.isPtrLikeOptional()) {1629 if (ty.isPtrLikeOptional()) {
...@@ -1532,10 +1635,12 @@ fn store(self: *Self, lhs: WValue, rhs: WValue, ty: Type, offset: u32) InnerErro...@@ -1532,10 +1635,12 @@ fn store(self: *Self, lhs: WValue, rhs: WValue, ty: Type, offset: u32) InnerErro
1532 return self.store(lhs, rhs, Type.u8, 0);1635 return self.store(lhs, rhs, Type.u8, 0);
1533 }1636 }
15341637
1535 return self.memCopy(ty, lhs, rhs);1638 const len = @intCast(u32, ty.abiSize(self.target));
1639 return self.memcpy(lhs, rhs, .{ .imm32 = len });
1536 },1640 },
1537 .Struct, .Array, .Union, .Vector => {1641 .Struct, .Array, .Union, .Vector => {
1538 return self.memCopy(ty, lhs, rhs);1642 const len = @intCast(u32, ty.abiSize(self.target));
1643 return self.memcpy(lhs, rhs, .{ .imm32 = len });
1539 },1644 },
1540 .Pointer => {1645 .Pointer => {
1541 if (ty.isSlice()) {1646 if (ty.isSlice()) {
...@@ -1550,7 +1655,8 @@ fn store(self: *Self, lhs: WValue, rhs: WValue, ty: Type, offset: u32) InnerErro...@@ -1550,7 +1655,8 @@ fn store(self: *Self, lhs: WValue, rhs: WValue, ty: Type, offset: u32) InnerErro
1550 }1655 }
1551 },1656 },
1552 .Int => if (ty.intInfo(self.target).bits > 64) {1657 .Int => if (ty.intInfo(self.target).bits > 64) {
1553 return self.memCopy(ty, lhs, rhs);1658 const len = @intCast(u32, ty.abiSize(self.target));
1659 return self.memcpy(lhs, rhs, .{ .imm32 = len });
1554 },1660 },
1555 else => {},1661 else => {},
1556 }1662 }
...@@ -2414,8 +2520,16 @@ fn airWrapErrUnionErr(self: *Self, inst: Air.Inst.Index) InnerError!WValue {...@@ -2414,8 +2520,16 @@ fn airWrapErrUnionErr(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
2414 if (!err_ty.errorUnionPayload().hasRuntimeBits()) return operand;2520 if (!err_ty.errorUnionPayload().hasRuntimeBits()) return operand;
24152521
2416 const err_union = try self.allocStack(err_ty);2522 const err_union = try self.allocStack(err_ty);
2417 // TODO: Also write 'undefined' to the payload
2418 try self.store(err_union, operand, err_ty.errorUnionSet(), 0);2523 try self.store(err_union, operand, err_ty.errorUnionSet(), 0);
2524
2525 // write 'undefined' to the payload
2526 const err_align = err_ty.abiAlignment(self.target);
2527 const set_size = err_ty.errorUnionSet().abiSize(self.target);
2528 const offset = mem.alignForwardGeneric(u64, set_size, err_align);
2529 const payload_ptr = try self.buildPointerOffset(err_union, offset, .new);
2530 const len = @intCast(u32, err_ty.errorUnionPayload().abiSize(self.target));
2531 try self.memset(payload_ptr, .{ .imm32 = len }, .{ .imm32 = 0xaaaaaaaa });
2532
2419 return err_union;2533 return err_union;
2420}2534}
24212535
...@@ -2867,7 +2981,7 @@ fn airMemset(self: *Self, inst: Air.Inst.Index) InnerError!WValue {...@@ -2867,7 +2981,7 @@ fn airMemset(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
2867 const ptr = try self.resolveInst(pl_op.operand);2981 const ptr = try self.resolveInst(pl_op.operand);
2868 const value = try self.resolveInst(bin_op.lhs);2982 const value = try self.resolveInst(bin_op.lhs);
2869 const len = try self.resolveInst(bin_op.rhs);2983 const len = try self.resolveInst(bin_op.rhs);
2870 try self.memSet(ptr, len, value);2984 try self.memset(ptr, len, value);
28712985
2872 return WValue{ .none = {} };2986 return WValue{ .none = {} };
2873}2987}
...@@ -2876,7 +2990,7 @@ fn airMemset(self: *Self, inst: Air.Inst.Index) InnerError!WValue {...@@ -2876,7 +2990,7 @@ fn airMemset(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
2876/// When the user has enabled the bulk_memory feature, we lower2990/// When the user has enabled the bulk_memory feature, we lower
2877/// this to wasm's memset instruction. When the feature is not present,2991/// this to wasm's memset instruction. When the feature is not present,
2878/// we implement it manually.2992/// we implement it manually.
2879fn memSet(self: *Self, ptr: WValue, len: WValue, value: WValue) InnerError!void {2993fn memset(self: *Self, ptr: WValue, len: WValue, value: WValue) InnerError!void {
2880 // When bulk_memory is enabled, we lower it to wasm's memset instruction.2994 // When bulk_memory is enabled, we lower it to wasm's memset instruction.
2881 // If not, we lower it ourselves2995 // If not, we lower it ourselves
2882 if (std.Target.wasm.featureSetHas(self.target.cpu.features, .bulk_memory)) {2996 if (std.Target.wasm.featureSetHas(self.target.cpu.features, .bulk_memory)) {
...@@ -2890,45 +3004,74 @@ fn memSet(self: *Self, ptr: WValue, len: WValue, value: WValue) InnerError!void...@@ -2890,45 +3004,74 @@ fn memSet(self: *Self, ptr: WValue, len: WValue, value: WValue) InnerError!void
2890 return;3004 return;
2891 }3005 }
28923006
2893 // TODO: We should probably lower this to a call to compiler_rt3007 // When the length is comptime-known we do the loop at codegen, rather
2894 // But for now, we implement it manually3008 // than emitting a runtime loop into the binary
2895 const offset = try self.allocLocal(Type.usize); // local for counter3009 switch (len) {
2896 // outer block to jump to when loop is done3010 .imm32, .imm64 => {
2897 try self.startBlock(.block, wasm.block_empty);3011 const length = switch (len) {
2898 try self.startBlock(.loop, wasm.block_empty);3012 .imm32 => |val| val,
2899 try self.emitWValue(offset);3013 .imm64 => |val| val,
2900 try self.emitWValue(len);3014 else => unreachable,
2901 switch (self.ptrSize()) {3015 };
2902 4 => try self.addTag(.i32_eq),3016
2903 8 => try self.addTag(.i64_eq),3017 var offset: u32 = 0;
2904 else => unreachable,3018 const base = ptr.offset();
2905 }3019 while (offset < length) : (offset += 1) {
2906 try self.addLabel(.br_if, 1); // jump out of loop into outer block (finished)3020 try self.emitWValue(ptr);
2907 try self.emitWValue(ptr);3021 try self.emitWValue(value);
2908 try self.emitWValue(offset);3022 switch (self.arch()) {
2909 switch (self.arch()) {3023 .wasm32 => {
2910 .wasm32 => try self.addTag(.i32_add),3024 try self.addMemArg(.i32_store8, .{ .offset = base + offset, .alignment = 1 });
2911 .wasm64 => try self.addTag(.i64_add),3025 },
2912 else => unreachable,3026 .wasm64 => {
2913 }3027 try self.addMemArg(.i64_store8, .{ .offset = base + offset, .alignment = 1 });
2914 try self.emitWValue(value);3028 },
2915 const mem_store_op: Mir.Inst.Tag = switch (self.arch()) {3029 else => unreachable,
2916 .wasm32 => .i32_store8,3030 }
2917 .wasm64 => .i64_store8,3031 }
2918 else => unreachable,3032 },
2919 };3033 else => {
2920 try self.addMemArg(mem_store_op, .{ .offset = ptr.offset(), .alignment = 1 });3034 // TODO: We should probably lower this to a call to compiler_rt
2921 try self.emitWValue(offset);3035 // But for now, we implement it manually
2922 try self.addImm32(1);3036 const offset = try self.allocLocal(Type.usize); // local for counter
2923 switch (self.ptrSize()) {3037 // outer block to jump to when loop is done
2924 4 => try self.addTag(.i32_add),3038 try self.startBlock(.block, wasm.block_empty);
2925 8 => try self.addTag(.i64_add),3039 try self.startBlock(.loop, wasm.block_empty);
2926 else => unreachable,3040 try self.emitWValue(offset);
3041 try self.emitWValue(len);
3042 switch (self.arch()) {
3043 .wasm32 => try self.addTag(.i32_eq),
3044 .wasm64 => try self.addTag(.i64_eq),
3045 else => unreachable,
3046 }
3047 try self.addLabel(.br_if, 1); // jump out of loop into outer block (finished)
3048 try self.emitWValue(ptr);
3049 try self.emitWValue(offset);
3050 switch (self.arch()) {
3051 .wasm32 => try self.addTag(.i32_add),
3052 .wasm64 => try self.addTag(.i64_add),
3053 else => unreachable,
3054 }
3055 try self.emitWValue(value);
3056 const mem_store_op: Mir.Inst.Tag = switch (self.arch()) {
3057 .wasm32 => .i32_store8,
3058 .wasm64 => .i64_store8,
3059 else => unreachable,
3060 };
3061 try self.addMemArg(mem_store_op, .{ .offset = ptr.offset(), .alignment = 1 });
3062 try self.emitWValue(offset);
3063 try self.addImm32(1);
3064 switch (self.arch()) {
3065 .wasm32 => try self.addTag(.i32_add),
3066 .wasm64 => try self.addTag(.i64_add),
3067 else => unreachable,
3068 }
3069 try self.addLabel(.local_set, offset.local);
3070 try self.addLabel(.br, 0); // jump to start of loop
3071 try self.endBlock();
3072 try self.endBlock();
3073 },
2927 }3074 }
2928 try self.addLabel(.local_set, offset.local);
2929 try self.addLabel(.br, 0); // jump to start of loop
2930 try self.endBlock();
2931 try self.endBlock();
2932}3075}
29333076
2934fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) InnerError!WValue {3077fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
...@@ -3309,3 +3452,13 @@ fn airFieldParentPtr(self: *Self, inst: Air.Inst.Index) InnerError!WValue {...@@ -3309,3 +3452,13 @@ fn airFieldParentPtr(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
3309 try self.addLabel(.local_set, base.local);3452 try self.addLabel(.local_set, base.local);
3310 return base;3453 return base;
3311}3454}
3455
3456fn airMemcpy(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
3457 const pl_op = self.air.instructions.items(.data)[inst].pl_op;
3458 const bin_op = self.air.extraData(Air.Bin, pl_op.payload).data;
3459 const dst = try self.resolveInst(pl_op.operand);
3460 const src = try self.resolveInst(bin_op.lhs);
3461 const len = try self.resolveInst(bin_op.rhs);
3462 try self.memcpy(dst, src, len);
3463 return WValue{ .none = {} };
3464}
test/behavior/basic.zig-1
...@@ -340,7 +340,6 @@ fn f2(x: bool) []const u8 {...@@ -340,7 +340,6 @@ fn f2(x: bool) []const u8 {
340test "memcpy and memset intrinsics" {340test "memcpy and memset intrinsics" {
341 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;341 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
342 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;342 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
343 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
344343
345 try testMemcpyMemset();344 try testMemcpyMemset();
346 // TODO add comptime test coverage345 // TODO add comptime test coverage