authorgravatar for luuk@degram.devLuuk de Gram <luuk@degram.dev> 2023-05-18 17:26:20+02:00
committergravatar for luuk@degram.devLuuk de Gram <luuk@degram.dev> 2023-05-19 20:22:45+02:00
log4a33aa922e90b76248b259a89be86966eb4898c2
tree9e0320d8d8cc8e4d63fc341b6f1db591a7d632c0
parent55a260c968aed32001df5355596331db38b13729
signaturelock-open Commit is signed but in an unrecognized format.

wasm: support `memset` for elem abi size > 1

Previously we incorrectly assumed all memset's to have its element abi-size be 1 byte. This would set the region of memory incorrectly. We now have a more efficient loop, as well as support any element type by re-using the `store` function for each element and moving the pointer by 1 element.

2 files changed, 108 insertions(+), 94 deletions(-)

src/arch/wasm/CodeGen.zig+108-80
......@@ -1605,10 +1605,16 @@ fn memcpy(func: *CodeGen, dst: WValue, src: WValue, len: WValue) !void {
16051605 else => {},
16061606 }
16071607
1608 // TODO: We should probably lower this to a call to compiler_rt
1609 // But for now, we implement it manually
1610 var offset = try func.ensureAllocLocal(Type.usize); // local for counter
1608 // allocate a local for the offset, and set it to 0.
1609 // This to ensure that inside loops we correctly re-set the counter.
1610 var offset = try func.allocLocal(Type.usize); // local for counter
16111611 defer offset.free(func);
1612 switch (func.arch()) {
1613 .wasm32 => try func.addImm32(0),
1614 .wasm64 => try func.addImm64(0),
1615 else => unreachable,
1616 }
1617 try func.addLabel(.local_set, offset.local.value);
16121618
16131619 // outer block to jump to when loop is done
16141620 try func.startBlock(.block, wasm.block_empty);
......@@ -3301,19 +3307,23 @@ fn airCondBr(func: *CodeGen, inst: Air.Inst.Index) InnerError!void {
33013307 {
33023308 func.branches.appendAssumeCapacity(.{});
33033309 try func.currentBranch().values.ensureUnusedCapacity(func.gpa, @intCast(u32, liveness_condbr.else_deaths.len));
3310 defer {
3311 var else_stack = func.branches.pop();
3312 else_stack.deinit(func.gpa);
3313 }
33043314 try func.genBody(else_body);
33053315 try func.endBlock();
3306 var else_stack = func.branches.pop();
3307 else_stack.deinit(func.gpa);
33083316 }
33093317
33103318 // Outer block that matches the condition
33113319 {
33123320 func.branches.appendAssumeCapacity(.{});
33133321 try func.currentBranch().values.ensureUnusedCapacity(func.gpa, @intCast(u32, liveness_condbr.then_deaths.len));
3322 defer {
3323 var then_stack = func.branches.pop();
3324 then_stack.deinit(func.gpa);
3325 }
33143326 try func.genBody(then_body);
3315 var then_stack = func.branches.pop();
3316 then_stack.deinit(func.gpa);
33173327 }
33183328
33193329 func.finishAir(inst, .none, &.{});
......@@ -3829,20 +3839,24 @@ fn airSwitchBr(func: *CodeGen, inst: Air.Inst.Index) InnerError!void {
38293839 }
38303840 func.branches.appendAssumeCapacity(.{});
38313841 try func.currentBranch().values.ensureUnusedCapacity(func.gpa, liveness.deaths[index].len);
3842 defer {
3843 var case_branch = func.branches.pop();
3844 case_branch.deinit(func.gpa);
3845 }
38323846 try func.genBody(case.body);
38333847 try func.endBlock();
3834 var case_branch = func.branches.pop();
3835 case_branch.deinit(func.gpa);
38363848 }
38373849
38383850 if (has_else_body) {
38393851 func.branches.appendAssumeCapacity(.{});
38403852 const else_deaths = liveness.deaths.len - 1;
38413853 try func.currentBranch().values.ensureUnusedCapacity(func.gpa, liveness.deaths[else_deaths].len);
3854 defer {
3855 var else_branch = func.branches.pop();
3856 else_branch.deinit(func.gpa);
3857 }
38423858 try func.genBody(else_body);
38433859 try func.endBlock();
3844 var else_branch = func.branches.pop();
3845 else_branch.deinit(func.gpa);
38463860 }
38473861 func.finishAir(inst, .none, &.{});
38483862}
......@@ -3971,7 +3985,7 @@ fn airWrapErrUnionErr(func: *CodeGen, inst: Air.Inst.Index) InnerError!void {
39713985 // write 'undefined' to the payload
39723986 const payload_ptr = try func.buildPointerOffset(err_union, @intCast(u32, errUnionPayloadOffset(pl_ty, func.target)), .new);
39733987 const len = @intCast(u32, err_ty.errorUnionPayload().abiSize(func.target));
3974 try func.memset(payload_ptr, .{ .imm32 = len }, .{ .imm32 = 0xaaaaaaaa });
3988 try func.memset(Type.u8, payload_ptr, .{ .imm32 = len }, .{ .imm32 = 0xaa });
39753989
39763990 break :result err_union;
39773991 };
......@@ -4466,8 +4480,13 @@ fn airMemset(func: *CodeGen, inst: Air.Inst.Index, safety: bool) InnerError!void
44664480 .C, .Many => unreachable,
44674481 };
44684482
4483 const elem_ty = if (ptr_ty.ptrSize() == .One)
4484 ptr_ty.childType().childType()
4485 else
4486 ptr_ty.childType();
4487
44694488 const dst_ptr = try func.sliceOrArrayPtr(ptr, ptr_ty);
4470 try func.memset(dst_ptr, len, value);
4489 try func.memset(elem_ty, dst_ptr, len, value);
44714490
44724491 func.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });
44734492}
......@@ -4476,10 +4495,12 @@ fn airMemset(func: *CodeGen, inst: Air.Inst.Index, safety: bool) InnerError!void
44764495/// When the user has enabled the bulk_memory feature, we lower
44774496/// this to wasm's memset instruction. When the feature is not present,
44784497/// we implement it manually.
4479fn memset(func: *CodeGen, ptr: WValue, len: WValue, value: WValue) InnerError!void {
4498fn memset(func: *CodeGen, elem_ty: Type, ptr: WValue, len: WValue, value: WValue) InnerError!void {
4499 const abi_size = @intCast(u32, elem_ty.abiSize(func.target));
4500
44804501 // When bulk_memory is enabled, we lower it to wasm's memset instruction.
4481 // If not, we lower it ourselves
4482 if (std.Target.wasm.featureSetHas(func.target.cpu.features, .bulk_memory)) {
4502 // If not, we lower it ourselves.
4503 if (std.Target.wasm.featureSetHas(func.target.cpu.features, .bulk_memory) and abi_size == 1) {
44834504 try func.lowerToStack(ptr);
44844505 try func.emitWValue(value);
44854506 try func.emitWValue(len);
......@@ -4487,74 +4508,79 @@ fn memset(func: *CodeGen, ptr: WValue, len: WValue, value: WValue) InnerError!vo
44874508 return;
44884509 }
44894510
4490 // When the length is comptime-known we do the loop at codegen, rather
4491 // than emitting a runtime loop into the binary
4492 switch (len) {
4493 .imm32, .imm64 => {
4494 const length = switch (len) {
4495 .imm32 => |val| val,
4496 .imm64 => |val| val,
4497 else => unreachable,
4498 };
4499
4500 var offset: u32 = 0;
4501 const base = ptr.offset();
4502 while (offset < length) : (offset += 1) {
4503 try func.emitWValue(ptr);
4504 try func.emitWValue(value);
4505 switch (func.arch()) {
4506 .wasm32 => {
4507 try func.addMemArg(.i32_store8, .{ .offset = base + offset, .alignment = 1 });
4508 },
4509 .wasm64 => {
4510 try func.addMemArg(.i64_store8, .{ .offset = base + offset, .alignment = 1 });
4511 },
4512 else => unreachable,
4513 }
4514 }
4515 },
4516 else => {
4517 // TODO: We should probably lower this to a call to compiler_rt
4518 // But for now, we implement it manually
4519 const offset = try func.ensureAllocLocal(Type.usize); // local for counter
4520 // outer block to jump to when loop is done
4521 try func.startBlock(.block, wasm.block_empty);
4522 try func.startBlock(.loop, wasm.block_empty);
4523 try func.emitWValue(offset);
4511 const final_len = switch (len) {
4512 .imm32 => |val| WValue{ .imm32 = val * abi_size },
4513 .imm64 => |val| WValue{ .imm64 = val * abi_size },
4514 else => if (abi_size != 1) blk: {
4515 const new_len = try func.ensureAllocLocal(Type.usize);
45244516 try func.emitWValue(len);
45254517 switch (func.arch()) {
4526 .wasm32 => try func.addTag(.i32_eq),
4527 .wasm64 => try func.addTag(.i64_eq),
4528 else => unreachable,
4529 }
4530 try func.addLabel(.br_if, 1); // jump out of loop into outer block (finished)
4531 try func.emitWValue(ptr);
4532 try func.emitWValue(offset);
4533 switch (func.arch()) {
4534 .wasm32 => try func.addTag(.i32_add),
4535 .wasm64 => try func.addTag(.i64_add),
4536 else => unreachable,
4537 }
4538 try func.emitWValue(value);
4539 const mem_store_op: Mir.Inst.Tag = switch (func.arch()) {
4540 .wasm32 => .i32_store8,
4541 .wasm64 => .i64_store8,
4542 else => unreachable,
4543 };
4544 try func.addMemArg(mem_store_op, .{ .offset = ptr.offset(), .alignment = 1 });
4545 try func.emitWValue(offset);
4546 try func.addImm32(1);
4547 switch (func.arch()) {
4548 .wasm32 => try func.addTag(.i32_add),
4549 .wasm64 => try func.addTag(.i64_add),
4518 .wasm32 => {
4519 try func.emitWValue(.{ .imm32 = abi_size });
4520 try func.addTag(.i32_mul);
4521 },
4522 .wasm64 => {
4523 try func.emitWValue(.{ .imm64 = abi_size });
4524 try func.addTag(.i64_mul);
4525 },
45504526 else => unreachable,
45514527 }
4552 try func.addLabel(.local_set, offset.local.value);
4553 try func.addLabel(.br, 0); // jump to start of loop
4554 try func.endBlock();
4555 try func.endBlock();
4528 try func.addLabel(.local_set, new_len.local.value);
4529 break :blk new_len;
4530 } else len,
4531 };
4532
4533 var end_ptr = try func.allocLocal(Type.usize);
4534 defer end_ptr.free(func);
4535 var new_ptr = try func.buildPointerOffset(ptr, 0, .new);
4536 defer new_ptr.free(func);
4537
4538 // get the loop conditional: if current pointer address equals final pointer's address
4539 try func.lowerToStack(ptr);
4540 try func.emitWValue(final_len);
4541 switch (func.arch()) {
4542 .wasm32 => try func.addTag(.i32_add),
4543 .wasm64 => try func.addTag(.i64_add),
4544 else => unreachable,
4545 }
4546 try func.addLabel(.local_set, end_ptr.local.value);
4547
4548 // outer block to jump to when loop is done
4549 try func.startBlock(.block, wasm.block_empty);
4550 try func.startBlock(.loop, wasm.block_empty);
4551
4552 // check for codition for loop end
4553 try func.emitWValue(new_ptr);
4554 try func.emitWValue(end_ptr);
4555 switch (func.arch()) {
4556 .wasm32 => try func.addTag(.i32_eq),
4557 .wasm64 => try func.addTag(.i64_eq),
4558 else => unreachable,
4559 }
4560 try func.addLabel(.br_if, 1); // jump out of loop into outer block (finished)
4561
4562 // store the value at the current position of the pointer
4563 try func.store(new_ptr, value, elem_ty, 0);
4564
4565 // move the pointer to the next element
4566 try func.emitWValue(new_ptr);
4567 switch (func.arch()) {
4568 .wasm32 => {
4569 try func.emitWValue(.{ .imm32 = abi_size });
4570 try func.addTag(.i32_add);
45564571 },
4572 .wasm64 => {
4573 try func.emitWValue(.{ .imm64 = abi_size });
4574 try func.addTag(.i64_add);
4575 },
4576 else => unreachable,
45574577 }
4578 try func.addLabel(.local_set, new_ptr.local.value);
4579
4580 // end of loop
4581 try func.addLabel(.br, 0); // jump to start of loop
4582 try func.endBlock();
4583 try func.endBlock();
45584584}
45594585
45604586fn airArrayElemVal(func: *CodeGen, inst: Air.Inst.Index) InnerError!void {
......@@ -6007,10 +6033,12 @@ fn lowerTry(
60076033 const liveness = func.liveness.getCondBr(inst);
60086034 try func.branches.append(func.gpa, .{});
60096035 try func.currentBranch().values.ensureUnusedCapacity(func.gpa, liveness.else_deaths.len + liveness.then_deaths.len);
6036 defer {
6037 var branch = func.branches.pop();
6038 branch.deinit(func.gpa);
6039 }
60106040 try func.genBody(body);
60116041 try func.endBlock();
6012 var branch = func.branches.pop();
6013 branch.deinit(func.gpa);
60146042 }
60156043
60166044 // if we reach here it means error was not set, and we want the payload
test/behavior/memset.zig-14
......@@ -7,10 +7,6 @@ test "@memset on array pointers" {
77 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
88 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest;
99 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
10 if (builtin.zig_backend == .stage2_wasm) {
11 // TODO: implement memset when element ABI size > 1
12 return error.SkipZigTest;
13 }
1410
1511 try testMemsetArray();
1612 try comptime testMemsetArray();
......@@ -40,11 +36,6 @@ test "@memset on slices" {
4036 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
4137 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest;
4238 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
43 if (builtin.zig_backend == .stage2_wasm) {
44 // TODO: implement memset when element ABI size > 1
45 // TODO: implement memset on slices
46 return error.SkipZigTest;
47 }
4839
4940 try testMemsetSlice();
5041 try comptime testMemsetSlice();
......@@ -78,7 +69,6 @@ test "memset with bool element" {
7869 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
7970 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest;
8071 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
81 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;
8272
8373 var buf: [5]bool = undefined;
8474 @memset(&buf, true);
......@@ -91,7 +81,6 @@ test "memset with 1-byte struct element" {
9181 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
9282 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest;
9383 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
94 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;
9584
9685 const S = struct { x: bool };
9786 var buf: [5]S = undefined;
......@@ -105,7 +94,6 @@ test "memset with 1-byte array element" {
10594 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
10695 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest;
10796 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
108 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;
10997
11098 const A = [1]bool;
11199 var buf: [5]A = undefined;
......@@ -119,7 +107,6 @@ test "memset with large array element, runtime known" {
119107 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
120108 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest;
121109 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
122 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;
123110
124111 const A = [128]u64;
125112 var buf: [5]A = undefined;
......@@ -137,7 +124,6 @@ test "memset with large array element, comptime known" {
137124 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
138125 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest;
139126 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
140 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;
141127
142128 const A = [128]u64;
143129 var buf: [5]A = undefined;