authorgravatar for paul.verigo@gmail.comPavel Verigo <paul.verigo@gmail.com> 2026-08-22 00:41:59+02:00
committergravatar for paul.verigo@gmail.comPavel Verigo <paul.verigo@gmail.com> 2026-09-17 00:04:51+02:00
log88c44442aed3c6b5d7b49e0978b80c5efdfab5f8
tree5f6e006e65390493a1ed3e1d214b8b5e6a1997de
parentb819da2d57316993c5bb77c93d50f626e32eea91

wasm: better assembly for offset and memset

1 files changed, 70 insertions(+), 85 deletions(-)

src/codegen/wasm/CodeGen.zig+70-85
......@@ -668,6 +668,41 @@ fn offsetPointer(cg: *CodeGen, ptr: WValue, offset: u64) InnerError!WValue {
668668 };
669669}
670670
671const PointerOffsetOp = enum { add, sub };
672
673/// The offset may only be known at runtime. Base pointer must be already on the stack. Result is left on it also.
674fn emitScaledPointerOffset(cg: *CodeGen, offset: WValue, scale: u64, op: PointerOffsetOp) InnerError!void {
675 if (scale == 0 or switch (offset) {
676 .imm32 => |imm| imm == 0,
677 .imm64 => |imm| imm == 0,
678 else => false,
679 }) return;
680
681 try cg.emitWValue(offset);
682 switch (cg.ptr_size) {
683 .wasm32 => {
684 if (scale != 1) {
685 try cg.addImm32(@intCast(scale));
686 try cg.addTag(.i32_mul);
687 }
688 try cg.addTag(switch (op) {
689 .add => .i32_add,
690 .sub => .i32_sub,
691 });
692 },
693 .wasm64 => {
694 if (scale != 1) {
695 try cg.addImm64(scale);
696 try cg.addTag(.i64_mul);
697 }
698 try cg.addTag(switch (op) {
699 .add => .i64_add,
700 .sub => .i64_sub,
701 });
702 },
703 }
704}
705
671706/// Writes the bytecode depending on the given `WValue` in `val`
672707fn emitWValue(cg: *CodeGen, value: WValue) InnerError!void {
673708 switch (value) {
......@@ -1187,7 +1222,7 @@ fn emitMemoryCopy(cg: *CodeGen, dst: WValue, src: WValue, len: WValue) !void {
11871222}
11881223
11891224fn memcpy(cg: *CodeGen, dst: WValue, src: WValue, len: WValue) !void {
1190 if (cg.target.cpu.has(.wasm, .bulk_memory)) {
1225 if (cg.target.cpu.has(.wasm, .bulk_memory_opt)) {
11911226 try cg.emitMemoryCopy(dst, src, len);
11921227 return;
11931228 }
......@@ -1200,7 +1235,7 @@ fn memcpy(cg: *CodeGen, dst: WValue, src: WValue, len: WValue) !void {
12001235}
12011236
12021237fn memmove(cg: *CodeGen, dst: WValue, src: WValue, len: WValue) !void {
1203 if (cg.target.cpu.has(.wasm, .bulk_memory)) {
1238 if (cg.target.cpu.has(.wasm, .bulk_memory_opt)) {
12041239 try cg.emitMemoryCopy(dst, src, len);
12051240 return;
12061241 }
......@@ -1309,10 +1344,7 @@ fn genInst(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
13091344 const elem_size = elem_ty.abiSize(zcu);
13101345
13111346 try cg.emitWValue(vec_ptr);
1312 try cg.emitWValue(elem_idx);
1313 try cg.addImm32(@intCast(elem_size));
1314 try cg.addTag(.i32_mul);
1315 try cg.addTag(.i32_add);
1347 try cg.emitScaledPointerOffset(elem_idx, elem_size, .add);
13161348 const ptr = try WValue.toLocal(.stack, cg, Type.usize);
13171349
13181350 try cg.store(ptr, elem_val, elem_ty, .{});
......@@ -2258,15 +2290,17 @@ fn airStore(cg: *CodeGen, inst: Air.Inst.Index, safety: bool) InnerError!void {
22582290 const bin_op = cg.air.instructions.items(.data)[@backingInt(inst)].bin_op;
22592291
22602292 const lhs = try cg.resolveInst(bin_op.lhs);
2261 const rhs = try cg.resolveInst(bin_op.rhs);
22622293 const ptr_ty = cg.typeOf(bin_op.lhs);
22632294 const ptr_info = ptr_ty.ptrInfo(zcu);
22642295 const elem_ty = ptr_ty.childType(zcu);
2296 const rhs_is_undef = if (bin_op.rhs.toInterned()) |i| Value.fromInterned(i).isUndef(zcu) else false;
22652297
2266 if (!safety and bin_op.rhs == .undef) {
2298 if (!safety and rhs_is_undef) {
22672299 return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });
22682300 }
22692301
2302 const rhs = try cg.resolveInst(bin_op.rhs);
2303
22702304 const offset: u64 = switch (ptr_info.flags.vector_index) {
22712305 .none => offset: {
22722306 assert(ptr_info.packed_offset.host_size == 0); // legalize .expand_packed_store
......@@ -2274,6 +2308,13 @@ fn airStore(cg: *CodeGen, inst: Air.Inst.Index, safety: bool) InnerError!void {
22742308 },
22752309 else => |index| @backingInt(index) * elem_ty.abiSize(zcu),
22762310 };
2311
2312 if (safety and rhs_is_undef and isByRef(elem_ty, zcu, cg.target)) {
2313 const dst = try cg.offsetPointer(lhs, offset);
2314 try cg.memset(.u8, dst, .{ .imm32 = @intCast(elem_ty.abiSize(zcu)) }, .{ .imm32 = 0xaa });
2315 return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });
2316 }
2317
22772318 try cg.store(lhs, rhs, elem_ty, .{
22782319 .offset = offset,
22792320 .alignment = ptr_ty.ptrAlignment(zcu),
......@@ -6368,10 +6409,7 @@ fn airSliceElemVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
63686409 _ = try cg.load(slice, .usize, .{});
63696410
63706411 // calculate index into slice
6371 try cg.emitWValue(index);
6372 try cg.addImm32(@intCast(elem_size));
6373 try cg.addTag(.i32_mul);
6374 try cg.addTag(.i32_add);
6412 try cg.emitScaledPointerOffset(index, elem_size, .add);
63756413
63766414 const elem_result = try cg.load(.stack, elem_ty, .{});
63776415
......@@ -6392,10 +6430,7 @@ fn airSliceElemPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
63926430 _ = try cg.load(slice, .usize, .{});
63936431
63946432 // calculate index into slice
6395 try cg.emitWValue(index);
6396 try cg.addImm32(@intCast(elem_size));
6397 try cg.addTag(.i32_mul);
6398 try cg.addTag(.i32_add);
6433 try cg.emitScaledPointerOffset(index, elem_size, .add);
63996434
64006435 return cg.finishAir(inst, .stack, &.{ bin_op.lhs, bin_op.rhs });
64016436}
......@@ -6454,10 +6489,7 @@ fn airPtrElemVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
64546489 }
64556490
64566491 // calculate index into slice
6457 try cg.emitWValue(index);
6458 try cg.addImm32(@intCast(elem_size));
6459 try cg.addTag(.i32_mul);
6460 try cg.addTag(.i32_add);
6492 try cg.emitScaledPointerOffset(index, elem_size, .add);
64616493
64626494 const elem_result = try cg.load(.stack, elem_ty, .{});
64636495
......@@ -6484,15 +6516,12 @@ fn airPtrElemPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
64846516 }
64856517
64866518 // calculate index into ptr
6487 try cg.emitWValue(index);
6488 try cg.addImm32(@intCast(elem_size));
6489 try cg.addTag(.i32_mul);
6490 try cg.addTag(.i32_add);
6519 try cg.emitScaledPointerOffset(index, elem_size, .add);
64916520
64926521 return cg.finishAir(inst, .stack, &.{ bin_op.lhs, bin_op.rhs });
64936522}
64946523
6495fn airPtrBinOp(cg: *CodeGen, inst: Air.Inst.Index, op: enum { add, sub }) InnerError!void {
6524fn airPtrBinOp(cg: *CodeGen, inst: Air.Inst.Index, op: PointerOffsetOp) InnerError!void {
64966525 const zcu = cg.pt.zcu;
64976526 const ty_pl = cg.air.instructions.items(.data)[@backingInt(inst)].ty_pl;
64986527 const bin_op = cg.air.extraData(Air.Bin, ty_pl.payload).data;
......@@ -6506,26 +6535,7 @@ fn airPtrBinOp(cg: *CodeGen, inst: Air.Inst.Index, op: enum { add, sub }) InnerE
65066535 };
65076536
65086537 try cg.emitWValue(ptr);
6509 try cg.emitWValue(offset);
6510
6511 switch (cg.ptr_size) {
6512 .wasm32 => {
6513 try cg.addImm32(@intCast(pointee_ty.abiSize(zcu)));
6514 try cg.addTag(.i32_mul);
6515 try cg.addTag(switch (op) {
6516 .add => .i32_add,
6517 .sub => .i32_sub,
6518 });
6519 },
6520 .wasm64 => {
6521 try cg.addImm64(pointee_ty.abiSize(zcu));
6522 try cg.addTag(.i64_mul);
6523 try cg.addTag(switch (op) {
6524 .add => .i64_add,
6525 .sub => .i64_sub,
6526 });
6527 },
6528 }
6538 try cg.emitScaledPointerOffset(offset, pointee_ty.abiSize(zcu), op);
65296539
65306540 return cg.finishAir(inst, .stack, &.{ bin_op.lhs, bin_op.rhs });
65316541}
......@@ -6559,19 +6569,24 @@ fn airMemset(cg: *CodeGen, inst: Air.Inst.Index, safety: bool) InnerError!void {
65596569}
65606570
65616571/// Sets a region of memory at `ptr` to the value of `value`
6562/// When the user has enabled the bulk_memory feature, we lower
6572/// When the user has enabled the bulk_memory_opt feature, we lower
65636573/// this to wasm's memset instruction. When the feature is not present,
65646574/// we implement it manually.
65656575fn memset(cg: *CodeGen, elem_ty: Type, ptr: WValue, len: WValue, value: WValue) InnerError!void {
65666576 const zcu = cg.pt.zcu;
65676577 const abi_size = @as(u32, @intCast(elem_ty.abiSize(zcu)));
6568
6569 // When bulk_memory is enabled, we lower it to wasm's memset instruction.
6578 const len_known_neq_0 = switch (len) {
6579 .imm32 => |imm| if (imm != 0) true else return,
6580 .imm64 => |imm| if (imm != 0) true else return,
6581 else => false,
6582 };
6583 // When bulk_memory_opt is enabled, we lower it to wasm's memset instruction.
65706584 // If not, we lower it ourselves.
6571 if (cg.target.cpu.has(.wasm, .bulk_memory) and abi_size == 1) {
6585 if (cg.target.cpu.has(.wasm, .bulk_memory_opt) and abi_size == 1) {
65726586 const len0_ok = cg.target.cpu.has(.wasm, .nontrapping_bulk_memory_len0);
6587 const emit_check = !(len0_ok or len_known_neq_0);
65736588
6574 if (!len0_ok) {
6589 if (emit_check) {
65756590 try cg.startBlock(.block, .empty);
65766591
65776592 // Even if `len` is zero, the spec requires an implementation to trap if `ptr + len` is
......@@ -6592,7 +6607,7 @@ fn memset(cg: *CodeGen, elem_ty: Type, ptr: WValue, len: WValue, value: WValue)
65926607 try cg.emitWValue(len);
65936608 try cg.addExtended(.memory_fill);
65946609
6595 if (!len0_ok) {
6610 if (emit_check) {
65966611 try cg.endBlock();
65976612 }
65986613
......@@ -6684,10 +6699,7 @@ fn airArrayElemVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
66846699
66856700 if (isByRef(array_ty, zcu, cg.target)) {
66866701 try cg.emitWValue(array);
6687 try cg.emitWValue(index);
6688 try cg.addImm32(@intCast(elem_size));
6689 try cg.addTag(.i32_mul);
6690 try cg.addTag(.i32_add);
6702 try cg.emitScaledPointerOffset(index, elem_size, .add);
66916703 } else {
66926704 assert(array_ty.zigTypeTag(zcu) == .vector);
66936705
......@@ -6717,10 +6729,7 @@ fn airArrayElemVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
67176729
67186730 // Is a non-unrolled vector (v128)
67196731 try cg.emitWValue(stack_vec);
6720 try cg.emitWValue(index);
6721 try cg.addImm32(@intCast(elem_size));
6722 try cg.addTag(.i32_mul);
6723 try cg.addTag(.i32_add);
6732 try cg.emitScaledPointerOffset(index, elem_size, .add);
67246733 },
67256734 }
67266735 }
......@@ -7249,19 +7258,7 @@ fn airErrorName(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
72497258 // Lowers to a i32.const or i64.const with the error table memory address.
72507259 cg.error_name_table_ref_count += 1;
72517260 try cg.addTag(.error_name_table_ref);
7252 try cg.emitWValue(operand);
7253 switch (cg.ptr_size) {
7254 .wasm32 => {
7255 try cg.addImm32(@intCast(abi_size));
7256 try cg.addTag(.i32_mul);
7257 try cg.addTag(.i32_add);
7258 },
7259 .wasm64 => {
7260 try cg.addImm64(abi_size);
7261 try cg.addTag(.i64_mul);
7262 try cg.addTag(.i64_add);
7263 },
7264 }
7261 try cg.emitScaledPointerOffset(operand, abi_size, .add);
72657262
72667263 return cg.finishAir(inst, .stack, &.{un_op});
72677264}
......@@ -7427,19 +7424,7 @@ fn airTagName(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
74277424 try cg.addInst(.{ .tag = .enum_tag_name_table_ref, .data = .{ .ip_index = enum_ty.toIntern() } });
74287425 try cg.emitWValue(operand);
74297426 try cg.addInst(.{ .tag = .call_tag_index, .data = .{ .ip_index = enum_ty.toIntern() } });
7430
7431 switch (cg.ptr_size) {
7432 .wasm32 => {
7433 try cg.addImm32(@intCast(8));
7434 try cg.addTag(.i32_mul);
7435 try cg.addTag(.i32_add);
7436 },
7437 .wasm64 => {
7438 try cg.addImm64(8);
7439 try cg.addTag(.i64_mul);
7440 try cg.addTag(.i64_add);
7441 },
7442 }
7427 try cg.emitScaledPointerOffset(.stack, 8, .add);
74437428
74447429 return cg.finishAir(inst, .stack, &.{un_op});
74457430}