authorgravatar for paul.verigo@gmail.comPavel Verigo <paul.verigo@gmail.com> 2026-08-22 00:41:59+02:00
committergravatar for paul.verigo@gmail.comPavel Verigo <paul.verigo@gmail.com> 2026-09-17 00:04:51+02:00
log88c44442aed3c6b5d7b49e0978b80c5efdfab5f8
tree5f6e006e65390493a1ed3e1d214b8b5e6a1997de
parentb819da2d57316993c5bb77c93d50f626e32eea91

wasm: better assembly for offset and memset


1 files changed, 70 insertions(+), 85 deletions(-)

src/codegen/wasm/CodeGen.zig+70-85
...@@ -668,6 +668,41 @@ fn offsetPointer(cg: *CodeGen, ptr: WValue, offset: u64) InnerError!WValue {...@@ -668,6 +668,41 @@ fn offsetPointer(cg: *CodeGen, ptr: WValue, offset: u64) InnerError!WValue {
668 };668 };
669}669}
670670
671const PointerOffsetOp = enum { add, sub };
672
673/// The offset may only be known at runtime. Base pointer must be already on the stack. Result is left on it also.
674fn emitScaledPointerOffset(cg: *CodeGen, offset: WValue, scale: u64, op: PointerOffsetOp) InnerError!void {
675 if (scale == 0 or switch (offset) {
676 .imm32 => |imm| imm == 0,
677 .imm64 => |imm| imm == 0,
678 else => false,
679 }) return;
680
681 try cg.emitWValue(offset);
682 switch (cg.ptr_size) {
683 .wasm32 => {
684 if (scale != 1) {
685 try cg.addImm32(@intCast(scale));
686 try cg.addTag(.i32_mul);
687 }
688 try cg.addTag(switch (op) {
689 .add => .i32_add,
690 .sub => .i32_sub,
691 });
692 },
693 .wasm64 => {
694 if (scale != 1) {
695 try cg.addImm64(scale);
696 try cg.addTag(.i64_mul);
697 }
698 try cg.addTag(switch (op) {
699 .add => .i64_add,
700 .sub => .i64_sub,
701 });
702 },
703 }
704}
705
671/// Writes the bytecode depending on the given `WValue` in `val`706/// Writes the bytecode depending on the given `WValue` in `val`
672fn emitWValue(cg: *CodeGen, value: WValue) InnerError!void {707fn emitWValue(cg: *CodeGen, value: WValue) InnerError!void {
673 switch (value) {708 switch (value) {
...@@ -1187,7 +1222,7 @@ fn emitMemoryCopy(cg: *CodeGen, dst: WValue, src: WValue, len: WValue) !void {...@@ -1187,7 +1222,7 @@ fn emitMemoryCopy(cg: *CodeGen, dst: WValue, src: WValue, len: WValue) !void {
1187}1222}
11881223
1189fn memcpy(cg: *CodeGen, dst: WValue, src: WValue, len: WValue) !void {1224fn memcpy(cg: *CodeGen, dst: WValue, src: WValue, len: WValue) !void {
1190 if (cg.target.cpu.has(.wasm, .bulk_memory)) {1225 if (cg.target.cpu.has(.wasm, .bulk_memory_opt)) {
1191 try cg.emitMemoryCopy(dst, src, len);1226 try cg.emitMemoryCopy(dst, src, len);
1192 return;1227 return;
1193 }1228 }
...@@ -1200,7 +1235,7 @@ fn memcpy(cg: *CodeGen, dst: WValue, src: WValue, len: WValue) !void {...@@ -1200,7 +1235,7 @@ fn memcpy(cg: *CodeGen, dst: WValue, src: WValue, len: WValue) !void {
1200}1235}
12011236
1202fn memmove(cg: *CodeGen, dst: WValue, src: WValue, len: WValue) !void {1237fn memmove(cg: *CodeGen, dst: WValue, src: WValue, len: WValue) !void {
1203 if (cg.target.cpu.has(.wasm, .bulk_memory)) {1238 if (cg.target.cpu.has(.wasm, .bulk_memory_opt)) {
1204 try cg.emitMemoryCopy(dst, src, len);1239 try cg.emitMemoryCopy(dst, src, len);
1205 return;1240 return;
1206 }1241 }
...@@ -1309,10 +1344,7 @@ fn genInst(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -1309,10 +1344,7 @@ fn genInst(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
1309 const elem_size = elem_ty.abiSize(zcu);1344 const elem_size = elem_ty.abiSize(zcu);
13101345
1311 try cg.emitWValue(vec_ptr);1346 try cg.emitWValue(vec_ptr);
1312 try cg.emitWValue(elem_idx);1347 try cg.emitScaledPointerOffset(elem_idx, elem_size, .add);
1313 try cg.addImm32(@intCast(elem_size));
1314 try cg.addTag(.i32_mul);
1315 try cg.addTag(.i32_add);
1316 const ptr = try WValue.toLocal(.stack, cg, Type.usize);1348 const ptr = try WValue.toLocal(.stack, cg, Type.usize);
13171349
1318 try cg.store(ptr, elem_val, elem_ty, .{});1350 try cg.store(ptr, elem_val, elem_ty, .{});
...@@ -2258,15 +2290,17 @@ fn airStore(cg: *CodeGen, inst: Air.Inst.Index, safety: bool) InnerError!void {...@@ -2258,15 +2290,17 @@ fn airStore(cg: *CodeGen, inst: Air.Inst.Index, safety: bool) InnerError!void {
2258 const bin_op = cg.air.instructions.items(.data)[@backingInt(inst)].bin_op;2290 const bin_op = cg.air.instructions.items(.data)[@backingInt(inst)].bin_op;
22592291
2260 const lhs = try cg.resolveInst(bin_op.lhs);2292 const lhs = try cg.resolveInst(bin_op.lhs);
2261 const rhs = try cg.resolveInst(bin_op.rhs);
2262 const ptr_ty = cg.typeOf(bin_op.lhs);2293 const ptr_ty = cg.typeOf(bin_op.lhs);
2263 const ptr_info = ptr_ty.ptrInfo(zcu);2294 const ptr_info = ptr_ty.ptrInfo(zcu);
2264 const elem_ty = ptr_ty.childType(zcu);2295 const elem_ty = ptr_ty.childType(zcu);
2296 const rhs_is_undef = if (bin_op.rhs.toInterned()) |i| Value.fromInterned(i).isUndef(zcu) else false;
22652297
2266 if (!safety and bin_op.rhs == .undef) {2298 if (!safety and rhs_is_undef) {
2267 return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });2299 return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });
2268 }2300 }
22692301
2302 const rhs = try cg.resolveInst(bin_op.rhs);
2303
2270 const offset: u64 = switch (ptr_info.flags.vector_index) {2304 const offset: u64 = switch (ptr_info.flags.vector_index) {
2271 .none => offset: {2305 .none => offset: {
2272 assert(ptr_info.packed_offset.host_size == 0); // legalize .expand_packed_store2306 assert(ptr_info.packed_offset.host_size == 0); // legalize .expand_packed_store
...@@ -2274,6 +2308,13 @@ fn airStore(cg: *CodeGen, inst: Air.Inst.Index, safety: bool) InnerError!void {...@@ -2274,6 +2308,13 @@ fn airStore(cg: *CodeGen, inst: Air.Inst.Index, safety: bool) InnerError!void {
2274 },2308 },
2275 else => |index| @backingInt(index) * elem_ty.abiSize(zcu),2309 else => |index| @backingInt(index) * elem_ty.abiSize(zcu),
2276 };2310 };
2311
2312 if (safety and rhs_is_undef and isByRef(elem_ty, zcu, cg.target)) {
2313 const dst = try cg.offsetPointer(lhs, offset);
2314 try cg.memset(.u8, dst, .{ .imm32 = @intCast(elem_ty.abiSize(zcu)) }, .{ .imm32 = 0xaa });
2315 return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });
2316 }
2317
2277 try cg.store(lhs, rhs, elem_ty, .{2318 try cg.store(lhs, rhs, elem_ty, .{
2278 .offset = offset,2319 .offset = offset,
2279 .alignment = ptr_ty.ptrAlignment(zcu),2320 .alignment = ptr_ty.ptrAlignment(zcu),
...@@ -6368,10 +6409,7 @@ fn airSliceElemVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -6368,10 +6409,7 @@ fn airSliceElemVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6368 _ = try cg.load(slice, .usize, .{});6409 _ = try cg.load(slice, .usize, .{});
63696410
6370 // calculate index into slice6411 // calculate index into slice
6371 try cg.emitWValue(index);6412 try cg.emitScaledPointerOffset(index, elem_size, .add);
6372 try cg.addImm32(@intCast(elem_size));
6373 try cg.addTag(.i32_mul);
6374 try cg.addTag(.i32_add);
63756413
6376 const elem_result = try cg.load(.stack, elem_ty, .{});6414 const elem_result = try cg.load(.stack, elem_ty, .{});
63776415
...@@ -6392,10 +6430,7 @@ fn airSliceElemPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -6392,10 +6430,7 @@ fn airSliceElemPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6392 _ = try cg.load(slice, .usize, .{});6430 _ = try cg.load(slice, .usize, .{});
63936431
6394 // calculate index into slice6432 // calculate index into slice
6395 try cg.emitWValue(index);6433 try cg.emitScaledPointerOffset(index, elem_size, .add);
6396 try cg.addImm32(@intCast(elem_size));
6397 try cg.addTag(.i32_mul);
6398 try cg.addTag(.i32_add);
63996434
6400 return cg.finishAir(inst, .stack, &.{ bin_op.lhs, bin_op.rhs });6435 return cg.finishAir(inst, .stack, &.{ bin_op.lhs, bin_op.rhs });
6401}6436}
...@@ -6454,10 +6489,7 @@ fn airPtrElemVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -6454,10 +6489,7 @@ fn airPtrElemVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6454 }6489 }
64556490
6456 // calculate index into slice6491 // calculate index into slice
6457 try cg.emitWValue(index);6492 try cg.emitScaledPointerOffset(index, elem_size, .add);
6458 try cg.addImm32(@intCast(elem_size));
6459 try cg.addTag(.i32_mul);
6460 try cg.addTag(.i32_add);
64616493
6462 const elem_result = try cg.load(.stack, elem_ty, .{});6494 const elem_result = try cg.load(.stack, elem_ty, .{});
64636495
...@@ -6484,15 +6516,12 @@ fn airPtrElemPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -6484,15 +6516,12 @@ fn airPtrElemPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6484 }6516 }
64856517
6486 // calculate index into ptr6518 // calculate index into ptr
6487 try cg.emitWValue(index);6519 try cg.emitScaledPointerOffset(index, elem_size, .add);
6488 try cg.addImm32(@intCast(elem_size));
6489 try cg.addTag(.i32_mul);
6490 try cg.addTag(.i32_add);
64916520
6492 return cg.finishAir(inst, .stack, &.{ bin_op.lhs, bin_op.rhs });6521 return cg.finishAir(inst, .stack, &.{ bin_op.lhs, bin_op.rhs });
6493}6522}
64946523
6495fn airPtrBinOp(cg: *CodeGen, inst: Air.Inst.Index, op: enum { add, sub }) InnerError!void {6524fn airPtrBinOp(cg: *CodeGen, inst: Air.Inst.Index, op: PointerOffsetOp) InnerError!void {
6496 const zcu = cg.pt.zcu;6525 const zcu = cg.pt.zcu;
6497 const ty_pl = cg.air.instructions.items(.data)[@backingInt(inst)].ty_pl;6526 const ty_pl = cg.air.instructions.items(.data)[@backingInt(inst)].ty_pl;
6498 const bin_op = cg.air.extraData(Air.Bin, ty_pl.payload).data;6527 const bin_op = cg.air.extraData(Air.Bin, ty_pl.payload).data;
...@@ -6506,26 +6535,7 @@ fn airPtrBinOp(cg: *CodeGen, inst: Air.Inst.Index, op: enum { add, sub }) InnerE...@@ -6506,26 +6535,7 @@ fn airPtrBinOp(cg: *CodeGen, inst: Air.Inst.Index, op: enum { add, sub }) InnerE
6506 };6535 };
65076536
6508 try cg.emitWValue(ptr);6537 try cg.emitWValue(ptr);
6509 try cg.emitWValue(offset);6538 try cg.emitScaledPointerOffset(offset, pointee_ty.abiSize(zcu), op);
6510
6511 switch (cg.ptr_size) {
6512 .wasm32 => {
6513 try cg.addImm32(@intCast(pointee_ty.abiSize(zcu)));
6514 try cg.addTag(.i32_mul);
6515 try cg.addTag(switch (op) {
6516 .add => .i32_add,
6517 .sub => .i32_sub,
6518 });
6519 },
6520 .wasm64 => {
6521 try cg.addImm64(pointee_ty.abiSize(zcu));
6522 try cg.addTag(.i64_mul);
6523 try cg.addTag(switch (op) {
6524 .add => .i64_add,
6525 .sub => .i64_sub,
6526 });
6527 },
6528 }
65296539
6530 return cg.finishAir(inst, .stack, &.{ bin_op.lhs, bin_op.rhs });6540 return cg.finishAir(inst, .stack, &.{ bin_op.lhs, bin_op.rhs });
6531}6541}
...@@ -6559,19 +6569,24 @@ fn airMemset(cg: *CodeGen, inst: Air.Inst.Index, safety: bool) InnerError!void {...@@ -6559,19 +6569,24 @@ fn airMemset(cg: *CodeGen, inst: Air.Inst.Index, safety: bool) InnerError!void {
6559}6569}
65606570
6561/// Sets a region of memory at `ptr` to the value of `value`6571/// Sets a region of memory at `ptr` to the value of `value`
6562/// When the user has enabled the bulk_memory feature, we lower6572/// When the user has enabled the bulk_memory_opt feature, we lower
6563/// this to wasm's memset instruction. When the feature is not present,6573/// this to wasm's memset instruction. When the feature is not present,
6564/// we implement it manually.6574/// we implement it manually.
6565fn memset(cg: *CodeGen, elem_ty: Type, ptr: WValue, len: WValue, value: WValue) InnerError!void {6575fn memset(cg: *CodeGen, elem_ty: Type, ptr: WValue, len: WValue, value: WValue) InnerError!void {
6566 const zcu = cg.pt.zcu;6576 const zcu = cg.pt.zcu;
6567 const abi_size = @as(u32, @intCast(elem_ty.abiSize(zcu)));6577 const abi_size = @as(u32, @intCast(elem_ty.abiSize(zcu)));
65686578 const len_known_neq_0 = switch (len) {
6569 // When bulk_memory is enabled, we lower it to wasm's memset instruction.6579 .imm32 => |imm| if (imm != 0) true else return,
6580 .imm64 => |imm| if (imm != 0) true else return,
6581 else => false,
6582 };
6583 // When bulk_memory_opt is enabled, we lower it to wasm's memset instruction.
6570 // If not, we lower it ourselves.6584 // If not, we lower it ourselves.
6571 if (cg.target.cpu.has(.wasm, .bulk_memory) and abi_size == 1) {6585 if (cg.target.cpu.has(.wasm, .bulk_memory_opt) and abi_size == 1) {
6572 const len0_ok = cg.target.cpu.has(.wasm, .nontrapping_bulk_memory_len0);6586 const len0_ok = cg.target.cpu.has(.wasm, .nontrapping_bulk_memory_len0);
6587 const emit_check = !(len0_ok or len_known_neq_0);
65736588
6574 if (!len0_ok) {6589 if (emit_check) {
6575 try cg.startBlock(.block, .empty);6590 try cg.startBlock(.block, .empty);
65766591
6577 // Even if `len` is zero, the spec requires an implementation to trap if `ptr + len` is6592 // Even if `len` is zero, the spec requires an implementation to trap if `ptr + len` is
...@@ -6592,7 +6607,7 @@ fn memset(cg: *CodeGen, elem_ty: Type, ptr: WValue, len: WValue, value: WValue)...@@ -6592,7 +6607,7 @@ fn memset(cg: *CodeGen, elem_ty: Type, ptr: WValue, len: WValue, value: WValue)
6592 try cg.emitWValue(len);6607 try cg.emitWValue(len);
6593 try cg.addExtended(.memory_fill);6608 try cg.addExtended(.memory_fill);
65946609
6595 if (!len0_ok) {6610 if (emit_check) {
6596 try cg.endBlock();6611 try cg.endBlock();
6597 }6612 }
65986613
...@@ -6684,10 +6699,7 @@ fn airArrayElemVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -6684,10 +6699,7 @@ fn airArrayElemVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
66846699
6685 if (isByRef(array_ty, zcu, cg.target)) {6700 if (isByRef(array_ty, zcu, cg.target)) {
6686 try cg.emitWValue(array);6701 try cg.emitWValue(array);
6687 try cg.emitWValue(index);6702 try cg.emitScaledPointerOffset(index, elem_size, .add);
6688 try cg.addImm32(@intCast(elem_size));
6689 try cg.addTag(.i32_mul);
6690 try cg.addTag(.i32_add);
6691 } else {6703 } else {
6692 assert(array_ty.zigTypeTag(zcu) == .vector);6704 assert(array_ty.zigTypeTag(zcu) == .vector);
66936705
...@@ -6717,10 +6729,7 @@ fn airArrayElemVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -6717,10 +6729,7 @@ fn airArrayElemVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
67176729
6718 // Is a non-unrolled vector (v128)6730 // Is a non-unrolled vector (v128)
6719 try cg.emitWValue(stack_vec);6731 try cg.emitWValue(stack_vec);
6720 try cg.emitWValue(index);6732 try cg.emitScaledPointerOffset(index, elem_size, .add);
6721 try cg.addImm32(@intCast(elem_size));
6722 try cg.addTag(.i32_mul);
6723 try cg.addTag(.i32_add);
6724 },6733 },
6725 }6734 }
6726 }6735 }
...@@ -7249,19 +7258,7 @@ fn airErrorName(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -7249,19 +7258,7 @@ fn airErrorName(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7249 // Lowers to a i32.const or i64.const with the error table memory address.7258 // Lowers to a i32.const or i64.const with the error table memory address.
7250 cg.error_name_table_ref_count += 1;7259 cg.error_name_table_ref_count += 1;
7251 try cg.addTag(.error_name_table_ref);7260 try cg.addTag(.error_name_table_ref);
7252 try cg.emitWValue(operand);7261 try cg.emitScaledPointerOffset(operand, abi_size, .add);
7253 switch (cg.ptr_size) {
7254 .wasm32 => {
7255 try cg.addImm32(@intCast(abi_size));
7256 try cg.addTag(.i32_mul);
7257 try cg.addTag(.i32_add);
7258 },
7259 .wasm64 => {
7260 try cg.addImm64(abi_size);
7261 try cg.addTag(.i64_mul);
7262 try cg.addTag(.i64_add);
7263 },
7264 }
72657262
7266 return cg.finishAir(inst, .stack, &.{un_op});7263 return cg.finishAir(inst, .stack, &.{un_op});
7267}7264}
...@@ -7427,19 +7424,7 @@ fn airTagName(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -7427,19 +7424,7 @@ fn airTagName(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7427 try cg.addInst(.{ .tag = .enum_tag_name_table_ref, .data = .{ .ip_index = enum_ty.toIntern() } });7424 try cg.addInst(.{ .tag = .enum_tag_name_table_ref, .data = .{ .ip_index = enum_ty.toIntern() } });
7428 try cg.emitWValue(operand);7425 try cg.emitWValue(operand);
7429 try cg.addInst(.{ .tag = .call_tag_index, .data = .{ .ip_index = enum_ty.toIntern() } });7426 try cg.addInst(.{ .tag = .call_tag_index, .data = .{ .ip_index = enum_ty.toIntern() } });
74307427 try cg.emitScaledPointerOffset(.stack, 8, .add);
7431 switch (cg.ptr_size) {
7432 .wasm32 => {
7433 try cg.addImm32(@intCast(8));
7434 try cg.addTag(.i32_mul);
7435 try cg.addTag(.i32_add);
7436 },
7437 .wasm64 => {
7438 try cg.addImm64(8);
7439 try cg.addTag(.i64_mul);
7440 try cg.addTag(.i64_add);
7441 },
7442 }
74437428
7444 return cg.finishAir(inst, .stack, &.{un_op});7429 return cg.finishAir(inst, .stack, &.{un_op});
7445}7430}