authorgravatar for luuk@degram.devLuuk de Gram <luuk@degram.dev> 2023-04-22 15:40:53+02:00
committergravatar for luuk@degram.devLuuk de Gram <luuk@degram.dev> 2023-04-26 16:28:40+02:00
logb19c258f045aa1653b4e38ba2ff96302d914eea2
tree581bb2724a6b27589fe7f9ce377aab68eb7fd58c
parentfd47eddc862b0a5bd90949fe21cf87a716c5464c
signaturelock-open Commit is signed but in an unrecognized format.

wasm: implement `@atomicRmw`

Implements the lowering of the `@atomicRmw` builtin. Uses the atomic opcodes when the cpu feature `atomics` is enabled. Otherwise lowers it to regular instructions. For the operations that do not lower to a direct atomic opcode, we use a loop in combiantion with a cmpxchg to ensure the swapping of values is doing atomically.

2 files changed, 233 insertions(+), 5 deletions(-)

src/arch/wasm/CodeGen.zig+176-3
...@@ -1969,7 +1969,6 @@ fn genInst(func: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -1969,7 +1969,6 @@ fn genInst(func: *CodeGen, inst: Air.Inst.Index) InnerError!void {
1969 .atomic_store_monotonic,1969 .atomic_store_monotonic,
1970 .atomic_store_release,1970 .atomic_store_release,
1971 .atomic_store_seq_cst,1971 .atomic_store_seq_cst,
1972 .atomic_rmw,
1973 .err_return_trace,1972 .err_return_trace,
1974 .set_err_return_trace,1973 .set_err_return_trace,
1975 .save_err_return_trace_index,1974 .save_err_return_trace_index,
...@@ -1983,6 +1982,7 @@ fn genInst(func: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -1983,6 +1982,7 @@ fn genInst(func: *CodeGen, inst: Air.Inst.Index) InnerError!void {
1983 => |tag| return func.fail("TODO: Implement wasm inst: {s}", .{@tagName(tag)}),1982 => |tag| return func.fail("TODO: Implement wasm inst: {s}", .{@tagName(tag)}),
19841983
1985 .atomic_load => func.airAtomicLoad(inst),1984 .atomic_load => func.airAtomicLoad(inst),
1985 .atomic_rmw => func.airAtomicRmw(inst),
1986 .cmpxchg_weak => func.airCmpxchg(inst),1986 .cmpxchg_weak => func.airCmpxchg(inst),
1987 .cmpxchg_strong => func.airCmpxchg(inst),1987 .cmpxchg_strong => func.airCmpxchg(inst),
19881988
...@@ -6526,7 +6526,6 @@ fn getTagNameFunction(func: *CodeGen, enum_ty: Type) InnerError!u32 {...@@ -6526,7 +6526,6 @@ fn getTagNameFunction(func: *CodeGen, enum_ty: Type) InnerError!u32 {
6526 return func.bin_file.createFunction(func_name, func_type, &body_list, &relocs);6526 return func.bin_file.createFunction(func_name, func_type, &body_list, &relocs);
6527}6527}
65286528
6529<<<<<<< HEAD
6530fn airErrorSetHasValue(func: *CodeGen, inst: Air.Inst.Index) InnerError!void {6529fn airErrorSetHasValue(func: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6531 const ty_op = func.air.instructions.items(.data)[inst].ty_op;6530 const ty_op = func.air.instructions.items(.data)[inst].ty_op;
65326531
...@@ -6676,7 +6675,7 @@ fn airAtomicLoad(func: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -6676,7 +6675,7 @@ fn airAtomicLoad(func: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6676 2 => .i32_atomic_load16_u,6675 2 => .i32_atomic_load16_u,
6677 4 => .i32_atomic_load,6676 4 => .i32_atomic_load,
6678 8 => .i64_atomic_load,6677 8 => .i64_atomic_load,
6679 else => |size| return func.fail("TODO: @atomicLoad for integers with abi size {d}", .{size}),6678 else => |size| return func.fail("TODO: @atomicLoad for types with abi size {d}", .{size}),
6680 };6679 };
6681 try func.emitWValue(ptr);6680 try func.emitWValue(ptr);
6682 try func.addAtomicMemArg(tag, .{6681 try func.addAtomicMemArg(tag, .{
...@@ -6690,3 +6689,177 @@ fn airAtomicLoad(func: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -6690,3 +6689,177 @@ fn airAtomicLoad(func: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6690 const result = try WValue.toLocal(.stack, func, ty);6689 const result = try WValue.toLocal(.stack, func, ty);
6691 return func.finishAir(inst, result, &.{atomic_load.ptr});6690 return func.finishAir(inst, result, &.{atomic_load.ptr});
6692}6691}
6692
6693fn airAtomicRmw(func: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6694 const pl_op = func.air.instructions.items(.data)[inst].pl_op;
6695 const extra = func.air.extraData(Air.AtomicRmw, pl_op.payload).data;
6696
6697 const ptr = try func.resolveInst(pl_op.operand);
6698 const operand = try func.resolveInst(extra.operand);
6699 const ty = func.air.typeOfIndex(inst);
6700 const op: std.builtin.AtomicRmwOp = extra.op();
6701
6702 if (func.useAtomicFeature()) {
6703 switch (op) {
6704 .Max,
6705 .Min,
6706 .Nand,
6707 => {
6708 const tmp = try func.load(ptr, ty, 0);
6709 const value = try tmp.toLocal(func, ty);
6710
6711 // create a loop to cmpxchg the new value
6712 try func.startBlock(.loop, wasm.block_empty);
6713
6714 try func.emitWValue(ptr);
6715 try func.emitWValue(value);
6716 if (op == .Nand) {
6717 const wasm_bits = toWasmBits(@intCast(u16, ty.bitSize(func.target))).?;
6718
6719 const and_res = try func.binOp(value, operand, ty, .@"and");
6720 if (wasm_bits == 32)
6721 try func.addImm32(-1)
6722 else if (wasm_bits == 64)
6723 try func.addImm64(@bitCast(u64, @as(i64, -1)))
6724 else
6725 return func.fail("TODO: `@atomicRmw` with operator `Nand` for types larger than 64 bits", .{});
6726 _ = try func.binOp(and_res, .stack, ty, .xor);
6727 } else {
6728 try func.emitWValue(value);
6729 try func.emitWValue(operand);
6730 _ = try func.cmp(value, operand, ty, if (op == .Max) .gt else .lt);
6731 try func.addTag(.select);
6732 }
6733 try func.addAtomicMemArg(
6734 switch (ty.abiSize(func.target)) {
6735 1 => .i32_atomic_rmw8_cmpxchg_u,
6736 2 => .i32_atomic_rmw16_cmpxchg_u,
6737 4 => .i32_atomic_rmw_cmpxchg,
6738 8 => .i64_atomic_rmw_cmpxchg,
6739 else => return func.fail("TODO: implement `@atomicRmw` with operation `{s}` for types larger than 64 bits", .{@tagName(op)}),
6740 },
6741 .{
6742 .offset = ptr.offset(),
6743 .alignment = ty.abiAlignment(func.target),
6744 },
6745 );
6746 const select_res = try func.allocLocal(ty);
6747 try func.addLabel(.local_tee, select_res.local.value);
6748 _ = try func.cmp(.stack, value, ty, .neq); // leave on stack so we can use it for br_if
6749
6750 try func.emitWValue(select_res);
6751 try func.addLabel(.local_set, value.local.value);
6752
6753 try func.addLabel(.br_if, 0);
6754 try func.endBlock();
6755 return func.finishAir(inst, value, &.{ pl_op.operand, extra.operand });
6756 },
6757
6758 // the other operations have their own instructions for Wasm.
6759 else => {
6760 try func.emitWValue(ptr);
6761 try func.emitWValue(operand);
6762 const tag: wasm.AtomicsOpcode = switch (ty.abiSize(func.target)) {
6763 1 => switch (op) {
6764 .Xchg => .i32_atomic_rmw8_xchg_u,
6765 .Add => .i32_atomic_rmw8_add_u,
6766 .Sub => .i32_atomic_rmw8_sub_u,
6767 .And => .i32_atomic_rmw8_and_u,
6768 .Or => .i32_atomic_rmw8_or_u,
6769 .Xor => .i32_atomic_rmw8_xor_u,
6770 else => unreachable,
6771 },
6772 2 => switch (op) {
6773 .Xchg => .i32_atomic_rmw16_xchg_u,
6774 .Add => .i32_atomic_rmw16_add_u,
6775 .Sub => .i32_atomic_rmw16_sub_u,
6776 .And => .i32_atomic_rmw16_and_u,
6777 .Or => .i32_atomic_rmw16_or_u,
6778 .Xor => .i32_atomic_rmw16_xor_u,
6779 else => unreachable,
6780 },
6781 4 => switch (op) {
6782 .Xchg => .i32_atomic_rmw_xchg,
6783 .Add => .i32_atomic_rmw_add,
6784 .Sub => .i32_atomic_rmw_sub,
6785 .And => .i32_atomic_rmw_and,
6786 .Or => .i32_atomic_rmw_or,
6787 .Xor => .i32_atomic_rmw_xor,
6788 else => unreachable,
6789 },
6790 8 => switch (op) {
6791 .Xchg => .i64_atomic_rmw_xchg,
6792 .Add => .i64_atomic_rmw_add,
6793 .Sub => .i64_atomic_rmw_sub,
6794 .And => .i64_atomic_rmw_and,
6795 .Or => .i64_atomic_rmw_or,
6796 .Xor => .i64_atomic_rmw_xor,
6797 else => unreachable,
6798 },
6799 else => |size| return func.fail("TODO: Implement `@atomicRmw` for types with abi size {d}", .{size}),
6800 };
6801 try func.addAtomicMemArg(tag, .{
6802 .offset = ptr.offset(),
6803 .alignment = ty.abiAlignment(func.target),
6804 });
6805 const result = try WValue.toLocal(.stack, func, ty);
6806 return func.finishAir(inst, result, &.{ pl_op.operand, extra.operand });
6807 },
6808 }
6809 } else {
6810 const loaded = try func.load(ptr, ty, 0);
6811 const result = try loaded.toLocal(func, ty);
6812
6813 switch (op) {
6814 .Xchg => {
6815 try func.store(ptr, operand, ty, 0);
6816 },
6817 .Add,
6818 .Sub,
6819 .And,
6820 .Or,
6821 .Xor,
6822 => {
6823 try func.emitWValue(ptr);
6824 _ = try func.binOp(result, operand, ty, switch (op) {
6825 .Add => .add,
6826 .Sub => .sub,
6827 .And => .@"and",
6828 .Or => .@"or",
6829 .Xor => .xor,
6830 else => unreachable,
6831 });
6832 if (ty.isInt() and (op == .Add or op == .Sub)) {
6833 _ = try func.wrapOperand(.stack, ty);
6834 }
6835 try func.store(.stack, .stack, ty, ptr.offset());
6836 },
6837 .Max,
6838 .Min,
6839 => {
6840 try func.emitWValue(ptr);
6841 try func.emitWValue(result);
6842 try func.emitWValue(operand);
6843 _ = try func.cmp(result, operand, ty, if (op == .Max) .gt else .lt);
6844 try func.addTag(.select);
6845 try func.store(.stack, .stack, ty, ptr.offset());
6846 },
6847 .Nand => {
6848 const wasm_bits = toWasmBits(@intCast(u16, ty.bitSize(func.target))).?;
6849
6850 try func.emitWValue(ptr);
6851 const and_res = try func.binOp(result, operand, ty, .@"and");
6852 if (wasm_bits == 32)
6853 try func.addImm32(-1)
6854 else if (wasm_bits == 64)
6855 try func.addImm64(@bitCast(u64, @as(i64, -1)))
6856 else
6857 return func.fail("TODO: `@atomicRmw` with operator `Nand` for types larger than 64 bits", .{});
6858 _ = try func.binOp(and_res, .stack, ty, .xor);
6859 try func.store(.stack, .stack, ty, ptr.offset());
6860 },
6861 }
6862
6863 return func.finishAir(inst, result, &.{ pl_op.operand, extra.operand });
6864 }
6865}
src/arch/wasm/Emit.zig+57-2
...@@ -527,8 +527,6 @@ fn emitAtomic(emit: *Emit, inst: Mir.Inst.Index) !void {...@@ -527,8 +527,6 @@ fn emitAtomic(emit: *Emit, inst: Mir.Inst.Index) !void {
527 try emit.code.append(std.wasm.opcode(.atomics_prefix));527 try emit.code.append(std.wasm.opcode(.atomics_prefix));
528 try leb128.writeULEB128(writer, opcode);528 try leb128.writeULEB128(writer, opcode);
529 switch (@intToEnum(std.wasm.AtomicsOpcode, opcode)) {529 switch (@intToEnum(std.wasm.AtomicsOpcode, opcode)) {
530 .i32_atomic_rmw_cmpxchg,
531 .i64_atomic_rmw_cmpxchg,
532 .i32_atomic_load,530 .i32_atomic_load,
533 .i64_atomic_load,531 .i64_atomic_load,
534 .i32_atomic_load8_u,532 .i32_atomic_load8_u,
...@@ -536,6 +534,63 @@ fn emitAtomic(emit: *Emit, inst: Mir.Inst.Index) !void {...@@ -536,6 +534,63 @@ fn emitAtomic(emit: *Emit, inst: Mir.Inst.Index) !void {
536 .i64_atomic_load8_u,534 .i64_atomic_load8_u,
537 .i64_atomic_load16_u,535 .i64_atomic_load16_u,
538 .i64_atomic_load32_u,536 .i64_atomic_load32_u,
537 .i32_atomic_store,
538 .i64_atomic_store,
539 .i32_atomic_store8,
540 .i32_atomic_store16,
541 .i64_atomic_store8,
542 .i64_atomic_store16,
543 .i64_atomic_store32,
544 .i32_atomic_rmw_add,
545 .i64_atomic_rmw_add,
546 .i32_atomic_rmw8_add_u,
547 .i32_atomic_rmw16_add_u,
548 .i64_atomic_rmw8_add_u,
549 .i64_atomic_rmw16_add_u,
550 .i64_atomic_rmw32_add_u,
551 .i32_atomic_rmw_sub,
552 .i64_atomic_rmw_sub,
553 .i32_atomic_rmw8_sub_u,
554 .i32_atomic_rmw16_sub_u,
555 .i64_atomic_rmw8_sub_u,
556 .i64_atomic_rmw16_sub_u,
557 .i64_atomic_rmw32_sub_u,
558 .i32_atomic_rmw_and,
559 .i64_atomic_rmw_and,
560 .i32_atomic_rmw8_and_u,
561 .i32_atomic_rmw16_and_u,
562 .i64_atomic_rmw8_and_u,
563 .i64_atomic_rmw16_and_u,
564 .i64_atomic_rmw32_and_u,
565 .i32_atomic_rmw_or,
566 .i64_atomic_rmw_or,
567 .i32_atomic_rmw8_or_u,
568 .i32_atomic_rmw16_or_u,
569 .i64_atomic_rmw8_or_u,
570 .i64_atomic_rmw16_or_u,
571 .i64_atomic_rmw32_or_u,
572 .i32_atomic_rmw_xor,
573 .i64_atomic_rmw_xor,
574 .i32_atomic_rmw8_xor_u,
575 .i32_atomic_rmw16_xor_u,
576 .i64_atomic_rmw8_xor_u,
577 .i64_atomic_rmw16_xor_u,
578 .i64_atomic_rmw32_xor_u,
579 .i32_atomic_rmw_xchg,
580 .i64_atomic_rmw_xchg,
581 .i32_atomic_rmw8_xchg_u,
582 .i32_atomic_rmw16_xchg_u,
583 .i64_atomic_rmw8_xchg_u,
584 .i64_atomic_rmw16_xchg_u,
585 .i64_atomic_rmw32_xchg_u,
586
587 .i32_atomic_rmw_cmpxchg,
588 .i64_atomic_rmw_cmpxchg,
589 .i32_atomic_rmw8_cmpxchg_u,
590 .i32_atomic_rmw16_cmpxchg_u,
591 .i64_atomic_rmw8_cmpxchg_u,
592 .i64_atomic_rmw16_cmpxchg_u,
593 .i64_atomic_rmw32_cmpxchg_u,
539 => {594 => {
540 const mem_arg = emit.mir.extraData(Mir.MemArg, extra_index + 1).data;595 const mem_arg = emit.mir.extraData(Mir.MemArg, extra_index + 1).data;
541 try encodeMemArg(mem_arg, writer);596 try encodeMemArg(mem_arg, writer);