| ... | ... | @@ -940,6 +940,20 @@ fn addMemArg(func: *CodeGen, tag: Mir.Inst.Tag, mem_arg: Mir.MemArg) error{OutOf |
| 940 | 940 | try func.addInst(.{ .tag = tag, .data = .{ .payload = extra_index } }); |
| 941 | 941 | } |
| 942 | 942 | |
| 943 | /// Inserts an instruction from the 'atomics' feature which accesses wasm's linear memory dependent on the |
| 944 | /// given `tag`. |
| 945 | fn addAtomicMemArg(func: *CodeGen, tag: wasm.AtomicsOpcode, mem_arg: Mir.MemArg) error{OutOfMemory}!void { |
| 946 | const extra_index = try func.addExtra(@as(struct { val: u32 }, .{ .val = wasm.atomicsOpcode(tag) })); |
| 947 | _ = try func.addExtra(mem_arg); |
| 948 | try func.addInst(.{ .tag = .atomics_prefix, .data = .{ .payload = extra_index } }); |
| 949 | } |
| 950 | |
| 951 | /// Helper function to emit atomic mir opcodes. |
| 952 | fn addAtomicTag(func: *CodeGen, tag: wasm.AtomicsOpcode) error{OutOfMemory}!void { |
| 953 | const extra_index = try func.addExtra(@as(struct { val: u32 }, .{ .val = wasm.atomicsOpcode(tag) })); |
| 954 | try func.addInst(.{ .tag = .atomics_prefix, .data = .{ .payload = extra_index } }); |
| 955 | } |
| 956 | |
| 943 | 957 | /// Appends entries to `mir_extra` based on the type of `extra`. |
| 944 | 958 | /// Returns the index into `mir_extra` |
| 945 | 959 | fn addExtra(func: *CodeGen, extra: anytype) error{OutOfMemory}!u32 { |
| ... | ... | @@ -1958,15 +1972,6 @@ fn genInst(func: *CodeGen, inst: Air.Inst.Index) InnerError!void { |
| 1958 | 1972 | .is_err_ptr, |
| 1959 | 1973 | .is_non_err_ptr, |
| 1960 | 1974 | |
| 1961 | | .cmpxchg_weak, |
| 1962 | | .cmpxchg_strong, |
| 1963 | | .fence, |
| 1964 | | .atomic_load, |
| 1965 | | .atomic_store_unordered, |
| 1966 | | .atomic_store_monotonic, |
| 1967 | | .atomic_store_release, |
| 1968 | | .atomic_store_seq_cst, |
| 1969 | | .atomic_rmw, |
| 1970 | 1975 | .err_return_trace, |
| 1971 | 1976 | .set_err_return_trace, |
| 1972 | 1977 | .save_err_return_trace_index, |
| ... | ... | @@ -1979,6 +1984,18 @@ fn genInst(func: *CodeGen, inst: Air.Inst.Index) InnerError!void { |
| 1979 | 1984 | .c_va_start, |
| 1980 | 1985 | => |tag| return func.fail("TODO: Implement wasm inst: {s}", .{@tagName(tag)}), |
| 1981 | 1986 | |
| 1987 | .atomic_load => func.airAtomicLoad(inst), |
| 1988 | .atomic_store_unordered, |
| 1989 | .atomic_store_monotonic, |
| 1990 | .atomic_store_release, |
| 1991 | .atomic_store_seq_cst, |
| 1992 | // in WebAssembly, all atomic instructions are sequentially ordered. |
| 1993 | => func.airAtomicStore(inst), |
| 1994 | .atomic_rmw => func.airAtomicRmw(inst), |
| 1995 | .cmpxchg_weak => func.airCmpxchg(inst), |
| 1996 | .cmpxchg_strong => func.airCmpxchg(inst), |
| 1997 | .fence => func.airFence(inst), |
| 1998 | |
| 1982 | 1999 | .add_optimized, |
| 1983 | 2000 | .addwrap_optimized, |
| 1984 | 2001 | .sub_optimized, |
| ... | ... | @@ -6636,3 +6653,319 @@ fn airErrorSetHasValue(func: *CodeGen, inst: Air.Inst.Index) InnerError!void { |
| 6636 | 6653 | |
| 6637 | 6654 | return func.finishAir(inst, result, &.{ty_op.operand}); |
| 6638 | 6655 | } |
| 6656 | |
| 6657 | inline fn useAtomicFeature(func: *const CodeGen) bool { |
| 6658 | return std.Target.wasm.featureSetHas(func.target.cpu.features, .atomics); |
| 6659 | } |
| 6660 | |
| 6661 | fn airCmpxchg(func: *CodeGen, inst: Air.Inst.Index) InnerError!void { |
| 6662 | const ty_pl = func.air.instructions.items(.data)[inst].ty_pl; |
| 6663 | const extra = func.air.extraData(Air.Cmpxchg, ty_pl.payload).data; |
| 6664 | |
| 6665 | const ptr_ty = func.air.typeOf(extra.ptr); |
| 6666 | const ty = ptr_ty.childType(); |
| 6667 | const result_ty = func.air.typeOfIndex(inst); |
| 6668 | |
| 6669 | const ptr_operand = try func.resolveInst(extra.ptr); |
| 6670 | const expected_val = try func.resolveInst(extra.expected_value); |
| 6671 | const new_val = try func.resolveInst(extra.new_value); |
| 6672 | |
| 6673 | const cmp_result = try func.allocLocal(Type.bool); |
| 6674 | |
| 6675 | const ptr_val = if (func.useAtomicFeature()) val: { |
| 6676 | const val_local = try func.allocLocal(ty); |
| 6677 | try func.emitWValue(ptr_operand); |
| 6678 | try func.lowerToStack(expected_val); |
| 6679 | try func.lowerToStack(new_val); |
| 6680 | try func.addAtomicMemArg(switch (ty.abiSize(func.target)) { |
| 6681 | 1 => .i32_atomic_rmw8_cmpxchg_u, |
| 6682 | 2 => .i32_atomic_rmw16_cmpxchg_u, |
| 6683 | 4 => .i32_atomic_rmw_cmpxchg, |
| 6684 | 8 => .i32_atomic_rmw_cmpxchg, |
| 6685 | else => |size| return func.fail("TODO: implement `@cmpxchg` for types with abi size '{d}'", .{size}), |
| 6686 | }, .{ |
| 6687 | .offset = ptr_operand.offset(), |
| 6688 | .alignment = ty.abiAlignment(func.target), |
| 6689 | }); |
| 6690 | try func.addLabel(.local_tee, val_local.local.value); |
| 6691 | _ = try func.cmp(.stack, expected_val, ty, .eq); |
| 6692 | try func.addLabel(.local_set, cmp_result.local.value); |
| 6693 | break :val val_local; |
| 6694 | } else val: { |
| 6695 | if (ty.abiSize(func.target) > 8) { |
| 6696 | return func.fail("TODO: Implement `@cmpxchg` for types larger than abi size of 8 bytes", .{}); |
| 6697 | } |
| 6698 | const ptr_val = try WValue.toLocal(try func.load(ptr_operand, ty, 0), func, ty); |
| 6699 | |
| 6700 | try func.lowerToStack(ptr_operand); |
| 6701 | try func.lowerToStack(new_val); |
| 6702 | try func.emitWValue(ptr_val); |
| 6703 | _ = try func.cmp(ptr_val, expected_val, ty, .eq); |
| 6704 | try func.addLabel(.local_tee, cmp_result.local.value); |
| 6705 | try func.addTag(.select); |
| 6706 | try func.store(.stack, .stack, ty, 0); |
| 6707 | |
| 6708 | break :val ptr_val; |
| 6709 | }; |
| 6710 | |
| 6711 | const result_ptr = if (isByRef(result_ty, func.target)) val: { |
| 6712 | try func.emitWValue(cmp_result); |
| 6713 | try func.addImm32(-1); |
| 6714 | try func.addTag(.i32_xor); |
| 6715 | try func.addImm32(1); |
| 6716 | try func.addTag(.i32_and); |
| 6717 | const and_result = try WValue.toLocal(.stack, func, Type.bool); |
| 6718 | const result_ptr = try func.allocStack(result_ty); |
| 6719 | try func.store(result_ptr, and_result, Type.bool, @intCast(u32, ty.abiSize(func.target))); |
| 6720 | try func.store(result_ptr, ptr_val, ty, 0); |
| 6721 | break :val result_ptr; |
| 6722 | } else val: { |
| 6723 | try func.addImm32(0); |
| 6724 | try func.emitWValue(ptr_val); |
| 6725 | try func.emitWValue(cmp_result); |
| 6726 | try func.addTag(.select); |
| 6727 | break :val try WValue.toLocal(.stack, func, result_ty); |
| 6728 | }; |
| 6729 | |
| 6730 | return func.finishAir(inst, result_ptr, &.{ extra.ptr, extra.new_value, extra.expected_value }); |
| 6731 | } |
| 6732 | |
| 6733 | fn airAtomicLoad(func: *CodeGen, inst: Air.Inst.Index) InnerError!void { |
| 6734 | const atomic_load = func.air.instructions.items(.data)[inst].atomic_load; |
| 6735 | const ptr = try func.resolveInst(atomic_load.ptr); |
| 6736 | const ty = func.air.typeOfIndex(inst); |
| 6737 | |
| 6738 | if (func.useAtomicFeature()) { |
| 6739 | const tag: wasm.AtomicsOpcode = switch (ty.abiSize(func.target)) { |
| 6740 | 1 => .i32_atomic_load8_u, |
| 6741 | 2 => .i32_atomic_load16_u, |
| 6742 | 4 => .i32_atomic_load, |
| 6743 | 8 => .i64_atomic_load, |
| 6744 | else => |size| return func.fail("TODO: @atomicLoad for types with abi size {d}", .{size}), |
| 6745 | }; |
| 6746 | try func.emitWValue(ptr); |
| 6747 | try func.addAtomicMemArg(tag, .{ |
| 6748 | .offset = ptr.offset(), |
| 6749 | .alignment = ty.abiAlignment(func.target), |
| 6750 | }); |
| 6751 | } else { |
| 6752 | _ = try func.load(ptr, ty, 0); |
| 6753 | } |
| 6754 | |
| 6755 | const result = try WValue.toLocal(.stack, func, ty); |
| 6756 | return func.finishAir(inst, result, &.{atomic_load.ptr}); |
| 6757 | } |
| 6758 | |
| 6759 | fn airAtomicRmw(func: *CodeGen, inst: Air.Inst.Index) InnerError!void { |
| 6760 | const pl_op = func.air.instructions.items(.data)[inst].pl_op; |
| 6761 | const extra = func.air.extraData(Air.AtomicRmw, pl_op.payload).data; |
| 6762 | |
| 6763 | const ptr = try func.resolveInst(pl_op.operand); |
| 6764 | const operand = try func.resolveInst(extra.operand); |
| 6765 | const ty = func.air.typeOfIndex(inst); |
| 6766 | const op: std.builtin.AtomicRmwOp = extra.op(); |
| 6767 | |
| 6768 | if (func.useAtomicFeature()) { |
| 6769 | switch (op) { |
| 6770 | .Max, |
| 6771 | .Min, |
| 6772 | .Nand, |
| 6773 | => { |
| 6774 | const tmp = try func.load(ptr, ty, 0); |
| 6775 | const value = try tmp.toLocal(func, ty); |
| 6776 | |
| 6777 | // create a loop to cmpxchg the new value |
| 6778 | try func.startBlock(.loop, wasm.block_empty); |
| 6779 | |
| 6780 | try func.emitWValue(ptr); |
| 6781 | try func.emitWValue(value); |
| 6782 | if (op == .Nand) { |
| 6783 | const wasm_bits = toWasmBits(@intCast(u16, ty.bitSize(func.target))).?; |
| 6784 | |
| 6785 | const and_res = try func.binOp(value, operand, ty, .@"and"); |
| 6786 | if (wasm_bits == 32) |
| 6787 | try func.addImm32(-1) |
| 6788 | else if (wasm_bits == 64) |
| 6789 | try func.addImm64(@bitCast(u64, @as(i64, -1))) |
| 6790 | else |
| 6791 | return func.fail("TODO: `@atomicRmw` with operator `Nand` for types larger than 64 bits", .{}); |
| 6792 | _ = try func.binOp(and_res, .stack, ty, .xor); |
| 6793 | } else { |
| 6794 | try func.emitWValue(value); |
| 6795 | try func.emitWValue(operand); |
| 6796 | _ = try func.cmp(value, operand, ty, if (op == .Max) .gt else .lt); |
| 6797 | try func.addTag(.select); |
| 6798 | } |
| 6799 | try func.addAtomicMemArg( |
| 6800 | switch (ty.abiSize(func.target)) { |
| 6801 | 1 => .i32_atomic_rmw8_cmpxchg_u, |
| 6802 | 2 => .i32_atomic_rmw16_cmpxchg_u, |
| 6803 | 4 => .i32_atomic_rmw_cmpxchg, |
| 6804 | 8 => .i64_atomic_rmw_cmpxchg, |
| 6805 | else => return func.fail("TODO: implement `@atomicRmw` with operation `{s}` for types larger than 64 bits", .{@tagName(op)}), |
| 6806 | }, |
| 6807 | .{ |
| 6808 | .offset = ptr.offset(), |
| 6809 | .alignment = ty.abiAlignment(func.target), |
| 6810 | }, |
| 6811 | ); |
| 6812 | const select_res = try func.allocLocal(ty); |
| 6813 | try func.addLabel(.local_tee, select_res.local.value); |
| 6814 | _ = try func.cmp(.stack, value, ty, .neq); // leave on stack so we can use it for br_if |
| 6815 | |
| 6816 | try func.emitWValue(select_res); |
| 6817 | try func.addLabel(.local_set, value.local.value); |
| 6818 | |
| 6819 | try func.addLabel(.br_if, 0); |
| 6820 | try func.endBlock(); |
| 6821 | return func.finishAir(inst, value, &.{ pl_op.operand, extra.operand }); |
| 6822 | }, |
| 6823 | |
| 6824 | // the other operations have their own instructions for Wasm. |
| 6825 | else => { |
| 6826 | try func.emitWValue(ptr); |
| 6827 | try func.emitWValue(operand); |
| 6828 | const tag: wasm.AtomicsOpcode = switch (ty.abiSize(func.target)) { |
| 6829 | 1 => switch (op) { |
| 6830 | .Xchg => .i32_atomic_rmw8_xchg_u, |
| 6831 | .Add => .i32_atomic_rmw8_add_u, |
| 6832 | .Sub => .i32_atomic_rmw8_sub_u, |
| 6833 | .And => .i32_atomic_rmw8_and_u, |
| 6834 | .Or => .i32_atomic_rmw8_or_u, |
| 6835 | .Xor => .i32_atomic_rmw8_xor_u, |
| 6836 | else => unreachable, |
| 6837 | }, |
| 6838 | 2 => switch (op) { |
| 6839 | .Xchg => .i32_atomic_rmw16_xchg_u, |
| 6840 | .Add => .i32_atomic_rmw16_add_u, |
| 6841 | .Sub => .i32_atomic_rmw16_sub_u, |
| 6842 | .And => .i32_atomic_rmw16_and_u, |
| 6843 | .Or => .i32_atomic_rmw16_or_u, |
| 6844 | .Xor => .i32_atomic_rmw16_xor_u, |
| 6845 | else => unreachable, |
| 6846 | }, |
| 6847 | 4 => switch (op) { |
| 6848 | .Xchg => .i32_atomic_rmw_xchg, |
| 6849 | .Add => .i32_atomic_rmw_add, |
| 6850 | .Sub => .i32_atomic_rmw_sub, |
| 6851 | .And => .i32_atomic_rmw_and, |
| 6852 | .Or => .i32_atomic_rmw_or, |
| 6853 | .Xor => .i32_atomic_rmw_xor, |
| 6854 | else => unreachable, |
| 6855 | }, |
| 6856 | 8 => switch (op) { |
| 6857 | .Xchg => .i64_atomic_rmw_xchg, |
| 6858 | .Add => .i64_atomic_rmw_add, |
| 6859 | .Sub => .i64_atomic_rmw_sub, |
| 6860 | .And => .i64_atomic_rmw_and, |
| 6861 | .Or => .i64_atomic_rmw_or, |
| 6862 | .Xor => .i64_atomic_rmw_xor, |
| 6863 | else => unreachable, |
| 6864 | }, |
| 6865 | else => |size| return func.fail("TODO: Implement `@atomicRmw` for types with abi size {d}", .{size}), |
| 6866 | }; |
| 6867 | try func.addAtomicMemArg(tag, .{ |
| 6868 | .offset = ptr.offset(), |
| 6869 | .alignment = ty.abiAlignment(func.target), |
| 6870 | }); |
| 6871 | const result = try WValue.toLocal(.stack, func, ty); |
| 6872 | return func.finishAir(inst, result, &.{ pl_op.operand, extra.operand }); |
| 6873 | }, |
| 6874 | } |
| 6875 | } else { |
| 6876 | const loaded = try func.load(ptr, ty, 0); |
| 6877 | const result = try loaded.toLocal(func, ty); |
| 6878 | |
| 6879 | switch (op) { |
| 6880 | .Xchg => { |
| 6881 | try func.store(ptr, operand, ty, 0); |
| 6882 | }, |
| 6883 | .Add, |
| 6884 | .Sub, |
| 6885 | .And, |
| 6886 | .Or, |
| 6887 | .Xor, |
| 6888 | => { |
| 6889 | try func.emitWValue(ptr); |
| 6890 | _ = try func.binOp(result, operand, ty, switch (op) { |
| 6891 | .Add => .add, |
| 6892 | .Sub => .sub, |
| 6893 | .And => .@"and", |
| 6894 | .Or => .@"or", |
| 6895 | .Xor => .xor, |
| 6896 | else => unreachable, |
| 6897 | }); |
| 6898 | if (ty.isInt() and (op == .Add or op == .Sub)) { |
| 6899 | _ = try func.wrapOperand(.stack, ty); |
| 6900 | } |
| 6901 | try func.store(.stack, .stack, ty, ptr.offset()); |
| 6902 | }, |
| 6903 | .Max, |
| 6904 | .Min, |
| 6905 | => { |
| 6906 | try func.emitWValue(ptr); |
| 6907 | try func.emitWValue(result); |
| 6908 | try func.emitWValue(operand); |
| 6909 | _ = try func.cmp(result, operand, ty, if (op == .Max) .gt else .lt); |
| 6910 | try func.addTag(.select); |
| 6911 | try func.store(.stack, .stack, ty, ptr.offset()); |
| 6912 | }, |
| 6913 | .Nand => { |
| 6914 | const wasm_bits = toWasmBits(@intCast(u16, ty.bitSize(func.target))).?; |
| 6915 | |
| 6916 | try func.emitWValue(ptr); |
| 6917 | const and_res = try func.binOp(result, operand, ty, .@"and"); |
| 6918 | if (wasm_bits == 32) |
| 6919 | try func.addImm32(-1) |
| 6920 | else if (wasm_bits == 64) |
| 6921 | try func.addImm64(@bitCast(u64, @as(i64, -1))) |
| 6922 | else |
| 6923 | return func.fail("TODO: `@atomicRmw` with operator `Nand` for types larger than 64 bits", .{}); |
| 6924 | _ = try func.binOp(and_res, .stack, ty, .xor); |
| 6925 | try func.store(.stack, .stack, ty, ptr.offset()); |
| 6926 | }, |
| 6927 | } |
| 6928 | |
| 6929 | return func.finishAir(inst, result, &.{ pl_op.operand, extra.operand }); |
| 6930 | } |
| 6931 | } |
| 6932 | |
| 6933 | fn airFence(func: *CodeGen, inst: Air.Inst.Index) InnerError!void { |
| 6934 | // Only when the atomic feature is enabled, and we're not building |
| 6935 | // for a single-threaded build, can we emit the `fence` instruction. |
| 6936 | // In all other cases, we emit no instructions for a fence. |
| 6937 | if (func.useAtomicFeature() and !func.bin_file.base.options.single_threaded) { |
| 6938 | try func.addAtomicTag(.atomic_fence); |
| 6939 | } |
| 6940 | |
| 6941 | return func.finishAir(inst, .none, &.{}); |
| 6942 | } |
| 6943 | |
| 6944 | fn airAtomicStore(func: *CodeGen, inst: Air.Inst.Index) InnerError!void { |
| 6945 | const bin_op = func.air.instructions.items(.data)[inst].bin_op; |
| 6946 | |
| 6947 | const ptr = try func.resolveInst(bin_op.lhs); |
| 6948 | const operand = try func.resolveInst(bin_op.rhs); |
| 6949 | const ptr_ty = func.air.typeOf(bin_op.lhs); |
| 6950 | const ty = ptr_ty.childType(); |
| 6951 | |
| 6952 | if (func.useAtomicFeature()) { |
| 6953 | const tag: wasm.AtomicsOpcode = switch (ty.abiSize(func.target)) { |
| 6954 | 1 => .i32_atomic_store8, |
| 6955 | 2 => .i32_atomic_store16, |
| 6956 | 4 => .i32_atomic_store, |
| 6957 | 8 => .i64_atomic_store, |
| 6958 | else => |size| return func.fail("TODO: @atomicLoad for types with abi size {d}", .{size}), |
| 6959 | }; |
| 6960 | try func.emitWValue(ptr); |
| 6961 | try func.lowerToStack(operand); |
| 6962 | try func.addAtomicMemArg(tag, .{ |
| 6963 | .offset = ptr.offset(), |
| 6964 | .alignment = ty.abiAlignment(func.target), |
| 6965 | }); |
| 6966 | } else { |
| 6967 | try func.store(ptr, operand, ty, 0); |
| 6968 | } |
| 6969 | |
| 6970 | return func.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs }); |
| 6971 | } |