authorgravatar for paul.verigo@gmail.comPavel Verigo <paul.verigo@gmail.com> 2026-04-07 00:08:17+02:00
committergravatar for paul.verigo@gmail.comPavel Verigo <paul.verigo@gmail.com> 2026-04-09 00:14:08+02:00
log0ebd270d907a3fb7e2e600891fe2f455b7b2d4ad
tree8226d1e1f7b31939be1f47b8b1acd3b897de72cb
parent58944586beca0a771ed03a9e6b1f281d9e23cc57

stage2-wasm: bigint mulo

5 files changed, 193 insertions(+), 66 deletions(-)

lib/compiler_rt/limb64.zig+138
...@@ -839,3 +839,141 @@ test __byteswap_limb64 {...@@ -839,3 +839,141 @@ test __byteswap_limb64 {
839 try test__byteswap_limb64(i128, 1 << 56, 1 << 64);839 try test__byteswap_limb64(i128, 1 << 56, 1 << 64);
840 try test__byteswap_limb64(i248, minInt(i248), 128);840 try test__byteswap_limb64(i248, minInt(i248), 128);
841}841}
842
843comptime {
844 symbol(&__mulo_limb64, "__mulo_limb64");
845}
846
847inline fn add3(x: *[3]u64, start: usize, v0: u64) void {
848 var i = start;
849 var v = v0;
850 while (i < 3) : (i += 1) {
851 const s = @addWithOverflow(x[i], v);
852 x[i] = s[0];
853 if (s[1] == 0) break;
854 v = 1;
855 }
856}
857
858fn mulwide(a: u64, b: u64) [2]u64 {
859 const muldXi = @import("mulXi3.zig").muldXi;
860 return @bitCast(muldXi(u64, a, b));
861}
862
863fn __mulo_limb64(out_ptr: [*]u64, a_ptr: [*]const u64, b_ptr: [*]const u64, is_signed: bool, bits: u16) callconv(.c) bool {
864 const limb_cnt = limbCount(bits);
865
866 const out = out_ptr[0..limb_cnt];
867 const a = a_ptr[0..limb_cnt];
868 const b = b_ptr[0..limb_cnt];
869
870 @memset(out, 0);
871
872 const all_ones = ~@as(u64, 0);
873 const a_neg = is_signed and ((limbGet(a, limb_cnt - 1) >> 63) != 0);
874 const b_neg = is_signed and ((limbGet(b, limb_cnt - 1) >> 63) != 0);
875
876 var carry: [3]u64 = @splat(0);
877 var hi_zero = true;
878 var hi_ones = true;
879 var hi_borrow: u1 = 0;
880 var raw_last: u64 = 0;
881
882 var k: usize = 0;
883 while (k < 2 * limb_cnt) : (k += 1) {
884 var acc = carry;
885
886 var i: usize = if (k < limb_cnt) 0 else k - (limb_cnt - 1);
887 while (i < limb_cnt and i <= k) : (i += 1) {
888 const j = k - i;
889 if (j >= limb_cnt) continue;
890
891 const p = mulwide(limbGet(a, i), limbGet(b, j));
892 add3(&acc, 0, p[0]);
893 add3(&acc, 1, p[1]);
894 }
895
896 var limb = acc[0];
897 if (k < limb_cnt) {
898 limbSet(out, k, limb);
899 if (k == limb_cnt - 1) raw_last = limb;
900 } else {
901 if (is_signed) {
902 const h = k - limb_cnt;
903
904 const s0 = @subWithOverflow(limb, if (a_neg) limbGet(b, h) else 0);
905 const s1 = @subWithOverflow(s0[0], if (b_neg) limbGet(a, h) else 0);
906 const s2 = @subWithOverflow(s1[0], hi_borrow);
907
908 limb = s2[0];
909 hi_borrow = @intFromBool(s0[1] != 0 or s1[1] != 0 or s2[1] != 0);
910 }
911
912 hi_zero = hi_zero and limb == 0;
913 hi_ones = hi_ones and limb == all_ones;
914 }
915
916 carry = .{ acc[1], acc[2], 0 };
917 }
918
919 const last = if (bits % 64 == 0) raw_last else limbWrap(raw_last, is_signed, bits);
920 if (bits % 64 != 0) {
921 limbSet(out, limb_cnt - 1, last);
922 }
923
924 if (!is_signed) {
925 return !hi_zero or raw_last != last;
926 }
927
928 const sign_extend: u64 = if ((last >> 63) == 1) all_ones else 0;
929 return (raw_last != last) or if (sign_extend == 0) !hi_zero else !hi_ones;
930}
931
932fn test__mulo_limb64(comptime T: type, a: T, b: T, expected: struct { T, bool }) !void {
933 const int_info = @typeInfo(T).int;
934 const is_signed = int_info.signedness == .signed;
935
936 var a_limbs = asLimbs(a);
937 var b_limbs = asLimbs(b);
938 var out: Limbs(T) = undefined;
939 const overflow = __mulo_limb64(&out, &a_limbs, &b_limbs, is_signed, int_info.bits);
940
941 const expected_limbs = asLimbs(expected[0]);
942 try testing.expectEqual(expected_limbs, out);
943 try testing.expectEqual(expected[1], overflow);
944}
945
946test __mulo_limb64 {
947 try test__mulo_limb64(u64, 3, 5, .{ 15, false });
948 try test__mulo_limb64(u64, maxInt(u64), 2, .{ maxInt(u64) - 1, true });
949 try test__mulo_limb64(u65, 1 << 32, 1 << 32, .{ 1 << 64, false });
950 try test__mulo_limb64(u65, 1 << 64, 2, .{ 0, true });
951 try test__mulo_limb64(u128, 1 << 80, 1 << 40, .{ 1 << 120, false });
952 try test__mulo_limb64(u128, 1 << 100, 1 << 40, .{ 0, true });
953 try test__mulo_limb64(u255, 7, 9, .{ 63, false });
954 try test__mulo_limb64(u255, maxInt(u255), 2, .{ maxInt(u255) - 1, true });
955
956 try test__mulo_limb64(i64, -3, 2, .{ -6, false });
957 try test__mulo_limb64(i64, maxInt(i64), 2, .{ -2, true });
958 try test__mulo_limb64(i65, 1 << 63, 2, .{ minInt(i65), true });
959 try test__mulo_limb64(i65, -1 << 32, 1 << 16, .{ -1 << 48, false });
960 try test__mulo_limb64(i128, 1 << 100, 1 << 27, .{ minInt(i128), true });
961 try test__mulo_limb64(i128, -1 << 80, 1 << 40, .{ -1 << 120, false });
962 try test__mulo_limb64(i255, -3, 2, .{ -6, false });
963 try test__mulo_limb64(i255, maxInt(i255), 2, .{ -2, true });
964
965 try test__mulo_limb64(u200, 0, maxInt(u200), .{ 0, false });
966 try test__mulo_limb64(u200, 1, maxInt(u200), .{ maxInt(u200), false });
967 try test__mulo_limb64(u200, 1 << 100, 1 << 99, .{ 1 << 199, false });
968 try test__mulo_limb64(u200, 1 << 100, 1 << 100, .{ 0, true });
969 try test__mulo_limb64(u200, maxInt(u200), maxInt(u200), .{ 1, true });
970
971 try test__mulo_limb64(i200, 0, -1, .{ 0, false });
972 try test__mulo_limb64(i200, -1, -1, .{ 1, false });
973 try test__mulo_limb64(i200, -1, minInt(i200), .{ minInt(i200), true });
974 try test__mulo_limb64(i200, maxInt(i200), 2, .{ -2, true });
975 try test__mulo_limb64(i200, 1 << 100, 1 << 98, .{ 1 << 198, false });
976 try test__mulo_limb64(i200, 1 << 100, 1 << 99, .{ minInt(i200), true });
977 try test__mulo_limb64(i200, maxInt(i200), maxInt(i200), .{ 1, true });
978 try test__mulo_limb64(i200, minInt(i200), minInt(i200), .{ 0, true });
979}
lib/compiler_rt/mulXi3.zig+1-1
...@@ -63,7 +63,7 @@ fn DoubleInt(comptime T: type) type {...@@ -63,7 +63,7 @@ fn DoubleInt(comptime T: type) type {
63 };63 };
64}64}
6565
66fn muldXi(comptime T: type, a: T, b: T) DoubleInt(T) {66pub fn muldXi(comptime T: type, a: T, b: T) DoubleInt(T) {
67 const DT = DoubleInt(T);67 const DT = DoubleInt(T);
68 const word_t = compiler_rt.HalveInt(DT, false);68 const word_t = compiler_rt.HalveInt(DT, false);
69 const bits_in_word_2 = @sizeOf(T) * 8 / 2;69 const bits_in_word_2 = @sizeOf(T) * 8 / 2;
src/codegen/wasm/CodeGen.zig+25-64
...@@ -2480,7 +2480,19 @@ fn intMul(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue...@@ -2480,7 +2480,19 @@ fn intMul(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue
2480 return .stack;2480 return .stack;
2481 },2481 },
2482 65...128 => return cg.callIntrinsic(.__multi3, &.{ .i128_type, .i128_type }, Type.i128, &.{ lhs, rhs }),2482 65...128 => return cg.callIntrinsic(.__multi3, &.{ .i128_type, .i128_type }, Type.i128, &.{ lhs, rhs }),
2483 else => return cg.fail("TODO: Support intMul for integer bitsize: {d}", .{ty.bits}),2483 else => {
2484 const result = try cg.allocInt(ty);
2485
2486 try cg.lowerToStack(result);
2487 try cg.lowerToStack(lhs);
2488 try cg.lowerToStack(rhs);
2489 try cg.addImm32(@intFromBool(ty.is_signed));
2490 try cg.addImm32(ty.bits);
2491 try cg.addCallIntrinsic(.__mulo_limb64);
2492 try cg.addTag(.drop);
2493
2494 return result;
2495 },
2484 }2496 }
2485}2497}
24862498
...@@ -3680,68 +3692,6 @@ fn intMulOverflow(cg: *CodeGen, int_ty: IntType, lhs: WValue, rhs: WValue) Inner...@@ -3680,68 +3692,6 @@ fn intMulOverflow(cg: *CodeGen, int_ty: IntType, lhs: WValue, rhs: WValue) Inner
3680 _ = try cg.intCmp(new_ty, .neq, res_upcast, bin_op);3692 _ = try cg.intCmp(new_ty, .neq, res_upcast, bin_op);
3681 try cg.addLocal(.local_set, overflow_bit.local.value);3693 try cg.addLocal(.local_set, overflow_bit.local.value);
3682 break :blk res_tmp;3694 break :blk res_tmp;
3683 } else if (int_ty.bits == 128 and !int_ty.is_signed) blk: {
3684 var lhs_lsb = try (try cg.load(lhs, Type.u64, 0)).toLocal(cg, Type.u64);
3685 defer lhs_lsb.free(cg);
3686 var lhs_msb = try (try cg.load(lhs, Type.u64, 8)).toLocal(cg, Type.u64);
3687 defer lhs_msb.free(cg);
3688 var rhs_lsb = try (try cg.load(rhs, Type.u64, 0)).toLocal(cg, Type.u64);
3689 defer rhs_lsb.free(cg);
3690 var rhs_msb = try (try cg.load(rhs, Type.u64, 8)).toLocal(cg, Type.u64);
3691 defer rhs_msb.free(cg);
3692
3693 const zero: WValue = .{ .imm64 = 0 };
3694
3695 const cross_1 = try cg.callIntrinsic(
3696 .__multi3,
3697 &[_]InternPool.Index{.i64_type} ** 4,
3698 Type.i128,
3699 &.{ lhs_msb, zero, rhs_lsb, zero },
3700 );
3701 const cross_2 = try cg.callIntrinsic(
3702 .__multi3,
3703 &[_]InternPool.Index{.i64_type} ** 4,
3704 Type.i128,
3705 &.{ rhs_msb, zero, lhs_lsb, zero },
3706 );
3707 const mul_lsb = try cg.callIntrinsic(
3708 .__multi3,
3709 &[_]InternPool.Index{.i64_type} ** 4,
3710 Type.i128,
3711 &.{ rhs_lsb, zero, lhs_lsb, zero },
3712 );
3713
3714 const rhs_msb_not_zero = try cg.intCmp(.u64, .neq, rhs_msb, zero);
3715 const lhs_msb_not_zero = try cg.intCmp(.u64, .neq, lhs_msb, zero);
3716 const both_msb_not_zero = try cg.intAnd(.u32, rhs_msb_not_zero, lhs_msb_not_zero);
3717
3718 const cross_1_msb = try cg.load(cross_1, .u64, 8);
3719 const cross_1_msb_not_zero = try cg.intCmp(.u64, .neq, cross_1_msb, zero);
3720 const cond_1 = try cg.intOr(.u32, both_msb_not_zero, cross_1_msb_not_zero);
3721
3722 const cross_2_msb = try cg.load(cross_2, Type.u64, 8);
3723 const cross_2_msb_not_zero = try cg.intCmp(.u64, .neq, cross_2_msb, zero);
3724 const cond_2 = try cg.intOr(.u32, cond_1, cross_2_msb_not_zero);
3725
3726 const cross_1_lsb = try cg.load(cross_1, Type.u64, 0);
3727 const cross_2_lsb = try cg.load(cross_2, Type.u64, 0);
3728 const cross_add = try cg.intAdd(.u64, cross_1_lsb, cross_2_lsb);
3729
3730 var mul_lsb_msb = try (try cg.load(mul_lsb, Type.u64, 8)).toLocal(cg, Type.u64);
3731 defer mul_lsb_msb.free(cg);
3732 var all_add = try (try cg.intAdd(.u64, cross_add, mul_lsb_msb)).toLocal(cg, Type.u64);
3733 defer all_add.free(cg);
3734 const add_overflow = try cg.intCmp(.u64, .lt, all_add, mul_lsb_msb);
3735
3736 _ = try cg.intOr(.u32, cond_2, add_overflow);
3737 try cg.addLocal(.local_set, overflow_bit.local.value);
3738
3739 const tmp_result = try cg.allocStack(Type.u128);
3740 try cg.emitWValue(tmp_result);
3741 const mul_lsb_lsb = try cg.load(mul_lsb, Type.u64, 0);
3742 try cg.store(.stack, mul_lsb_lsb, Type.u64, tmp_result.offset());
3743 try cg.store(tmp_result, all_add, Type.u64, 8);
3744 break :blk tmp_result;
3745 } else if (int_ty.bits == 128 and int_ty.is_signed) blk: {3695 } else if (int_ty.bits == 128 and int_ty.is_signed) blk: {
3746 const overflow_ret = try cg.allocStack(Type.i32);3696 const overflow_ret = try cg.allocStack(Type.i32);
3747 const res = try cg.callIntrinsic(3697 const res = try cg.callIntrinsic(
...@@ -3753,7 +3703,18 @@ fn intMulOverflow(cg: *CodeGen, int_ty: IntType, lhs: WValue, rhs: WValue) Inner...@@ -3753,7 +3703,18 @@ fn intMulOverflow(cg: *CodeGen, int_ty: IntType, lhs: WValue, rhs: WValue) Inner
3753 _ = try cg.load(overflow_ret, Type.i32, 0);3703 _ = try cg.load(overflow_ret, Type.i32, 0);
3754 try cg.addLocal(.local_set, overflow_bit.local.value);3704 try cg.addLocal(.local_set, overflow_bit.local.value);
3755 break :blk res;3705 break :blk res;
3756 } else return cg.fail("TODO: intMulOverflow for bitsize {d}", .{int_ty.bits});3706 } else {
3707 const result = try cg.allocInt(int_ty);
3708
3709 try cg.lowerToStack(result);
3710 try cg.lowerToStack(lhs);
3711 try cg.lowerToStack(rhs);
3712 try cg.addImm32(@intFromBool(int_ty.is_signed));
3713 try cg.addImm32(int_ty.bits);
3714 try cg.addCallIntrinsic(.__mulo_limb64);
3715
3716 return .{ .result = result, .ov = .stack };
3717 };
37573718
3758 return .{ .result = result_val, .ov = .{ .local = overflow_bit.local } };3719 return .{ .result = result_val, .ov = .{ .local = overflow_bit.local } };
3759}3720}
src/codegen/wasm/Mir.zig+1
...@@ -1018,4 +1018,5 @@ pub const Intrinsic = enum(u32) {...@@ -1018,4 +1018,5 @@ pub const Intrinsic = enum(u32) {
1018 __popcount_limb64,1018 __popcount_limb64,
1019 __bitreverse_limb64,1019 __bitreverse_limb64,
1020 __byteswap_limb64,1020 __byteswap_limb64,
1021 __mulo_limb64,
1021};1022};
test/behavior/math.zig+28-1
...@@ -1100,10 +1100,37 @@ test "@mulWithOverflow bitsize 128 bits" {...@@ -1100,10 +1100,37 @@ test "@mulWithOverflow bitsize 128 bits" {
1100 try testMulWithOverflow(i128, -1 << 63, -1 << 64, -1 << 127, 1);1100 try testMulWithOverflow(i128, -1 << 63, -1 << 64, -1 << 127, 1);
1101}1101}
11021102
1103test "@mulWithOverflow > 128 bits" {
1104 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
1105
1106 try testMulWithOverflow(u140, 0, maxInt(u140), 0, 0);
1107 try testMulWithOverflow(u140, 1, maxInt(u140), maxInt(u140), 0);
1108 try testMulWithOverflow(u140, 1 << 70, 1 << 69, 1 << 139, 0);
1109 try testMulWithOverflow(u140, 1 << 70, 1 << 70, 0, 1);
1110
1111 try testMulWithOverflow(u200, 1 << 100, 1 << 99, 1 << 199, 0);
1112 try testMulWithOverflow(u200, 1 << 100, 1 << 100, 0, 1);
1113 try testMulWithOverflow(u200, maxInt(u200), maxInt(u200), 1, 1);
1114 try testMulWithOverflow(u200, maxInt(u200) - 1, 2, maxInt(u200) - 3, 1);
1115
1116 try testMulWithOverflow(i140, 0, -1, 0, 0);
1117 try testMulWithOverflow(i140, -1, -1, 1, 0);
1118 try testMulWithOverflow(i140, 1 << 69, 1 << 69, 1 << 138, 0);
1119 try testMulWithOverflow(i140, 1 << 69, 1 << 70, minInt(i140), 1);
1120 try testMulWithOverflow(i140, -1 << 70, 1 << 20, -1 << 90, 0);
1121 try testMulWithOverflow(i140, minInt(i140), -1, minInt(i140), 1);
1122
1123 try testMulWithOverflow(i200, 1 << 100, 1 << 98, 1 << 198, 0);
1124 try testMulWithOverflow(i200, 1 << 100, 1 << 99, minInt(i200), 1);
1125 try testMulWithOverflow(i200, -1 << 120, 1 << 30, -1 << 150, 0);
1126 try testMulWithOverflow(i200, minInt(i200), minInt(i200), 0, 1);
1127 try testMulWithOverflow(i200, maxInt(i200), 2, -2, 1);
1128 try testMulWithOverflow(i200, maxInt(i200), maxInt(i200), 1, 1);
1129}
1130
1103test "@mulWithOverflow bitsize 256 bits" {1131test "@mulWithOverflow bitsize 256 bits" {
1104 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;1132 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
1105 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest;1133 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest;
1106 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;
1107 if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest;1134 if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest;
1108 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;1135 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
11091136