| ... | @@ -638,6 +638,28 @@ pub fn Modulus(comptime max_bits: comptime_int) type { | ... | @@ -638,6 +638,28 @@ pub fn Modulus(comptime max_bits: comptime_int) type { |
| 638 | return x.eql(mo); | 638 | return x.eql(mo); |
| 639 | } | 639 | } |
| 640 | | 640 | |
| | 641 | // Computes -lo^-1 mod 2^t_bits for an odd `lo`. |
| | 642 | fn computeM0Inv(lo: Limb) Limb { |
| | 643 | var y = lo; |
| | 644 | inline for (0..comptime math.log2_int(usize, t_bits)) |_| { |
| | 645 | y = y *% (2 -% lo *% y); |
| | 646 | } |
| | 647 | return (@as(Limb, 1) << t_bits) - (@as(TLimb, @truncate(y))); |
| | 648 | } |
| | 649 | |
| | 650 | // Initialize without R2 |
| | 651 | fn initWithoutRR(v: FeUint, leading: usize, m0inv: Limb) Self { |
| | 652 | var m = Self{ |
| | 653 | .zero = Fe{ .v = FeUint.zero }, |
| | 654 | .v = v, |
| | 655 | .leading = leading, |
| | 656 | .m0inv = m0inv, |
| | 657 | .rr = undefined, |
| | 658 | }; |
| | 659 | m.shrink(&m.zero) catch unreachable; |
| | 660 | return m; |
| | 661 | } |
| | 662 | |
| 641 | /// Creates a new modulus from a `Uint` value. | 663 | /// Creates a new modulus from a `Uint` value. |
| 642 | /// The modulus must be odd and larger than 2. | 664 | /// The modulus must be odd and larger than 2. |
| 643 | pub fn fromUint(v_: FeUint) InvalidModulusError!Self { | 665 | pub fn fromUint(v_: FeUint) InvalidModulusError!Self { |
| ... | @@ -651,25 +673,8 @@ pub fn Modulus(comptime max_bits: comptime_int) type { | ... | @@ -651,25 +673,8 @@ pub fn Modulus(comptime max_bits: comptime_int) type { |
| 651 | return error.ModulusTooSmall; | 673 | return error.ModulusTooSmall; |
| 652 | } | 674 | } |
| 653 | | 675 | |
| 654 | const leading = @clz(hi) - carry_bits; | 676 | var m = initWithoutRR(v, @clz(hi) - carry_bits, computeM0Inv(lo)); |
| 655 | | 677 | |
| 656 | var y = lo; | | |
| 657 | | | |
| 658 | inline for (0..comptime math.log2_int(usize, t_bits)) |_| { | | |
| 659 | y = y *% (2 -% lo *% y); | | |
| 660 | } | | |
| 661 | const m0inv = (@as(Limb, 1) << t_bits) - (@as(TLimb, @truncate(y))); | | |
| 662 | | | |
| 663 | const zero = Fe{ .v = FeUint.zero }; | | |
| 664 | | | |
| 665 | var m = Self{ | | |
| 666 | .zero = zero, | | |
| 667 | .v = v, | | |
| 668 | .leading = leading, | | |
| 669 | .m0inv = m0inv, | | |
| 670 | .rr = undefined, // will be computed right after | | |
| 671 | }; | | |
| 672 | m.shrink(&m.zero) catch unreachable; | | |
| 673 | computeRR(&m); | 678 | computeRR(&m); |
| 674 | | 679 | |
| 675 | return m; | 680 | return m; |
| ... | @@ -759,15 +764,21 @@ pub fn Modulus(comptime max_bits: comptime_int) type { | ... | @@ -759,15 +764,21 @@ pub fn Modulus(comptime max_bits: comptime_int) type { |
| 759 | x.v.cmov(need_sub, d.v); | 764 | x.v.cmov(need_sub, d.v); |
| 760 | } | 765 | } |
| 761 | | 766 | |
| | 767 | // Adds y to x. Returns whether reduction was needed. |
| | 768 | fn addAndReduce(x: *FeUint, y: FeUint, bound: FeUint) bool { |
| | 769 | const overflow = x.addWithOverflow(y); |
| | 770 | const underflow: u1 = @bitCast(ct.limbsCmpLt(x.*, bound)); |
| | 771 | const need_sub = ct.eql(overflow, underflow); |
| | 772 | _ = x.conditionalSubWithOverflow(need_sub, bound); |
| | 773 | return need_sub; |
| | 774 | } |
| | 775 | |
| 762 | /// Adds two field elements (mod m). | 776 | /// Adds two field elements (mod m). |
| 763 | pub fn add(self: Self, x: Fe, y: Fe) Fe { | 777 | pub fn add(self: Self, x: Fe, y: Fe) Fe { |
| 764 | var out = x; | 778 | var out = x; |
| 765 | if (x.montgomery == y.montgomery) { | 779 | if (x.montgomery == y.montgomery) { |
| 766 | @branchHint(.likely); | 780 | @branchHint(.likely); |
| 767 | const overflow = out.v.addWithOverflow(y.v); | 781 | _ = addAndReduce(&out.v, y.v, self.v); |
| 768 | const underflow: u1 = @bitCast(ct.limbsCmpLt(out.v, self.v)); | | |
| 769 | const need_sub = ct.eql(overflow, underflow); | | |
| 770 | _ = out.v.conditionalSubWithOverflow(need_sub, self.v); | | |
| 771 | return out; | 782 | return out; |
| 772 | } else { | 783 | } else { |
| 773 | var y_ = y; | 784 | var y_ = y; |
| ... | @@ -776,10 +787,7 @@ pub fn Modulus(comptime max_bits: comptime_int) type { | ... | @@ -776,10 +787,7 @@ pub fn Modulus(comptime max_bits: comptime_int) type { |
| 776 | } else { | 787 | } else { |
| 777 | self.toMontgomery(&y_) catch unreachable; | 788 | self.toMontgomery(&y_) catch unreachable; |
| 778 | } | 789 | } |
| 779 | const overflow = out.v.addWithOverflow(y_.v); | 790 | _ = addAndReduce(&out.v, y_.v, self.v); |
| 780 | const underflow: u1 = @bitCast(ct.limbsCmpLt(out.v, self.v)); | | |
| 781 | const need_sub = ct.eql(overflow, underflow); | | |
| 782 | _ = out.v.conditionalSubWithOverflow(need_sub, self.v); | | |
| 783 | return out; | 791 | return out; |
| 784 | } | 792 | } |
| 785 | } | 793 | } |