authorgravatar for 124872+jedisct1@users.noreply.github.comFrank Denis <124872+jedisct1@users.noreply.github.com> 2026-09-07 21:03:33+02:00
committergravatar for 124872+jedisct1@users.noreply.github.comFrank Denis <124872+jedisct1@users.noreply.github.com> 2026-10-12 09:55:45+02:00
logbfb20fe2163a54a96d962b6985e75d480648ab60
treedba8f6cb9e78b9c7ad82f39b41151537bfa01880
parentab3825e026d96d99527e701674aabe3b9bae810f

crypto.ff: specialize full-width Montgomery multiplication

When the modulus takes its full limb capacity, we can take advantage of the fact that this is a comptime value to get specialized code with better vectorization. Also avoid expensive copies. This is all internal; no changes to the public APIs.

1 files changed, 22 insertions(+), 14 deletions(-)

lib/std/crypto/ff.zig+22-14
...@@ -857,23 +857,33 @@ pub fn Modulus(comptime max_bits: comptime_int) type {...@@ -857,23 +857,33 @@ pub fn Modulus(comptime max_bits: comptime_int) type {
857 return out;857 return out;
858 }858 }
859859
860 fn montgomeryLoop(self: Self, d: *Fe, x: Fe, y: Fe) u1 {860 fn montgomeryLoop(self: *const Self, noalias d: *Fe, x: *const Fe, y: *const Fe) u1 {
861 if (self.limbs_count() == FeUint.max_limbs_count) {
862 return montgomeryLoopWithWidth(true, self, d, x, y);
863 }
864 return montgomeryLoopWithWidth(false, self, d, x, y);
865 }
866
867 fn montgomeryLoopWithWidth(comptime full_width: bool, self: *const Self, noalias d: *Fe, x: *const Fe, y: *const Fe) u1 {
868 assert(d != x and d != y);
861 assert(d.limbs_count() == x.limbs_count());869 assert(d.limbs_count() == x.limbs_count());
862 assert(d.limbs_count() == y.limbs_count());870 assert(d.limbs_count() == y.limbs_count());
863 assert(d.limbs_count() == self.limbs_count());871 assert(d.limbs_count() == self.limbs_count());
872 assert(self.v.isOdd());
864873
865 const a_limbs = x.v.limbsConst();874 const n = if (full_width) FeUint.max_limbs_count else self.limbs_count();
866 const b_limbs = y.v.limbsConst();875 const a_limbs = x.v.limbs_buffer[0..n];
867 const d_limbs = d.v.limbs();876 const b_limbs = y.v.limbs_buffer[0..n];
868 const m_limbs = self.v.limbsConst();877 const d_limbs = d.v.limbs_buffer[0..n];
878 const m_limbs = self.v.limbs_buffer[0..n];
869879
870 var overflow: u1 = 0;880 var overflow: u1 = 0;
871 for (0..self.limbs_count()) |i| {881 for (0..n) |i| {
872 var carry: Limb = 0;882 var carry: Limb = 0;
873883
874 var wide = ct.mulWide(a_limbs[i], b_limbs[0]);884 var wide = ct.mulWide(a_limbs[i], b_limbs[0]);
875 var z_lo = @addWithOverflow(d_limbs[0], wide.lo);885 var z_lo = @addWithOverflow(d_limbs[0], wide.lo);
876 const f = @as(TLimb, @truncate(z_lo[0] *% self.m0inv));886 const f: TLimb = @truncate(z_lo[0] *% self.m0inv);
877 var z_hi = wide.hi +% z_lo[1];887 var z_hi = wide.hi +% z_lo[1];
878 wide = ct.mulWide(f, m_limbs[0]);888 wide = ct.mulWide(f, m_limbs[0]);
879 z_lo = @addWithOverflow(z_lo[0], wide.lo);889 z_lo = @addWithOverflow(z_lo[0], wide.lo);
...@@ -881,7 +891,7 @@ pub fn Modulus(comptime max_bits: comptime_int) type {...@@ -881,7 +891,7 @@ pub fn Modulus(comptime max_bits: comptime_int) type {
881 z_hi +%= wide.hi;891 z_hi +%= wide.hi;
882 carry = (z_hi << 1) | (z_lo[0] >> t_bits);892 carry = (z_hi << 1) | (z_lo[0] >> t_bits);
883893
884 for (1..self.limbs_count()) |j| {894 for (1..n) |j| {
885 wide = ct.mulWide(a_limbs[i], b_limbs[j]);895 wide = ct.mulWide(a_limbs[i], b_limbs[j]);
886 z_lo = @addWithOverflow(d_limbs[j], wide.lo);896 z_lo = @addWithOverflow(d_limbs[j], wide.lo);
887 z_hi = wide.hi +% z_lo[1];897 z_hi = wide.hi +% z_lo[1];
...@@ -897,18 +907,17 @@ pub fn Modulus(comptime max_bits: comptime_int) type {...@@ -897,18 +907,17 @@ pub fn Modulus(comptime max_bits: comptime_int) type {
897 carry = (z_hi << 1) | (z_lo[0] >> t_bits);907 carry = (z_hi << 1) | (z_lo[0] >> t_bits);
898 }908 }
899 const z = overflow + carry;909 const z = overflow + carry;
900 d_limbs[self.limbs_count() - 1] = @as(TLimb, @truncate(z));910 d_limbs[n - 1] = @as(TLimb, @truncate(z));
901 overflow = @as(u1, @truncate(z >> t_bits));911 overflow = @truncate(z >> t_bits);
902 }912 }
903 return overflow;913 return overflow;
904 }914 }
905915
906 // Montgomery multiplication.
907 fn montgomeryMul(self: Self, x: Fe, y: Fe) Fe {916 fn montgomeryMul(self: Self, x: Fe, y: Fe) Fe {
908 var d = self.zero;917 var d = self.zero;
909 assert(x.limbs_count() == self.limbs_count());918 assert(x.limbs_count() == self.limbs_count());
910 assert(y.limbs_count() == self.limbs_count());919 assert(y.limbs_count() == self.limbs_count());
911 const overflow = self.montgomeryLoop(&d, x, y);920 const overflow = self.montgomeryLoop(&d, &x, &y);
912 const underflow = 1 -% @intFromBool(ct.limbsCmpGeq(d.v, self.v));921 const underflow = 1 -% @intFromBool(ct.limbsCmpGeq(d.v, self.v));
913 const need_sub = ct.eql(overflow, underflow);922 const need_sub = ct.eql(overflow, underflow);
914 _ = d.v.conditionalSubWithOverflow(need_sub, self.v);923 _ = d.v.conditionalSubWithOverflow(need_sub, self.v);
...@@ -916,11 +925,10 @@ pub fn Modulus(comptime max_bits: comptime_int) type {...@@ -916,11 +925,10 @@ pub fn Modulus(comptime max_bits: comptime_int) type {
916 return d;925 return d;
917 }926 }
918927
919 // Montgomery squaring.
920 fn montgomerySq(self: Self, x: Fe) Fe {928 fn montgomerySq(self: Self, x: Fe) Fe {
921 var d = self.zero;929 var d = self.zero;
922 assert(x.limbs_count() == self.limbs_count());930 assert(x.limbs_count() == self.limbs_count());
923 const overflow = self.montgomeryLoop(&d, x, x);931 const overflow = self.montgomeryLoop(&d, &x, &x);
924 const underflow = 1 -% @intFromBool(ct.limbsCmpGeq(d.v, self.v));932 const underflow = 1 -% @intFromBool(ct.limbsCmpGeq(d.v, self.v));
925 const need_sub = ct.eql(overflow, underflow);933 const need_sub = ct.eql(overflow, underflow);
926 _ = d.v.conditionalSubWithOverflow(need_sub, self.v);934 _ = d.v.conditionalSubWithOverflow(need_sub, self.v);