authorgravatar for 124872+jedisct1@users.noreply.github.comFrank Denis <124872+jedisct1@users.noreply.github.com> 2022-11-07 21:45:29+01:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2022-11-07 21:45:29+01:00
log7d48cb11389f5dfe73d25e59c1038398a8ba9ca9
tree7a06b19b14b978ac1ed6fd0a6afb642017d1c982
parent88d2e4f66a988dabca45c9add992c9f029c2176f
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

std.crypto: make ghash faster, esp. for small messages (#13464)

* std.crypto: make ghash faster, esp. for small messages Aggregated reduction requires 5 additional multiplications (to precompute the powers of H), in order to save 2 multiplications per batch. So, only use large batches when it's actually interesting to do so. For the last blocks, reuse the precomputations in order to perform a single reduction. Also, even in .ReleaseSmall, allow 2-block aggregation. The speedup is worth it, and the code increase is reasonable. And in .ReleaseFast, bump the upper batch size up to 16. Leverage comptime by the way instead of duplicating code. std/crypto/benchmark.zig on Apple M1: Zig 0.10.0: 2769 MiB/s Before: 6014 MiB/s After: 7334 MiB/s Normalize function names by the way. * Change clmul() to accept the half to be processed This avoids a bunch of truncate() calls. * Add more ghash tests to check all code paths

1 files changed, 152 insertions(+), 116 deletions(-)

lib/std/crypto/ghash.zig+152-116
......@@ -18,7 +18,11 @@ pub const Ghash = struct {
1818 pub const mac_length = 16;
1919 pub const key_length = 16;
2020
21 const pc_count = if (builtin.mode != .ReleaseSmall) 8 else 1;
21 const pc_count = if (builtin.mode != .ReleaseSmall) 16 else 2;
22 const agg_2_treshold = 5 * block_length;
23 const agg_4_treshold = 22 * block_length;
24 const agg_8_treshold = 84 * block_length;
25 const agg_16_treshold = 328 * block_length;
2226
2327 hx: [pc_count]Precomp,
2428 acc: u128 = 0,
......@@ -38,19 +42,26 @@ pub const Ghash = struct {
3842
3943 var hx: [pc_count]Precomp = undefined;
4044 hx[0] = h;
45 if (block_count >= agg_2_treshold) {
46 hx[1] = gcmReduce(clsq128(hx[0])); // h^2
47 }
4148 if (builtin.mode != .ReleaseSmall) {
42 if (block_count > 2) {
43 hx[1] = gcm_reduce(clsq128(hx[0])); // h^2
49 if (block_count >= agg_4_treshold) {
50 hx[2] = gcmReduce(clmul128(hx[1], h)); // h^3
51 hx[3] = gcmReduce(clsq128(hx[1])); // h^4 = h^2^2
4452 }
45 if (block_count > 4) {
46 hx[2] = gcm_reduce(clmul128(hx[1], h)); // h^3
47 hx[3] = gcm_reduce(clsq128(hx[1])); // h^4
53 if (block_count >= agg_8_treshold) {
54 hx[4] = gcmReduce(clmul128(hx[3], h)); // h^5
55 hx[5] = gcmReduce(clsq128(hx[2])); // h^6 = h^3^2
56 hx[6] = gcmReduce(clmul128(hx[5], h)); // h^7
57 hx[7] = gcmReduce(clsq128(hx[3])); // h^8 = h^4^2
4858 }
49 if (block_count > 8) {
50 hx[4] = gcm_reduce(clmul128(hx[3], h)); // h^5
51 hx[5] = gcm_reduce(clmul128(hx[4], h)); // h^6
52 hx[6] = gcm_reduce(clmul128(hx[5], h)); // h^7
53 hx[7] = gcm_reduce(clsq128(hx[3])); // h^8
59 if (block_count >= agg_16_treshold) {
60 var i: usize = 8;
61 while (i < 16) : (i += 2) {
62 hx[i] = gcmReduce(clmul128(hx[i - 1], h));
63 hx[i + 1] = gcmReduce(clsq128(hx[i / 2]));
64 }
5465 }
5566 }
5667 return Ghash{ .hx = hx };
......@@ -62,29 +73,52 @@ pub const Ghash = struct {
6273 }
6374
6475 // Carryless multiplication of two 64-bit integers for x86_64.
65 inline fn clmul_pclmul(x: u64, y: u64) u128 {
66 const product = asm (
67 \\ vpclmulqdq $0x00, %[x], %[y], %[out]
68 : [out] "=x" (-> @Vector(2, u64)),
69 : [x] "x" (@bitCast(@Vector(2, u64), @as(u128, x))),
70 [y] "x" (@bitCast(@Vector(2, u64), @as(u128, y))),
71 );
72 return (@as(u128, product[1]) << 64) | product[0];
76 inline fn clmulPclmul(x: u128, y: u128, comptime half: enum { lo, hi }) u128 {
77 if (half == .hi) {
78 const product = asm (
79 \\ vpclmulqdq $0x11, %[x], %[y], %[out]
80 : [out] "=x" (-> @Vector(2, u64)),
81 : [x] "x" (@bitCast(@Vector(2, u64), @as(u128, x))),
82 [y] "x" (@bitCast(@Vector(2, u64), @as(u128, y))),
83 );
84 return @bitCast(u128, product);
85 } else {
86 const product = asm (
87 \\ vpclmulqdq $0x00, %[x], %[y], %[out]
88 : [out] "=x" (-> @Vector(2, u64)),
89 : [x] "x" (@bitCast(@Vector(2, u64), @as(u128, x))),
90 [y] "x" (@bitCast(@Vector(2, u64), @as(u128, y))),
91 );
92 return @bitCast(u128, product);
93 }
7394 }
7495
7596 // Carryless multiplication of two 64-bit integers for ARM crypto.
76 inline fn clmul_pmull(x: u64, y: u64) u128 {
77 const product = asm (
78 \\ pmull %[out].1q, %[x].1d, %[y].1d
79 : [out] "=w" (-> @Vector(2, u64)),
80 : [x] "w" (@bitCast(@Vector(2, u64), @as(u128, x))),
81 [y] "w" (@bitCast(@Vector(2, u64), @as(u128, y))),
82 );
83 return (@as(u128, product[1]) << 64) | product[0];
97 inline fn clmulPmull(x: u128, y: u128, comptime half: enum { lo, hi }) u128 {
98 if (half == .hi) {
99 const product = asm (
100 \\ pmull2 %[out].1q, %[x].2d, %[y].2d
101 : [out] "=w" (-> @Vector(2, u64)),
102 : [x] "w" (@bitCast(@Vector(2, u64), @as(u128, x))),
103 [y] "w" (@bitCast(@Vector(2, u64), @as(u128, y))),
104 );
105 return @bitCast(u128, product);
106 } else {
107 const product = asm (
108 \\ pmull %[out].1q, %[x].1d, %[y].1d
109 : [out] "=w" (-> @Vector(2, u64)),
110 : [x] "w" (@bitCast(@Vector(2, u64), @as(u128, x))),
111 [y] "w" (@bitCast(@Vector(2, u64), @as(u128, y))),
112 );
113 return @bitCast(u128, product);
114 }
84115 }
85116
86117 // Software carryless multiplication of two 64-bit integers.
87 fn clmul_soft(x: u64, y: u64) u128 {
118 fn clmulSoft(x_: u128, y_: u128, comptime half: enum { lo, hi }) u128 {
119 const x = @truncate(u64, if (half == .hi) x_ >> 64 else x_);
120 const y = @truncate(u64, if (half == .hi) y_ >> 64 else y_);
121
88122 const x0 = x & 0x1111111111111110;
89123 const x1 = x & 0x2222222222222220;
90124 const x2 = x & 0x4444444444444440;
......@@ -116,32 +150,31 @@ pub const Ghash = struct {
116150 const lo = @truncate(u64, x);
117151 const hi = @truncate(u64, x >> 64);
118152 const mid = lo ^ hi;
119 const r_lo = clmul(lo, lo);
120 const r_hi = clmul(hi, hi);
121 const r_mid = clmul(mid, mid) ^ r_lo ^ r_hi;
153 const r_lo = clmul(x, x, .lo);
154 const r_hi = clmul(x, x, .hi);
155 const r_mid = clmul(mid, mid, .lo) ^ r_lo ^ r_hi;
122156 return (@as(u256, r_hi) << 128) ^ (@as(u256, r_mid) << 64) ^ r_lo;
123157 }
124158
125159 // Multiply two 128-bit integers in GF(2^128).
126160 inline fn clmul128(x: u128, y: u128) u256 {
127 const x_lo = @truncate(u64, x);
128161 const x_hi = @truncate(u64, x >> 64);
129 const y_lo = @truncate(u64, y);
130162 const y_hi = @truncate(u64, y >> 64);
131 const r_lo = clmul(x_lo, y_lo);
132 const r_hi = clmul(x_hi, y_hi);
133 const r_mid = clmul(x_lo ^ x_hi, y_lo ^ y_hi) ^ r_lo ^ r_hi;
163 const r_lo = clmul(x, y, .lo);
164 const r_hi = clmul(x, y, .hi);
165 const r_mid = clmul(x ^ x_hi, y ^ y_hi, .lo) ^ r_lo ^ r_hi;
134166 return (@as(u256, r_hi) << 128) ^ (@as(u256, r_mid) << 64) ^ r_lo;
135167 }
136168
137169 // Reduce a 256-bit representative of a polynomial modulo the irreducible polynomial x^128 + x^127 + x^126 + x^121 + 1.
138170 // This is done *without reversing the bits*, using Shay Gueron's black magic demysticated here:
139171 // https://blog.quarkslab.com/reversing-a-finite-field-multiplication-optimization.html
140 inline fn gcm_reduce(x: u256) u128 {
172 inline fn gcmReduce(x: u256) u128 {
141173 const p64 = (((1 << 121) | (1 << 126) | (1 << 127)) >> 64);
142 const a = clmul(@truncate(u64, x), p64);
143 const b = ((@truncate(u128, x) << 64) | (@truncate(u128, x) >> 64)) ^ a;
144 const c = clmul(@truncate(u64, b), p64);
174 const lo = @truncate(u128, x);
175 const a = clmul(lo, p64, .lo);
176 const b = ((lo << 64) | (lo >> 64)) ^ a;
177 const c = clmul(b, p64, .lo);
145178 const d = ((b << 64) | (b >> 64)) ^ c;
146179 return d ^ @truncate(u128, x >> 128);
147180 }
......@@ -150,103 +183,73 @@ pub const Ghash = struct {
150183 const has_avx = std.Target.x86.featureSetHas(builtin.cpu.features, .avx);
151184 const has_armaes = std.Target.aarch64.featureSetHas(builtin.cpu.features, .aes);
152185 const clmul = if (builtin.cpu.arch == .x86_64 and has_pclmul and has_avx) impl: {
153 break :impl clmul_pclmul;
186 break :impl clmulPclmul;
154187 } else if (builtin.cpu.arch == .aarch64 and has_armaes) impl: {
155 break :impl clmul_pmull;
188 break :impl clmulPmull;
156189 } else impl: {
157 break :impl clmul_soft;
190 break :impl clmulSoft;
158191 };
159192
160 // Process a block of 16 bytes.
193 // Process 16 byte blocks.
161194 fn blocks(st: *Ghash, msg: []const u8) void {
162195 assert(msg.len % 16 == 0); // GHASH blocks() expects full blocks
163196 var acc = st.acc;
164197
165198 var i: usize = 0;
166199
167 if (builtin.mode != .ReleaseSmall) {
200 if (builtin.mode != .ReleaseSmall and msg.len >= agg_16_treshold) {
201 // 16-blocks aggregated reduction
202 while (i + 256 <= msg.len) : (i += 256) {
203 var u = clmul128(acc ^ mem.readIntBig(u128, msg[i..][0..16]), st.hx[15 - 0]);
204 comptime var j = 1;
205 inline while (j < 16) : (j += 1) {
206 u ^= clmul128(mem.readIntBig(u128, msg[i..][j * 16 ..][0..16]), st.hx[15 - j]);
207 }
208 acc = gcmReduce(u);
209 }
210 } else if (builtin.mode != .ReleaseSmall and msg.len >= agg_8_treshold) {
168211 // 8-blocks aggregated reduction
169212 while (i + 128 <= msg.len) : (i += 128) {
170 const b0 = mem.readIntBig(u128, msg[i..][0..16]);
171 const z0 = acc ^ b0;
172 const z0h = clmul128(z0, st.hx[7]);
173
174 const b1 = mem.readIntBig(u128, msg[i..][16..32]);
175 const b1h = clmul128(b1, st.hx[6]);
176
177 const b2 = mem.readIntBig(u128, msg[i..][32..48]);
178 const b2h = clmul128(b2, st.hx[5]);
179
180 const b3 = mem.readIntBig(u128, msg[i..][48..64]);
181 const b3h = clmul128(b3, st.hx[4]);
182
183 const b4 = mem.readIntBig(u128, msg[i..][64..80]);
184 const b4h = clmul128(b4, st.hx[3]);
185
186 const b5 = mem.readIntBig(u128, msg[i..][80..96]);
187 const b5h = clmul128(b5, st.hx[2]);
188
189 const b6 = mem.readIntBig(u128, msg[i..][96..112]);
190 const b6h = clmul128(b6, st.hx[1]);
191
192 const b7 = mem.readIntBig(u128, msg[i..][112..128]);
193 const b7h = clmul128(b7, st.hx[0]);
194
195 const u = z0h ^ b1h ^ b2h ^ b3h ^ b4h ^ b5h ^ b6h ^ b7h;
196 acc = gcm_reduce(u);
213 var u = clmul128(acc ^ mem.readIntBig(u128, msg[i..][0..16]), st.hx[7 - 0]);
214 comptime var j = 1;
215 inline while (j < 8) : (j += 1) {
216 u ^= clmul128(mem.readIntBig(u128, msg[i..][j * 16 ..][0..16]), st.hx[7 - j]);
217 }
218 acc = gcmReduce(u);
197219 }
198
220 } else if (builtin.mode != .ReleaseSmall and msg.len >= agg_4_treshold) {
199221 // 4-blocks aggregated reduction
200222 while (i + 64 <= msg.len) : (i += 64) {
201 // (acc + b0) * H^4 unreduced
202 const b0 = mem.readIntBig(u128, msg[i..][0..16]);
203 const z0 = acc ^ b0;
204 const z0h = clmul128(z0, st.hx[3]);
205
206 // b1 * H^3 unreduced
207 const b1 = mem.readIntBig(u128, msg[i..][16..32]);
208 const b1h = clmul128(b1, st.hx[2]);
209
210 // b2 * H^2 unreduced
211 const b2 = mem.readIntBig(u128, msg[i..][32..48]);
212 const b2h = clmul128(b2, st.hx[1]);
213
214 // b3 * H unreduced
215 const b3 = mem.readIntBig(u128, msg[i..][48..64]);
216 const b3h = clmul128(b3, st.hx[0]);
217
218 // (((acc + b0) * H^4) + B1 * H^3 + B2 * H^2 + B3 * H) (mod P)
219 const u = z0h ^ b1h ^ b2h ^ b3h;
220 acc = gcm_reduce(u);
223 var u = clmul128(acc ^ mem.readIntBig(u128, msg[i..][0..16]), st.hx[3 - 0]);
224 comptime var j = 1;
225 inline while (j < 4) : (j += 1) {
226 u ^= clmul128(mem.readIntBig(u128, msg[i..][j * 16 ..][0..16]), st.hx[3 - j]);
227 }
228 acc = gcmReduce(u);
221229 }
222
230 } else if (msg.len >= agg_2_treshold) {
223231 // 2-blocks aggregated reduction
224232 while (i + 32 <= msg.len) : (i += 32) {
225 // (acc + b0) * H^2 unreduced
226 const b0 = mem.readIntBig(u128, msg[i..][0..16]);
227 const z0 = acc ^ b0;
228 const z0h = clmul128(z0, st.hx[1]);
229
230 // b1 * H unreduced
231 const b1 = mem.readIntBig(u128, msg[i..][16..32]);
232 const b1h = clmul128(b1, st.hx[0]);
233
234 // (((acc + b0) * H^2) + B1 * H) (mod P)
235 const u = z0h ^ b1h;
236 acc = gcm_reduce(u);
233 var u = clmul128(acc ^ mem.readIntBig(u128, msg[i..][0..16]), st.hx[1 - 0]);
234 comptime var j = 1;
235 inline while (j < 2) : (j += 1) {
236 u ^= clmul128(mem.readIntBig(u128, msg[i..][j * 16 ..][0..16]), st.hx[1 - j]);
237 }
238 acc = gcmReduce(u);
237239 }
238240 }
239
240 // single block
241 while (i + 16 <= msg.len) : (i += 16) {
242 // (acc + b0) * H unreduced
243 const b0 = mem.readIntBig(u128, msg[i..][0..16]);
244 const z0 = acc ^ b0;
245 const z0h = clmul128(z0, st.hx[0]);
246
247 // (acc + b0) * H (mod P)
248 acc = gcm_reduce(z0h);
241 // remaining blocks
242 if (i < msg.len) {
243 const n = (msg.len - i) / 16;
244 var u = clmul128(acc ^ mem.readIntBig(u128, msg[i..][0..16]), st.hx[n - 1 - 0]);
245 var j: usize = 1;
246 while (j < n) : (j += 1) {
247 u ^= clmul128(mem.readIntBig(u128, msg[i..][j * 16 ..][0..16]), st.hx[n - 1 - j]);
248 }
249 i += n * 16;
250 acc = gcmReduce(u);
249251 }
252 assert(i == msg.len);
250253 st.acc = acc;
251254 }
252255
......@@ -328,3 +331,36 @@ test "ghash" {
328331 st.final(&out);
329332 try htest.assertEqual("889295fa746e8b174bf4ec80a65dea41", &out);
330333}
334
335test "ghash2" {
336 var key: [16]u8 = undefined;
337 var i: usize = 0;
338 while (i < key.len) : (i += 1) {
339 key[i] = @intCast(u8, i * 15 + 1);
340 }
341 const tvs = [_]struct { len: usize, hash: [:0]const u8 }{
342 .{ .len = 5263, .hash = "b9395f37c131cd403a327ccf82ec016a" },
343 .{ .len = 1361, .hash = "8c24cb3664e9a36e32ddef0c8178ab33" },
344 .{ .len = 1344, .hash = "015d7243b52d62eee8be33a66a9658cc" },
345 .{ .len = 1000, .hash = "56e148799944193f351f2014ef9dec9d" },
346 .{ .len = 512, .hash = "ca4882ce40d37546185c57709d17d1ca" },
347 .{ .len = 128, .hash = "d36dc3aac16cfe21a75cd5562d598c1c" },
348 .{ .len = 111, .hash = "6e2bea99700fd19cf1694e7b56543320" },
349 .{ .len = 80, .hash = "aa28f4092a7cca155f3de279cf21aa17" },
350 .{ .len = 16, .hash = "9d7eb5ed121a52a4b0996e4ec9b98911" },
351 .{ .len = 1, .hash = "968a203e5c7a98b6d4f3112f4d6b89a7" },
352 .{ .len = 0, .hash = "00000000000000000000000000000000" },
353 };
354 inline for (tvs) |tv| {
355 var m: [tv.len]u8 = undefined;
356 i = 0;
357 while (i < m.len) : (i += 1) {
358 m[i] = @truncate(u8, i % 254 + 1);
359 }
360 var st = Ghash.init(&key);
361 st.update(&m);
362 var out: [16]u8 = undefined;
363 st.final(&out);
364 try htest.assertEqual(tv.hash, &out);
365 }
366}