authorgravatar for 124872+jedisct1@users.noreply.github.comFrank Denis <124872+jedisct1@users.noreply.github.com> 2022-11-07 21:45:29+01:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2022-11-07 21:45:29+01:00
log7d48cb11389f5dfe73d25e59c1038398a8ba9ca9
tree7a06b19b14b978ac1ed6fd0a6afb642017d1c982
parent88d2e4f66a988dabca45c9add992c9f029c2176f
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

std.crypto: make ghash faster, esp. for small messages (#13464)

* std.crypto: make ghash faster, esp. for small messages Aggregated reduction requires 5 additional multiplications (to precompute the powers of H), in order to save 2 multiplications per batch. So, only use large batches when it's actually interesting to do so. For the last blocks, reuse the precomputations in order to perform a single reduction. Also, even in .ReleaseSmall, allow 2-block aggregation. The speedup is worth it, and the code increase is reasonable. And in .ReleaseFast, bump the upper batch size up to 16. Leverage comptime by the way instead of duplicating code. std/crypto/benchmark.zig on Apple M1: Zig 0.10.0: 2769 MiB/s Before: 6014 MiB/s After: 7334 MiB/s Normalize function names by the way. * Change clmul() to accept the half to be processed This avoids a bunch of truncate() calls. * Add more ghash tests to check all code paths

1 files changed, 152 insertions(+), 116 deletions(-)

lib/std/crypto/ghash.zig+152-116
...@@ -18,7 +18,11 @@ pub const Ghash = struct {...@@ -18,7 +18,11 @@ pub const Ghash = struct {
18 pub const mac_length = 16;18 pub const mac_length = 16;
19 pub const key_length = 16;19 pub const key_length = 16;
2020
21 const pc_count = if (builtin.mode != .ReleaseSmall) 8 else 1;21 const pc_count = if (builtin.mode != .ReleaseSmall) 16 else 2;
22 const agg_2_treshold = 5 * block_length;
23 const agg_4_treshold = 22 * block_length;
24 const agg_8_treshold = 84 * block_length;
25 const agg_16_treshold = 328 * block_length;
2226
23 hx: [pc_count]Precomp,27 hx: [pc_count]Precomp,
24 acc: u128 = 0,28 acc: u128 = 0,
...@@ -38,19 +42,26 @@ pub const Ghash = struct {...@@ -38,19 +42,26 @@ pub const Ghash = struct {
3842
39 var hx: [pc_count]Precomp = undefined;43 var hx: [pc_count]Precomp = undefined;
40 hx[0] = h;44 hx[0] = h;
45 if (block_count >= agg_2_treshold) {
46 hx[1] = gcmReduce(clsq128(hx[0])); // h^2
47 }
41 if (builtin.mode != .ReleaseSmall) {48 if (builtin.mode != .ReleaseSmall) {
42 if (block_count > 2) {49 if (block_count >= agg_4_treshold) {
43 hx[1] = gcm_reduce(clsq128(hx[0])); // h^250 hx[2] = gcmReduce(clmul128(hx[1], h)); // h^3
51 hx[3] = gcmReduce(clsq128(hx[1])); // h^4 = h^2^2
44 }52 }
45 if (block_count > 4) {53 if (block_count >= agg_8_treshold) {
46 hx[2] = gcm_reduce(clmul128(hx[1], h)); // h^354 hx[4] = gcmReduce(clmul128(hx[3], h)); // h^5
47 hx[3] = gcm_reduce(clsq128(hx[1])); // h^455 hx[5] = gcmReduce(clsq128(hx[2])); // h^6 = h^3^2
56 hx[6] = gcmReduce(clmul128(hx[5], h)); // h^7
57 hx[7] = gcmReduce(clsq128(hx[3])); // h^8 = h^4^2
48 }58 }
49 if (block_count > 8) {59 if (block_count >= agg_16_treshold) {
50 hx[4] = gcm_reduce(clmul128(hx[3], h)); // h^560 var i: usize = 8;
51 hx[5] = gcm_reduce(clmul128(hx[4], h)); // h^661 while (i < 16) : (i += 2) {
52 hx[6] = gcm_reduce(clmul128(hx[5], h)); // h^762 hx[i] = gcmReduce(clmul128(hx[i - 1], h));
53 hx[7] = gcm_reduce(clsq128(hx[3])); // h^863 hx[i + 1] = gcmReduce(clsq128(hx[i / 2]));
64 }
54 }65 }
55 }66 }
56 return Ghash{ .hx = hx };67 return Ghash{ .hx = hx };
...@@ -62,29 +73,52 @@ pub const Ghash = struct {...@@ -62,29 +73,52 @@ pub const Ghash = struct {
62 }73 }
6374
64 // Carryless multiplication of two 64-bit integers for x86_64.75 // Carryless multiplication of two 64-bit integers for x86_64.
65 inline fn clmul_pclmul(x: u64, y: u64) u128 {76 inline fn clmulPclmul(x: u128, y: u128, comptime half: enum { lo, hi }) u128 {
66 const product = asm (77 if (half == .hi) {
67 \\ vpclmulqdq $0x00, %[x], %[y], %[out]78 const product = asm (
68 : [out] "=x" (-> @Vector(2, u64)),79 \\ vpclmulqdq $0x11, %[x], %[y], %[out]
69 : [x] "x" (@bitCast(@Vector(2, u64), @as(u128, x))),80 : [out] "=x" (-> @Vector(2, u64)),
70 [y] "x" (@bitCast(@Vector(2, u64), @as(u128, y))),81 : [x] "x" (@bitCast(@Vector(2, u64), @as(u128, x))),
71 );82 [y] "x" (@bitCast(@Vector(2, u64), @as(u128, y))),
72 return (@as(u128, product[1]) << 64) | product[0];83 );
84 return @bitCast(u128, product);
85 } else {
86 const product = asm (
87 \\ vpclmulqdq $0x00, %[x], %[y], %[out]
88 : [out] "=x" (-> @Vector(2, u64)),
89 : [x] "x" (@bitCast(@Vector(2, u64), @as(u128, x))),
90 [y] "x" (@bitCast(@Vector(2, u64), @as(u128, y))),
91 );
92 return @bitCast(u128, product);
93 }
73 }94 }
7495
75 // Carryless multiplication of two 64-bit integers for ARM crypto.96 // Carryless multiplication of two 64-bit integers for ARM crypto.
76 inline fn clmul_pmull(x: u64, y: u64) u128 {97 inline fn clmulPmull(x: u128, y: u128, comptime half: enum { lo, hi }) u128 {
77 const product = asm (98 if (half == .hi) {
78 \\ pmull %[out].1q, %[x].1d, %[y].1d99 const product = asm (
79 : [out] "=w" (-> @Vector(2, u64)),100 \\ pmull2 %[out].1q, %[x].2d, %[y].2d
80 : [x] "w" (@bitCast(@Vector(2, u64), @as(u128, x))),101 : [out] "=w" (-> @Vector(2, u64)),
81 [y] "w" (@bitCast(@Vector(2, u64), @as(u128, y))),102 : [x] "w" (@bitCast(@Vector(2, u64), @as(u128, x))),
82 );103 [y] "w" (@bitCast(@Vector(2, u64), @as(u128, y))),
83 return (@as(u128, product[1]) << 64) | product[0];104 );
105 return @bitCast(u128, product);
106 } else {
107 const product = asm (
108 \\ pmull %[out].1q, %[x].1d, %[y].1d
109 : [out] "=w" (-> @Vector(2, u64)),
110 : [x] "w" (@bitCast(@Vector(2, u64), @as(u128, x))),
111 [y] "w" (@bitCast(@Vector(2, u64), @as(u128, y))),
112 );
113 return @bitCast(u128, product);
114 }
84 }115 }
85116
86 // Software carryless multiplication of two 64-bit integers.117 // Software carryless multiplication of two 64-bit integers.
87 fn clmul_soft(x: u64, y: u64) u128 {118 fn clmulSoft(x_: u128, y_: u128, comptime half: enum { lo, hi }) u128 {
119 const x = @truncate(u64, if (half == .hi) x_ >> 64 else x_);
120 const y = @truncate(u64, if (half == .hi) y_ >> 64 else y_);
121
88 const x0 = x & 0x1111111111111110;122 const x0 = x & 0x1111111111111110;
89 const x1 = x & 0x2222222222222220;123 const x1 = x & 0x2222222222222220;
90 const x2 = x & 0x4444444444444440;124 const x2 = x & 0x4444444444444440;
...@@ -116,32 +150,31 @@ pub const Ghash = struct {...@@ -116,32 +150,31 @@ pub const Ghash = struct {
116 const lo = @truncate(u64, x);150 const lo = @truncate(u64, x);
117 const hi = @truncate(u64, x >> 64);151 const hi = @truncate(u64, x >> 64);
118 const mid = lo ^ hi;152 const mid = lo ^ hi;
119 const r_lo = clmul(lo, lo);153 const r_lo = clmul(x, x, .lo);
120 const r_hi = clmul(hi, hi);154 const r_hi = clmul(x, x, .hi);
121 const r_mid = clmul(mid, mid) ^ r_lo ^ r_hi;155 const r_mid = clmul(mid, mid, .lo) ^ r_lo ^ r_hi;
122 return (@as(u256, r_hi) << 128) ^ (@as(u256, r_mid) << 64) ^ r_lo;156 return (@as(u256, r_hi) << 128) ^ (@as(u256, r_mid) << 64) ^ r_lo;
123 }157 }
124158
125 // Multiply two 128-bit integers in GF(2^128).159 // Multiply two 128-bit integers in GF(2^128).
126 inline fn clmul128(x: u128, y: u128) u256 {160 inline fn clmul128(x: u128, y: u128) u256 {
127 const x_lo = @truncate(u64, x);
128 const x_hi = @truncate(u64, x >> 64);161 const x_hi = @truncate(u64, x >> 64);
129 const y_lo = @truncate(u64, y);
130 const y_hi = @truncate(u64, y >> 64);162 const y_hi = @truncate(u64, y >> 64);
131 const r_lo = clmul(x_lo, y_lo);163 const r_lo = clmul(x, y, .lo);
132 const r_hi = clmul(x_hi, y_hi);164 const r_hi = clmul(x, y, .hi);
133 const r_mid = clmul(x_lo ^ x_hi, y_lo ^ y_hi) ^ r_lo ^ r_hi;165 const r_mid = clmul(x ^ x_hi, y ^ y_hi, .lo) ^ r_lo ^ r_hi;
134 return (@as(u256, r_hi) << 128) ^ (@as(u256, r_mid) << 64) ^ r_lo;166 return (@as(u256, r_hi) << 128) ^ (@as(u256, r_mid) << 64) ^ r_lo;
135 }167 }
136168
137 // Reduce a 256-bit representative of a polynomial modulo the irreducible polynomial x^128 + x^127 + x^126 + x^121 + 1.169 // Reduce a 256-bit representative of a polynomial modulo the irreducible polynomial x^128 + x^127 + x^126 + x^121 + 1.
138 // This is done *without reversing the bits*, using Shay Gueron's black magic demysticated here:170 // This is done *without reversing the bits*, using Shay Gueron's black magic demysticated here:
139 // https://blog.quarkslab.com/reversing-a-finite-field-multiplication-optimization.html171 // https://blog.quarkslab.com/reversing-a-finite-field-multiplication-optimization.html
140 inline fn gcm_reduce(x: u256) u128 {172 inline fn gcmReduce(x: u256) u128 {
141 const p64 = (((1 << 121) | (1 << 126) | (1 << 127)) >> 64);173 const p64 = (((1 << 121) | (1 << 126) | (1 << 127)) >> 64);
142 const a = clmul(@truncate(u64, x), p64);174 const lo = @truncate(u128, x);
143 const b = ((@truncate(u128, x) << 64) | (@truncate(u128, x) >> 64)) ^ a;175 const a = clmul(lo, p64, .lo);
144 const c = clmul(@truncate(u64, b), p64);176 const b = ((lo << 64) | (lo >> 64)) ^ a;
177 const c = clmul(b, p64, .lo);
145 const d = ((b << 64) | (b >> 64)) ^ c;178 const d = ((b << 64) | (b >> 64)) ^ c;
146 return d ^ @truncate(u128, x >> 128);179 return d ^ @truncate(u128, x >> 128);
147 }180 }
...@@ -150,103 +183,73 @@ pub const Ghash = struct {...@@ -150,103 +183,73 @@ pub const Ghash = struct {
150 const has_avx = std.Target.x86.featureSetHas(builtin.cpu.features, .avx);183 const has_avx = std.Target.x86.featureSetHas(builtin.cpu.features, .avx);
151 const has_armaes = std.Target.aarch64.featureSetHas(builtin.cpu.features, .aes);184 const has_armaes = std.Target.aarch64.featureSetHas(builtin.cpu.features, .aes);
152 const clmul = if (builtin.cpu.arch == .x86_64 and has_pclmul and has_avx) impl: {185 const clmul = if (builtin.cpu.arch == .x86_64 and has_pclmul and has_avx) impl: {
153 break :impl clmul_pclmul;186 break :impl clmulPclmul;
154 } else if (builtin.cpu.arch == .aarch64 and has_armaes) impl: {187 } else if (builtin.cpu.arch == .aarch64 and has_armaes) impl: {
155 break :impl clmul_pmull;188 break :impl clmulPmull;
156 } else impl: {189 } else impl: {
157 break :impl clmul_soft;190 break :impl clmulSoft;
158 };191 };
159192
160 // Process a block of 16 bytes.193 // Process 16 byte blocks.
161 fn blocks(st: *Ghash, msg: []const u8) void {194 fn blocks(st: *Ghash, msg: []const u8) void {
162 assert(msg.len % 16 == 0); // GHASH blocks() expects full blocks195 assert(msg.len % 16 == 0); // GHASH blocks() expects full blocks
163 var acc = st.acc;196 var acc = st.acc;
164197
165 var i: usize = 0;198 var i: usize = 0;
166199
167 if (builtin.mode != .ReleaseSmall) {200 if (builtin.mode != .ReleaseSmall and msg.len >= agg_16_treshold) {
201 // 16-blocks aggregated reduction
202 while (i + 256 <= msg.len) : (i += 256) {
203 var u = clmul128(acc ^ mem.readIntBig(u128, msg[i..][0..16]), st.hx[15 - 0]);
204 comptime var j = 1;
205 inline while (j < 16) : (j += 1) {
206 u ^= clmul128(mem.readIntBig(u128, msg[i..][j * 16 ..][0..16]), st.hx[15 - j]);
207 }
208 acc = gcmReduce(u);
209 }
210 } else if (builtin.mode != .ReleaseSmall and msg.len >= agg_8_treshold) {
168 // 8-blocks aggregated reduction211 // 8-blocks aggregated reduction
169 while (i + 128 <= msg.len) : (i += 128) {212 while (i + 128 <= msg.len) : (i += 128) {
170 const b0 = mem.readIntBig(u128, msg[i..][0..16]);213 var u = clmul128(acc ^ mem.readIntBig(u128, msg[i..][0..16]), st.hx[7 - 0]);
171 const z0 = acc ^ b0;214 comptime var j = 1;
172 const z0h = clmul128(z0, st.hx[7]);215 inline while (j < 8) : (j += 1) {
173216 u ^= clmul128(mem.readIntBig(u128, msg[i..][j * 16 ..][0..16]), st.hx[7 - j]);
174 const b1 = mem.readIntBig(u128, msg[i..][16..32]);217 }
175 const b1h = clmul128(b1, st.hx[6]);218 acc = gcmReduce(u);
176
177 const b2 = mem.readIntBig(u128, msg[i..][32..48]);
178 const b2h = clmul128(b2, st.hx[5]);
179
180 const b3 = mem.readIntBig(u128, msg[i..][48..64]);
181 const b3h = clmul128(b3, st.hx[4]);
182
183 const b4 = mem.readIntBig(u128, msg[i..][64..80]);
184 const b4h = clmul128(b4, st.hx[3]);
185
186 const b5 = mem.readIntBig(u128, msg[i..][80..96]);
187 const b5h = clmul128(b5, st.hx[2]);
188
189 const b6 = mem.readIntBig(u128, msg[i..][96..112]);
190 const b6h = clmul128(b6, st.hx[1]);
191
192 const b7 = mem.readIntBig(u128, msg[i..][112..128]);
193 const b7h = clmul128(b7, st.hx[0]);
194
195 const u = z0h ^ b1h ^ b2h ^ b3h ^ b4h ^ b5h ^ b6h ^ b7h;
196 acc = gcm_reduce(u);
197 }219 }
198220 } else if (builtin.mode != .ReleaseSmall and msg.len >= agg_4_treshold) {
199 // 4-blocks aggregated reduction221 // 4-blocks aggregated reduction
200 while (i + 64 <= msg.len) : (i += 64) {222 while (i + 64 <= msg.len) : (i += 64) {
201 // (acc + b0) * H^4 unreduced223 var u = clmul128(acc ^ mem.readIntBig(u128, msg[i..][0..16]), st.hx[3 - 0]);
202 const b0 = mem.readIntBig(u128, msg[i..][0..16]);224 comptime var j = 1;
203 const z0 = acc ^ b0;225 inline while (j < 4) : (j += 1) {
204 const z0h = clmul128(z0, st.hx[3]);226 u ^= clmul128(mem.readIntBig(u128, msg[i..][j * 16 ..][0..16]), st.hx[3 - j]);
205227 }
206 // b1 * H^3 unreduced228 acc = gcmReduce(u);
207 const b1 = mem.readIntBig(u128, msg[i..][16..32]);
208 const b1h = clmul128(b1, st.hx[2]);
209
210 // b2 * H^2 unreduced
211 const b2 = mem.readIntBig(u128, msg[i..][32..48]);
212 const b2h = clmul128(b2, st.hx[1]);
213
214 // b3 * H unreduced
215 const b3 = mem.readIntBig(u128, msg[i..][48..64]);
216 const b3h = clmul128(b3, st.hx[0]);
217
218 // (((acc + b0) * H^4) + B1 * H^3 + B2 * H^2 + B3 * H) (mod P)
219 const u = z0h ^ b1h ^ b2h ^ b3h;
220 acc = gcm_reduce(u);
221 }229 }
222230 } else if (msg.len >= agg_2_treshold) {
223 // 2-blocks aggregated reduction231 // 2-blocks aggregated reduction
224 while (i + 32 <= msg.len) : (i += 32) {232 while (i + 32 <= msg.len) : (i += 32) {
225 // (acc + b0) * H^2 unreduced233 var u = clmul128(acc ^ mem.readIntBig(u128, msg[i..][0..16]), st.hx[1 - 0]);
226 const b0 = mem.readIntBig(u128, msg[i..][0..16]);234 comptime var j = 1;
227 const z0 = acc ^ b0;235 inline while (j < 2) : (j += 1) {
228 const z0h = clmul128(z0, st.hx[1]);236 u ^= clmul128(mem.readIntBig(u128, msg[i..][j * 16 ..][0..16]), st.hx[1 - j]);
229237 }
230 // b1 * H unreduced238 acc = gcmReduce(u);
231 const b1 = mem.readIntBig(u128, msg[i..][16..32]);
232 const b1h = clmul128(b1, st.hx[0]);
233
234 // (((acc + b0) * H^2) + B1 * H) (mod P)
235 const u = z0h ^ b1h;
236 acc = gcm_reduce(u);
237 }239 }
238 }240 }
239241 // remaining blocks
240 // single block242 if (i < msg.len) {
241 while (i + 16 <= msg.len) : (i += 16) {243 const n = (msg.len - i) / 16;
242 // (acc + b0) * H unreduced244 var u = clmul128(acc ^ mem.readIntBig(u128, msg[i..][0..16]), st.hx[n - 1 - 0]);
243 const b0 = mem.readIntBig(u128, msg[i..][0..16]);245 var j: usize = 1;
244 const z0 = acc ^ b0;246 while (j < n) : (j += 1) {
245 const z0h = clmul128(z0, st.hx[0]);247 u ^= clmul128(mem.readIntBig(u128, msg[i..][j * 16 ..][0..16]), st.hx[n - 1 - j]);
246248 }
247 // (acc + b0) * H (mod P)249 i += n * 16;
248 acc = gcm_reduce(z0h);250 acc = gcmReduce(u);
249 }251 }
252 assert(i == msg.len);
250 st.acc = acc;253 st.acc = acc;
251 }254 }
252255
...@@ -328,3 +331,36 @@ test "ghash" {...@@ -328,3 +331,36 @@ test "ghash" {
328 st.final(&out);331 st.final(&out);
329 try htest.assertEqual("889295fa746e8b174bf4ec80a65dea41", &out);332 try htest.assertEqual("889295fa746e8b174bf4ec80a65dea41", &out);
330}333}
334
335test "ghash2" {
336 var key: [16]u8 = undefined;
337 var i: usize = 0;
338 while (i < key.len) : (i += 1) {
339 key[i] = @intCast(u8, i * 15 + 1);
340 }
341 const tvs = [_]struct { len: usize, hash: [:0]const u8 }{
342 .{ .len = 5263, .hash = "b9395f37c131cd403a327ccf82ec016a" },
343 .{ .len = 1361, .hash = "8c24cb3664e9a36e32ddef0c8178ab33" },
344 .{ .len = 1344, .hash = "015d7243b52d62eee8be33a66a9658cc" },
345 .{ .len = 1000, .hash = "56e148799944193f351f2014ef9dec9d" },
346 .{ .len = 512, .hash = "ca4882ce40d37546185c57709d17d1ca" },
347 .{ .len = 128, .hash = "d36dc3aac16cfe21a75cd5562d598c1c" },
348 .{ .len = 111, .hash = "6e2bea99700fd19cf1694e7b56543320" },
349 .{ .len = 80, .hash = "aa28f4092a7cca155f3de279cf21aa17" },
350 .{ .len = 16, .hash = "9d7eb5ed121a52a4b0996e4ec9b98911" },
351 .{ .len = 1, .hash = "968a203e5c7a98b6d4f3112f4d6b89a7" },
352 .{ .len = 0, .hash = "00000000000000000000000000000000" },
353 };
354 inline for (tvs) |tv| {
355 var m: [tv.len]u8 = undefined;
356 i = 0;
357 while (i < m.len) : (i += 1) {
358 m[i] = @truncate(u8, i % 254 + 1);
359 }
360 var st = Ghash.init(&key);
361 st.update(&m);
362 var out: [16]u8 = undefined;
363 st.final(&out);
364 try htest.assertEqual(tv.hash, &out);
365 }
366}