authorgravatar for 124872+jedisct1@users.noreply.github.comFrank Denis <124872+jedisct1@users.noreply.github.com> 2022-11-01 18:49:13+01:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2022-11-01 13:49:13-04:00
log0d192ee9ef6a69ca4841b1932579b9178938a6d5
treef1050372fe82bfa47b0da196605167254c4c5d71
parent1780d7a348e0f4abfaa71e70eb4cf410a89c00c0
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

std.crypto.onetimeauth.Ghash: make GHASH 2 - 2.5x faster (#13374)

Rewrite GHASH to use 128-bit multiplication over non-reversed integers, and up to 8 blocks aggregated reduction. lib/std/crypto/benchmark.zig results: Xeon E5: Before: 1604 MiB/s After: 4005 MiB/s Apple M1: Before: 2769 MiB/s After: 6014 MiB/s This also makes AES-GCM faster by the way.

2 files changed, 188 insertions(+), 185 deletions(-)

lib/std/crypto/aes_gcm.zig+5-2
...@@ -3,6 +3,7 @@ const assert = std.debug.assert;...@@ -3,6 +3,7 @@ const assert = std.debug.assert;
3const crypto = std.crypto;3const crypto = std.crypto;
4const debug = std.debug;4const debug = std.debug;
5const Ghash = std.crypto.onetimeauth.Ghash;5const Ghash = std.crypto.onetimeauth.Ghash;
6const math = std.math;
6const mem = std.mem;7const mem = std.mem;
7const modes = crypto.core.modes;8const modes = crypto.core.modes;
8const AuthenticationError = crypto.errors.AuthenticationError;9const AuthenticationError = crypto.errors.AuthenticationError;
...@@ -34,7 +35,8 @@ fn AesGcm(comptime Aes: anytype) type {...@@ -34,7 +35,8 @@ fn AesGcm(comptime Aes: anytype) type {
34 mem.writeIntBig(u32, j[nonce_length..][0..4], 1);35 mem.writeIntBig(u32, j[nonce_length..][0..4], 1);
35 aes.encrypt(&t, &j);36 aes.encrypt(&t, &j);
3637
37 var mac = Ghash.init(&h);38 const block_count = (math.divCeil(usize, ad.len, Ghash.block_length) catch unreachable) + (math.divCeil(usize, c.len, Ghash.block_length) catch unreachable);
39 var mac = Ghash.initForBlockCount(&h, block_count);
38 mac.update(ad);40 mac.update(ad);
39 mac.pad();41 mac.pad();
4042
...@@ -66,7 +68,8 @@ fn AesGcm(comptime Aes: anytype) type {...@@ -66,7 +68,8 @@ fn AesGcm(comptime Aes: anytype) type {
66 mem.writeIntBig(u32, j[nonce_length..][0..4], 1);68 mem.writeIntBig(u32, j[nonce_length..][0..4], 1);
67 aes.encrypt(&t, &j);69 aes.encrypt(&t, &j);
6870
69 var mac = Ghash.init(&h);71 const block_count = (math.divCeil(usize, ad.len, Ghash.block_length) catch unreachable) + (math.divCeil(usize, c.len, Ghash.block_length) catch unreachable) + 1;
72 var mac = Ghash.initForBlockCount(&h, block_count);
70 mac.update(ad);73 mac.update(ad);
71 mac.pad();74 mac.pad();
7275
lib/std/crypto/ghash.zig+183-183
...@@ -1,6 +1,3 @@...@@ -1,6 +1,3 @@
1//
2// Adapted from BearSSL's ctmul64 implementation originally written by Thomas Pornin <pornin@bolet.org>
3
4const std = @import("../std.zig");1const std = @import("../std.zig");
5const builtin = @import("builtin");2const builtin = @import("builtin");
6const assert = std.debug.assert;3const assert = std.debug.assert;
...@@ -8,6 +5,8 @@ const math = std.math;...@@ -8,6 +5,8 @@ const math = std.math;
8const mem = std.mem;5const mem = std.mem;
9const utils = std.crypto.utils;6const utils = std.crypto.utils;
107
8const Precomp = u128;
9
11/// GHASH is a universal hash function that features multiplication10/// GHASH is a universal hash function that features multiplication
12/// by a fixed parameter within a Galois field.11/// by a fixed parameter within a Galois field.
13///12///
...@@ -19,116 +18,132 @@ pub const Ghash = struct {...@@ -19,116 +18,132 @@ pub const Ghash = struct {
19 pub const mac_length = 16;18 pub const mac_length = 16;
20 pub const key_length = 16;19 pub const key_length = 16;
2120
22 y0: u64 = 0,21 const pc_count = if (builtin.mode != .ReleaseSmall) 8 else 1;
23 y1: u64 = 0,22
24 h0: u64,23 hx: [pc_count]Precomp,
25 h1: u64,24 acc: u128 = 0,
26 h2: u64,
27 h0r: u64,
28 h1r: u64,
29 h2r: u64,
30
31 hh0: u64 = undefined,
32 hh1: u64 = undefined,
33 hh2: u64 = undefined,
34 hh0r: u64 = undefined,
35 hh1r: u64 = undefined,
36 hh2r: u64 = undefined,
3725
38 leftover: usize = 0,26 leftover: usize = 0,
39 buf: [block_length]u8 align(16) = undefined,27 buf: [block_length]u8 align(16) = undefined,
4028
41 pub fn init(key: *const [key_length]u8) Ghash {29 /// Initialize the GHASH state with a key, and a minimum number of block count.
42 const h1 = mem.readIntBig(u64, key[0..8]);30 pub fn initForBlockCount(key: *const [key_length]u8, block_count: usize) Ghash {
43 const h0 = mem.readIntBig(u64, key[8..16]);31 const h0 = mem.readIntBig(u128, key[0..16]);
44 const h1r = @bitReverse(h1);32
45 const h0r = @bitReverse(h0);33 // We keep the values encoded as in GCM, not Polyval, i.e. without reversing the bits.
46 const h2 = h0 ^ h1;34 // This is fine, but the reversed result would be shifted by 1 bit. So, we shift h
47 const h2r = h0r ^ h1r;35 // to compensate.
4836 const carry = ((@as(u128, 0xc2) << 120) | 1) & (@as(u128, 0) -% (h0 >> 127));
49 if (builtin.mode == .ReleaseSmall) {37 const h = (h0 << 1) ^ carry;
50 return Ghash{38
51 .h0 = h0,39 var hx: [pc_count]Precomp = undefined;
52 .h1 = h1,40 hx[0] = h;
53 .h2 = h2,41 if (builtin.mode != .ReleaseSmall) {
54 .h0r = h0r,42 if (block_count > 2) {
55 .h1r = h1r,43 hx[1] = gcm_reduce(clsq128(hx[0])); // h^2
56 .h2r = h2r,44 }
57 };45 if (block_count > 4) {
58 } else {46 hx[2] = gcm_reduce(clmul128(hx[1], h)); // h^3
59 // Precompute H^247 hx[3] = gcm_reduce(clsq128(hx[1])); // h^4
60 var hh = Ghash{48 }
61 .h0 = h0,49 if (block_count > 8) {
62 .h1 = h1,50 hx[4] = gcm_reduce(clmul128(hx[3], h)); // h^5
63 .h2 = h2,51 hx[5] = gcm_reduce(clmul128(hx[4], h)); // h^6
64 .h0r = h0r,52 hx[6] = gcm_reduce(clmul128(hx[5], h)); // h^7
65 .h1r = h1r,53 hx[7] = gcm_reduce(clsq128(hx[3])); // h^8
66 .h2r = h2r,54 }
67 };
68 hh.update(key);
69 const hh1 = hh.y1;
70 const hh0 = hh.y0;
71 const hh1r = @bitReverse(hh1);
72 const hh0r = @bitReverse(hh0);
73 const hh2 = hh0 ^ hh1;
74 const hh2r = hh0r ^ hh1r;
75
76 return Ghash{
77 .h0 = h0,
78 .h1 = h1,
79 .h2 = h2,
80 .h0r = h0r,
81 .h1r = h1r,
82 .h2r = h2r,
83
84 .hh0 = hh0,
85 .hh1 = hh1,
86 .hh2 = hh2,
87 .hh0r = hh0r,
88 .hh1r = hh1r,
89 .hh2r = hh2r,
90 };
91 }55 }
56 return Ghash{ .hx = hx };
92 }57 }
9358
94 inline fn clmul_pclmul(x: u64, y: u64) u64 {59 /// Initialize the GHASH state with a key.
60 pub fn init(key: *const [key_length]u8) Ghash {
61 return Ghash.initForBlockCount(key, math.maxInt(usize));
62 }
63
64 // Carryless multiplication of two 64-bit integers for x86_64.
65 inline fn clmul_pclmul(x: u64, y: u64) u128 {
95 const product = asm (66 const product = asm (
96 \\ vpclmulqdq $0x00, %[x], %[y], %[out]67 \\ vpclmulqdq $0x00, %[x], %[y], %[out]
97 : [out] "=x" (-> @Vector(2, u64)),68 : [out] "=x" (-> @Vector(2, u64)),
98 : [x] "x" (@bitCast(@Vector(2, u64), @as(u128, x))),69 : [x] "x" (@bitCast(@Vector(2, u64), @as(u128, x))),
99 [y] "x" (@bitCast(@Vector(2, u64), @as(u128, y))),70 [y] "x" (@bitCast(@Vector(2, u64), @as(u128, y))),
100 );71 );
101 return product[0];72 return (@as(u128, product[1]) << 64) | product[0];
102 }73 }
10374
104 inline fn clmul_pmull(x: u64, y: u64) u64 {75 // Carryless multiplication of two 64-bit integers for ARM crypto.
76 inline fn clmul_pmull(x: u64, y: u64) u128 {
105 const product = asm (77 const product = asm (
106 \\ pmull %[out].1q, %[x].1d, %[y].1d78 \\ pmull %[out].1q, %[x].1d, %[y].1d
107 : [out] "=w" (-> @Vector(2, u64)),79 : [out] "=w" (-> @Vector(2, u64)),
108 : [x] "w" (@bitCast(@Vector(2, u64), @as(u128, x))),80 : [x] "w" (@bitCast(@Vector(2, u64), @as(u128, x))),
109 [y] "w" (@bitCast(@Vector(2, u64), @as(u128, y))),81 [y] "w" (@bitCast(@Vector(2, u64), @as(u128, y))),
110 );82 );
111 return product[0];83 return (@as(u128, product[1]) << 64) | product[0];
112 }84 }
11385
114 fn clmul_soft(x: u64, y: u64) u64 {86 // Software carryless multiplication of two 64-bit integers.
115 const x0 = x & 0x1111111111111111;87 fn clmul_soft(x: u64, y: u64) u128 {
116 const x1 = x & 0x2222222222222222;88 const x0 = x & 0x1111111111111110;
117 const x2 = x & 0x4444444444444444;89 const x1 = x & 0x2222222222222220;
118 const x3 = x & 0x8888888888888888;90 const x2 = x & 0x4444444444444440;
91 const x3 = x & 0x8888888888888880;
119 const y0 = y & 0x1111111111111111;92 const y0 = y & 0x1111111111111111;
120 const y1 = y & 0x2222222222222222;93 const y1 = y & 0x2222222222222222;
121 const y2 = y & 0x4444444444444444;94 const y2 = y & 0x4444444444444444;
122 const y3 = y & 0x8888888888888888;95 const y3 = y & 0x8888888888888888;
123 var z0 = (x0 *% y0) ^ (x1 *% y3) ^ (x2 *% y2) ^ (x3 *% y1);96 const z0 = (x0 * @as(u128, y0)) ^ (x1 * @as(u128, y3)) ^ (x2 * @as(u128, y2)) ^ (x3 * @as(u128, y1));
124 var z1 = (x0 *% y1) ^ (x1 *% y0) ^ (x2 *% y3) ^ (x3 *% y2);97 const z1 = (x0 * @as(u128, y1)) ^ (x1 * @as(u128, y0)) ^ (x2 * @as(u128, y3)) ^ (x3 * @as(u128, y2));
125 var z2 = (x0 *% y2) ^ (x1 *% y1) ^ (x2 *% y0) ^ (x3 *% y3);98 const z2 = (x0 * @as(u128, y2)) ^ (x1 * @as(u128, y1)) ^ (x2 * @as(u128, y0)) ^ (x3 * @as(u128, y3));
126 var z3 = (x0 *% y3) ^ (x1 *% y2) ^ (x2 *% y1) ^ (x3 *% y0);99 const z3 = (x0 * @as(u128, y3)) ^ (x1 * @as(u128, y2)) ^ (x2 * @as(u128, y1)) ^ (x3 * @as(u128, y0));
127 z0 &= 0x1111111111111111;100
128 z1 &= 0x2222222222222222;101 const x0_mask = @as(u64, 0) -% (x & 1);
129 z2 &= 0x4444444444444444;102 const x1_mask = @as(u64, 0) -% ((x >> 1) & 1);
130 z3 &= 0x8888888888888888;103 const x2_mask = @as(u64, 0) -% ((x >> 2) & 1);
131 return z0 | z1 | z2 | z3;104 const x3_mask = @as(u64, 0) -% ((x >> 3) & 1);
105 const extra = (x0_mask & y) ^ (@as(u128, x1_mask & y) << 1) ^
106 (@as(u128, x2_mask & y) << 2) ^ (@as(u128, x3_mask & y) << 3);
107
108 return (z0 & 0x11111111111111111111111111111111) ^
109 (z1 & 0x22222222222222222222222222222222) ^
110 (z2 & 0x44444444444444444444444444444444) ^
111 (z3 & 0x88888888888888888888888888888888) ^ extra;
112 }
113
114 // Square a 128-bit integer in GF(2^128).
115 fn clsq128(x: u128) u256 {
116 const lo = @truncate(u64, x);
117 const hi = @truncate(u64, x >> 64);
118 const mid = lo ^ hi;
119 const r_lo = clmul(lo, lo);
120 const r_hi = clmul(hi, hi);
121 const r_mid = clmul(mid, mid) ^ r_lo ^ r_hi;
122 return (@as(u256, r_hi) << 128) ^ (@as(u256, r_mid) << 64) ^ r_lo;
123 }
124
125 // Multiply two 128-bit integers in GF(2^128).
126 inline fn clmul128(x: u128, y: u128) u256 {
127 const x_lo = @truncate(u64, x);
128 const x_hi = @truncate(u64, x >> 64);
129 const y_lo = @truncate(u64, y);
130 const y_hi = @truncate(u64, y >> 64);
131 const r_lo = clmul(x_lo, y_lo);
132 const r_hi = clmul(x_hi, y_hi);
133 const r_mid = clmul(x_lo ^ x_hi, y_lo ^ y_hi) ^ r_lo ^ r_hi;
134 return (@as(u256, r_hi) << 128) ^ (@as(u256, r_mid) << 64) ^ r_lo;
135 }
136
137 // Reduce a 256-bit representative of a polynomial modulo the irreducible polynomial x^128 + x^127 + x^126 + x^121 + 1.
138 // This is done *without reversing the bits*, using Shay Gueron's black magic demysticated here:
139 // https://blog.quarkslab.com/reversing-a-finite-field-multiplication-optimization.html
140 inline fn gcm_reduce(x: u256) u128 {
141 const p64 = (((1 << 121) | (1 << 126) | (1 << 127)) >> 64);
142 const a = clmul(@truncate(u64, x), p64);
143 const b = ((@truncate(u128, x) << 64) | (@truncate(u128, x) >> 64)) ^ a;
144 const c = clmul(@truncate(u64, b), p64);
145 const d = ((b << 64) | (b >> 64)) ^ c;
146 return d ^ @truncate(u128, x >> 128);
132 }147 }
133148
134 const has_pclmul = std.Target.x86.featureSetHas(builtin.cpu.features, .pclmul);149 const has_pclmul = std.Target.x86.featureSetHas(builtin.cpu.features, .pclmul);
...@@ -142,116 +157,100 @@ pub const Ghash = struct {...@@ -142,116 +157,100 @@ pub const Ghash = struct {
142 break :impl clmul_soft;157 break :impl clmul_soft;
143 };158 };
144159
160 // Process a block of 16 bytes.
145 fn blocks(st: *Ghash, msg: []const u8) void {161 fn blocks(st: *Ghash, msg: []const u8) void {
146 assert(msg.len % 16 == 0); // GHASH blocks() expects full blocks162 assert(msg.len % 16 == 0); // GHASH blocks() expects full blocks
147 var y1 = st.y1;163 var acc = st.acc;
148 var y0 = st.y0;
149164
150 var i: usize = 0;165 var i: usize = 0;
151166
152 // 2-blocks aggregated reduction
153 if (builtin.mode != .ReleaseSmall) {167 if (builtin.mode != .ReleaseSmall) {
168 // 8-blocks aggregated reduction
169 while (i + 128 <= msg.len) : (i += 128) {
170 const b0 = mem.readIntBig(u128, msg[i..][0..16]);
171 const z0 = acc ^ b0;
172 const z0h = clmul128(z0, st.hx[7]);
173
174 const b1 = mem.readIntBig(u128, msg[i..][16..32]);
175 const b1h = clmul128(b1, st.hx[6]);
176
177 const b2 = mem.readIntBig(u128, msg[i..][32..48]);
178 const b2h = clmul128(b2, st.hx[5]);
179
180 const b3 = mem.readIntBig(u128, msg[i..][48..64]);
181 const b3h = clmul128(b3, st.hx[4]);
182
183 const b4 = mem.readIntBig(u128, msg[i..][64..80]);
184 const b4h = clmul128(b4, st.hx[3]);
185
186 const b5 = mem.readIntBig(u128, msg[i..][80..96]);
187 const b5h = clmul128(b5, st.hx[2]);
188
189 const b6 = mem.readIntBig(u128, msg[i..][96..112]);
190 const b6h = clmul128(b6, st.hx[1]);
191
192 const b7 = mem.readIntBig(u128, msg[i..][112..128]);
193 const b7h = clmul128(b7, st.hx[0]);
194
195 const u = z0h ^ b1h ^ b2h ^ b3h ^ b4h ^ b5h ^ b6h ^ b7h;
196 acc = gcm_reduce(u);
197 }
198
199 // 4-blocks aggregated reduction
200 while (i + 64 <= msg.len) : (i += 64) {
201 // (acc + b0) * H^4 unreduced
202 const b0 = mem.readIntBig(u128, msg[i..][0..16]);
203 const z0 = acc ^ b0;
204 const z0h = clmul128(z0, st.hx[3]);
205
206 // b1 * H^3 unreduced
207 const b1 = mem.readIntBig(u128, msg[i..][16..32]);
208 const b1h = clmul128(b1, st.hx[2]);
209
210 // b2 * H^2 unreduced
211 const b2 = mem.readIntBig(u128, msg[i..][32..48]);
212 const b2h = clmul128(b2, st.hx[1]);
213
214 // b3 * H unreduced
215 const b3 = mem.readIntBig(u128, msg[i..][48..64]);
216 const b3h = clmul128(b3, st.hx[0]);
217
218 // (((acc + b0) * H^4) + B1 * H^3 + B2 * H^2 + B3 * H) (mod P)
219 const u = z0h ^ b1h ^ b2h ^ b3h;
220 acc = gcm_reduce(u);
221 }
222
223 // 2-blocks aggregated reduction
154 while (i + 32 <= msg.len) : (i += 32) {224 while (i + 32 <= msg.len) : (i += 32) {
155 // B0 * H^2 unreduced225 // (acc + b0) * H^2 unreduced
156 y1 ^= mem.readIntBig(u64, msg[i..][0..8]);226 const b0 = mem.readIntBig(u128, msg[i..][0..16]);
157 y0 ^= mem.readIntBig(u64, msg[i..][8..16]);227 const z0 = acc ^ b0;
158228 const z0h = clmul128(z0, st.hx[1]);
159 const y1r = @bitReverse(y1);229
160 const y0r = @bitReverse(y0);230 // b1 * H unreduced
161 const y2 = y0 ^ y1;231 const b1 = mem.readIntBig(u128, msg[i..][16..32]);
162 const y2r = y0r ^ y1r;232 const b1h = clmul128(b1, st.hx[0]);
163233
164 var z0 = clmul(y0, st.hh0);234 // (((acc + b0) * H^2) + B1 * H) (mod P)
165 var z1 = clmul(y1, st.hh1);235 const u = z0h ^ b1h;
166 var z2 = clmul(y2, st.hh2) ^ z0 ^ z1;236 acc = gcm_reduce(u);
167 var z0h = clmul(y0r, st.hh0r);
168 var z1h = clmul(y1r, st.hh1r);
169 var z2h = clmul(y2r, st.hh2r) ^ z0h ^ z1h;
170
171 // B1 * H unreduced
172 const sy1 = mem.readIntBig(u64, msg[i..][16..24]);
173 const sy0 = mem.readIntBig(u64, msg[i..][24..32]);
174
175 const sy1r = @bitReverse(sy1);
176 const sy0r = @bitReverse(sy0);
177 const sy2 = sy0 ^ sy1;
178 const sy2r = sy0r ^ sy1r;
179
180 const sz0 = clmul(sy0, st.h0);
181 const sz1 = clmul(sy1, st.h1);
182 const sz2 = clmul(sy2, st.h2) ^ sz0 ^ sz1;
183 const sz0h = clmul(sy0r, st.h0r);
184 const sz1h = clmul(sy1r, st.h1r);
185 const sz2h = clmul(sy2r, st.h2r) ^ sz0h ^ sz1h;
186
187 // ((B0 * H^2) + B1 * H) (mod M)
188 z0 ^= sz0;
189 z1 ^= sz1;
190 z2 ^= sz2;
191 z0h ^= sz0h;
192 z1h ^= sz1h;
193 z2h ^= sz2h;
194 z0h = @bitReverse(z0h) >> 1;
195 z1h = @bitReverse(z1h) >> 1;
196 z2h = @bitReverse(z2h) >> 1;
197
198 var v3 = z1h;
199 var v2 = z1 ^ z2h;
200 var v1 = z0h ^ z2;
201 var v0 = z0;
202
203 v3 = (v3 << 1) | (v2 >> 63);
204 v2 = (v2 << 1) | (v1 >> 63);
205 v1 = (v1 << 1) | (v0 >> 63);
206 v0 = (v0 << 1);
207
208 v2 ^= v0 ^ (v0 >> 1) ^ (v0 >> 2) ^ (v0 >> 7);
209 v1 ^= (v0 << 63) ^ (v0 << 62) ^ (v0 << 57);
210 y1 = v3 ^ v1 ^ (v1 >> 1) ^ (v1 >> 2) ^ (v1 >> 7);
211 y0 = v2 ^ (v1 << 63) ^ (v1 << 62) ^ (v1 << 57);
212 }237 }
213 }238 }
214239
215 // single block240 // single block
216 while (i + 16 <= msg.len) : (i += 16) {241 while (i + 16 <= msg.len) : (i += 16) {
217 y1 ^= mem.readIntBig(u64, msg[i..][0..8]);242 // (acc + b0) * H unreduced
218 y0 ^= mem.readIntBig(u64, msg[i..][8..16]);243 const b0 = mem.readIntBig(u128, msg[i..][0..16]);
219244 const z0 = acc ^ b0;
220 const y1r = @bitReverse(y1);245 const z0h = clmul128(z0, st.hx[0]);
221 const y0r = @bitReverse(y0);246
222 const y2 = y0 ^ y1;247 // (acc + b0) * H (mod P)
223 const y2r = y0r ^ y1r;248 acc = gcm_reduce(z0h);
224
225 const z0 = clmul(y0, st.h0);
226 const z1 = clmul(y1, st.h1);
227 var z2 = clmul(y2, st.h2) ^ z0 ^ z1;
228 var z0h = clmul(y0r, st.h0r);
229 var z1h = clmul(y1r, st.h1r);
230 var z2h = clmul(y2r, st.h2r) ^ z0h ^ z1h;
231 z0h = @bitReverse(z0h) >> 1;
232 z1h = @bitReverse(z1h) >> 1;
233 z2h = @bitReverse(z2h) >> 1;
234
235 // shift & reduce
236 var v3 = z1h;
237 var v2 = z1 ^ z2h;
238 var v1 = z0h ^ z2;
239 var v0 = z0;
240
241 v3 = (v3 << 1) | (v2 >> 63);
242 v2 = (v2 << 1) | (v1 >> 63);
243 v1 = (v1 << 1) | (v0 >> 63);
244 v0 = (v0 << 1);
245
246 v2 ^= v0 ^ (v0 >> 1) ^ (v0 >> 2) ^ (v0 >> 7);
247 v1 ^= (v0 << 63) ^ (v0 << 62) ^ (v0 << 57);
248 y1 = v3 ^ v1 ^ (v1 >> 1) ^ (v1 >> 2) ^ (v1 >> 7);
249 y0 = v2 ^ (v1 << 63) ^ (v1 << 62) ^ (v1 << 57);
250 }249 }
251 st.y1 = y1;250 st.acc = acc;
252 st.y0 = y0;
253 }251 }
254252
253 /// Absorb a message into the GHASH state.
255 pub fn update(st: *Ghash, m: []const u8) void {254 pub fn update(st: *Ghash, m: []const u8) void {
256 var mb = m;255 var mb = m;
257256
...@@ -295,14 +294,15 @@ pub const Ghash = struct {...@@ -295,14 +294,15 @@ pub const Ghash = struct {
295 st.leftover = 0;294 st.leftover = 0;
296 }295 }
297296
297 /// Compute the GHASH of the entire input.
298 pub fn final(st: *Ghash, out: *[mac_length]u8) void {298 pub fn final(st: *Ghash, out: *[mac_length]u8) void {
299 st.pad();299 st.pad();
300 mem.writeIntBig(u64, out[0..8], st.y1);300 mem.writeIntBig(u128, out[0..16], st.acc);
301 mem.writeIntBig(u64, out[8..16], st.y0);
302301
303 utils.secureZero(u8, @ptrCast([*]u8, st)[0..@sizeOf(Ghash)]);302 utils.secureZero(u8, @ptrCast([*]u8, st)[0..@sizeOf(Ghash)]);
304 }303 }
305304
305 /// Compute the GHASH of a message.
306 pub fn create(out: *[mac_length]u8, msg: []const u8, key: *const [key_length]u8) void {306 pub fn create(out: *[mac_length]u8, msg: []const u8, key: *const [key_length]u8) void {
307 var st = Ghash.init(key);307 var st = Ghash.init(key);
308 st.update(msg);308 st.update(msg);