authorgravatar for 124872+jedisct1@users.noreply.github.comFrank Denis <124872+jedisct1@users.noreply.github.com> 2026-07-06 11:09:46+02:00
committergravatar for 124872+jedisct1@users.noreply.github.comFrank Denis <124872+jedisct1@users.noreply.github.com> 2026-09-23 13:25:56+02:00
log4d06435fe3a03227c7b899436efde3ee1bd0dcb9
tree92116101a05c2e3329dfd71c0ef87066e0e292e3
parent55fe0ad7da5b1b3b946d81a66ade1f0e1797706c

crypto.aes.soft: use a bitsliced representation

AES is nice but has a problem when implemented in pure software (on CPUs without AES instructions). In order to not be insanely slow, the non-linear operation is usually implemented with a lookup tables. But memory lookups with a secret index is a side channel that can be exploited for key recovery. The current implementation of software AES mitigates this by doing multiple passes over tiny lookup tables that fit in a single cache line. It works. Intra-cacheline attacks leveraging SMT exist, but the number of CPUs with SMT but no AES instructions is pretty small (old Intel Atom Medfield CPUs). But we can do better. Bitslicing is another way to mitigate these issues, but packing all the bits that will go through the same circuit into the same register, essentially emulating a hardware implementation. For single-block AES, this is quite slow, but for any algorithm that can leverage parallelization (AES-CTR, AES-OCB, etc), it's not bad at all. Long story short: - Constant time everywhere, including weird old CPUs - About 4x faster than the current cacheline-tables code for inputs > 32 bytes. On modern CPUs, the speedup is significant for WebAssembly. Note that when side channels are not part of the threat model, the unprotected lookup table remains faster. The packing/unpacking/round operations are intentionally public. Because in order to get decent performance, some AES-based ciphers really need to control when they want to use the regular word representation vs the packed representation.

4 files changed, 652 insertions(+), 76 deletions(-)

lib/std/crypto/aes/bitsliced.zig created+561
...@@ -0,0 +1,561 @@
1//! Bitsliced AES round function, for targets without hardware AES support.
2//! It has no data-dependent memory access, so it is constant time.
3
4const std = @import("../../std.zig");
5const builtin = @import("builtin");
6const mem = std.mem;
7
8/// The widest batch of AES blocks the target supports
9pub const Wide = if (builtin.cpu.arch == .wasm32 or builtin.cpu.arch == .wasm64 or @bitSizeOf(usize) >= 64) Batch(2) else Batch(1);
10
11/// Number of AES blocks we can process together
12pub const width = Wide.nblocks;
13
14/// A state holding a whole `width`-block batch
15pub const State = Wide.Bitsliced;
16
17/// A batch of `8 * lanes` blocks, one 32-bit lane per group of eight blocks
18pub fn Batch(comptime lanes: usize) type {
19 return struct {
20 pub const Word = @Int(.unsigned, 32 * lanes);
21 const Shift = std.math.Log2Int(Word);
22
23 const nblocks = 8 * lanes;
24 pub const Bitsliced = [32]Word;
25
26 /// Splat a 32-bit pattern across all lanes of a word
27 pub fn splatWord(x: u32) Word {
28 const t: [lanes]u32 = @splat(x);
29 return @bitCast(t);
30 }
31
32 fn laneRotl(x: Word, comptime n: u5) Word {
33 if (lanes == 1) return std.math.rotl(u32, x, n);
34 const lo_bits = (@as(u32, 1) << n) - 1;
35 const mask_lo = comptime splatWord(lo_bits);
36 const mask_hi = comptime splatWord(~lo_bits);
37 const l: Shift = n;
38 const r: Shift = @intCast(32 - @as(u32, n));
39 return ((x << l) & mask_hi) | ((x >> r) & mask_lo);
40 }
41
42 pub fn swapMove(a: *Word, b: *Word, comptime mask32: u32, comptime n: comptime_int) void {
43 const shift: Shift = n;
44 const mask = comptime splatWord(mask32);
45 const tmp = (b.* ^ (a.* >> shift)) & mask;
46 b.* ^= tmp;
47 a.* ^= tmp << shift;
48 }
49
50 fn sbox(u: []Word) void {
51 const s0 = u[4] ^ u[16];
52 const s1 = u[20] ^ u[28];
53 const s2 = u[12] ^ s0;
54 const s3 = u[0] ^ u[8];
55 const q0 = s1 ^ s2;
56 const s4 = u[0] ^ u[24];
57 const s5 = u[8] ^ u[24];
58 const s6 = u[12] ^ s1;
59 const s7 = u[20] ^ s3;
60 const q1 = s1 ^ s5;
61 const q2 = u[8] ^ q0;
62 const q3 = s4 ^ s2;
63 const q4 = s3 ^ q0;
64 const s8 = u[16] ^ s3;
65 const q5 = s6 ^ s8;
66 const q6 = u[8] ^ u[12];
67 const q7 = u[24] ^ s2;
68 const s9 = u[24] ^ s0;
69 const q8 = s3 ^ s9;
70 const q9 = s4 ^ s6;
71 const q10 = s0 ^ s5;
72 const q12 = u[28] ^ s2;
73 const q13 = u[4] ^ s7;
74 const q14 = u[28] ^ s3;
75 const q15 = s2 ^ s7;
76 const q16 = u[4] ^ s1;
77 const q17 = u[4] ^ u[28];
78 const q11 = u[20];
79
80 const t20 = q6 & q12;
81 const t21 = q3 & q14;
82 const t22 = q1 & q16;
83 const t23 = q2 & q17;
84 const x0 = ((q3 | q14) ^ (q0 & q7)) ^ (t20 ^ t22);
85 const x1 = ((q4 | q13) ^ (q10 & q11)) ^ (t21 ^ t20);
86 const x2 = ((q2 | q17) ^ (q5 & q9)) ^ (t21 ^ t22);
87 const x3 = ((q8 | q15) ^ t23) ^ (t21 ^ (q4 & q13));
88
89 const a = x1 & ~x3;
90 const b = x0 & ~x3;
91 const c = x3 & ~x1;
92 const d = x2 & ~x1;
93 const e = x0 ^ a;
94 const y0 = x3 ^ (x2 & ~e);
95 const f = x1 ^ b;
96 const y1 = c ^ (x2 & f);
97 const g = x2 ^ c;
98 const y2 = x1 ^ (x0 & ~g);
99 const h = x3 ^ d;
100 const y3 = a ^ (x0 & h);
101 const y02 = y2 ^ y0;
102 const y13 = y3 ^ y1;
103 const y23 = y3 ^ y2;
104 const y01 = y1 ^ y0;
105 const y00 = y02 ^ y13;
106
107 const a0 = y01 & q11;
108 const a1 = y0 & q12;
109 const a2 = y1 & q0;
110 const a3 = y23 & q17;
111 const a4 = y2 & q5;
112 const a5 = y3 & q15;
113 const a6 = y13 & q14;
114 const a7 = y00 & q16;
115 const a8 = y02 & q13;
116 const a9 = y01 & q7;
117 const a10 = y0 & q10;
118 const a11 = y1 & q6;
119 const a12 = y23 & q2;
120 const a13 = y2 & q9;
121 const a14 = y3 & q8;
122 const a15 = y13 & q3;
123 const a16 = y00 & q1;
124 const a17 = y02 & q4;
125
126 const r0 = a1 ^ a5;
127 const r1 = a9 ^ a15;
128 const r2 = a4 ^ r0;
129 const r3 = a2 ^ a10;
130 const r4 = a11 ^ a17;
131 const r5 = a8 ^ r1;
132 const r6 = a0 ^ a16;
133 const r7 = a7 ^ a13;
134 const r8 = a11 ^ a14;
135 const r9 = r3 ^ r4;
136 const r10 = r5 ^ r6;
137 const r11 = r2 ^ r9;
138 const r12 = a3 ^ r0;
139 const r13 = r7 ^ r8;
140 const r14 = r12 ^ r13;
141 u[0] = r10 ^ r14;
142 const r15 = a6 ^ a10;
143 const r16 = r15 ^ r2;
144 u[4] = ~(r10 ^ r16);
145 u[8] = ~(a2 ^ r2);
146 const r17 = a12 ^ a13;
147 const r18 = a15 ^ r17;
148 u[12] = r18 ^ r11;
149 const r19 = a1 ^ a14;
150 const r20 = a17 ^ r3;
151 const r21 = r7 ^ r19;
152 const r22 = r5 ^ r20;
153 u[16] = r21 ^ r22;
154 const r23 = a9 ^ a12;
155 u[20] = r8 ^ r23;
156 u[24] = ~(r1 ^ r4);
157 u[28] = ~(a16 ^ r11);
158 }
159
160 fn subBytes(st: *Bitsliced) void {
161 sbox(st[0..]);
162 sbox(st[1..]);
163 sbox(st[2..]);
164 sbox(st[3..]);
165 }
166
167 fn shiftRows(st: *Bitsliced) void {
168 var i: usize = 0;
169 while (i < 32) : (i += 4) {
170 st[i + 1] = laneRotl(st[i + 1], 24);
171 st[i + 2] = laneRotl(st[i + 2], 16);
172 st[i + 3] = laneRotl(st[i + 3], 8);
173 }
174 }
175
176 fn mixColumns(st: *Bitsliced) void {
177 const t2_0 = st[0] ^ st[1];
178 const t2_1 = st[1] ^ st[2];
179 const t2_2 = st[2] ^ st[3];
180 const t2_3 = st[3] ^ st[0];
181 var t0_0 = st[28] ^ st[29];
182 var t0_1 = st[29] ^ st[30];
183 var t0_2 = st[30] ^ st[31];
184 var t0_3 = st[31] ^ st[28];
185 var t = st[28];
186 st[28] = t2_0 ^ t0_2 ^ st[29];
187 st[29] = t2_1 ^ t0_2 ^ t;
188 t = st[30];
189 st[30] = t2_2 ^ t0_0 ^ st[31];
190 st[31] = t2_3 ^ t0_0 ^ t;
191 var t1_0 = st[24] ^ st[25];
192 var t1_1 = st[25] ^ st[26];
193 var t1_2 = st[26] ^ st[27];
194 var t1_3 = st[27] ^ st[24];
195 t = st[24];
196 st[24] = t0_0 ^ t2_0 ^ st[25] ^ t1_2;
197 var t_bis = st[25];
198 st[25] = t0_1 ^ t2_1 ^ t1_2 ^ t;
199 t = st[26];
200 st[26] = t0_2 ^ t2_2 ^ t1_3 ^ t_bis;
201 st[27] = t0_3 ^ t2_3 ^ t1_0 ^ t;
202 t0_0 = st[20] ^ st[21];
203 t0_1 = st[21] ^ st[22];
204 t0_2 = st[22] ^ st[23];
205 t0_3 = st[23] ^ st[20];
206 t = st[20];
207 st[20] = t1_0 ^ t0_1 ^ st[23];
208 t_bis = st[21];
209 st[21] = t1_1 ^ t0_2 ^ t;
210 t = st[22];
211 st[22] = t1_2 ^ t0_3 ^ t_bis;
212 st[23] = t1_3 ^ t0_0 ^ t;
213 t1_0 = st[16] ^ st[17];
214 t1_1 = st[17] ^ st[18];
215 t1_2 = st[18] ^ st[19];
216 t1_3 = st[19] ^ st[16];
217 t = st[16];
218 st[16] = t0_0 ^ t2_0 ^ t1_1 ^ st[19];
219 t_bis = st[17];
220 st[17] = t0_1 ^ t2_1 ^ t1_2 ^ t;
221 t = st[18];
222 st[18] = t0_2 ^ t2_2 ^ t1_3 ^ t_bis;
223 st[19] = t0_3 ^ t2_3 ^ t1_0 ^ t;
224 t0_0 = st[12] ^ st[13];
225 t0_1 = st[13] ^ st[14];
226 t0_2 = st[14] ^ st[15];
227 t0_3 = st[15] ^ st[12];
228 t = st[12];
229 st[12] = t1_0 ^ t2_0 ^ t0_1 ^ st[15];
230 t_bis = st[13];
231 st[13] = t1_1 ^ t2_1 ^ t0_2 ^ t;
232 t = st[14];
233 st[14] = t1_2 ^ t2_2 ^ t0_3 ^ t_bis;
234 st[15] = t1_3 ^ t2_3 ^ t0_0 ^ t;
235 t1_0 = st[8] ^ st[9];
236 t1_1 = st[9] ^ st[10];
237 t1_2 = st[10] ^ st[11];
238 t1_3 = st[11] ^ st[8];
239 t = st[8];
240 st[8] = t0_0 ^ t1_1 ^ st[11];
241 t_bis = st[9];
242 st[9] = t0_1 ^ t1_2 ^ t;
243 t = st[10];
244 st[10] = t0_2 ^ t1_3 ^ t_bis;
245 st[11] = t0_3 ^ t1_0 ^ t;
246 t0_0 = st[4] ^ st[5];
247 t0_1 = st[5] ^ st[6];
248 t0_2 = st[6] ^ st[7];
249 t0_3 = st[7] ^ st[4];
250 t = st[4];
251 st[4] = t1_0 ^ t0_1 ^ st[7];
252 t_bis = st[5];
253 st[5] = t1_1 ^ t0_2 ^ t;
254 t = st[6];
255 st[6] = t1_2 ^ t0_3 ^ t_bis;
256 st[7] = t1_3 ^ t0_0 ^ t;
257 t = st[0];
258 st[0] = t0_0 ^ t2_1 ^ st[3];
259 t_bis = st[1];
260 st[1] = t0_1 ^ t2_2 ^ t;
261 t = st[2];
262 st[2] = t0_2 ^ t2_3 ^ t_bis;
263 st[3] = t0_3 ^ t2_0 ^ t;
264 }
265
266 /// One keyless AES round (SubBytes, ShiftRows, MixColumns) over the whole batch
267 pub fn round(st: *Bitsliced) void {
268 subBytes(st);
269 shiftRows(st);
270 mixColumns(st);
271 }
272
273 /// Add a round key, which must also be in bitsliced form
274 pub fn addRoundKey(st: *Bitsliced, round_key: *const Bitsliced) void {
275 for (0..32) |i| st[i] ^= round_key[i];
276 }
277
278 /// Orthogonalization (converts blocks-in-words to bit slices)
279 pub fn packState(st: *Bitsliced) void {
280 var i: usize = 0;
281 while (i < 32) : (i += 4) {
282 swapMove(&st[i], &st[i + 1], 0x00ff00ff, 8);
283 swapMove(&st[i + 2], &st[i + 3], 0x00ff00ff, 8);
284 swapMove(&st[i], &st[i + 2], 0x0000ffff, 16);
285 swapMove(&st[i + 1], &st[i + 3], 0x0000ffff, 16);
286 }
287 for (0..4) |k| {
288 swapMove(&st[k + 4], &st[k], 0x55555555, 1);
289 swapMove(&st[k + 12], &st[k + 8], 0x55555555, 1);
290 swapMove(&st[k + 20], &st[k + 16], 0x55555555, 1);
291 swapMove(&st[k + 28], &st[k + 24], 0x55555555, 1);
292 swapMove(&st[k + 8], &st[k], 0x33333333, 2);
293 swapMove(&st[k + 12], &st[k + 4], 0x33333333, 2);
294 swapMove(&st[k + 24], &st[k + 16], 0x33333333, 2);
295 swapMove(&st[k + 28], &st[k + 20], 0x33333333, 2);
296 swapMove(&st[k + 16], &st[k], 0x0f0f0f0f, 4);
297 swapMove(&st[k + 20], &st[k + 4], 0x0f0f0f0f, 4);
298 swapMove(&st[k + 24], &st[k + 8], 0x0f0f0f0f, 4);
299 swapMove(&st[k + 28], &st[k + 12], 0x0f0f0f0f, 4);
300 }
301 }
302
303 /// Inverse of `packState`
304 pub fn unpackState(st: *Bitsliced) void {
305 for (0..4) |k| {
306 swapMove(&st[k + 4], &st[k], 0x55555555, 1);
307 swapMove(&st[k + 12], &st[k + 8], 0x55555555, 1);
308 swapMove(&st[k + 20], &st[k + 16], 0x55555555, 1);
309 swapMove(&st[k + 28], &st[k + 24], 0x55555555, 1);
310 swapMove(&st[k + 8], &st[k], 0x33333333, 2);
311 swapMove(&st[k + 12], &st[k + 4], 0x33333333, 2);
312 swapMove(&st[k + 24], &st[k + 16], 0x33333333, 2);
313 swapMove(&st[k + 28], &st[k + 20], 0x33333333, 2);
314 swapMove(&st[k + 16], &st[k], 0x0f0f0f0f, 4);
315 swapMove(&st[k + 20], &st[k + 4], 0x0f0f0f0f, 4);
316 swapMove(&st[k + 24], &st[k + 8], 0x0f0f0f0f, 4);
317 swapMove(&st[k + 28], &st[k + 12], 0x0f0f0f0f, 4);
318 }
319 var i: usize = 0;
320 while (i < 32) : (i += 4) {
321 swapMove(&st[i], &st[i + 2], 0x0000ffff, 16);
322 swapMove(&st[i + 1], &st[i + 3], 0x0000ffff, 16);
323 swapMove(&st[i], &st[i + 1], 0x00ff00ff, 8);
324 swapMove(&st[i + 2], &st[i + 3], 0x00ff00ff, 8);
325 }
326 }
327
328 /// Convert `nblocks` little-endian blocks into the packed state.
329 /// Blocks`8*g .. 8*g+7` land in lane `g` of each word.
330 fn pack(bytes: *const [nblocks * 16]u8) Bitsliced {
331 var st: Bitsliced = @splat(0);
332 inline for (0..nblocks) |bl| {
333 const g = bl / 8;
334 const p = bl % 8;
335 inline for (0..4) |wd| {
336 const v = mem.readInt(u32, bytes[bl * 16 + wd * 4 ..][0..4], .little);
337 st[p * 4 + wd] |= @as(Word, v) << (32 * g);
338 }
339 }
340 packState(&st);
341 return st;
342 }
343
344 /// Inverse of `pack`
345 fn unpack(st: *Bitsliced) [nblocks * 16]u8 {
346 unpackState(st);
347 var bytes: [nblocks * 16]u8 = undefined;
348 inline for (0..nblocks) |bl| {
349 const g = bl / 8;
350 const p = bl % 8;
351 inline for (0..4) |wd| {
352 const v: u32 = @truncate(st[p * 4 + wd] >> (32 * g));
353 mem.writeInt(u32, bytes[bl * 16 + wd * 4 ..][0..4], v, .little);
354 }
355 }
356 return bytes;
357 }
358
359 /// Precompute where key bit `i` of a round key lands after packing
360 const broadcast_loc = blk: {
361 @setEvalBranchQuota(200000);
362 var loc: [128]struct { word: u8, shift: u5 } = undefined;
363 for (0..128) |i| {
364 var st: Bitsliced = @splat(0);
365 const word = (i / 8) / 4;
366 const bit: u5 = @intCast(((i / 8) % 4) * 8 + (i % 8));
367 for (0..8) |b| st[b * 4 + word] |= @as(Word, 1) << bit;
368 packState(&st);
369 found: for (0..32) |w| {
370 for (0..4) |byte| {
371 if ((st[w] >> @intCast(byte * 8)) & 0xff == 0xff) {
372 loc[i] = .{ .word = @intCast(w), .shift = @intCast(byte * 8) };
373 break :found;
374 }
375 }
376 }
377 }
378 break :blk loc;
379 };
380
381 /// Pack a round key and broadcast it to all blocks, so it can be directly XORed into a packed state
382 fn packRoundKey(round_key: *const [16]u8) Bitsliced {
383 var st: Bitsliced = @splat(0);
384 inline for (0..16) |pos| {
385 const byte = round_key[pos];
386 inline for (0..8) |bit| {
387 const smear: Word = 0 -% @as(Word, (byte >> bit) & 1);
388 const loc = broadcast_loc[pos * 8 + bit];
389 st[loc.word] |= smear & (splatWord(0xff) << loc.shift);
390 }
391 }
392 return st;
393 }
394
395 /// Pack a whole key schedule into the bitsliced representation.
396 pub fn packKeys(round_keys: anytype) [round_keys.len]Bitsliced {
397 var keys: [round_keys.len]Bitsliced = undefined;
398 inline for (&keys, 0..) |*k, r| {
399 const bytes = round_keys[r].toBytes();
400 k.* = packRoundKey(&bytes);
401 }
402 return keys;
403 }
404
405 /// Run the full AES cipher over the whole batch in place
406 pub fn encrypt(comptime rounds: usize, keys: *const [rounds + 1]Bitsliced, data: *[nblocks * 16]u8) void {
407 var st = pack(data);
408 addRoundKey(&st, &keys[0]);
409 inline for (1..rounds) |r| {
410 round(&st);
411 addRoundKey(&st, &keys[r]);
412 }
413 subBytes(&st);
414 shiftRows(&st);
415 addRoundKey(&st, &keys[rounds]);
416 data.* = unpack(&st);
417 }
418
419 fn invShiftRows(st: *Bitsliced) void {
420 var i: usize = 0;
421 while (i < 32) : (i += 4) {
422 st[i + 1] = laneRotl(st[i + 1], 8);
423 st[i + 2] = laneRotl(st[i + 2], 16);
424 st[i + 3] = laneRotl(st[i + 3], 24);
425 }
426 }
427
428 /// Precompute the `mixColumns` inverse matrix
429 const mix_inv: [32]u32 = blk: {
430 @setEvalBranchQuota(200000);
431 var m: [32]u32 = @splat(0);
432 for (0..32) |i| {
433 var st: Bitsliced = @splat(0);
434 st[i] = 1;
435 mixColumns(&st);
436 for (0..32) |o| {
437 if (st[o] & 1 == 1) m[o] |= @as(u32, 1) << @intCast(i);
438 }
439 }
440 var inv: [32]u32 = undefined;
441 for (0..32) |k| inv[k] = @as(u32, 1) << @intCast(k);
442 for (0..32) |col| {
443 var piv = col;
444 while (m[piv] & (@as(u32, 1) << @intCast(col)) == 0) piv += 1;
445 const tm = m[col];
446 m[col] = m[piv];
447 m[piv] = tm;
448 const ti = inv[col];
449 inv[col] = inv[piv];
450 inv[piv] = ti;
451 for (0..32) |r| {
452 if (r != col and (m[r] & (@as(u32, 1) << @intCast(col))) != 0) {
453 m[r] ^= m[col];
454 inv[r] ^= inv[col];
455 }
456 }
457 }
458 break :blk inv;
459 };
460
461 fn invMixColumns(st: *Bitsliced) void {
462 @setEvalBranchQuota(20000);
463 var out: Bitsliced = @splat(0);
464 inline for (0..32) |o| {
465 inline for (0..32) |j| {
466 if (mix_inv[o] & (@as(u32, 1) << @intCast(j)) != 0) out[o] ^= st[j];
467 }
468 }
469 st.* = out;
470 }
471
472 /// Linear map for the inverse S-box
473 fn invAffineGroup(st: *Bitsliced, comptime g: usize) void {
474 const p = [8]usize{ g + 28, g + 24, g + 20, g + 16, g + 12, g + 8, g + 4, g };
475 const a0 = ~st[p[0]];
476 const a1 = ~st[p[1]];
477 const a2 = st[p[2]];
478 const a3 = st[p[3]];
479 const a4 = st[p[4]];
480 const a5 = ~st[p[5]];
481 const a6 = ~st[p[6]];
482 const a7 = st[p[7]];
483 st[p[7]] = a1 ^ a4 ^ a6;
484 st[p[6]] = a0 ^ a3 ^ a5;
485 st[p[5]] = a7 ^ a2 ^ a4;
486 st[p[4]] = a6 ^ a1 ^ a3;
487 st[p[3]] = a5 ^ a0 ^ a2;
488 st[p[2]] = a4 ^ a7 ^ a1;
489 st[p[1]] = a3 ^ a6 ^ a0;
490 st[p[0]] = a2 ^ a5 ^ a7;
491 }
492
493 /// Compact inverse S-box: we apply a fixed linear map, the forward S-box, then the same linear map
494 /// Trick from Thomas Pornin
495 fn invSubBytes(st: *Bitsliced) void {
496 inline for (0..4) |g| invAffineGroup(st, g);
497 subBytes(st);
498 inline for (0..4) |g| invAffineGroup(st, g);
499 }
500
501 /// Full inverse AES cipher over the whole batch in bitsliced representation
502 pub fn decrypt(comptime rounds: usize, keys: *const [rounds + 1]Bitsliced, data: *[nblocks * 16]u8) void {
503 var st = pack(data);
504 addRoundKey(&st, &keys[rounds]);
505 comptime var r: usize = rounds - 1;
506 inline while (r > 0) : (r -= 1) {
507 invShiftRows(&st);
508 invSubBytes(&st);
509 addRoundKey(&st, &keys[r]);
510 invMixColumns(&st);
511 }
512 invShiftRows(&st);
513 invSubBytes(&st);
514 addRoundKey(&st, &keys[0]);
515 data.* = unpack(&st);
516 }
517 };
518}
519
520const testing = std.testing;
521
522test "bitslice cross-check against table AES" {
523 const key: [16]u8 = @splat(0x37);
524 const ref = std.crypto.core.aes.Aes128.initEnc(key);
525 const keys = Wide.packKeys(ref.key_schedule.round_keys[0..]);
526
527 var data: [width * 16]u8 = undefined;
528 for (0..width * 16) |i| data[i] = @truncate(i *% 101 +% 7);
529
530 var expected: [width * 16]u8 = undefined;
531 for (0..width) |b| {
532 var blk: [16]u8 = data[b * 16 ..][0..16].*;
533 ref.encrypt(&blk, &blk);
534 expected[b * 16 ..][0..16].* = blk;
535 }
536
537 Wide.encrypt(10, &keys, &data);
538 try testing.expectEqualSlices(u8, &expected, &data);
539}
540
541test "bitslice decrypt cross-check against table AES" {
542 inline for (.{ std.crypto.core.aes.Aes128, std.crypto.core.aes.Aes256 }) |Aes| {
543 const key: [Aes.key_bits / 8]u8 = @splat(0x4d);
544 const enc = Aes.initEnc(key);
545 const dec = Aes.initDec(key);
546 const keys = Wide.packKeys(enc.key_schedule.round_keys[0..]);
547
548 var data: [width * 16]u8 = undefined;
549 for (0..width * 16) |i| data[i] = @truncate(i *% 211 +% 13);
550
551 var expected: [width * 16]u8 = undefined;
552 for (0..width) |b| {
553 var blk: [16]u8 = data[b * 16 ..][0..16].*;
554 dec.decrypt(&blk, &blk);
555 expected[b * 16 ..][0..16].* = blk;
556 }
557
558 Wide.decrypt(Aes.rounds, &keys, &data);
559 try testing.expectEqualSlices(u8, &expected, &data);
560 }
561}
lib/std/crypto/aes/soft.zig+74-59
...@@ -1,9 +1,12 @@...@@ -1,9 +1,12 @@
1const std = @import("../../std.zig");1const std = @import("../../std.zig");
2const math = std.math;2const math = std.math;
3const mem = std.mem;3const mem = std.mem;
4const bitsliced = @import("bitsliced.zig");
45
5const side_channels_mitigations = std.options.side_channels_mitigations;6const side_channels_mitigations = std.options.side_channels_mitigations;
67
8const protected = side_channels_mitigations != .none;
9
7/// A single AES block.10/// A single AES block.
8pub const Block = struct {11pub const Block = struct {
9 const Repr = [4]u32;12 const Repr = [4]u32;
...@@ -288,7 +291,7 @@ pub const Block = struct {...@@ -288,7 +291,7 @@ pub const Block = struct {
288 /// Perform operations on multiple blocks in parallel.291 /// Perform operations on multiple blocks in parallel.
289 pub const parallel = struct {292 pub const parallel = struct {
290 /// The recommended number of AES encryption/decryption to perform in parallel for the chosen implementation.293 /// The recommended number of AES encryption/decryption to perform in parallel for the chosen implementation.
291 pub const optimal_parallel_blocks = 1;294 pub const optimal_parallel_blocks = if (protected) bitsliced.width else 1;
292295
293 /// Encrypt multiple blocks in parallel, each their own round key.296 /// Encrypt multiple blocks in parallel, each their own round key.
294 pub fn encryptParallel(comptime count: usize, blocks: [count]Block, round_keys: [count]Block) [count]Block {297 pub fn encryptParallel(comptime count: usize, blocks: [count]Block, round_keys: [count]Block) [count]Block {
...@@ -539,6 +542,38 @@ fn KeySchedule(comptime Aes: type) type {...@@ -539,6 +542,38 @@ fn KeySchedule(comptime Aes: type) type {
539 };542 };
540}543}
541544
545/// Packed forward round keys for the bitsliced implementation
546fn PackedKeys(comptime rounds: usize) type {
547 return if (protected) [rounds + 1]bitsliced.State else void;
548}
549
550/// Bitsliced operation over `count` blocks
551fn bitslicedOperation(
552 comptime rounds: usize,
553 comptime operation: enum { encrypt, decrypt },
554 keys: *const [rounds + 1]bitsliced.State,
555 comptime count: usize,
556 dst: *[16 * count]u8,
557 src: *const [16 * count]u8,
558) void {
559 const w = bitsliced.width;
560 const chunks = (count + w - 1) / w;
561 inline for (0..chunks) |c| {
562 const off: usize = c * w;
563 const n: usize = @min(w, count - off);
564 const lo = off * 16;
565 const hi = lo + n * 16;
566 var buf: [w * 16]u8 = undefined;
567 @memcpy(buf[0 .. n * 16], src[lo..hi]);
568 if (operation == .decrypt) {
569 bitsliced.Wide.decrypt(rounds, keys, &buf);
570 } else {
571 bitsliced.Wide.encrypt(rounds, keys, &buf);
572 }
573 @memcpy(dst[lo..hi], buf[0 .. n * 16]);
574 }
575}
576
542/// A context to perform encryption using the standard AES key schedule.577/// A context to perform encryption using the standard AES key schedule.
543pub fn AesEncryptCtx(comptime Aes: type) type {578pub fn AesEncryptCtx(comptime Aes: type) type {
544 std.debug.assert(Aes.key_bits == 128 or Aes.key_bits == 256);579 std.debug.assert(Aes.key_bits == 128 or Aes.key_bits == 256);
...@@ -549,73 +584,54 @@ pub fn AesEncryptCtx(comptime Aes: type) type {...@@ -549,73 +584,54 @@ pub fn AesEncryptCtx(comptime Aes: type) type {
549 pub const block = Aes.block;584 pub const block = Aes.block;
550 pub const block_length = block.block_length;585 pub const block_length = block.block_length;
551 key_schedule: KeySchedule(Aes),586 key_schedule: KeySchedule(Aes),
587 packed_keys: PackedKeys(rounds),
552588
553 /// Create a new encryption context with the given key.589 /// Create a new encryption context with the given key.
554 pub fn init(key: [Aes.key_bits / 8]u8) Self {590 pub fn init(key: [Aes.key_bits / 8]u8) Self {
555 const key_schedule = KeySchedule(Aes).expandKey(key);591 const key_schedule = KeySchedule(Aes).expandKey(key);
556 return Self{592 return Self{
557 .key_schedule = key_schedule,593 .key_schedule = key_schedule,
594 .packed_keys = if (protected) bitsliced.Wide.packKeys(key_schedule.round_keys[0..]) else {},
558 };595 };
559 }596 }
560597
561 /// Encrypt a single block.598 /// Encrypt a single block, without any protection against side channels.
562 pub fn encrypt(ctx: Self, dst: *[16]u8, src: *const [16]u8) void {599 fn encryptUnprotectedBlock(ctx: Self, dst: *[16]u8, src: *const [16]u8) void {
563 const round_keys = ctx.key_schedule.round_keys;600 const round_keys = ctx.key_schedule.round_keys;
564 var t = Block.fromBytes(src).xorBlocks(round_keys[0]);601 var t = Block.fromBytes(src).xorBlocks(round_keys[0]);
565 comptime var i = 1;602 inline for (1..rounds) |i| {
566 if (side_channels_mitigations == .full) {603 t = t.encryptUnprotected(round_keys[i]);
567 inline while (i < rounds) : (i += 1) {
568 t = t.encrypt(round_keys[i]);
569 }
570 } else {
571 inline while (i < 5) : (i += 1) {
572 t = t.encrypt(round_keys[i]);
573 }
574 inline while (i < rounds - 1) : (i += 1) {
575 t = t.encryptUnprotected(round_keys[i]);
576 }
577 t = t.encrypt(round_keys[i]);
578 }604 }
579 t = t.encryptLast(round_keys[rounds]);605 t = t.encryptLast(round_keys[rounds]);
580 dst.* = t.toBytes();606 dst.* = t.toBytes();
581 }607 }
582608
609 /// Encrypt a single block.
610 pub fn encrypt(ctx: Self, dst: *[16]u8, src: *const [16]u8) void {
611 ctx.encryptWide(1, dst, src);
612 }
613
583 /// Encrypt+XOR a single block.614 /// Encrypt+XOR a single block.
584 pub fn xor(ctx: Self, dst: *[16]u8, src: *const [16]u8, counter: [16]u8) void {615 pub fn xor(ctx: Self, dst: *[16]u8, src: *const [16]u8, counter: [16]u8) void {
585 const round_keys = ctx.key_schedule.round_keys;616 ctx.xorWide(1, dst, src, counter);
586 var t = Block.fromBytes(&counter).xorBlocks(round_keys[0]);
587 comptime var i = 1;
588 if (side_channels_mitigations == .full) {
589 inline while (i < rounds) : (i += 1) {
590 t = t.encrypt(round_keys[i]);
591 }
592 } else {
593 inline while (i < 5) : (i += 1) {
594 t = t.encrypt(round_keys[i]);
595 }
596 inline while (i < rounds - 1) : (i += 1) {
597 t = t.encryptUnprotected(round_keys[i]);
598 }
599 t = t.encrypt(round_keys[i]);
600 }
601 t = t.encryptLast(round_keys[rounds]);
602 dst.* = t.xorBytes(src);
603 }617 }
604618
605 /// Encrypt multiple blocks, possibly leveraging parallelization.619 /// Encrypt multiple blocks, possibly leveraging parallelization.
606 pub fn encryptWide(ctx: Self, comptime count: usize, dst: *[16 * count]u8, src: *const [16 * count]u8) void {620 pub fn encryptWide(ctx: Self, comptime count: usize, dst: *[16 * count]u8, src: *const [16 * count]u8) void {
607 var i: usize = 0;621 if (protected) {
608 while (i < count) : (i += 1) {622 bitslicedOperation(rounds, .encrypt, &ctx.packed_keys, count, dst, src);
609 ctx.encrypt(dst[16 * i .. 16 * i + 16][0..16], src[16 * i .. 16 * i + 16][0..16]);623 } else {
624 for (0..count) |i| {
625 ctx.encryptUnprotectedBlock(dst[16 * i .. 16 * i + 16][0..16], src[16 * i .. 16 * i + 16][0..16]);
626 }
610 }627 }
611 }628 }
612629
613 /// Encrypt+XOR multiple blocks, possibly leveraging parallelization.630 /// Encrypt+XOR multiple blocks, possibly leveraging parallelization.
614 pub fn xorWide(ctx: Self, comptime count: usize, dst: *[16 * count]u8, src: *const [16 * count]u8, counters: [16 * count]u8) void {631 pub fn xorWide(ctx: Self, comptime count: usize, dst: *[16 * count]u8, src: *const [16 * count]u8, counters: [16 * count]u8) void {
615 var i: usize = 0;632 var ks: [16 * count]u8 = undefined;
616 while (i < count) : (i += 1) {633 ctx.encryptWide(count, &ks, &counters);
617 ctx.xor(dst[16 * i .. 16 * i + 16][0..16], src[16 * i .. 16 * i + 16][0..16], counters[16 * i .. 16 * i + 16][0..16].*);634 for (dst, ks, src) |*d, k, s| d.* = k ^ s;
618 }
619 }635 }
620 };636 };
621}637}
...@@ -630,11 +646,13 @@ pub fn AesDecryptCtx(comptime Aes: type) type {...@@ -630,11 +646,13 @@ pub fn AesDecryptCtx(comptime Aes: type) type {
630 pub const block = Aes.block;646 pub const block = Aes.block;
631 pub const block_length = block.block_length;647 pub const block_length = block.block_length;
632 key_schedule: KeySchedule(Aes),648 key_schedule: KeySchedule(Aes),
649 packed_keys: PackedKeys(rounds),
633650
634 /// Create a decryption context from an existing encryption context.651 /// Create a decryption context from an existing encryption context.
635 pub fn initFromEnc(ctx: AesEncryptCtx(Aes)) Self {652 pub fn initFromEnc(ctx: AesEncryptCtx(Aes)) Self {
636 return Self{653 return Self{
637 .key_schedule = ctx.key_schedule.invert(),654 .key_schedule = ctx.key_schedule.invert(),
655 .packed_keys = ctx.packed_keys,
638 };656 };
639 }657 }
640658
...@@ -644,33 +662,30 @@ pub fn AesDecryptCtx(comptime Aes: type) type {...@@ -644,33 +662,30 @@ pub fn AesDecryptCtx(comptime Aes: type) type {
644 return initFromEnc(enc_ctx);662 return initFromEnc(enc_ctx);
645 }663 }
646664
647 /// Decrypt a single block.665 /// Decrypt a single block, without any protection against side channels.
648 pub fn decrypt(ctx: Self, dst: *[16]u8, src: *const [16]u8) void {666 fn decryptUnprotectedBlock(ctx: Self, dst: *[16]u8, src: *const [16]u8) void {
649 const inv_round_keys = ctx.key_schedule.round_keys;667 const inv_round_keys = ctx.key_schedule.round_keys;
650 var t = Block.fromBytes(src).xorBlocks(inv_round_keys[0]);668 var t = Block.fromBytes(src).xorBlocks(inv_round_keys[0]);
651 comptime var i = 1;669 inline for (1..rounds) |i| {
652 if (side_channels_mitigations == .full) {670 t = t.decryptUnprotected(inv_round_keys[i]);
653 inline while (i < rounds) : (i += 1) {
654 t = t.decrypt(inv_round_keys[i]);
655 }
656 } else {
657 inline while (i < 5) : (i += 1) {
658 t = t.decrypt(inv_round_keys[i]);
659 }
660 inline while (i < rounds - 1) : (i += 1) {
661 t = t.decryptUnprotected(inv_round_keys[i]);
662 }
663 t = t.decrypt(inv_round_keys[i]);
664 }671 }
665 t = t.decryptLast(inv_round_keys[rounds]);672 t = t.decryptLast(inv_round_keys[rounds]);
666 dst.* = t.toBytes();673 dst.* = t.toBytes();
667 }674 }
668675
676 /// Decrypt a single block.
677 pub fn decrypt(ctx: Self, dst: *[16]u8, src: *const [16]u8) void {
678 ctx.decryptWide(1, dst, src);
679 }
680
669 /// Decrypt multiple blocks, possibly leveraging parallelization.681 /// Decrypt multiple blocks, possibly leveraging parallelization.
670 pub fn decryptWide(ctx: Self, comptime count: usize, dst: *[16 * count]u8, src: *const [16 * count]u8) void {682 pub fn decryptWide(ctx: Self, comptime count: usize, dst: *[16 * count]u8, src: *const [16 * count]u8) void {
671 var i: usize = 0;683 if (protected) {
672 while (i < count) : (i += 1) {684 bitslicedOperation(rounds, .decrypt, &ctx.packed_keys, count, dst, src);
673 ctx.decrypt(dst[16 * i .. 16 * i + 16][0..16], src[16 * i .. 16 * i + 16][0..16]);685 } else {
686 for (0..count) |i| {
687 ctx.decryptUnprotectedBlock(dst[16 * i .. 16 * i + 16][0..16], src[16 * i .. 16 * i + 16][0..16]);
688 }
674 }689 }
675 }690 }
676 };691 };
lib/std/crypto/aes_ocb.zig+6-3
...@@ -103,7 +103,10 @@ fn AesOcb(comptime Aes: anytype) type {...@@ -103,7 +103,10 @@ fn AesOcb(comptime Aes: anytype) type {
103103
104 const has_aesni = builtin.cpu.has(.x86, .aes);104 const has_aesni = builtin.cpu.has(.x86, .aes);
105 const has_armaes = builtin.cpu.has(.aarch64, .aes);105 const has_armaes = builtin.cpu.has(.aarch64, .aes);
106 const wb: usize = if ((builtin.cpu.arch == .x86_64 and has_aesni) or (builtin.cpu.arch == .aarch64 and has_armaes)) 4 else 0;106 const wb: usize = if ((builtin.cpu.arch == .x86_64 and has_aesni) or (builtin.cpu.arch == .aarch64 and has_armaes))
107 4 // empirically what works best on x86_64 and aarch64 with AES-NI/ARM Crypto
108 else
109 Aes.block.parallel.optimal_parallel_blocks;
107110
108 /// c: ciphertext: output buffer should be of size m.len111 /// c: ciphertext: output buffer should be of size m.len
109 /// tag: authentication tag: output MAC112 /// tag: authentication tag: output MAC
...@@ -124,7 +127,7 @@ fn AesOcb(comptime Aes: anytype) type {...@@ -124,7 +127,7 @@ fn AesOcb(comptime Aes: anytype) type {
124 var sum: [16]u8 = @splat(0);127 var sum: [16]u8 = @splat(0);
125 var i: usize = 0;128 var i: usize = 0;
126129
127 while (wb > 0 and i + wb <= full_blocks) : (i += wb) {130 while (i + wb <= full_blocks) : (i += wb) {
128 var offsets: [wb]Block align(16) = undefined;131 var offsets: [wb]Block align(16) = undefined;
129 var es: [16 * wb]u8 align(16) = undefined;132 var es: [16 * wb]u8 align(16) = undefined;
130 var j: usize = 0;133 var j: usize = 0;
...@@ -191,7 +194,7 @@ fn AesOcb(comptime Aes: anytype) type {...@@ -191,7 +194,7 @@ fn AesOcb(comptime Aes: anytype) type {
191 var sum: [16]u8 = @splat(0);194 var sum: [16]u8 = @splat(0);
192 var i: usize = 0;195 var i: usize = 0;
193196
194 while (wb > 0 and i + wb <= full_blocks) : (i += wb) {197 while (i + wb <= full_blocks) : (i += wb) {
195 var offsets: [wb]Block align(16) = undefined;198 var offsets: [wb]Block align(16) = undefined;
196 var es: [16 * wb]u8 align(16) = undefined;199 var es: [16 * wb]u8 align(16) = undefined;
197 var j: usize = 0;200 var j: usize = 0;
lib/std/crypto/modes.zig+11-14
...@@ -37,14 +37,13 @@ pub fn ctrSlice(...@@ -37,14 +37,13 @@ pub fn ctrSlice(
37 debug.assert(counter_offset + counter_size <= block_length);37 debug.assert(counter_offset + counter_size <= block_length);
38 debug.assert(counter_size > 0 and counter_size <= block_length);38 debug.assert(counter_size > 0 and counter_size <= block_length);
3939
40 var counterBlock = iv;
41 var i: usize = 0;40 var i: usize = 0;
4241
43 const CounterInt = @Int(.unsigned, counter_size * 8);42 const CounterInt = @Int(.unsigned, counter_size * 8);
4443
45 const parallel_count = BlockCipher.block.parallel.optimal_parallel_blocks;44 const parallel_count = BlockCipher.block.parallel.optimal_parallel_blocks;
46 const wide_block_length = parallel_count * block_length;45 const wide_block_length = parallel_count * block_length;
47 var cnt_val = mem.readInt(CounterInt, counterBlock[counter_offset..][0..counter_size], endian);46 var cnt_val = mem.readInt(CounterInt, iv[counter_offset..][0..counter_size], endian);
48 if (src.len >= wide_block_length) {47 if (src.len >= wide_block_length) {
49 var counters: [parallel_count * block_length]u8 = undefined;48 var counters: [parallel_count * block_length]u8 = undefined;
50 inline for (0..parallel_count) |j| {49 inline for (0..parallel_count) |j| {
...@@ -58,20 +57,18 @@ pub fn ctrSlice(...@@ -58,20 +57,18 @@ pub fn ctrSlice(
58 cnt_val +%= parallel_count;57 cnt_val +%= parallel_count;
59 block_cipher.xorWide(parallel_count, dst[i .. i + wide_block_length][0..wide_block_length], src[i .. i + wide_block_length][0..wide_block_length], counters);58 block_cipher.xorWide(parallel_count, dst[i .. i + wide_block_length][0..wide_block_length], src[i .. i + wide_block_length][0..wide_block_length], counters);
60 }59 }
61 mem.writeInt(CounterInt, counterBlock[counter_offset..][0..counter_size], cnt_val, endian);
62 }
63 while (i + block_length <= src.len) : (i += block_length) {
64 block_cipher.xor(dst[i .. i + block_length][0..block_length], src[i .. i + block_length][0..block_length], counterBlock);
65 cnt_val +%= 1;
66 mem.writeInt(CounterInt, counterBlock[counter_offset..][0..counter_size], cnt_val, endian);
67 }60 }
68 if (i < src.len) {61 if (i < src.len) {
69 var pad: [block_length]u8 = @splat(0);62 const rem = src.len - i;
70 const src_slice = src[i..];63 var counters: [wide_block_length]u8 = undefined;
71 @memcpy(pad[0..src_slice.len], src_slice);64 inline for (0..parallel_count) |j| {
72 block_cipher.xor(&pad, &pad, counterBlock);65 counters[j * block_length ..][0..block_length].* = iv;
73 const pad_slice = pad[0 .. src.len - i];66 const block_ctr = cnt_val +% @as(CounterInt, j);
74 @memcpy(dst[i..][0..pad_slice.len], pad_slice);67 mem.writeInt(CounterInt, counters[j * block_length + counter_offset ..][0..counter_size], block_ctr, endian);
68 }
69 var keystream: [wide_block_length]u8 = undefined;
70 block_cipher.encryptWide(parallel_count, &keystream, &counters);
71 for (dst[i..][0..rem], src[i..][0..rem], keystream[0..rem]) |*d, s, k| d.* = s ^ k;
75 }72 }
76}73}
7774