authorgravatar for 124872+jedisct1@users.noreply.github.comFrank Denis <124872+jedisct1@users.noreply.github.com> 2026-09-23 13:25:56+02:00
committergravatar for 124872+jedisct1@users.noreply.github.comFrank Denis <124872+jedisct1@users.noreply.github.com> 2026-09-23 13:25:56+02:00
log55b12af06a0c8b5bb42b93db69ef632058665acf
tree92116101a05c2e3329dfd71c0ef87066e0e292e3
parent55fe0ad7da5b1b3b946d81a66ade1f0e1797706c
parent4d06435fe3a03227c7b899436efde3ee1bd0dcb9

Merge pull request 'crypto.aes.soft: use a bitsliced representation' (#36110) from jedisct1/zig:bitsliced into master

Reviewed-on: https://codeberg.org/ziglang/zig/pulls/36110

4 files changed, 652 insertions(+), 76 deletions(-)

lib/std/crypto/aes/bitsliced.zig created+561
...@@ -0,0 +1,561 @@
1//! Bitsliced AES round function, for targets without hardware AES support.
2//! It has no data-dependent memory access, so it is constant time.
3
4const std = @import("../../std.zig");
5const builtin = @import("builtin");
6const mem = std.mem;
7
8/// The widest batch of AES blocks the target supports
9pub const Wide = if (builtin.cpu.arch == .wasm32 or builtin.cpu.arch == .wasm64 or @bitSizeOf(usize) >= 64) Batch(2) else Batch(1);
10
11/// Number of AES blocks we can process together
12pub const width = Wide.nblocks;
13
14/// A state holding a whole `width`-block batch
15pub const State = Wide.Bitsliced;
16
17/// A batch of `8 * lanes` blocks, one 32-bit lane per group of eight blocks
18pub fn Batch(comptime lanes: usize) type {
19 return struct {
20 pub const Word = @Int(.unsigned, 32 * lanes);
21 const Shift = std.math.Log2Int(Word);
22
23 const nblocks = 8 * lanes;
24 pub const Bitsliced = [32]Word;
25
26 /// Splat a 32-bit pattern across all lanes of a word
27 pub fn splatWord(x: u32) Word {
28 const t: [lanes]u32 = @splat(x);
29 return @bitCast(t);
30 }
31
32 fn laneRotl(x: Word, comptime n: u5) Word {
33 if (lanes == 1) return std.math.rotl(u32, x, n);
34 const lo_bits = (@as(u32, 1) << n) - 1;
35 const mask_lo = comptime splatWord(lo_bits);
36 const mask_hi = comptime splatWord(~lo_bits);
37 const l: Shift = n;
38 const r: Shift = @intCast(32 - @as(u32, n));
39 return ((x << l) & mask_hi) | ((x >> r) & mask_lo);
40 }
41
42 pub fn swapMove(a: *Word, b: *Word, comptime mask32: u32, comptime n: comptime_int) void {
43 const shift: Shift = n;
44 const mask = comptime splatWord(mask32);
45 const tmp = (b.* ^ (a.* >> shift)) & mask;
46 b.* ^= tmp;
47 a.* ^= tmp << shift;
48 }
49
50 fn sbox(u: []Word) void {
51 const s0 = u[4] ^ u[16];
52 const s1 = u[20] ^ u[28];
53 const s2 = u[12] ^ s0;
54 const s3 = u[0] ^ u[8];
55 const q0 = s1 ^ s2;
56 const s4 = u[0] ^ u[24];
57 const s5 = u[8] ^ u[24];
58 const s6 = u[12] ^ s1;
59 const s7 = u[20] ^ s3;
60 const q1 = s1 ^ s5;
61 const q2 = u[8] ^ q0;
62 const q3 = s4 ^ s2;
63 const q4 = s3 ^ q0;
64 const s8 = u[16] ^ s3;
65 const q5 = s6 ^ s8;
66 const q6 = u[8] ^ u[12];
67 const q7 = u[24] ^ s2;
68 const s9 = u[24] ^ s0;
69 const q8 = s3 ^ s9;
70 const q9 = s4 ^ s6;
71 const q10 = s0 ^ s5;
72 const q12 = u[28] ^ s2;
73 const q13 = u[4] ^ s7;
74 const q14 = u[28] ^ s3;
75 const q15 = s2 ^ s7;
76 const q16 = u[4] ^ s1;
77 const q17 = u[4] ^ u[28];
78 const q11 = u[20];
79
80 const t20 = q6 & q12;
81 const t21 = q3 & q14;
82 const t22 = q1 & q16;
83 const t23 = q2 & q17;
84 const x0 = ((q3 | q14) ^ (q0 & q7)) ^ (t20 ^ t22);
85 const x1 = ((q4 | q13) ^ (q10 & q11)) ^ (t21 ^ t20);
86 const x2 = ((q2 | q17) ^ (q5 & q9)) ^ (t21 ^ t22);
87 const x3 = ((q8 | q15) ^ t23) ^ (t21 ^ (q4 & q13));
88
89 const a = x1 & ~x3;
90 const b = x0 & ~x3;
91 const c = x3 & ~x1;
92 const d = x2 & ~x1;
93 const e = x0 ^ a;
94 const y0 = x3 ^ (x2 & ~e);
95 const f = x1 ^ b;
96 const y1 = c ^ (x2 & f);
97 const g = x2 ^ c;
98 const y2 = x1 ^ (x0 & ~g);
99 const h = x3 ^ d;
100 const y3 = a ^ (x0 & h);
101 const y02 = y2 ^ y0;
102 const y13 = y3 ^ y1;
103 const y23 = y3 ^ y2;
104 const y01 = y1 ^ y0;
105 const y00 = y02 ^ y13;
106
107 const a0 = y01 & q11;
108 const a1 = y0 & q12;
109 const a2 = y1 & q0;
110 const a3 = y23 & q17;
111 const a4 = y2 & q5;
112 const a5 = y3 & q15;
113 const a6 = y13 & q14;
114 const a7 = y00 & q16;
115 const a8 = y02 & q13;
116 const a9 = y01 & q7;
117 const a10 = y0 & q10;
118 const a11 = y1 & q6;
119 const a12 = y23 & q2;
120 const a13 = y2 & q9;
121 const a14 = y3 & q8;
122 const a15 = y13 & q3;
123 const a16 = y00 & q1;
124 const a17 = y02 & q4;
125
126 const r0 = a1 ^ a5;
127 const r1 = a9 ^ a15;
128 const r2 = a4 ^ r0;
129 const r3 = a2 ^ a10;
130 const r4 = a11 ^ a17;
131 const r5 = a8 ^ r1;
132 const r6 = a0 ^ a16;
133 const r7 = a7 ^ a13;
134 const r8 = a11 ^ a14;
135 const r9 = r3 ^ r4;
136 const r10 = r5 ^ r6;
137 const r11 = r2 ^ r9;
138 const r12 = a3 ^ r0;
139 const r13 = r7 ^ r8;
140 const r14 = r12 ^ r13;
141 u[0] = r10 ^ r14;
142 const r15 = a6 ^ a10;
143 const r16 = r15 ^ r2;
144 u[4] = ~(r10 ^ r16);
145 u[8] = ~(a2 ^ r2);
146 const r17 = a12 ^ a13;
147 const r18 = a15 ^ r17;
148 u[12] = r18 ^ r11;
149 const r19 = a1 ^ a14;
150 const r20 = a17 ^ r3;
151 const r21 = r7 ^ r19;
152 const r22 = r5 ^ r20;
153 u[16] = r21 ^ r22;
154 const r23 = a9 ^ a12;
155 u[20] = r8 ^ r23;
156 u[24] = ~(r1 ^ r4);
157 u[28] = ~(a16 ^ r11);
158 }
159
160 fn subBytes(st: *Bitsliced) void {
161 sbox(st[0..]);
162 sbox(st[1..]);
163 sbox(st[2..]);
164 sbox(st[3..]);
165 }
166
167 fn shiftRows(st: *Bitsliced) void {
168 var i: usize = 0;
169 while (i < 32) : (i += 4) {
170 st[i + 1] = laneRotl(st[i + 1], 24);
171 st[i + 2] = laneRotl(st[i + 2], 16);
172 st[i + 3] = laneRotl(st[i + 3], 8);
173 }
174 }
175
176 fn mixColumns(st: *Bitsliced) void {
177 const t2_0 = st[0] ^ st[1];
178 const t2_1 = st[1] ^ st[2];
179 const t2_2 = st[2] ^ st[3];
180 const t2_3 = st[3] ^ st[0];
181 var t0_0 = st[28] ^ st[29];
182 var t0_1 = st[29] ^ st[30];
183 var t0_2 = st[30] ^ st[31];
184 var t0_3 = st[31] ^ st[28];
185 var t = st[28];
186 st[28] = t2_0 ^ t0_2 ^ st[29];
187 st[29] = t2_1 ^ t0_2 ^ t;
188 t = st[30];
189 st[30] = t2_2 ^ t0_0 ^ st[31];
190 st[31] = t2_3 ^ t0_0 ^ t;
191 var t1_0 = st[24] ^ st[25];
192 var t1_1 = st[25] ^ st[26];
193 var t1_2 = st[26] ^ st[27];
194 var t1_3 = st[27] ^ st[24];
195 t = st[24];
196 st[24] = t0_0 ^ t2_0 ^ st[25] ^ t1_2;
197 var t_bis = st[25];
198 st[25] = t0_1 ^ t2_1 ^ t1_2 ^ t;
199 t = st[26];
200 st[26] = t0_2 ^ t2_2 ^ t1_3 ^ t_bis;
201 st[27] = t0_3 ^ t2_3 ^ t1_0 ^ t;
202 t0_0 = st[20] ^ st[21];
203 t0_1 = st[21] ^ st[22];
204 t0_2 = st[22] ^ st[23];
205 t0_3 = st[23] ^ st[20];
206 t = st[20];
207 st[20] = t1_0 ^ t0_1 ^ st[23];
208 t_bis = st[21];
209 st[21] = t1_1 ^ t0_2 ^ t;
210 t = st[22];
211 st[22] = t1_2 ^ t0_3 ^ t_bis;
212 st[23] = t1_3 ^ t0_0 ^ t;
213 t1_0 = st[16] ^ st[17];
214 t1_1 = st[17] ^ st[18];
215 t1_2 = st[18] ^ st[19];
216 t1_3 = st[19] ^ st[16];
217 t = st[16];
218 st[16] = t0_0 ^ t2_0 ^ t1_1 ^ st[19];
219 t_bis = st[17];
220 st[17] = t0_1 ^ t2_1 ^ t1_2 ^ t;
221 t = st[18];
222 st[18] = t0_2 ^ t2_2 ^ t1_3 ^ t_bis;
223 st[19] = t0_3 ^ t2_3 ^ t1_0 ^ t;
224 t0_0 = st[12] ^ st[13];
225 t0_1 = st[13] ^ st[14];
226 t0_2 = st[14] ^ st[15];
227 t0_3 = st[15] ^ st[12];
228 t = st[12];
229 st[12] = t1_0 ^ t2_0 ^ t0_1 ^ st[15];
230 t_bis = st[13];
231 st[13] = t1_1 ^ t2_1 ^ t0_2 ^ t;
232 t = st[14];
233 st[14] = t1_2 ^ t2_2 ^ t0_3 ^ t_bis;
234 st[15] = t1_3 ^ t2_3 ^ t0_0 ^ t;
235 t1_0 = st[8] ^ st[9];
236 t1_1 = st[9] ^ st[10];
237 t1_2 = st[10] ^ st[11];
238 t1_3 = st[11] ^ st[8];
239 t = st[8];
240 st[8] = t0_0 ^ t1_1 ^ st[11];
241 t_bis = st[9];
242 st[9] = t0_1 ^ t1_2 ^ t;
243 t = st[10];
244 st[10] = t0_2 ^ t1_3 ^ t_bis;
245 st[11] = t0_3 ^ t1_0 ^ t;
246 t0_0 = st[4] ^ st[5];
247 t0_1 = st[5] ^ st[6];
248 t0_2 = st[6] ^ st[7];
249 t0_3 = st[7] ^ st[4];
250 t = st[4];
251 st[4] = t1_0 ^ t0_1 ^ st[7];
252 t_bis = st[5];
253 st[5] = t1_1 ^ t0_2 ^ t;
254 t = st[6];
255 st[6] = t1_2 ^ t0_3 ^ t_bis;
256 st[7] = t1_3 ^ t0_0 ^ t;
257 t = st[0];
258 st[0] = t0_0 ^ t2_1 ^ st[3];
259 t_bis = st[1];
260 st[1] = t0_1 ^ t2_2 ^ t;
261 t = st[2];
262 st[2] = t0_2 ^ t2_3 ^ t_bis;
263 st[3] = t0_3 ^ t2_0 ^ t;
264 }
265
266 /// One keyless AES round (SubBytes, ShiftRows, MixColumns) over the whole batch
267 pub fn round(st: *Bitsliced) void {
268 subBytes(st);
269 shiftRows(st);
270 mixColumns(st);
271 }
272
273 /// Add a round key, which must also be in bitsliced form
274 pub fn addRoundKey(st: *Bitsliced, round_key: *const Bitsliced) void {
275 for (0..32) |i| st[i] ^= round_key[i];
276 }
277
278 /// Orthogonalization (converts blocks-in-words to bit slices)
279 pub fn packState(st: *Bitsliced) void {
280 var i: usize = 0;
281 while (i < 32) : (i += 4) {
282 swapMove(&st[i], &st[i + 1], 0x00ff00ff, 8);
283 swapMove(&st[i + 2], &st[i + 3], 0x00ff00ff, 8);
284 swapMove(&st[i], &st[i + 2], 0x0000ffff, 16);
285 swapMove(&st[i + 1], &st[i + 3], 0x0000ffff, 16);
286 }
287 for (0..4) |k| {
288 swapMove(&st[k + 4], &st[k], 0x55555555, 1);
289 swapMove(&st[k + 12], &st[k + 8], 0x55555555, 1);
290 swapMove(&st[k + 20], &st[k + 16], 0x55555555, 1);
291 swapMove(&st[k + 28], &st[k + 24], 0x55555555, 1);
292 swapMove(&st[k + 8], &st[k], 0x33333333, 2);
293 swapMove(&st[k + 12], &st[k + 4], 0x33333333, 2);
294 swapMove(&st[k + 24], &st[k + 16], 0x33333333, 2);
295 swapMove(&st[k + 28], &st[k + 20], 0x33333333, 2);
296 swapMove(&st[k + 16], &st[k], 0x0f0f0f0f, 4);
297 swapMove(&st[k + 20], &st[k + 4], 0x0f0f0f0f, 4);
298 swapMove(&st[k + 24], &st[k + 8], 0x0f0f0f0f, 4);
299 swapMove(&st[k + 28], &st[k + 12], 0x0f0f0f0f, 4);
300 }
301 }
302
303 /// Inverse of `packState`
304 pub fn unpackState(st: *Bitsliced) void {
305 for (0..4) |k| {
306 swapMove(&st[k + 4], &st[k], 0x55555555, 1);
307 swapMove(&st[k + 12], &st[k + 8], 0x55555555, 1);
308 swapMove(&st[k + 20], &st[k + 16], 0x55555555, 1);
309 swapMove(&st[k + 28], &st[k + 24], 0x55555555, 1);
310 swapMove(&st[k + 8], &st[k], 0x33333333, 2);
311 swapMove(&st[k + 12], &st[k + 4], 0x33333333, 2);
312 swapMove(&st[k + 24], &st[k + 16], 0x33333333, 2);
313 swapMove(&st[k + 28], &st[k + 20], 0x33333333, 2);
314 swapMove(&st[k + 16], &st[k], 0x0f0f0f0f, 4);
315 swapMove(&st[k + 20], &st[k + 4], 0x0f0f0f0f, 4);
316 swapMove(&st[k + 24], &st[k + 8], 0x0f0f0f0f, 4);
317 swapMove(&st[k + 28], &st[k + 12], 0x0f0f0f0f, 4);
318 }
319 var i: usize = 0;
320 while (i < 32) : (i += 4) {
321 swapMove(&st[i], &st[i + 2], 0x0000ffff, 16);
322 swapMove(&st[i + 1], &st[i + 3], 0x0000ffff, 16);
323 swapMove(&st[i], &st[i + 1], 0x00ff00ff, 8);
324 swapMove(&st[i + 2], &st[i + 3], 0x00ff00ff, 8);
325 }
326 }
327
328 /// Convert `nblocks` little-endian blocks into the packed state.
329 /// Blocks`8*g .. 8*g+7` land in lane `g` of each word.
330 fn pack(bytes: *const [nblocks * 16]u8) Bitsliced {
331 var st: Bitsliced = @splat(0);
332 inline for (0..nblocks) |bl| {
333 const g = bl / 8;
334 const p = bl % 8;
335 inline for (0..4) |wd| {
336 const v = mem.readInt(u32, bytes[bl * 16 + wd * 4 ..][0..4], .little);
337 st[p * 4 + wd] |= @as(Word, v) << (32 * g);
338 }
339 }
340 packState(&st);
341 return st;
342 }
343
344 /// Inverse of `pack`
345 fn unpack(st: *Bitsliced) [nblocks * 16]u8 {
346 unpackState(st);
347 var bytes: [nblocks * 16]u8 = undefined;
348 inline for (0..nblocks) |bl| {
349 const g = bl / 8;
350 const p = bl % 8;
351 inline for (0..4) |wd| {
352 const v: u32 = @truncate(st[p * 4 + wd] >> (32 * g));
353 mem.writeInt(u32, bytes[bl * 16 + wd * 4 ..][0..4], v, .little);
354 }
355 }
356 return bytes;
357 }
358
359 /// Precompute where key bit `i` of a round key lands after packing
360 const broadcast_loc = blk: {
361 @setEvalBranchQuota(200000);
362 var loc: [128]struct { word: u8, shift: u5 } = undefined;
363 for (0..128) |i| {
364 var st: Bitsliced = @splat(0);
365 const word = (i / 8) / 4;
366 const bit: u5 = @intCast(((i / 8) % 4) * 8 + (i % 8));
367 for (0..8) |b| st[b * 4 + word] |= @as(Word, 1) << bit;
368 packState(&st);
369 found: for (0..32) |w| {
370 for (0..4) |byte| {
371 if ((st[w] >> @intCast(byte * 8)) & 0xff == 0xff) {
372 loc[i] = .{ .word = @intCast(w), .shift = @intCast(byte * 8) };
373 break :found;
374 }
375 }
376 }
377 }
378 break :blk loc;
379 };
380
381 /// Pack a round key and broadcast it to all blocks, so it can be directly XORed into a packed state
382 fn packRoundKey(round_key: *const [16]u8) Bitsliced {
383 var st: Bitsliced = @splat(0);
384 inline for (0..16) |pos| {
385 const byte = round_key[pos];
386 inline for (0..8) |bit| {
387 const smear: Word = 0 -% @as(Word, (byte >> bit) & 1);
388 const loc = broadcast_loc[pos * 8 + bit];
389 st[loc.word] |= smear & (splatWord(0xff) << loc.shift);
390 }
391 }
392 return st;
393 }
394
395 /// Pack a whole key schedule into the bitsliced representation.
396 pub fn packKeys(round_keys: anytype) [round_keys.len]Bitsliced {
397 var keys: [round_keys.len]Bitsliced = undefined;
398 inline for (&keys, 0..) |*k, r| {
399 const bytes = round_keys[r].toBytes();
400 k.* = packRoundKey(&bytes);
401 }
402 return keys;
403 }
404
405 /// Run the full AES cipher over the whole batch in place
406 pub fn encrypt(comptime rounds: usize, keys: *const [rounds + 1]Bitsliced, data: *[nblocks * 16]u8) void {
407 var st = pack(data);
408 addRoundKey(&st, &keys[0]);
409 inline for (1..rounds) |r| {
410 round(&st);
411 addRoundKey(&st, &keys[r]);
412 }
413 subBytes(&st);
414 shiftRows(&st);
415 addRoundKey(&st, &keys[rounds]);
416 data.* = unpack(&st);
417 }
418
419 fn invShiftRows(st: *Bitsliced) void {
420 var i: usize = 0;
421 while (i < 32) : (i += 4) {
422 st[i + 1] = laneRotl(st[i + 1], 8);
423 st[i + 2] = laneRotl(st[i + 2], 16);
424 st[i + 3] = laneRotl(st[i + 3], 24);
425 }
426 }
427
428 /// Precompute the `mixColumns` inverse matrix
429 const mix_inv: [32]u32 = blk: {
430 @setEvalBranchQuota(200000);
431 var m: [32]u32 = @splat(0);
432 for (0..32) |i| {
433 var st: Bitsliced = @splat(0);
434 st[i] = 1;
435 mixColumns(&st);
436 for (0..32) |o| {
437 if (st[o] & 1 == 1) m[o] |= @as(u32, 1) << @intCast(i);
438 }
439 }
440 var inv: [32]u32 = undefined;
441 for (0..32) |k| inv[k] = @as(u32, 1) << @intCast(k);
442 for (0..32) |col| {
443 var piv = col;
444 while (m[piv] & (@as(u32, 1) << @intCast(col)) == 0) piv += 1;
445 const tm = m[col];
446 m[col] = m[piv];
447 m[piv] = tm;
448 const ti = inv[col];
449 inv[col] = inv[piv];
450 inv[piv] = ti;
451 for (0..32) |r| {
452 if (r != col and (m[r] & (@as(u32, 1) << @intCast(col))) != 0) {
453 m[r] ^= m[col];
454 inv[r] ^= inv[col];
455 }
456 }
457 }
458 break :blk inv;
459 };
460
461 fn invMixColumns(st: *Bitsliced) void {
462 @setEvalBranchQuota(20000);
463 var out: Bitsliced = @splat(0);
464 inline for (0..32) |o| {
465 inline for (0..32) |j| {
466 if (mix_inv[o] & (@as(u32, 1) << @intCast(j)) != 0) out[o] ^= st[j];
467 }
468 }
469 st.* = out;
470 }
471
472 /// Linear map for the inverse S-box
473 fn invAffineGroup(st: *Bitsliced, comptime g: usize) void {
474 const p = [8]usize{ g + 28, g + 24, g + 20, g + 16, g + 12, g + 8, g + 4, g };
475 const a0 = ~st[p[0]];
476 const a1 = ~st[p[1]];
477 const a2 = st[p[2]];
478 const a3 = st[p[3]];
479 const a4 = st[p[4]];
480 const a5 = ~st[p[5]];
481 const a6 = ~st[p[6]];
482 const a7 = st[p[7]];
483 st[p[7]] = a1 ^ a4 ^ a6;
484 st[p[6]] = a0 ^ a3 ^ a5;
485 st[p[5]] = a7 ^ a2 ^ a4;
486 st[p[4]] = a6 ^ a1 ^ a3;
487 st[p[3]] = a5 ^ a0 ^ a2;
488 st[p[2]] = a4 ^ a7 ^ a1;
489 st[p[1]] = a3 ^ a6 ^ a0;
490 st[p[0]] = a2 ^ a5 ^ a7;
491 }
492
493 /// Compact inverse S-box: we apply a fixed linear map, the forward S-box, then the same linear map
494 /// Trick from Thomas Pornin
495 fn invSubBytes(st: *Bitsliced) void {
496 inline for (0..4) |g| invAffineGroup(st, g);
497 subBytes(st);
498 inline for (0..4) |g| invAffineGroup(st, g);
499 }
500
501 /// Full inverse AES cipher over the whole batch in bitsliced representation
502 pub fn decrypt(comptime rounds: usize, keys: *const [rounds + 1]Bitsliced, data: *[nblocks * 16]u8) void {
503 var st = pack(data);
504 addRoundKey(&st, &keys[rounds]);
505 comptime var r: usize = rounds - 1;
506 inline while (r > 0) : (r -= 1) {
507 invShiftRows(&st);
508 invSubBytes(&st);
509 addRoundKey(&st, &keys[r]);
510 invMixColumns(&st);
511 }
512 invShiftRows(&st);
513 invSubBytes(&st);
514 addRoundKey(&st, &keys[0]);
515 data.* = unpack(&st);
516 }
517 };
518}
519
520const testing = std.testing;
521
522test "bitslice cross-check against table AES" {
523 const key: [16]u8 = @splat(0x37);
524 const ref = std.crypto.core.aes.Aes128.initEnc(key);
525 const keys = Wide.packKeys(ref.key_schedule.round_keys[0..]);
526
527 var data: [width * 16]u8 = undefined;
528 for (0..width * 16) |i| data[i] = @truncate(i *% 101 +% 7);
529
530 var expected: [width * 16]u8 = undefined;
531 for (0..width) |b| {
532 var blk: [16]u8 = data[b * 16 ..][0..16].*;
533 ref.encrypt(&blk, &blk);
534 expected[b * 16 ..][0..16].* = blk;
535 }
536
537 Wide.encrypt(10, &keys, &data);
538 try testing.expectEqualSlices(u8, &expected, &data);
539}
540
541test "bitslice decrypt cross-check against table AES" {
542 inline for (.{ std.crypto.core.aes.Aes128, std.crypto.core.aes.Aes256 }) |Aes| {
543 const key: [Aes.key_bits / 8]u8 = @splat(0x4d);
544 const enc = Aes.initEnc(key);
545 const dec = Aes.initDec(key);
546 const keys = Wide.packKeys(enc.key_schedule.round_keys[0..]);
547
548 var data: [width * 16]u8 = undefined;
549 for (0..width * 16) |i| data[i] = @truncate(i *% 211 +% 13);
550
551 var expected: [width * 16]u8 = undefined;
552 for (0..width) |b| {
553 var blk: [16]u8 = data[b * 16 ..][0..16].*;
554 dec.decrypt(&blk, &blk);
555 expected[b * 16 ..][0..16].* = blk;
556 }
557
558 Wide.decrypt(Aes.rounds, &keys, &data);
559 try testing.expectEqualSlices(u8, &expected, &data);
560 }
561}
lib/std/crypto/aes/soft.zig+74-59
...@@ -1,9 +1,12 @@...@@ -1,9 +1,12 @@
1const std = @import("../../std.zig");1const std = @import("../../std.zig");
2const math = std.math;2const math = std.math;
3const mem = std.mem;3const mem = std.mem;
4const bitsliced = @import("bitsliced.zig");
45
5const side_channels_mitigations = std.options.side_channels_mitigations;6const side_channels_mitigations = std.options.side_channels_mitigations;
67
8const protected = side_channels_mitigations != .none;
9
7/// A single AES block.10/// A single AES block.
8pub const Block = struct {11pub const Block = struct {
9 const Repr = [4]u32;12 const Repr = [4]u32;
...@@ -288,7 +291,7 @@ pub const Block = struct {...@@ -288,7 +291,7 @@ pub const Block = struct {
288 /// Perform operations on multiple blocks in parallel.291 /// Perform operations on multiple blocks in parallel.
289 pub const parallel = struct {292 pub const parallel = struct {
290 /// The recommended number of AES encryption/decryption to perform in parallel for the chosen implementation.293 /// The recommended number of AES encryption/decryption to perform in parallel for the chosen implementation.
291 pub const optimal_parallel_blocks = 1;294 pub const optimal_parallel_blocks = if (protected) bitsliced.width else 1;
292295
293 /// Encrypt multiple blocks in parallel, each their own round key.296 /// Encrypt multiple blocks in parallel, each their own round key.
294 pub fn encryptParallel(comptime count: usize, blocks: [count]Block, round_keys: [count]Block) [count]Block {297 pub fn encryptParallel(comptime count: usize, blocks: [count]Block, round_keys: [count]Block) [count]Block {
...@@ -539,6 +542,38 @@ fn KeySchedule(comptime Aes: type) type {...@@ -539,6 +542,38 @@ fn KeySchedule(comptime Aes: type) type {
539 };542 };
540}543}
541544
545/// Packed forward round keys for the bitsliced implementation
546fn PackedKeys(comptime rounds: usize) type {
547 return if (protected) [rounds + 1]bitsliced.State else void;
548}
549
550/// Bitsliced operation over `count` blocks
551fn bitslicedOperation(
552 comptime rounds: usize,
553 comptime operation: enum { encrypt, decrypt },
554 keys: *const [rounds + 1]bitsliced.State,
555 comptime count: usize,
556 dst: *[16 * count]u8,
557 src: *const [16 * count]u8,
558) void {
559 const w = bitsliced.width;
560 const chunks = (count + w - 1) / w;
561 inline for (0..chunks) |c| {
562 const off: usize = c * w;
563 const n: usize = @min(w, count - off);
564 const lo = off * 16;
565 const hi = lo + n * 16;
566 var buf: [w * 16]u8 = undefined;
567 @memcpy(buf[0 .. n * 16], src[lo..hi]);
568 if (operation == .decrypt) {
569 bitsliced.Wide.decrypt(rounds, keys, &buf);
570 } else {
571 bitsliced.Wide.encrypt(rounds, keys, &buf);
572 }
573 @memcpy(dst[lo..hi], buf[0 .. n * 16]);
574 }
575}
576
542/// A context to perform encryption using the standard AES key schedule.577/// A context to perform encryption using the standard AES key schedule.
543pub fn AesEncryptCtx(comptime Aes: type) type {578pub fn AesEncryptCtx(comptime Aes: type) type {
544 std.debug.assert(Aes.key_bits == 128 or Aes.key_bits == 256);579 std.debug.assert(Aes.key_bits == 128 or Aes.key_bits == 256);
...@@ -549,73 +584,54 @@ pub fn AesEncryptCtx(comptime Aes: type) type {...@@ -549,73 +584,54 @@ pub fn AesEncryptCtx(comptime Aes: type) type {
549 pub const block = Aes.block;584 pub const block = Aes.block;
550 pub const block_length = block.block_length;585 pub const block_length = block.block_length;
551 key_schedule: KeySchedule(Aes),586 key_schedule: KeySchedule(Aes),
587 packed_keys: PackedKeys(rounds),
552588
553 /// Create a new encryption context with the given key.589 /// Create a new encryption context with the given key.
554 pub fn init(key: [Aes.key_bits / 8]u8) Self {590 pub fn init(key: [Aes.key_bits / 8]u8) Self {
555 const key_schedule = KeySchedule(Aes).expandKey(key);591 const key_schedule = KeySchedule(Aes).expandKey(key);
556 return Self{592 return Self{
557 .key_schedule = key_schedule,593 .key_schedule = key_schedule,
594 .packed_keys = if (protected) bitsliced.Wide.packKeys(key_schedule.round_keys[0..]) else {},
558 };595 };
559 }596 }
560597
561 /// Encrypt a single block.598 /// Encrypt a single block, without any protection against side channels.
562 pub fn encrypt(ctx: Self, dst: *[16]u8, src: *const [16]u8) void {599 fn encryptUnprotectedBlock(ctx: Self, dst: *[16]u8, src: *const [16]u8) void {
563 const round_keys = ctx.key_schedule.round_keys;600 const round_keys = ctx.key_schedule.round_keys;
564 var t = Block.fromBytes(src).xorBlocks(round_keys[0]);601 var t = Block.fromBytes(src).xorBlocks(round_keys[0]);
565 comptime var i = 1;602 inline for (1..rounds) |i| {
566 if (side_channels_mitigations == .full) {603 t = t.encryptUnprotected(round_keys[i]);
567 inline while (i < rounds) : (i += 1) {
568 t = t.encrypt(round_keys[i]);
569 }
570 } else {
571 inline while (i < 5) : (i += 1) {
572 t = t.encrypt(round_keys[i]);
573 }
574 inline while (i < rounds - 1) : (i += 1) {
575 t = t.encryptUnprotected(round_keys[i]);
576 }
577 t = t.encrypt(round_keys[i]);
578 }604 }
579 t = t.encryptLast(round_keys[rounds]);605 t = t.encryptLast(round_keys[rounds]);
580 dst.* = t.toBytes();606 dst.* = t.toBytes();
581 }607 }
582608
609 /// Encrypt a single block.
610 pub fn encrypt(ctx: Self, dst: *[16]u8, src: *const [16]u8) void {
611 ctx.encryptWide(1, dst, src);
612 }
613
583 /// Encrypt+XOR a single block.614 /// Encrypt+XOR a single block.
584 pub fn xor(ctx: Self, dst: *[16]u8, src: *const [16]u8, counter: [16]u8) void {615 pub fn xor(ctx: Self, dst: *[16]u8, src: *const [16]u8, counter: [16]u8) void {
585 const round_keys = ctx.key_schedule.round_keys;616 ctx.xorWide(1, dst, src, counter);
586 var t = Block.fromBytes(&counter).xorBlocks(round_keys[0]);
587 comptime var i = 1;
588 if (side_channels_mitigations == .full) {
589 inline while (i < rounds) : (i += 1) {
590 t = t.encrypt(round_keys[i]);
591 }
592 } else {
593 inline while (i < 5) : (i += 1) {
594 t = t.encrypt(round_keys[i]);
595 }
596 inline while (i < rounds - 1) : (i += 1) {
597 t = t.encryptUnprotected(round_keys[i]);
598 }
599 t = t.encrypt(round_keys[i]);
600 }
601 t = t.encryptLast(round_keys[rounds]);
602 dst.* = t.xorBytes(src);
603 }617 }
604618
605 /// Encrypt multiple blocks, possibly leveraging parallelization.619 /// Encrypt multiple blocks, possibly leveraging parallelization.
606 pub fn encryptWide(ctx: Self, comptime count: usize, dst: *[16 * count]u8, src: *const [16 * count]u8) void {620 pub fn encryptWide(ctx: Self, comptime count: usize, dst: *[16 * count]u8, src: *const [16 * count]u8) void {
607 var i: usize = 0;621 if (protected) {
608 while (i < count) : (i += 1) {622 bitslicedOperation(rounds, .encrypt, &ctx.packed_keys, count, dst, src);
609 ctx.encrypt(dst[16 * i .. 16 * i + 16][0..16], src[16 * i .. 16 * i + 16][0..16]);623 } else {
624 for (0..count) |i| {
625 ctx.encryptUnprotectedBlock(dst[16 * i .. 16 * i + 16][0..16], src[16 * i .. 16 * i + 16][0..16]);
626 }
610 }627 }
611 }628 }
612629
613 /// Encrypt+XOR multiple blocks, possibly leveraging parallelization.630 /// Encrypt+XOR multiple blocks, possibly leveraging parallelization.
614 pub fn xorWide(ctx: Self, comptime count: usize, dst: *[16 * count]u8, src: *const [16 * count]u8, counters: [16 * count]u8) void {631 pub fn xorWide(ctx: Self, comptime count: usize, dst: *[16 * count]u8, src: *const [16 * count]u8, counters: [16 * count]u8) void {
615 var i: usize = 0;632 var ks: [16 * count]u8 = undefined;
616 while (i < count) : (i += 1) {633 ctx.encryptWide(count, &ks, &counters);
617 ctx.xor(dst[16 * i .. 16 * i + 16][0..16], src[16 * i .. 16 * i + 16][0..16], counters[16 * i .. 16 * i + 16][0..16].*);634 for (dst, ks, src) |*d, k, s| d.* = k ^ s;
618 }
619 }635 }
620 };636 };
621}637}
...@@ -630,11 +646,13 @@ pub fn AesDecryptCtx(comptime Aes: type) type {...@@ -630,11 +646,13 @@ pub fn AesDecryptCtx(comptime Aes: type) type {
630 pub const block = Aes.block;646 pub const block = Aes.block;
631 pub const block_length = block.block_length;647 pub const block_length = block.block_length;
632 key_schedule: KeySchedule(Aes),648 key_schedule: KeySchedule(Aes),
649 packed_keys: PackedKeys(rounds),
633650
634 /// Create a decryption context from an existing encryption context.651 /// Create a decryption context from an existing encryption context.
635 pub fn initFromEnc(ctx: AesEncryptCtx(Aes)) Self {652 pub fn initFromEnc(ctx: AesEncryptCtx(Aes)) Self {
636 return Self{653 return Self{
637 .key_schedule = ctx.key_schedule.invert(),654 .key_schedule = ctx.key_schedule.invert(),
655 .packed_keys = ctx.packed_keys,
638 };656 };
639 }657 }
640658
...@@ -644,33 +662,30 @@ pub fn AesDecryptCtx(comptime Aes: type) type {...@@ -644,33 +662,30 @@ pub fn AesDecryptCtx(comptime Aes: type) type {
644 return initFromEnc(enc_ctx);662 return initFromEnc(enc_ctx);
645 }663 }
646664
647 /// Decrypt a single block.665 /// Decrypt a single block, without any protection against side channels.
648 pub fn decrypt(ctx: Self, dst: *[16]u8, src: *const [16]u8) void {666 fn decryptUnprotectedBlock(ctx: Self, dst: *[16]u8, src: *const [16]u8) void {
649 const inv_round_keys = ctx.key_schedule.round_keys;667 const inv_round_keys = ctx.key_schedule.round_keys;
650 var t = Block.fromBytes(src).xorBlocks(inv_round_keys[0]);668 var t = Block.fromBytes(src).xorBlocks(inv_round_keys[0]);
651 comptime var i = 1;669 inline for (1..rounds) |i| {
652 if (side_channels_mitigations == .full) {670 t = t.decryptUnprotected(inv_round_keys[i]);
653 inline while (i < rounds) : (i += 1) {
654 t = t.decrypt(inv_round_keys[i]);
655 }
656 } else {
657 inline while (i < 5) : (i += 1) {
658 t = t.decrypt(inv_round_keys[i]);
659 }
660 inline while (i < rounds - 1) : (i += 1) {
661 t = t.decryptUnprotected(inv_round_keys[i]);
662 }
663 t = t.decrypt(inv_round_keys[i]);
664 }671 }
665 t = t.decryptLast(inv_round_keys[rounds]);672 t = t.decryptLast(inv_round_keys[rounds]);
666 dst.* = t.toBytes();673 dst.* = t.toBytes();
667 }674 }
668675
676 /// Decrypt a single block.
677 pub fn decrypt(ctx: Self, dst: *[16]u8, src: *const [16]u8) void {
678 ctx.decryptWide(1, dst, src);
679 }
680
669 /// Decrypt multiple blocks, possibly leveraging parallelization.681 /// Decrypt multiple blocks, possibly leveraging parallelization.
670 pub fn decryptWide(ctx: Self, comptime count: usize, dst: *[16 * count]u8, src: *const [16 * count]u8) void {682 pub fn decryptWide(ctx: Self, comptime count: usize, dst: *[16 * count]u8, src: *const [16 * count]u8) void {
671 var i: usize = 0;683 if (protected) {
672 while (i < count) : (i += 1) {684 bitslicedOperation(rounds, .decrypt, &ctx.packed_keys, count, dst, src);
673 ctx.decrypt(dst[16 * i .. 16 * i + 16][0..16], src[16 * i .. 16 * i + 16][0..16]);685 } else {
686 for (0..count) |i| {
687 ctx.decryptUnprotectedBlock(dst[16 * i .. 16 * i + 16][0..16], src[16 * i .. 16 * i + 16][0..16]);
688 }
674 }689 }
675 }690 }
676 };691 };
lib/std/crypto/aes_ocb.zig+6-3
...@@ -103,7 +103,10 @@ fn AesOcb(comptime Aes: anytype) type {...@@ -103,7 +103,10 @@ fn AesOcb(comptime Aes: anytype) type {
103103
104 const has_aesni = builtin.cpu.has(.x86, .aes);104 const has_aesni = builtin.cpu.has(.x86, .aes);
105 const has_armaes = builtin.cpu.has(.aarch64, .aes);105 const has_armaes = builtin.cpu.has(.aarch64, .aes);
106 const wb: usize = if ((builtin.cpu.arch == .x86_64 and has_aesni) or (builtin.cpu.arch == .aarch64 and has_armaes)) 4 else 0;106 const wb: usize = if ((builtin.cpu.arch == .x86_64 and has_aesni) or (builtin.cpu.arch == .aarch64 and has_armaes))
107 4 // empirically what works best on x86_64 and aarch64 with AES-NI/ARM Crypto
108 else
109 Aes.block.parallel.optimal_parallel_blocks;
107110
108 /// c: ciphertext: output buffer should be of size m.len111 /// c: ciphertext: output buffer should be of size m.len
109 /// tag: authentication tag: output MAC112 /// tag: authentication tag: output MAC
...@@ -124,7 +127,7 @@ fn AesOcb(comptime Aes: anytype) type {...@@ -124,7 +127,7 @@ fn AesOcb(comptime Aes: anytype) type {
124 var sum: [16]u8 = @splat(0);127 var sum: [16]u8 = @splat(0);
125 var i: usize = 0;128 var i: usize = 0;
126129
127 while (wb > 0 and i + wb <= full_blocks) : (i += wb) {130 while (i + wb <= full_blocks) : (i += wb) {
128 var offsets: [wb]Block align(16) = undefined;131 var offsets: [wb]Block align(16) = undefined;
129 var es: [16 * wb]u8 align(16) = undefined;132 var es: [16 * wb]u8 align(16) = undefined;
130 var j: usize = 0;133 var j: usize = 0;
...@@ -191,7 +194,7 @@ fn AesOcb(comptime Aes: anytype) type {...@@ -191,7 +194,7 @@ fn AesOcb(comptime Aes: anytype) type {
191 var sum: [16]u8 = @splat(0);194 var sum: [16]u8 = @splat(0);
192 var i: usize = 0;195 var i: usize = 0;
193196
194 while (wb > 0 and i + wb <= full_blocks) : (i += wb) {197 while (i + wb <= full_blocks) : (i += wb) {
195 var offsets: [wb]Block align(16) = undefined;198 var offsets: [wb]Block align(16) = undefined;
196 var es: [16 * wb]u8 align(16) = undefined;199 var es: [16 * wb]u8 align(16) = undefined;
197 var j: usize = 0;200 var j: usize = 0;
lib/std/crypto/modes.zig+11-14
...@@ -37,14 +37,13 @@ pub fn ctrSlice(...@@ -37,14 +37,13 @@ pub fn ctrSlice(
37 debug.assert(counter_offset + counter_size <= block_length);37 debug.assert(counter_offset + counter_size <= block_length);
38 debug.assert(counter_size > 0 and counter_size <= block_length);38 debug.assert(counter_size > 0 and counter_size <= block_length);
3939
40 var counterBlock = iv;
41 var i: usize = 0;40 var i: usize = 0;
4241
43 const CounterInt = @Int(.unsigned, counter_size * 8);42 const CounterInt = @Int(.unsigned, counter_size * 8);
4443
45 const parallel_count = BlockCipher.block.parallel.optimal_parallel_blocks;44 const parallel_count = BlockCipher.block.parallel.optimal_parallel_blocks;
46 const wide_block_length = parallel_count * block_length;45 const wide_block_length = parallel_count * block_length;
47 var cnt_val = mem.readInt(CounterInt, counterBlock[counter_offset..][0..counter_size], endian);46 var cnt_val = mem.readInt(CounterInt, iv[counter_offset..][0..counter_size], endian);
48 if (src.len >= wide_block_length) {47 if (src.len >= wide_block_length) {
49 var counters: [parallel_count * block_length]u8 = undefined;48 var counters: [parallel_count * block_length]u8 = undefined;
50 inline for (0..parallel_count) |j| {49 inline for (0..parallel_count) |j| {
...@@ -58,20 +57,18 @@ pub fn ctrSlice(...@@ -58,20 +57,18 @@ pub fn ctrSlice(
58 cnt_val +%= parallel_count;57 cnt_val +%= parallel_count;
59 block_cipher.xorWide(parallel_count, dst[i .. i + wide_block_length][0..wide_block_length], src[i .. i + wide_block_length][0..wide_block_length], counters);58 block_cipher.xorWide(parallel_count, dst[i .. i + wide_block_length][0..wide_block_length], src[i .. i + wide_block_length][0..wide_block_length], counters);
60 }59 }
61 mem.writeInt(CounterInt, counterBlock[counter_offset..][0..counter_size], cnt_val, endian);
62 }
63 while (i + block_length <= src.len) : (i += block_length) {
64 block_cipher.xor(dst[i .. i + block_length][0..block_length], src[i .. i + block_length][0..block_length], counterBlock);
65 cnt_val +%= 1;
66 mem.writeInt(CounterInt, counterBlock[counter_offset..][0..counter_size], cnt_val, endian);
67 }60 }
68 if (i < src.len) {61 if (i < src.len) {
69 var pad: [block_length]u8 = @splat(0);62 const rem = src.len - i;
70 const src_slice = src[i..];63 var counters: [wide_block_length]u8 = undefined;
71 @memcpy(pad[0..src_slice.len], src_slice);64 inline for (0..parallel_count) |j| {
72 block_cipher.xor(&pad, &pad, counterBlock);65 counters[j * block_length ..][0..block_length].* = iv;
73 const pad_slice = pad[0 .. src.len - i];66 const block_ctr = cnt_val +% @as(CounterInt, j);
74 @memcpy(dst[i..][0..pad_slice.len], pad_slice);67 mem.writeInt(CounterInt, counters[j * block_length + counter_offset ..][0..counter_size], block_ctr, endian);
68 }
69 var keystream: [wide_block_length]u8 = undefined;
70 block_cipher.encryptWide(parallel_count, &keystream, &counters);
71 for (dst[i..][0..rem], src[i..][0..rem], keystream[0..rem]) |*d, s, k| d.* = s ^ k;
75 }72 }
76}73}
7774