From 4d06435fe3a03227c7b899436efde3ee1bd0dcb9 Mon Sep 17 00:00:00 2001 From: Frank Denis Date: Mon, 6 Jul 2026 11:09:46 +0200 Subject: [PATCH] crypto.aes.soft: use a bitsliced representation AES is nice but has a problem when implemented in pure software (on CPUs without AES instructions). In order to not be insanely slow, the non-linear operation is usually implemented with a lookup tables. But memory lookups with a secret index is a side channel that can be exploited for key recovery. The current implementation of software AES mitigates this by doing multiple passes over tiny lookup tables that fit in a single cache line. It works. Intra-cacheline attacks leveraging SMT exist, but the number of CPUs with SMT but no AES instructions is pretty small (old Intel Atom Medfield CPUs). But we can do better. Bitslicing is another way to mitigate these issues, but packing all the bits that will go through the same circuit into the same register, essentially emulating a hardware implementation. For single-block AES, this is quite slow, but for any algorithm that can leverage parallelization (AES-CTR, AES-OCB, etc), it's not bad at all. Long story short: - Constant time everywhere, including weird old CPUs - About 4x faster than the current cacheline-tables code for inputs > 32 bytes. On modern CPUs, the speedup is significant for WebAssembly. Note that when side channels are not part of the threat model, the unprotected lookup table remains faster. The packing/unpacking/round operations are intentionally public. Because in order to get decent performance, some AES-based ciphers really need to control when they want to use the regular word representation vs the packed representation. --- lib/std/crypto/aes/bitsliced.zig | 561 +++++++++++++++++++++++++++++++ lib/std/crypto/aes/soft.zig | 139 ++++---- lib/std/crypto/aes_ocb.zig | 9 +- lib/std/crypto/modes.zig | 25 +- 4 files changed, 655 insertions(+), 79 deletions(-) create mode 100644 lib/std/crypto/aes/bitsliced.zig diff --git a/lib/std/crypto/aes/bitsliced.zig b/lib/std/crypto/aes/bitsliced.zig new file mode 100644 index 0000000000000000000000000000000000000000..5b35c604fb8141c491713ec87b966d4c364acc12 --- /dev/null +++ b/lib/std/crypto/aes/bitsliced.zig @@ -0,0 +1,561 @@ +//! Bitsliced AES round function, for targets without hardware AES support. +//! It has no data-dependent memory access, so it is constant time. + +const std = @import("../../std.zig"); +const builtin = @import("builtin"); +const mem = std.mem; + +/// The widest batch of AES blocks the target supports +pub const Wide = if (builtin.cpu.arch == .wasm32 or builtin.cpu.arch == .wasm64 or @bitSizeOf(usize) >= 64) Batch(2) else Batch(1); + +/// Number of AES blocks we can process together +pub const width = Wide.nblocks; + +/// A state holding a whole `width`-block batch +pub const State = Wide.Bitsliced; + +/// A batch of `8 * lanes` blocks, one 32-bit lane per group of eight blocks +pub fn Batch(comptime lanes: usize) type { + return struct { + pub const Word = @Int(.unsigned, 32 * lanes); + const Shift = std.math.Log2Int(Word); + + const nblocks = 8 * lanes; + pub const Bitsliced = [32]Word; + + /// Splat a 32-bit pattern across all lanes of a word + pub fn splatWord(x: u32) Word { + const t: [lanes]u32 = @splat(x); + return @bitCast(t); + } + + fn laneRotl(x: Word, comptime n: u5) Word { + if (lanes == 1) return std.math.rotl(u32, x, n); + const lo_bits = (@as(u32, 1) << n) - 1; + const mask_lo = comptime splatWord(lo_bits); + const mask_hi = comptime splatWord(~lo_bits); + const l: Shift = n; + const r: Shift = @intCast(32 - @as(u32, n)); + return ((x << l) & mask_hi) | ((x >> r) & mask_lo); + } + + pub fn swapMove(a: *Word, b: *Word, comptime mask32: u32, comptime n: comptime_int) void { + const shift: Shift = n; + const mask = comptime splatWord(mask32); + const tmp = (b.* ^ (a.* >> shift)) & mask; + b.* ^= tmp; + a.* ^= tmp << shift; + } + + fn sbox(u: []Word) void { + const s0 = u[4] ^ u[16]; + const s1 = u[20] ^ u[28]; + const s2 = u[12] ^ s0; + const s3 = u[0] ^ u[8]; + const q0 = s1 ^ s2; + const s4 = u[0] ^ u[24]; + const s5 = u[8] ^ u[24]; + const s6 = u[12] ^ s1; + const s7 = u[20] ^ s3; + const q1 = s1 ^ s5; + const q2 = u[8] ^ q0; + const q3 = s4 ^ s2; + const q4 = s3 ^ q0; + const s8 = u[16] ^ s3; + const q5 = s6 ^ s8; + const q6 = u[8] ^ u[12]; + const q7 = u[24] ^ s2; + const s9 = u[24] ^ s0; + const q8 = s3 ^ s9; + const q9 = s4 ^ s6; + const q10 = s0 ^ s5; + const q12 = u[28] ^ s2; + const q13 = u[4] ^ s7; + const q14 = u[28] ^ s3; + const q15 = s2 ^ s7; + const q16 = u[4] ^ s1; + const q17 = u[4] ^ u[28]; + const q11 = u[20]; + + const t20 = q6 & q12; + const t21 = q3 & q14; + const t22 = q1 & q16; + const t23 = q2 & q17; + const x0 = ((q3 | q14) ^ (q0 & q7)) ^ (t20 ^ t22); + const x1 = ((q4 | q13) ^ (q10 & q11)) ^ (t21 ^ t20); + const x2 = ((q2 | q17) ^ (q5 & q9)) ^ (t21 ^ t22); + const x3 = ((q8 | q15) ^ t23) ^ (t21 ^ (q4 & q13)); + + const a = x1 & ~x3; + const b = x0 & ~x3; + const c = x3 & ~x1; + const d = x2 & ~x1; + const e = x0 ^ a; + const y0 = x3 ^ (x2 & ~e); + const f = x1 ^ b; + const y1 = c ^ (x2 & f); + const g = x2 ^ c; + const y2 = x1 ^ (x0 & ~g); + const h = x3 ^ d; + const y3 = a ^ (x0 & h); + const y02 = y2 ^ y0; + const y13 = y3 ^ y1; + const y23 = y3 ^ y2; + const y01 = y1 ^ y0; + const y00 = y02 ^ y13; + + const a0 = y01 & q11; + const a1 = y0 & q12; + const a2 = y1 & q0; + const a3 = y23 & q17; + const a4 = y2 & q5; + const a5 = y3 & q15; + const a6 = y13 & q14; + const a7 = y00 & q16; + const a8 = y02 & q13; + const a9 = y01 & q7; + const a10 = y0 & q10; + const a11 = y1 & q6; + const a12 = y23 & q2; + const a13 = y2 & q9; + const a14 = y3 & q8; + const a15 = y13 & q3; + const a16 = y00 & q1; + const a17 = y02 & q4; + + const r0 = a1 ^ a5; + const r1 = a9 ^ a15; + const r2 = a4 ^ r0; + const r3 = a2 ^ a10; + const r4 = a11 ^ a17; + const r5 = a8 ^ r1; + const r6 = a0 ^ a16; + const r7 = a7 ^ a13; + const r8 = a11 ^ a14; + const r9 = r3 ^ r4; + const r10 = r5 ^ r6; + const r11 = r2 ^ r9; + const r12 = a3 ^ r0; + const r13 = r7 ^ r8; + const r14 = r12 ^ r13; + u[0] = r10 ^ r14; + const r15 = a6 ^ a10; + const r16 = r15 ^ r2; + u[4] = ~(r10 ^ r16); + u[8] = ~(a2 ^ r2); + const r17 = a12 ^ a13; + const r18 = a15 ^ r17; + u[12] = r18 ^ r11; + const r19 = a1 ^ a14; + const r20 = a17 ^ r3; + const r21 = r7 ^ r19; + const r22 = r5 ^ r20; + u[16] = r21 ^ r22; + const r23 = a9 ^ a12; + u[20] = r8 ^ r23; + u[24] = ~(r1 ^ r4); + u[28] = ~(a16 ^ r11); + } + + fn subBytes(st: *Bitsliced) void { + sbox(st[0..]); + sbox(st[1..]); + sbox(st[2..]); + sbox(st[3..]); + } + + fn shiftRows(st: *Bitsliced) void { + var i: usize = 0; + while (i < 32) : (i += 4) { + st[i + 1] = laneRotl(st[i + 1], 24); + st[i + 2] = laneRotl(st[i + 2], 16); + st[i + 3] = laneRotl(st[i + 3], 8); + } + } + + fn mixColumns(st: *Bitsliced) void { + const t2_0 = st[0] ^ st[1]; + const t2_1 = st[1] ^ st[2]; + const t2_2 = st[2] ^ st[3]; + const t2_3 = st[3] ^ st[0]; + var t0_0 = st[28] ^ st[29]; + var t0_1 = st[29] ^ st[30]; + var t0_2 = st[30] ^ st[31]; + var t0_3 = st[31] ^ st[28]; + var t = st[28]; + st[28] = t2_0 ^ t0_2 ^ st[29]; + st[29] = t2_1 ^ t0_2 ^ t; + t = st[30]; + st[30] = t2_2 ^ t0_0 ^ st[31]; + st[31] = t2_3 ^ t0_0 ^ t; + var t1_0 = st[24] ^ st[25]; + var t1_1 = st[25] ^ st[26]; + var t1_2 = st[26] ^ st[27]; + var t1_3 = st[27] ^ st[24]; + t = st[24]; + st[24] = t0_0 ^ t2_0 ^ st[25] ^ t1_2; + var t_bis = st[25]; + st[25] = t0_1 ^ t2_1 ^ t1_2 ^ t; + t = st[26]; + st[26] = t0_2 ^ t2_2 ^ t1_3 ^ t_bis; + st[27] = t0_3 ^ t2_3 ^ t1_0 ^ t; + t0_0 = st[20] ^ st[21]; + t0_1 = st[21] ^ st[22]; + t0_2 = st[22] ^ st[23]; + t0_3 = st[23] ^ st[20]; + t = st[20]; + st[20] = t1_0 ^ t0_1 ^ st[23]; + t_bis = st[21]; + st[21] = t1_1 ^ t0_2 ^ t; + t = st[22]; + st[22] = t1_2 ^ t0_3 ^ t_bis; + st[23] = t1_3 ^ t0_0 ^ t; + t1_0 = st[16] ^ st[17]; + t1_1 = st[17] ^ st[18]; + t1_2 = st[18] ^ st[19]; + t1_3 = st[19] ^ st[16]; + t = st[16]; + st[16] = t0_0 ^ t2_0 ^ t1_1 ^ st[19]; + t_bis = st[17]; + st[17] = t0_1 ^ t2_1 ^ t1_2 ^ t; + t = st[18]; + st[18] = t0_2 ^ t2_2 ^ t1_3 ^ t_bis; + st[19] = t0_3 ^ t2_3 ^ t1_0 ^ t; + t0_0 = st[12] ^ st[13]; + t0_1 = st[13] ^ st[14]; + t0_2 = st[14] ^ st[15]; + t0_3 = st[15] ^ st[12]; + t = st[12]; + st[12] = t1_0 ^ t2_0 ^ t0_1 ^ st[15]; + t_bis = st[13]; + st[13] = t1_1 ^ t2_1 ^ t0_2 ^ t; + t = st[14]; + st[14] = t1_2 ^ t2_2 ^ t0_3 ^ t_bis; + st[15] = t1_3 ^ t2_3 ^ t0_0 ^ t; + t1_0 = st[8] ^ st[9]; + t1_1 = st[9] ^ st[10]; + t1_2 = st[10] ^ st[11]; + t1_3 = st[11] ^ st[8]; + t = st[8]; + st[8] = t0_0 ^ t1_1 ^ st[11]; + t_bis = st[9]; + st[9] = t0_1 ^ t1_2 ^ t; + t = st[10]; + st[10] = t0_2 ^ t1_3 ^ t_bis; + st[11] = t0_3 ^ t1_0 ^ t; + t0_0 = st[4] ^ st[5]; + t0_1 = st[5] ^ st[6]; + t0_2 = st[6] ^ st[7]; + t0_3 = st[7] ^ st[4]; + t = st[4]; + st[4] = t1_0 ^ t0_1 ^ st[7]; + t_bis = st[5]; + st[5] = t1_1 ^ t0_2 ^ t; + t = st[6]; + st[6] = t1_2 ^ t0_3 ^ t_bis; + st[7] = t1_3 ^ t0_0 ^ t; + t = st[0]; + st[0] = t0_0 ^ t2_1 ^ st[3]; + t_bis = st[1]; + st[1] = t0_1 ^ t2_2 ^ t; + t = st[2]; + st[2] = t0_2 ^ t2_3 ^ t_bis; + st[3] = t0_3 ^ t2_0 ^ t; + } + + /// One keyless AES round (SubBytes, ShiftRows, MixColumns) over the whole batch + pub fn round(st: *Bitsliced) void { + subBytes(st); + shiftRows(st); + mixColumns(st); + } + + /// Add a round key, which must also be in bitsliced form + pub fn addRoundKey(st: *Bitsliced, round_key: *const Bitsliced) void { + for (0..32) |i| st[i] ^= round_key[i]; + } + + /// Orthogonalization (converts blocks-in-words to bit slices) + pub fn packState(st: *Bitsliced) void { + var i: usize = 0; + while (i < 32) : (i += 4) { + swapMove(&st[i], &st[i + 1], 0x00ff00ff, 8); + swapMove(&st[i + 2], &st[i + 3], 0x00ff00ff, 8); + swapMove(&st[i], &st[i + 2], 0x0000ffff, 16); + swapMove(&st[i + 1], &st[i + 3], 0x0000ffff, 16); + } + for (0..4) |k| { + swapMove(&st[k + 4], &st[k], 0x55555555, 1); + swapMove(&st[k + 12], &st[k + 8], 0x55555555, 1); + swapMove(&st[k + 20], &st[k + 16], 0x55555555, 1); + swapMove(&st[k + 28], &st[k + 24], 0x55555555, 1); + swapMove(&st[k + 8], &st[k], 0x33333333, 2); + swapMove(&st[k + 12], &st[k + 4], 0x33333333, 2); + swapMove(&st[k + 24], &st[k + 16], 0x33333333, 2); + swapMove(&st[k + 28], &st[k + 20], 0x33333333, 2); + swapMove(&st[k + 16], &st[k], 0x0f0f0f0f, 4); + swapMove(&st[k + 20], &st[k + 4], 0x0f0f0f0f, 4); + swapMove(&st[k + 24], &st[k + 8], 0x0f0f0f0f, 4); + swapMove(&st[k + 28], &st[k + 12], 0x0f0f0f0f, 4); + } + } + + /// Inverse of `packState` + pub fn unpackState(st: *Bitsliced) void { + for (0..4) |k| { + swapMove(&st[k + 4], &st[k], 0x55555555, 1); + swapMove(&st[k + 12], &st[k + 8], 0x55555555, 1); + swapMove(&st[k + 20], &st[k + 16], 0x55555555, 1); + swapMove(&st[k + 28], &st[k + 24], 0x55555555, 1); + swapMove(&st[k + 8], &st[k], 0x33333333, 2); + swapMove(&st[k + 12], &st[k + 4], 0x33333333, 2); + swapMove(&st[k + 24], &st[k + 16], 0x33333333, 2); + swapMove(&st[k + 28], &st[k + 20], 0x33333333, 2); + swapMove(&st[k + 16], &st[k], 0x0f0f0f0f, 4); + swapMove(&st[k + 20], &st[k + 4], 0x0f0f0f0f, 4); + swapMove(&st[k + 24], &st[k + 8], 0x0f0f0f0f, 4); + swapMove(&st[k + 28], &st[k + 12], 0x0f0f0f0f, 4); + } + var i: usize = 0; + while (i < 32) : (i += 4) { + swapMove(&st[i], &st[i + 2], 0x0000ffff, 16); + swapMove(&st[i + 1], &st[i + 3], 0x0000ffff, 16); + swapMove(&st[i], &st[i + 1], 0x00ff00ff, 8); + swapMove(&st[i + 2], &st[i + 3], 0x00ff00ff, 8); + } + } + + /// Convert `nblocks` little-endian blocks into the packed state. + /// Blocks`8*g .. 8*g+7` land in lane `g` of each word. + fn pack(bytes: *const [nblocks * 16]u8) Bitsliced { + var st: Bitsliced = @splat(0); + inline for (0..nblocks) |bl| { + const g = bl / 8; + const p = bl % 8; + inline for (0..4) |wd| { + const v = mem.readInt(u32, bytes[bl * 16 + wd * 4 ..][0..4], .little); + st[p * 4 + wd] |= @as(Word, v) << (32 * g); + } + } + packState(&st); + return st; + } + + /// Inverse of `pack` + fn unpack(st: *Bitsliced) [nblocks * 16]u8 { + unpackState(st); + var bytes: [nblocks * 16]u8 = undefined; + inline for (0..nblocks) |bl| { + const g = bl / 8; + const p = bl % 8; + inline for (0..4) |wd| { + const v: u32 = @truncate(st[p * 4 + wd] >> (32 * g)); + mem.writeInt(u32, bytes[bl * 16 + wd * 4 ..][0..4], v, .little); + } + } + return bytes; + } + + /// Precompute where key bit `i` of a round key lands after packing + const broadcast_loc = blk: { + @setEvalBranchQuota(200000); + var loc: [128]struct { word: u8, shift: u5 } = undefined; + for (0..128) |i| { + var st: Bitsliced = @splat(0); + const word = (i / 8) / 4; + const bit: u5 = @intCast(((i / 8) % 4) * 8 + (i % 8)); + for (0..8) |b| st[b * 4 + word] |= @as(Word, 1) << bit; + packState(&st); + found: for (0..32) |w| { + for (0..4) |byte| { + if ((st[w] >> @intCast(byte * 8)) & 0xff == 0xff) { + loc[i] = .{ .word = @intCast(w), .shift = @intCast(byte * 8) }; + break :found; + } + } + } + } + break :blk loc; + }; + + /// Pack a round key and broadcast it to all blocks, so it can be directly XORed into a packed state + fn packRoundKey(round_key: *const [16]u8) Bitsliced { + var st: Bitsliced = @splat(0); + inline for (0..16) |pos| { + const byte = round_key[pos]; + inline for (0..8) |bit| { + const smear: Word = 0 -% @as(Word, (byte >> bit) & 1); + const loc = broadcast_loc[pos * 8 + bit]; + st[loc.word] |= smear & (splatWord(0xff) << loc.shift); + } + } + return st; + } + + /// Pack a whole key schedule into the bitsliced representation. + pub fn packKeys(round_keys: anytype) [round_keys.len]Bitsliced { + var keys: [round_keys.len]Bitsliced = undefined; + inline for (&keys, 0..) |*k, r| { + const bytes = round_keys[r].toBytes(); + k.* = packRoundKey(&bytes); + } + return keys; + } + + /// Run the full AES cipher over the whole batch in place + pub fn encrypt(comptime rounds: usize, keys: *const [rounds + 1]Bitsliced, data: *[nblocks * 16]u8) void { + var st = pack(data); + addRoundKey(&st, &keys[0]); + inline for (1..rounds) |r| { + round(&st); + addRoundKey(&st, &keys[r]); + } + subBytes(&st); + shiftRows(&st); + addRoundKey(&st, &keys[rounds]); + data.* = unpack(&st); + } + + fn invShiftRows(st: *Bitsliced) void { + var i: usize = 0; + while (i < 32) : (i += 4) { + st[i + 1] = laneRotl(st[i + 1], 8); + st[i + 2] = laneRotl(st[i + 2], 16); + st[i + 3] = laneRotl(st[i + 3], 24); + } + } + + /// Precompute the `mixColumns` inverse matrix + const mix_inv: [32]u32 = blk: { + @setEvalBranchQuota(200000); + var m: [32]u32 = @splat(0); + for (0..32) |i| { + var st: Bitsliced = @splat(0); + st[i] = 1; + mixColumns(&st); + for (0..32) |o| { + if (st[o] & 1 == 1) m[o] |= @as(u32, 1) << @intCast(i); + } + } + var inv: [32]u32 = undefined; + for (0..32) |k| inv[k] = @as(u32, 1) << @intCast(k); + for (0..32) |col| { + var piv = col; + while (m[piv] & (@as(u32, 1) << @intCast(col)) == 0) piv += 1; + const tm = m[col]; + m[col] = m[piv]; + m[piv] = tm; + const ti = inv[col]; + inv[col] = inv[piv]; + inv[piv] = ti; + for (0..32) |r| { + if (r != col and (m[r] & (@as(u32, 1) << @intCast(col))) != 0) { + m[r] ^= m[col]; + inv[r] ^= inv[col]; + } + } + } + break :blk inv; + }; + + fn invMixColumns(st: *Bitsliced) void { + @setEvalBranchQuota(20000); + var out: Bitsliced = @splat(0); + inline for (0..32) |o| { + inline for (0..32) |j| { + if (mix_inv[o] & (@as(u32, 1) << @intCast(j)) != 0) out[o] ^= st[j]; + } + } + st.* = out; + } + + /// Linear map for the inverse S-box + fn invAffineGroup(st: *Bitsliced, comptime g: usize) void { + const p = [8]usize{ g + 28, g + 24, g + 20, g + 16, g + 12, g + 8, g + 4, g }; + const a0 = ~st[p[0]]; + const a1 = ~st[p[1]]; + const a2 = st[p[2]]; + const a3 = st[p[3]]; + const a4 = st[p[4]]; + const a5 = ~st[p[5]]; + const a6 = ~st[p[6]]; + const a7 = st[p[7]]; + st[p[7]] = a1 ^ a4 ^ a6; + st[p[6]] = a0 ^ a3 ^ a5; + st[p[5]] = a7 ^ a2 ^ a4; + st[p[4]] = a6 ^ a1 ^ a3; + st[p[3]] = a5 ^ a0 ^ a2; + st[p[2]] = a4 ^ a7 ^ a1; + st[p[1]] = a3 ^ a6 ^ a0; + st[p[0]] = a2 ^ a5 ^ a7; + } + + /// Compact inverse S-box: we apply a fixed linear map, the forward S-box, then the same linear map + /// Trick from Thomas Pornin + fn invSubBytes(st: *Bitsliced) void { + inline for (0..4) |g| invAffineGroup(st, g); + subBytes(st); + inline for (0..4) |g| invAffineGroup(st, g); + } + + /// Full inverse AES cipher over the whole batch in bitsliced representation + pub fn decrypt(comptime rounds: usize, keys: *const [rounds + 1]Bitsliced, data: *[nblocks * 16]u8) void { + var st = pack(data); + addRoundKey(&st, &keys[rounds]); + comptime var r: usize = rounds - 1; + inline while (r > 0) : (r -= 1) { + invShiftRows(&st); + invSubBytes(&st); + addRoundKey(&st, &keys[r]); + invMixColumns(&st); + } + invShiftRows(&st); + invSubBytes(&st); + addRoundKey(&st, &keys[0]); + data.* = unpack(&st); + } + }; +} + +const testing = std.testing; + +test "bitslice cross-check against table AES" { + const key: [16]u8 = @splat(0x37); + const ref = std.crypto.core.aes.Aes128.initEnc(key); + const keys = Wide.packKeys(ref.key_schedule.round_keys[0..]); + + var data: [width * 16]u8 = undefined; + for (0..width * 16) |i| data[i] = @truncate(i *% 101 +% 7); + + var expected: [width * 16]u8 = undefined; + for (0..width) |b| { + var blk: [16]u8 = data[b * 16 ..][0..16].*; + ref.encrypt(&blk, &blk); + expected[b * 16 ..][0..16].* = blk; + } + + Wide.encrypt(10, &keys, &data); + try testing.expectEqualSlices(u8, &expected, &data); +} + +test "bitslice decrypt cross-check against table AES" { + inline for (.{ std.crypto.core.aes.Aes128, std.crypto.core.aes.Aes256 }) |Aes| { + const key: [Aes.key_bits / 8]u8 = @splat(0x4d); + const enc = Aes.initEnc(key); + const dec = Aes.initDec(key); + const keys = Wide.packKeys(enc.key_schedule.round_keys[0..]); + + var data: [width * 16]u8 = undefined; + for (0..width * 16) |i| data[i] = @truncate(i *% 211 +% 13); + + var expected: [width * 16]u8 = undefined; + for (0..width) |b| { + var blk: [16]u8 = data[b * 16 ..][0..16].*; + dec.decrypt(&blk, &blk); + expected[b * 16 ..][0..16].* = blk; + } + + Wide.decrypt(Aes.rounds, &keys, &data); + try testing.expectEqualSlices(u8, &expected, &data); + } +} diff --git a/lib/std/crypto/aes/soft.zig b/lib/std/crypto/aes/soft.zig index 28a0ac9f98efbffc6d2a34d9780617ed46afd512..35b9ed2cb5218b980037e4c146eebc74adb9dd26 100644 --- a/lib/std/crypto/aes/soft.zig +++ b/lib/std/crypto/aes/soft.zig @@ -1,9 +1,12 @@ const std = @import("../../std.zig"); const math = std.math; const mem = std.mem; +const bitsliced = @import("bitsliced.zig"); const side_channels_mitigations = std.options.side_channels_mitigations; +const protected = side_channels_mitigations != .none; + /// A single AES block. pub const Block = struct { const Repr = [4]u32; @@ -288,7 +291,7 @@ pub const Block = struct { /// Perform operations on multiple blocks in parallel. pub const parallel = struct { /// The recommended number of AES encryption/decryption to perform in parallel for the chosen implementation. - pub const optimal_parallel_blocks = 1; + pub const optimal_parallel_blocks = if (protected) bitsliced.width else 1; /// Encrypt multiple blocks in parallel, each their own round key. pub fn encryptParallel(comptime count: usize, blocks: [count]Block, round_keys: [count]Block) [count]Block { @@ -539,6 +542,38 @@ fn KeySchedule(comptime Aes: type) type { }; } +/// Packed forward round keys for the bitsliced implementation +fn PackedKeys(comptime rounds: usize) type { + return if (protected) [rounds + 1]bitsliced.State else void; +} + +/// Bitsliced operation over `count` blocks +fn bitslicedOperation( + comptime rounds: usize, + comptime operation: enum { encrypt, decrypt }, + keys: *const [rounds + 1]bitsliced.State, + comptime count: usize, + dst: *[16 * count]u8, + src: *const [16 * count]u8, +) void { + const w = bitsliced.width; + const chunks = (count + w - 1) / w; + inline for (0..chunks) |c| { + const off: usize = c * w; + const n: usize = @min(w, count - off); + const lo = off * 16; + const hi = lo + n * 16; + var buf: [w * 16]u8 = undefined; + @memcpy(buf[0 .. n * 16], src[lo..hi]); + if (operation == .decrypt) { + bitsliced.Wide.decrypt(rounds, keys, &buf); + } else { + bitsliced.Wide.encrypt(rounds, keys, &buf); + } + @memcpy(dst[lo..hi], buf[0 .. n * 16]); + } +} + /// A context to perform encryption using the standard AES key schedule. pub fn AesEncryptCtx(comptime Aes: type) type { std.debug.assert(Aes.key_bits == 128 or Aes.key_bits == 256); @@ -549,73 +584,54 @@ pub fn AesEncryptCtx(comptime Aes: type) type { pub const block = Aes.block; pub const block_length = block.block_length; key_schedule: KeySchedule(Aes), + packed_keys: PackedKeys(rounds), /// Create a new encryption context with the given key. pub fn init(key: [Aes.key_bits / 8]u8) Self { const key_schedule = KeySchedule(Aes).expandKey(key); return Self{ .key_schedule = key_schedule, + .packed_keys = if (protected) bitsliced.Wide.packKeys(key_schedule.round_keys[0..]) else {}, }; } + /// Encrypt a single block, without any protection against side channels. + fn encryptUnprotectedBlock(ctx: Self, dst: *[16]u8, src: *const [16]u8) void { + const round_keys = ctx.key_schedule.round_keys; + var t = Block.fromBytes(src).xorBlocks(round_keys[0]); + inline for (1..rounds) |i| { + t = t.encryptUnprotected(round_keys[i]); + } + t = t.encryptLast(round_keys[rounds]); + dst.* = t.toBytes(); + } + /// Encrypt a single block. pub fn encrypt(ctx: Self, dst: *[16]u8, src: *const [16]u8) void { - const round_keys = ctx.key_schedule.round_keys; - var t = Block.fromBytes(src).xorBlocks(round_keys[0]); - comptime var i = 1; - if (side_channels_mitigations == .full) { - inline while (i < rounds) : (i += 1) { - t = t.encrypt(round_keys[i]); - } - } else { - inline while (i < 5) : (i += 1) { - t = t.encrypt(round_keys[i]); - } - inline while (i < rounds - 1) : (i += 1) { - t = t.encryptUnprotected(round_keys[i]); - } - t = t.encrypt(round_keys[i]); - } - t = t.encryptLast(round_keys[rounds]); - dst.* = t.toBytes(); + ctx.encryptWide(1, dst, src); } /// Encrypt+XOR a single block. pub fn xor(ctx: Self, dst: *[16]u8, src: *const [16]u8, counter: [16]u8) void { - const round_keys = ctx.key_schedule.round_keys; - var t = Block.fromBytes(&counter).xorBlocks(round_keys[0]); - comptime var i = 1; - if (side_channels_mitigations == .full) { - inline while (i < rounds) : (i += 1) { - t = t.encrypt(round_keys[i]); - } - } else { - inline while (i < 5) : (i += 1) { - t = t.encrypt(round_keys[i]); - } - inline while (i < rounds - 1) : (i += 1) { - t = t.encryptUnprotected(round_keys[i]); - } - t = t.encrypt(round_keys[i]); - } - t = t.encryptLast(round_keys[rounds]); - dst.* = t.xorBytes(src); + ctx.xorWide(1, dst, src, counter); } /// Encrypt multiple blocks, possibly leveraging parallelization. pub fn encryptWide(ctx: Self, comptime count: usize, dst: *[16 * count]u8, src: *const [16 * count]u8) void { - var i: usize = 0; - while (i < count) : (i += 1) { - ctx.encrypt(dst[16 * i .. 16 * i + 16][0..16], src[16 * i .. 16 * i + 16][0..16]); + if (protected) { + bitslicedOperation(rounds, .encrypt, &ctx.packed_keys, count, dst, src); + } else { + for (0..count) |i| { + ctx.encryptUnprotectedBlock(dst[16 * i .. 16 * i + 16][0..16], src[16 * i .. 16 * i + 16][0..16]); + } } } /// Encrypt+XOR multiple blocks, possibly leveraging parallelization. pub fn xorWide(ctx: Self, comptime count: usize, dst: *[16 * count]u8, src: *const [16 * count]u8, counters: [16 * count]u8) void { - var i: usize = 0; - while (i < count) : (i += 1) { - ctx.xor(dst[16 * i .. 16 * i + 16][0..16], src[16 * i .. 16 * i + 16][0..16], counters[16 * i .. 16 * i + 16][0..16].*); - } + var ks: [16 * count]u8 = undefined; + ctx.encryptWide(count, &ks, &counters); + for (dst, ks, src) |*d, k, s| d.* = k ^ s; } }; } @@ -630,11 +646,13 @@ pub fn AesDecryptCtx(comptime Aes: type) type { pub const block = Aes.block; pub const block_length = block.block_length; key_schedule: KeySchedule(Aes), + packed_keys: PackedKeys(rounds), /// Create a decryption context from an existing encryption context. pub fn initFromEnc(ctx: AesEncryptCtx(Aes)) Self { return Self{ .key_schedule = ctx.key_schedule.invert(), + .packed_keys = ctx.packed_keys, }; } @@ -644,33 +662,30 @@ pub fn AesDecryptCtx(comptime Aes: type) type { return initFromEnc(enc_ctx); } - /// Decrypt a single block. - pub fn decrypt(ctx: Self, dst: *[16]u8, src: *const [16]u8) void { + /// Decrypt a single block, without any protection against side channels. + fn decryptUnprotectedBlock(ctx: Self, dst: *[16]u8, src: *const [16]u8) void { const inv_round_keys = ctx.key_schedule.round_keys; var t = Block.fromBytes(src).xorBlocks(inv_round_keys[0]); - comptime var i = 1; - if (side_channels_mitigations == .full) { - inline while (i < rounds) : (i += 1) { - t = t.decrypt(inv_round_keys[i]); - } - } else { - inline while (i < 5) : (i += 1) { - t = t.decrypt(inv_round_keys[i]); - } - inline while (i < rounds - 1) : (i += 1) { - t = t.decryptUnprotected(inv_round_keys[i]); - } - t = t.decrypt(inv_round_keys[i]); + inline for (1..rounds) |i| { + t = t.decryptUnprotected(inv_round_keys[i]); } t = t.decryptLast(inv_round_keys[rounds]); dst.* = t.toBytes(); } + /// Decrypt a single block. + pub fn decrypt(ctx: Self, dst: *[16]u8, src: *const [16]u8) void { + ctx.decryptWide(1, dst, src); + } + /// Decrypt multiple blocks, possibly leveraging parallelization. pub fn decryptWide(ctx: Self, comptime count: usize, dst: *[16 * count]u8, src: *const [16 * count]u8) void { - var i: usize = 0; - while (i < count) : (i += 1) { - ctx.decrypt(dst[16 * i .. 16 * i + 16][0..16], src[16 * i .. 16 * i + 16][0..16]); + if (protected) { + bitslicedOperation(rounds, .decrypt, &ctx.packed_keys, count, dst, src); + } else { + for (0..count) |i| { + ctx.decryptUnprotectedBlock(dst[16 * i .. 16 * i + 16][0..16], src[16 * i .. 16 * i + 16][0..16]); + } } } }; diff --git a/lib/std/crypto/aes_ocb.zig b/lib/std/crypto/aes_ocb.zig index 5c3af0615112822c58c892fa373150e23050dfcc..b83400e5c3bbd2443f03130f3fcb0fb4501d59db 100644 --- a/lib/std/crypto/aes_ocb.zig +++ b/lib/std/crypto/aes_ocb.zig @@ -103,7 +103,10 @@ fn AesOcb(comptime Aes: anytype) type { const has_aesni = builtin.cpu.has(.x86, .aes); const has_armaes = builtin.cpu.has(.aarch64, .aes); - const wb: usize = if ((builtin.cpu.arch == .x86_64 and has_aesni) or (builtin.cpu.arch == .aarch64 and has_armaes)) 4 else 0; + const wb: usize = if ((builtin.cpu.arch == .x86_64 and has_aesni) or (builtin.cpu.arch == .aarch64 and has_armaes)) + 4 // empirically what works best on x86_64 and aarch64 with AES-NI/ARM Crypto + else + Aes.block.parallel.optimal_parallel_blocks; /// c: ciphertext: output buffer should be of size m.len /// tag: authentication tag: output MAC @@ -124,7 +127,7 @@ fn AesOcb(comptime Aes: anytype) type { var sum: [16]u8 = @splat(0); var i: usize = 0; - while (wb > 0 and i + wb <= full_blocks) : (i += wb) { + while (i + wb <= full_blocks) : (i += wb) { var offsets: [wb]Block align(16) = undefined; var es: [16 * wb]u8 align(16) = undefined; var j: usize = 0; @@ -191,7 +194,7 @@ fn AesOcb(comptime Aes: anytype) type { var sum: [16]u8 = @splat(0); var i: usize = 0; - while (wb > 0 and i + wb <= full_blocks) : (i += wb) { + while (i + wb <= full_blocks) : (i += wb) { var offsets: [wb]Block align(16) = undefined; var es: [16 * wb]u8 align(16) = undefined; var j: usize = 0; diff --git a/lib/std/crypto/modes.zig b/lib/std/crypto/modes.zig index eeba224d2738706774da66af30c4700aa9d0831f..145a0e5d5ffe29b1246e86c2539c7e7877472cfc 100644 --- a/lib/std/crypto/modes.zig +++ b/lib/std/crypto/modes.zig @@ -37,14 +37,13 @@ pub fn ctrSlice( debug.assert(counter_offset + counter_size <= block_length); debug.assert(counter_size > 0 and counter_size <= block_length); - var counterBlock = iv; var i: usize = 0; const CounterInt = @Int(.unsigned, counter_size * 8); const parallel_count = BlockCipher.block.parallel.optimal_parallel_blocks; const wide_block_length = parallel_count * block_length; - var cnt_val = mem.readInt(CounterInt, counterBlock[counter_offset..][0..counter_size], endian); + var cnt_val = mem.readInt(CounterInt, iv[counter_offset..][0..counter_size], endian); if (src.len >= wide_block_length) { var counters: [parallel_count * block_length]u8 = undefined; inline for (0..parallel_count) |j| { @@ -58,20 +57,18 @@ pub fn ctrSlice( cnt_val +%= parallel_count; block_cipher.xorWide(parallel_count, dst[i .. i + wide_block_length][0..wide_block_length], src[i .. i + wide_block_length][0..wide_block_length], counters); } - mem.writeInt(CounterInt, counterBlock[counter_offset..][0..counter_size], cnt_val, endian); - } - while (i + block_length <= src.len) : (i += block_length) { - block_cipher.xor(dst[i .. i + block_length][0..block_length], src[i .. i + block_length][0..block_length], counterBlock); - cnt_val +%= 1; - mem.writeInt(CounterInt, counterBlock[counter_offset..][0..counter_size], cnt_val, endian); } if (i < src.len) { - var pad: [block_length]u8 = @splat(0); - const src_slice = src[i..]; - @memcpy(pad[0..src_slice.len], src_slice); - block_cipher.xor(&pad, &pad, counterBlock); - const pad_slice = pad[0 .. src.len - i]; - @memcpy(dst[i..][0..pad_slice.len], pad_slice); + const rem = src.len - i; + var counters: [wide_block_length]u8 = undefined; + inline for (0..parallel_count) |j| { + counters[j * block_length ..][0..block_length].* = iv; + const block_ctr = cnt_val +% @as(CounterInt, j); + mem.writeInt(CounterInt, counters[j * block_length + counter_offset ..][0..counter_size], block_ctr, endian); + } + var keystream: [wide_block_length]u8 = undefined; + block_cipher.encryptWide(parallel_count, &keystream, &counters); + for (dst[i..][0..rem], src[i..][0..rem], keystream[0..rem]) |*d, s, k| d.* = s ^ k; } } -- 2.54.0