authorgravatar for 124872+jedisct1@users.noreply.github.comFrank Denis <124872+jedisct1@users.noreply.github.com> 2026-09-21 03:04:38+02:00
committergravatar for 124872+jedisct1@users.noreply.github.comFrank Denis <124872+jedisct1@users.noreply.github.com> 2026-09-21 03:04:38+02:00
loge519494530089fd932a9819f6136d42cfef891d5
treecd6e8c5f5ed9d14fe6fb619707ce1a0d2a1bfaf3
parent3bfb2999471d921cfdf461c3c12f6caedb45904b
parentfb52b2dcdf9cef0ad1eb1e442072472113d1f6cc

Merge pull request 'std.crypto.argon2: vectorize the BlaMka round function' (#32202) from jedisct1/zig:argon2v into master

Reviewed-on: https://codeberg.org/ziglang/zig/pulls/32202

1 files changed, 83 insertions(+), 70 deletions(-)

lib/std/crypto/argon2.zig+83-70
......@@ -8,7 +8,6 @@ const std = @import("std");
88const blake2 = crypto.hash.blake2;
99const crypto = std.crypto;
1010const Io = std.Io;
11const math = std.math;
1211const mem = std.mem;
1312const phc_format = pwhash.phc_format;
1413const pwhash = crypto.pwhash;
......@@ -177,32 +176,6 @@ fn blake2bLong(out: []u8, in: []const u8) void {
177176 @memcpy(out_slice, out_buf[0..out_slice.len]);
178177}
179178
180fn initBlocks(
181 blocks: *Blocks,
182 h0: *H0,
183 memory: u32,
184 threads: u24,
185) void {
186 var block0: [1024]u8 = undefined;
187 var lane: u24 = 0;
188 while (lane < threads) : (lane += 1) {
189 const j = lane * (memory / threads);
190 mem.writeInt(u32, h0[Blake2b512.digest_length + 4 ..][0..4], lane, .little);
191
192 mem.writeInt(u32, h0[Blake2b512.digest_length..][0..4], 0, .little);
193 blake2bLong(&block0, h0);
194 for (&blocks.items[j + 0], 0..) |*v, i| {
195 v.* = mem.readInt(u64, block0[i * 8 ..][0..8], .little);
196 }
197
198 mem.writeInt(u32, h0[Blake2b512.digest_length..][0..4], 1, .little);
199 blake2bLong(&block0, h0);
200 for (&blocks.items[j + 1], 0..) |*v, i| {
201 v.* = mem.readInt(u64, block0[i * 8 ..][0..8], .little);
202 }
203 }
204}
205
206179fn processBlocks(
207180 blocks: *Blocks,
208181 time: u32,
......@@ -322,7 +295,11 @@ fn processSegment(
322295 random = blocks.items[prev][0];
323296 }
324297 const new_offset = indexAlpha(random, lanes, segments, threads, n, slice, lane, index);
325 processBlockXor(&blocks.items[offset], &blocks.items[prev], &blocks.items[new_offset]);
298 if (n == 0) {
299 processBlock(&blocks.items[offset], &blocks.items[prev], &blocks.items[new_offset]);
300 } else {
301 processBlockXor(&blocks.items[offset], &blocks.items[prev], &blocks.items[new_offset]);
302 }
326303 }
327304}
328305
......@@ -354,22 +331,25 @@ fn processBlockGeneric(
354331 }
355332 var i: usize = 0;
356333 while (i < block_length) : (i += 16) {
357 blamkaGeneric(t[i..][0..16]);
334 blamkaGeneric(.{
335 &t[i + 0], &t[i + 1], &t[i + 2], &t[i + 3],
336 &t[i + 4], &t[i + 5], &t[i + 6], &t[i + 7],
337 &t[i + 8], &t[i + 9], &t[i + 10], &t[i + 11],
338 &t[i + 12], &t[i + 13], &t[i + 14], &t[i + 15],
339 });
358340 }
359341 i = 0;
360 var buffer: [16]u64 = undefined;
361342 while (i < block_length / 8) : (i += 2) {
362 var j: usize = 0;
363 while (j < block_length / 8) : (j += 2) {
364 buffer[j] = t[j * 8 + i];
365 buffer[j + 1] = t[j * 8 + i + 1];
366 }
367 blamkaGeneric(&buffer);
368 j = 0;
369 while (j < block_length / 8) : (j += 2) {
370 t[j * 8 + i] = buffer[j];
371 t[j * 8 + i + 1] = buffer[j + 1];
372 }
343 blamkaGeneric(.{
344 &t[0 * 16 + i], &t[0 * 16 + i + 1],
345 &t[1 * 16 + i], &t[1 * 16 + i + 1],
346 &t[2 * 16 + i], &t[2 * 16 + i + 1],
347 &t[3 * 16 + i], &t[3 * 16 + i + 1],
348 &t[4 * 16 + i], &t[4 * 16 + i + 1],
349 &t[5 * 16 + i], &t[5 * 16 + i + 1],
350 &t[6 * 16 + i], &t[6 * 16 + i + 1],
351 &t[7 * 16 + i], &t[7 * 16 + i + 1],
352 });
373353 }
374354 if (xor) {
375355 for (t, 0..) |v, j| {
......@@ -382,38 +362,53 @@ fn processBlockGeneric(
382362 }
383363}
384364
385const QuarterRound = struct { a: usize, b: usize, c: usize, d: usize };
365const BlamkaVector = @Vector(4, u64);
386366
387fn Rp(a: usize, b: usize, c: usize, d: usize) QuarterRound {
388 return .{ .a = a, .b = b, .c = c, .d = d };
367fn fBlaMka(x: BlamkaVector, y: BlamkaVector) BlamkaVector {
368 const x_lo: @Vector(4, u32) = @truncate(x);
369 const y_lo: @Vector(4, u32) = @truncate(y);
370 const xy = @as(BlamkaVector, x_lo) * @as(BlamkaVector, y_lo);
371 return x +% y +% @as(BlamkaVector, @splat(2)) *% xy;
389372}
390373
391fn fBlaMka(x: u64, y: u64) u64 {
392 const xy = @as(u64, @as(u32, @truncate(x))) * @as(u64, @as(u32, @truncate(y)));
393 return x +% y +% 2 *% xy;
374fn rotrVector(x: BlamkaVector, comptime n: comptime_int) BlamkaVector {
375 return (x >> @splat(n)) | (x << @splat(64 - n));
394376}
395377
396fn blamkaGeneric(x: *[16]u64) void {
397 const rounds = comptime [_]QuarterRound{
398 Rp(0, 4, 8, 12),
399 Rp(1, 5, 9, 13),
400 Rp(2, 6, 10, 14),
401 Rp(3, 7, 11, 15),
402 Rp(0, 5, 10, 15),
403 Rp(1, 6, 11, 12),
404 Rp(2, 7, 8, 13),
405 Rp(3, 4, 9, 14),
406 };
407 inline for (rounds) |r| {
408 x[r.a] = fBlaMka(x[r.a], x[r.b]);
409 x[r.d] = math.rotr(u64, x[r.d] ^ x[r.a], 32);
410 x[r.c] = fBlaMka(x[r.c], x[r.d]);
411 x[r.b] = math.rotr(u64, x[r.b] ^ x[r.c], 24);
412 x[r.a] = fBlaMka(x[r.a], x[r.b]);
413 x[r.d] = math.rotr(u64, x[r.d] ^ x[r.a], 16);
414 x[r.c] = fBlaMka(x[r.c], x[r.d]);
415 x[r.b] = math.rotr(u64, x[r.b] ^ x[r.c], 63);
416 }
378// The `inline` is load-bearing: without it ReleaseSmall keeps this out of line
379// and shuffles the four vectors through memory across every call, costing
380// roughly 9% on Apple Silicon.
381inline fn blamkaRound(a: *BlamkaVector, b: *BlamkaVector, c: *BlamkaVector, d: *BlamkaVector) void {
382 a.* = fBlaMka(a.*, b.*);
383 d.* = rotrVector(d.* ^ a.*, 32);
384 c.* = fBlaMka(c.*, d.*);
385 b.* = rotrVector(b.* ^ c.*, 24);
386 a.* = fBlaMka(a.*, b.*);
387 d.* = rotrVector(d.* ^ a.*, 16);
388 c.* = fBlaMka(c.*, d.*);
389 b.* = rotrVector(b.* ^ c.*, 63);
390}
391
392fn blamkaGeneric(xs: [16]*u64) void {
393 var a: BlamkaVector = .{ xs[0].*, xs[1].*, xs[2].*, xs[3].* };
394 var b: BlamkaVector = .{ xs[4].*, xs[5].*, xs[6].*, xs[7].* };
395 var c: BlamkaVector = .{ xs[8].*, xs[9].*, xs[10].*, xs[11].* };
396 var d: BlamkaVector = .{ xs[12].*, xs[13].*, xs[14].*, xs[15].* };
397
398 blamkaRound(&a, &b, &c, &d);
399
400 var b_diag = @shuffle(u64, b, undefined, [_]i32{ 1, 2, 3, 0 });
401 var c_diag = @shuffle(u64, c, undefined, [_]i32{ 2, 3, 0, 1 });
402 var d_diag = @shuffle(u64, d, undefined, [_]i32{ 3, 0, 1, 2 });
403 blamkaRound(&a, &b_diag, &c_diag, &d_diag);
404 b = @shuffle(u64, b_diag, undefined, [_]i32{ 3, 0, 1, 2 });
405 c = @shuffle(u64, c_diag, undefined, [_]i32{ 2, 3, 0, 1 });
406 d = @shuffle(u64, d_diag, undefined, [_]i32{ 1, 2, 3, 0 });
407
408 inline for (0..4) |k| xs[k].* = a[k];
409 inline for (0..4) |k| xs[4 + k].* = b[k];
410 inline for (0..4) |k| xs[8 + k].* = c[k];
411 inline for (0..4) |k| xs[12 + k].* = d[k];
417412}
418413
419414fn finalize(
......@@ -511,9 +506,27 @@ pub fn kdf(
511506 var blocks = try Blocks.initCapacity(allocator, memory);
512507 defer blocks.deinit();
513508
514 blocks.appendNTimesAssumeCapacity(@splat(0), memory);
509 const items = blocks.addManyAsSliceAssumeCapacity(memory);
510
511 var block0: [1024]u8 = undefined;
512 var lane: u24 = 0;
513 while (lane < params.p) : (lane += 1) {
514 const j = lane * (memory / params.p);
515 mem.writeInt(u32, h0[Blake2b512.digest_length + 4 ..][0..4], lane, .little);
516
517 mem.writeInt(u32, h0[Blake2b512.digest_length..][0..4], 0, .little);
518 blake2bLong(&block0, &h0);
519 for (&items[j + 0], 0..) |*v, i| {
520 v.* = mem.readInt(u64, block0[i * 8 ..][0..8], .little);
521 }
522
523 mem.writeInt(u32, h0[Blake2b512.digest_length..][0..4], 1, .little);
524 blake2bLong(&block0, &h0);
525 for (&items[j + 1], 0..) |*v, i| {
526 v.* = mem.readInt(u64, block0[i * 8 ..][0..8], .little);
527 }
528 }
515529
516 initBlocks(&blocks, &h0, memory, params.p);
517530 try processBlocks(&blocks, params.t, memory, params.p, mode, io);
518531 finalize(&blocks, memory, params.p, derived_key);
519532}