authorgravatar for 124872+jedisct1@users.noreply.github.comFrank Denis <124872+jedisct1@users.noreply.github.com> 2026-05-02 14:35:10+02:00
committergravatar for 124872+jedisct1@users.noreply.github.comFrank Denis <124872+jedisct1@users.noreply.github.com> 2026-09-20 16:05:33+02:00
logcd14ed892b6b5786ae4498140e9ccddbfcc9b971
treecd3a4d89d6fa171bf0c9bf0d83ad084fd9659cf5
parent3bfb2999471d921cfdf461c3c12f6caedb45904b

std.crypto.argon2: vectorize the BlaMka round function

Use a vector+@shuffle isntead of the scalar loop. Also avoid the intermediate buffer copy in the column-wise blamka pass by passing pointers directly. This makes argon2 about 30% faster (tested on x86_64 and aarch64).

1 files changed, 61 insertions(+), 43 deletions(-)

lib/std/crypto/argon2.zig+61-43
......@@ -8,7 +8,6 @@ const std = @import("std");
88const blake2 = crypto.hash.blake2;
99const crypto = std.crypto;
1010const Io = std.Io;
11const math = std.math;
1211const mem = std.mem;
1312const phc_format = pwhash.phc_format;
1413const pwhash = crypto.pwhash;
......@@ -322,7 +321,11 @@ fn processSegment(
322321 random = blocks.items[prev][0];
323322 }
324323 const new_offset = indexAlpha(random, lanes, segments, threads, n, slice, lane, index);
325 processBlockXor(&blocks.items[offset], &blocks.items[prev], &blocks.items[new_offset]);
324 if (n == 0) {
325 processBlock(&blocks.items[offset], &blocks.items[prev], &blocks.items[new_offset]);
326 } else {
327 processBlockXor(&blocks.items[offset], &blocks.items[prev], &blocks.items[new_offset]);
328 }
326329 }
327330}
328331
......@@ -354,22 +357,25 @@ fn processBlockGeneric(
354357 }
355358 var i: usize = 0;
356359 while (i < block_length) : (i += 16) {
357 blamkaGeneric(t[i..][0..16]);
360 blamkaGeneric(.{
361 &t[i + 0], &t[i + 1], &t[i + 2], &t[i + 3],
362 &t[i + 4], &t[i + 5], &t[i + 6], &t[i + 7],
363 &t[i + 8], &t[i + 9], &t[i + 10], &t[i + 11],
364 &t[i + 12], &t[i + 13], &t[i + 14], &t[i + 15],
365 });
358366 }
359367 i = 0;
360 var buffer: [16]u64 = undefined;
361368 while (i < block_length / 8) : (i += 2) {
362 var j: usize = 0;
363 while (j < block_length / 8) : (j += 2) {
364 buffer[j] = t[j * 8 + i];
365 buffer[j + 1] = t[j * 8 + i + 1];
366 }
367 blamkaGeneric(&buffer);
368 j = 0;
369 while (j < block_length / 8) : (j += 2) {
370 t[j * 8 + i] = buffer[j];
371 t[j * 8 + i + 1] = buffer[j + 1];
372 }
369 blamkaGeneric(.{
370 &t[0 * 16 + i], &t[0 * 16 + i + 1],
371 &t[1 * 16 + i], &t[1 * 16 + i + 1],
372 &t[2 * 16 + i], &t[2 * 16 + i + 1],
373 &t[3 * 16 + i], &t[3 * 16 + i + 1],
374 &t[4 * 16 + i], &t[4 * 16 + i + 1],
375 &t[5 * 16 + i], &t[5 * 16 + i + 1],
376 &t[6 * 16 + i], &t[6 * 16 + i + 1],
377 &t[7 * 16 + i], &t[7 * 16 + i + 1],
378 });
373379 }
374380 if (xor) {
375381 for (t, 0..) |v, j| {
......@@ -382,38 +388,50 @@ fn processBlockGeneric(
382388 }
383389}
384390
385const QuarterRound = struct { a: usize, b: usize, c: usize, d: usize };
391const BlamkaVector = @Vector(4, u64);
386392
387fn Rp(a: usize, b: usize, c: usize, d: usize) QuarterRound {
388 return .{ .a = a, .b = b, .c = c, .d = d };
393inline fn fBlaMka(x: BlamkaVector, y: BlamkaVector) BlamkaVector {
394 const x_lo: @Vector(4, u32) = @truncate(x);
395 const y_lo: @Vector(4, u32) = @truncate(y);
396 const xy = @as(BlamkaVector, x_lo) * @as(BlamkaVector, y_lo);
397 return x +% y +% @as(BlamkaVector, @splat(2)) *% xy;
389398}
390399
391fn fBlaMka(x: u64, y: u64) u64 {
392 const xy = @as(u64, @as(u32, @truncate(x))) * @as(u64, @as(u32, @truncate(y)));
393 return x +% y +% 2 *% xy;
400inline fn rotrVector(x: BlamkaVector, comptime n: comptime_int) BlamkaVector {
401 return (x >> @splat(n)) | (x << @splat(64 - n));
394402}
395403
396fn blamkaGeneric(x: *[16]u64) void {
397 const rounds = comptime [_]QuarterRound{
398 Rp(0, 4, 8, 12),
399 Rp(1, 5, 9, 13),
400 Rp(2, 6, 10, 14),
401 Rp(3, 7, 11, 15),
402 Rp(0, 5, 10, 15),
403 Rp(1, 6, 11, 12),
404 Rp(2, 7, 8, 13),
405 Rp(3, 4, 9, 14),
406 };
407 inline for (rounds) |r| {
408 x[r.a] = fBlaMka(x[r.a], x[r.b]);
409 x[r.d] = math.rotr(u64, x[r.d] ^ x[r.a], 32);
410 x[r.c] = fBlaMka(x[r.c], x[r.d]);
411 x[r.b] = math.rotr(u64, x[r.b] ^ x[r.c], 24);
412 x[r.a] = fBlaMka(x[r.a], x[r.b]);
413 x[r.d] = math.rotr(u64, x[r.d] ^ x[r.a], 16);
414 x[r.c] = fBlaMka(x[r.c], x[r.d]);
415 x[r.b] = math.rotr(u64, x[r.b] ^ x[r.c], 63);
416 }
404inline fn blamkaRound(a: *BlamkaVector, b: *BlamkaVector, c: *BlamkaVector, d: *BlamkaVector) void {
405 a.* = fBlaMka(a.*, b.*);
406 d.* = rotrVector(d.* ^ a.*, 32);
407 c.* = fBlaMka(c.*, d.*);
408 b.* = rotrVector(b.* ^ c.*, 24);
409 a.* = fBlaMka(a.*, b.*);
410 d.* = rotrVector(d.* ^ a.*, 16);
411 c.* = fBlaMka(c.*, d.*);
412 b.* = rotrVector(b.* ^ c.*, 63);
413}
414
415fn blamkaGeneric(xs: [16]*u64) void {
416 var a: BlamkaVector = .{ xs[0].*, xs[1].*, xs[2].*, xs[3].* };
417 var b: BlamkaVector = .{ xs[4].*, xs[5].*, xs[6].*, xs[7].* };
418 var c: BlamkaVector = .{ xs[8].*, xs[9].*, xs[10].*, xs[11].* };
419 var d: BlamkaVector = .{ xs[12].*, xs[13].*, xs[14].*, xs[15].* };
420
421 blamkaRound(&a, &b, &c, &d);
422
423 var b_diag = @shuffle(u64, b, undefined, [_]i32{ 1, 2, 3, 0 });
424 var c_diag = @shuffle(u64, c, undefined, [_]i32{ 2, 3, 0, 1 });
425 var d_diag = @shuffle(u64, d, undefined, [_]i32{ 3, 0, 1, 2 });
426 blamkaRound(&a, &b_diag, &c_diag, &d_diag);
427 b = @shuffle(u64, b_diag, undefined, [_]i32{ 3, 0, 1, 2 });
428 c = @shuffle(u64, c_diag, undefined, [_]i32{ 2, 3, 0, 1 });
429 d = @shuffle(u64, d_diag, undefined, [_]i32{ 1, 2, 3, 0 });
430
431 inline for (0..4) |k| xs[k].* = a[k];
432 inline for (0..4) |k| xs[4 + k].* = b[k];
433 inline for (0..4) |k| xs[8 + k].* = c[k];
434 inline for (0..4) |k| xs[12 + k].* = d[k];
417435}
418436
419437fn finalize(
......@@ -511,7 +529,7 @@ pub fn kdf(
511529 var blocks = try Blocks.initCapacity(allocator, memory);
512530 defer blocks.deinit();
513531
514 blocks.appendNTimesAssumeCapacity(@splat(0), memory);
532 _ = blocks.addManyAsSliceAssumeCapacity(memory);
515533
516534 initBlocks(&blocks, &h0, memory, params.p);
517535 try processBlocks(&blocks, params.t, memory, params.p, mode, io);