authorgravatar for 124872+jedisct1@users.noreply.github.comFrank Denis <124872+jedisct1@users.noreply.github.com> 2026-05-02 14:35:10+02:00
committergravatar for 124872+jedisct1@users.noreply.github.comFrank Denis <124872+jedisct1@users.noreply.github.com> 2026-09-20 16:05:33+02:00
logcd14ed892b6b5786ae4498140e9ccddbfcc9b971
treecd3a4d89d6fa171bf0c9bf0d83ad084fd9659cf5
parent3bfb2999471d921cfdf461c3c12f6caedb45904b

std.crypto.argon2: vectorize the BlaMka round function

Use a vector+@shuffle isntead of the scalar loop. Also avoid the intermediate buffer copy in the column-wise blamka pass by passing pointers directly. This makes argon2 about 30% faster (tested on x86_64 and aarch64).

1 files changed, 61 insertions(+), 43 deletions(-)

lib/std/crypto/argon2.zig+61-43
...@@ -8,7 +8,6 @@ const std = @import("std");...@@ -8,7 +8,6 @@ const std = @import("std");
8const blake2 = crypto.hash.blake2;8const blake2 = crypto.hash.blake2;
9const crypto = std.crypto;9const crypto = std.crypto;
10const Io = std.Io;10const Io = std.Io;
11const math = std.math;
12const mem = std.mem;11const mem = std.mem;
13const phc_format = pwhash.phc_format;12const phc_format = pwhash.phc_format;
14const pwhash = crypto.pwhash;13const pwhash = crypto.pwhash;
...@@ -322,7 +321,11 @@ fn processSegment(...@@ -322,7 +321,11 @@ fn processSegment(
322 random = blocks.items[prev][0];321 random = blocks.items[prev][0];
323 }322 }
324 const new_offset = indexAlpha(random, lanes, segments, threads, n, slice, lane, index);323 const new_offset = indexAlpha(random, lanes, segments, threads, n, slice, lane, index);
325 processBlockXor(&blocks.items[offset], &blocks.items[prev], &blocks.items[new_offset]);324 if (n == 0) {
325 processBlock(&blocks.items[offset], &blocks.items[prev], &blocks.items[new_offset]);
326 } else {
327 processBlockXor(&blocks.items[offset], &blocks.items[prev], &blocks.items[new_offset]);
328 }
326 }329 }
327}330}
328331
...@@ -354,22 +357,25 @@ fn processBlockGeneric(...@@ -354,22 +357,25 @@ fn processBlockGeneric(
354 }357 }
355 var i: usize = 0;358 var i: usize = 0;
356 while (i < block_length) : (i += 16) {359 while (i < block_length) : (i += 16) {
357 blamkaGeneric(t[i..][0..16]);360 blamkaGeneric(.{
361 &t[i + 0], &t[i + 1], &t[i + 2], &t[i + 3],
362 &t[i + 4], &t[i + 5], &t[i + 6], &t[i + 7],
363 &t[i + 8], &t[i + 9], &t[i + 10], &t[i + 11],
364 &t[i + 12], &t[i + 13], &t[i + 14], &t[i + 15],
365 });
358 }366 }
359 i = 0;367 i = 0;
360 var buffer: [16]u64 = undefined;
361 while (i < block_length / 8) : (i += 2) {368 while (i < block_length / 8) : (i += 2) {
362 var j: usize = 0;369 blamkaGeneric(.{
363 while (j < block_length / 8) : (j += 2) {370 &t[0 * 16 + i], &t[0 * 16 + i + 1],
364 buffer[j] = t[j * 8 + i];371 &t[1 * 16 + i], &t[1 * 16 + i + 1],
365 buffer[j + 1] = t[j * 8 + i + 1];372 &t[2 * 16 + i], &t[2 * 16 + i + 1],
366 }373 &t[3 * 16 + i], &t[3 * 16 + i + 1],
367 blamkaGeneric(&buffer);374 &t[4 * 16 + i], &t[4 * 16 + i + 1],
368 j = 0;375 &t[5 * 16 + i], &t[5 * 16 + i + 1],
369 while (j < block_length / 8) : (j += 2) {376 &t[6 * 16 + i], &t[6 * 16 + i + 1],
370 t[j * 8 + i] = buffer[j];377 &t[7 * 16 + i], &t[7 * 16 + i + 1],
371 t[j * 8 + i + 1] = buffer[j + 1];378 });
372 }
373 }379 }
374 if (xor) {380 if (xor) {
375 for (t, 0..) |v, j| {381 for (t, 0..) |v, j| {
...@@ -382,38 +388,50 @@ fn processBlockGeneric(...@@ -382,38 +388,50 @@ fn processBlockGeneric(
382 }388 }
383}389}
384390
385const QuarterRound = struct { a: usize, b: usize, c: usize, d: usize };391const BlamkaVector = @Vector(4, u64);
386392
387fn Rp(a: usize, b: usize, c: usize, d: usize) QuarterRound {393inline fn fBlaMka(x: BlamkaVector, y: BlamkaVector) BlamkaVector {
388 return .{ .a = a, .b = b, .c = c, .d = d };394 const x_lo: @Vector(4, u32) = @truncate(x);
395 const y_lo: @Vector(4, u32) = @truncate(y);
396 const xy = @as(BlamkaVector, x_lo) * @as(BlamkaVector, y_lo);
397 return x +% y +% @as(BlamkaVector, @splat(2)) *% xy;
389}398}
390399
391fn fBlaMka(x: u64, y: u64) u64 {400inline fn rotrVector(x: BlamkaVector, comptime n: comptime_int) BlamkaVector {
392 const xy = @as(u64, @as(u32, @truncate(x))) * @as(u64, @as(u32, @truncate(y)));401 return (x >> @splat(n)) | (x << @splat(64 - n));
393 return x +% y +% 2 *% xy;
394}402}
395403
396fn blamkaGeneric(x: *[16]u64) void {404inline fn blamkaRound(a: *BlamkaVector, b: *BlamkaVector, c: *BlamkaVector, d: *BlamkaVector) void {
397 const rounds = comptime [_]QuarterRound{405 a.* = fBlaMka(a.*, b.*);
398 Rp(0, 4, 8, 12),406 d.* = rotrVector(d.* ^ a.*, 32);
399 Rp(1, 5, 9, 13),407 c.* = fBlaMka(c.*, d.*);
400 Rp(2, 6, 10, 14),408 b.* = rotrVector(b.* ^ c.*, 24);
401 Rp(3, 7, 11, 15),409 a.* = fBlaMka(a.*, b.*);
402 Rp(0, 5, 10, 15),410 d.* = rotrVector(d.* ^ a.*, 16);
403 Rp(1, 6, 11, 12),411 c.* = fBlaMka(c.*, d.*);
404 Rp(2, 7, 8, 13),412 b.* = rotrVector(b.* ^ c.*, 63);
405 Rp(3, 4, 9, 14),413}
406 };414
407 inline for (rounds) |r| {415fn blamkaGeneric(xs: [16]*u64) void {
408 x[r.a] = fBlaMka(x[r.a], x[r.b]);416 var a: BlamkaVector = .{ xs[0].*, xs[1].*, xs[2].*, xs[3].* };
409 x[r.d] = math.rotr(u64, x[r.d] ^ x[r.a], 32);417 var b: BlamkaVector = .{ xs[4].*, xs[5].*, xs[6].*, xs[7].* };
410 x[r.c] = fBlaMka(x[r.c], x[r.d]);418 var c: BlamkaVector = .{ xs[8].*, xs[9].*, xs[10].*, xs[11].* };
411 x[r.b] = math.rotr(u64, x[r.b] ^ x[r.c], 24);419 var d: BlamkaVector = .{ xs[12].*, xs[13].*, xs[14].*, xs[15].* };
412 x[r.a] = fBlaMka(x[r.a], x[r.b]);420
413 x[r.d] = math.rotr(u64, x[r.d] ^ x[r.a], 16);421 blamkaRound(&a, &b, &c, &d);
414 x[r.c] = fBlaMka(x[r.c], x[r.d]);422
415 x[r.b] = math.rotr(u64, x[r.b] ^ x[r.c], 63);423 var b_diag = @shuffle(u64, b, undefined, [_]i32{ 1, 2, 3, 0 });
416 }424 var c_diag = @shuffle(u64, c, undefined, [_]i32{ 2, 3, 0, 1 });
425 var d_diag = @shuffle(u64, d, undefined, [_]i32{ 3, 0, 1, 2 });
426 blamkaRound(&a, &b_diag, &c_diag, &d_diag);
427 b = @shuffle(u64, b_diag, undefined, [_]i32{ 3, 0, 1, 2 });
428 c = @shuffle(u64, c_diag, undefined, [_]i32{ 2, 3, 0, 1 });
429 d = @shuffle(u64, d_diag, undefined, [_]i32{ 1, 2, 3, 0 });
430
431 inline for (0..4) |k| xs[k].* = a[k];
432 inline for (0..4) |k| xs[4 + k].* = b[k];
433 inline for (0..4) |k| xs[8 + k].* = c[k];
434 inline for (0..4) |k| xs[12 + k].* = d[k];
417}435}
418436
419fn finalize(437fn finalize(
...@@ -511,7 +529,7 @@ pub fn kdf(...@@ -511,7 +529,7 @@ pub fn kdf(
511 var blocks = try Blocks.initCapacity(allocator, memory);529 var blocks = try Blocks.initCapacity(allocator, memory);
512 defer blocks.deinit();530 defer blocks.deinit();
513531
514 blocks.appendNTimesAssumeCapacity(@splat(0), memory);532 _ = blocks.addManyAsSliceAssumeCapacity(memory);
515533
516 initBlocks(&blocks, &h0, memory, params.p);534 initBlocks(&blocks, &h0, memory, params.p);
517 try processBlocks(&blocks, params.t, memory, params.p, mode, io);535 try processBlocks(&blocks, params.t, memory, params.p, mode, io);