| ... | @@ -8,7 +8,6 @@ const std = @import("std"); | ... | @@ -8,7 +8,6 @@ const std = @import("std"); |
| 8 | const blake2 = crypto.hash.blake2; | 8 | const blake2 = crypto.hash.blake2; |
| 9 | const crypto = std.crypto; | 9 | const crypto = std.crypto; |
| 10 | const Io = std.Io; | 10 | const Io = std.Io; |
| 11 | const math = std.math; | | |
| 12 | const mem = std.mem; | 11 | const mem = std.mem; |
| 13 | const phc_format = pwhash.phc_format; | 12 | const phc_format = pwhash.phc_format; |
| 14 | const pwhash = crypto.pwhash; | 13 | const pwhash = crypto.pwhash; |
| ... | @@ -177,32 +176,6 @@ fn blake2bLong(out: []u8, in: []const u8) void { | ... | @@ -177,32 +176,6 @@ fn blake2bLong(out: []u8, in: []const u8) void { |
| 177 | @memcpy(out_slice, out_buf[0..out_slice.len]); | 176 | @memcpy(out_slice, out_buf[0..out_slice.len]); |
| 178 | } | 177 | } |
| 179 | | 178 | |
| 180 | fn initBlocks( | | |
| 181 | blocks: *Blocks, | | |
| 182 | h0: *H0, | | |
| 183 | memory: u32, | | |
| 184 | threads: u24, | | |
| 185 | ) void { | | |
| 186 | var block0: [1024]u8 = undefined; | | |
| 187 | var lane: u24 = 0; | | |
| 188 | while (lane < threads) : (lane += 1) { | | |
| 189 | const j = lane * (memory / threads); | | |
| 190 | mem.writeInt(u32, h0[Blake2b512.digest_length + 4 ..][0..4], lane, .little); | | |
| 191 | | | |
| 192 | mem.writeInt(u32, h0[Blake2b512.digest_length..][0..4], 0, .little); | | |
| 193 | blake2bLong(&block0, h0); | | |
| 194 | for (&blocks.items[j + 0], 0..) |*v, i| { | | |
| 195 | v.* = mem.readInt(u64, block0[i * 8 ..][0..8], .little); | | |
| 196 | } | | |
| 197 | | | |
| 198 | mem.writeInt(u32, h0[Blake2b512.digest_length..][0..4], 1, .little); | | |
| 199 | blake2bLong(&block0, h0); | | |
| 200 | for (&blocks.items[j + 1], 0..) |*v, i| { | | |
| 201 | v.* = mem.readInt(u64, block0[i * 8 ..][0..8], .little); | | |
| 202 | } | | |
| 203 | } | | |
| 204 | } | | |
| 205 | | | |
| 206 | fn processBlocks( | 179 | fn processBlocks( |
| 207 | blocks: *Blocks, | 180 | blocks: *Blocks, |
| 208 | time: u32, | 181 | time: u32, |
| ... | @@ -322,7 +295,11 @@ fn processSegment( | ... | @@ -322,7 +295,11 @@ fn processSegment( |
| 322 | random = blocks.items[prev][0]; | 295 | random = blocks.items[prev][0]; |
| 323 | } | 296 | } |
| 324 | const new_offset = indexAlpha(random, lanes, segments, threads, n, slice, lane, index); | 297 | const new_offset = indexAlpha(random, lanes, segments, threads, n, slice, lane, index); |
| 325 | processBlockXor(&blocks.items[offset], &blocks.items[prev], &blocks.items[new_offset]); | 298 | if (n == 0) { |
| | 299 | processBlock(&blocks.items[offset], &blocks.items[prev], &blocks.items[new_offset]); |
| | 300 | } else { |
| | 301 | processBlockXor(&blocks.items[offset], &blocks.items[prev], &blocks.items[new_offset]); |
| | 302 | } |
| 326 | } | 303 | } |
| 327 | } | 304 | } |
| 328 | | 305 | |
| ... | @@ -354,22 +331,25 @@ fn processBlockGeneric( | ... | @@ -354,22 +331,25 @@ fn processBlockGeneric( |
| 354 | } | 331 | } |
| 355 | var i: usize = 0; | 332 | var i: usize = 0; |
| 356 | while (i < block_length) : (i += 16) { | 333 | while (i < block_length) : (i += 16) { |
| 357 | blamkaGeneric(t[i..][0..16]); | 334 | blamkaGeneric(.{ |
| | 335 | &t[i + 0], &t[i + 1], &t[i + 2], &t[i + 3], |
| | 336 | &t[i + 4], &t[i + 5], &t[i + 6], &t[i + 7], |
| | 337 | &t[i + 8], &t[i + 9], &t[i + 10], &t[i + 11], |
| | 338 | &t[i + 12], &t[i + 13], &t[i + 14], &t[i + 15], |
| | 339 | }); |
| 358 | } | 340 | } |
| 359 | i = 0; | 341 | i = 0; |
| 360 | var buffer: [16]u64 = undefined; | | |
| 361 | while (i < block_length / 8) : (i += 2) { | 342 | while (i < block_length / 8) : (i += 2) { |
| 362 | var j: usize = 0; | 343 | blamkaGeneric(.{ |
| 363 | while (j < block_length / 8) : (j += 2) { | 344 | &t[0 * 16 + i], &t[0 * 16 + i + 1], |
| 364 | buffer[j] = t[j * 8 + i]; | 345 | &t[1 * 16 + i], &t[1 * 16 + i + 1], |
| 365 | buffer[j + 1] = t[j * 8 + i + 1]; | 346 | &t[2 * 16 + i], &t[2 * 16 + i + 1], |
| 366 | } | 347 | &t[3 * 16 + i], &t[3 * 16 + i + 1], |
| 367 | blamkaGeneric(&buffer); | 348 | &t[4 * 16 + i], &t[4 * 16 + i + 1], |
| 368 | j = 0; | 349 | &t[5 * 16 + i], &t[5 * 16 + i + 1], |
| 369 | while (j < block_length / 8) : (j += 2) { | 350 | &t[6 * 16 + i], &t[6 * 16 + i + 1], |
| 370 | t[j * 8 + i] = buffer[j]; | 351 | &t[7 * 16 + i], &t[7 * 16 + i + 1], |
| 371 | t[j * 8 + i + 1] = buffer[j + 1]; | 352 | }); |
| 372 | } | | |
| 373 | } | 353 | } |
| 374 | if (xor) { | 354 | if (xor) { |
| 375 | for (t, 0..) |v, j| { | 355 | for (t, 0..) |v, j| { |
| ... | @@ -382,38 +362,53 @@ fn processBlockGeneric( | ... | @@ -382,38 +362,53 @@ fn processBlockGeneric( |
| 382 | } | 362 | } |
| 383 | } | 363 | } |
| 384 | | 364 | |
| 385 | const QuarterRound = struct { a: usize, b: usize, c: usize, d: usize }; | 365 | const BlamkaVector = @Vector(4, u64); |
| 386 | | 366 | |
| 387 | fn Rp(a: usize, b: usize, c: usize, d: usize) QuarterRound { | 367 | fn fBlaMka(x: BlamkaVector, y: BlamkaVector) BlamkaVector { |
| 388 | return .{ .a = a, .b = b, .c = c, .d = d }; | 368 | const x_lo: @Vector(4, u32) = @truncate(x); |
| | 369 | const y_lo: @Vector(4, u32) = @truncate(y); |
| | 370 | const xy = @as(BlamkaVector, x_lo) * @as(BlamkaVector, y_lo); |
| | 371 | return x +% y +% @as(BlamkaVector, @splat(2)) *% xy; |
| 389 | } | 372 | } |
| 390 | | 373 | |
| 391 | fn fBlaMka(x: u64, y: u64) u64 { | 374 | fn rotrVector(x: BlamkaVector, comptime n: comptime_int) BlamkaVector { |
| 392 | const xy = @as(u64, @as(u32, @truncate(x))) * @as(u64, @as(u32, @truncate(y))); | 375 | return (x >> @splat(n)) | (x << @splat(64 - n)); |
| 393 | return x +% y +% 2 *% xy; | | |
| 394 | } | 376 | } |
| 395 | | 377 | |
| 396 | fn blamkaGeneric(x: *[16]u64) void { | 378 | // The `inline` is load-bearing: without it ReleaseSmall keeps this out of line |
| 397 | const rounds = comptime [_]QuarterRound{ | 379 | // and shuffles the four vectors through memory across every call, costing |
| 398 | Rp(0, 4, 8, 12), | 380 | // roughly 9% on Apple Silicon. |
| 399 | Rp(1, 5, 9, 13), | 381 | inline fn blamkaRound(a: *BlamkaVector, b: *BlamkaVector, c: *BlamkaVector, d: *BlamkaVector) void { |
| 400 | Rp(2, 6, 10, 14), | 382 | a.* = fBlaMka(a.*, b.*); |
| 401 | Rp(3, 7, 11, 15), | 383 | d.* = rotrVector(d.* ^ a.*, 32); |
| 402 | Rp(0, 5, 10, 15), | 384 | c.* = fBlaMka(c.*, d.*); |
| 403 | Rp(1, 6, 11, 12), | 385 | b.* = rotrVector(b.* ^ c.*, 24); |
| 404 | Rp(2, 7, 8, 13), | 386 | a.* = fBlaMka(a.*, b.*); |
| 405 | Rp(3, 4, 9, 14), | 387 | d.* = rotrVector(d.* ^ a.*, 16); |
| 406 | }; | 388 | c.* = fBlaMka(c.*, d.*); |
| 407 | inline for (rounds) |r| { | 389 | b.* = rotrVector(b.* ^ c.*, 63); |
| 408 | x[r.a] = fBlaMka(x[r.a], x[r.b]); | 390 | } |
| 409 | x[r.d] = math.rotr(u64, x[r.d] ^ x[r.a], 32); | 391 | |
| 410 | x[r.c] = fBlaMka(x[r.c], x[r.d]); | 392 | fn blamkaGeneric(xs: [16]*u64) void { |
| 411 | x[r.b] = math.rotr(u64, x[r.b] ^ x[r.c], 24); | 393 | var a: BlamkaVector = .{ xs[0].*, xs[1].*, xs[2].*, xs[3].* }; |
| 412 | x[r.a] = fBlaMka(x[r.a], x[r.b]); | 394 | var b: BlamkaVector = .{ xs[4].*, xs[5].*, xs[6].*, xs[7].* }; |
| 413 | x[r.d] = math.rotr(u64, x[r.d] ^ x[r.a], 16); | 395 | var c: BlamkaVector = .{ xs[8].*, xs[9].*, xs[10].*, xs[11].* }; |
| 414 | x[r.c] = fBlaMka(x[r.c], x[r.d]); | 396 | var d: BlamkaVector = .{ xs[12].*, xs[13].*, xs[14].*, xs[15].* }; |
| 415 | x[r.b] = math.rotr(u64, x[r.b] ^ x[r.c], 63); | 397 | |
| 416 | } | 398 | blamkaRound(&a, &b, &c, &d); |
| | 399 | |
| | 400 | var b_diag = @shuffle(u64, b, undefined, [_]i32{ 1, 2, 3, 0 }); |
| | 401 | var c_diag = @shuffle(u64, c, undefined, [_]i32{ 2, 3, 0, 1 }); |
| | 402 | var d_diag = @shuffle(u64, d, undefined, [_]i32{ 3, 0, 1, 2 }); |
| | 403 | blamkaRound(&a, &b_diag, &c_diag, &d_diag); |
| | 404 | b = @shuffle(u64, b_diag, undefined, [_]i32{ 3, 0, 1, 2 }); |
| | 405 | c = @shuffle(u64, c_diag, undefined, [_]i32{ 2, 3, 0, 1 }); |
| | 406 | d = @shuffle(u64, d_diag, undefined, [_]i32{ 1, 2, 3, 0 }); |
| | 407 | |
| | 408 | inline for (0..4) |k| xs[k].* = a[k]; |
| | 409 | inline for (0..4) |k| xs[4 + k].* = b[k]; |
| | 410 | inline for (0..4) |k| xs[8 + k].* = c[k]; |
| | 411 | inline for (0..4) |k| xs[12 + k].* = d[k]; |
| 417 | } | 412 | } |
| 418 | | 413 | |
| 419 | fn finalize( | 414 | fn finalize( |
| ... | @@ -511,9 +506,27 @@ pub fn kdf( | ... | @@ -511,9 +506,27 @@ pub fn kdf( |
| 511 | var blocks = try Blocks.initCapacity(allocator, memory); | 506 | var blocks = try Blocks.initCapacity(allocator, memory); |
| 512 | defer blocks.deinit(); | 507 | defer blocks.deinit(); |
| 513 | | 508 | |
| 514 | blocks.appendNTimesAssumeCapacity(@splat(0), memory); | 509 | const items = blocks.addManyAsSliceAssumeCapacity(memory); |
| | 510 | |
| | 511 | var block0: [1024]u8 = undefined; |
| | 512 | var lane: u24 = 0; |
| | 513 | while (lane < params.p) : (lane += 1) { |
| | 514 | const j = lane * (memory / params.p); |
| | 515 | mem.writeInt(u32, h0[Blake2b512.digest_length + 4 ..][0..4], lane, .little); |
| | 516 | |
| | 517 | mem.writeInt(u32, h0[Blake2b512.digest_length..][0..4], 0, .little); |
| | 518 | blake2bLong(&block0, &h0); |
| | 519 | for (&items[j + 0], 0..) |*v, i| { |
| | 520 | v.* = mem.readInt(u64, block0[i * 8 ..][0..8], .little); |
| | 521 | } |
| | 522 | |
| | 523 | mem.writeInt(u32, h0[Blake2b512.digest_length..][0..4], 1, .little); |
| | 524 | blake2bLong(&block0, &h0); |
| | 525 | for (&items[j + 1], 0..) |*v, i| { |
| | 526 | v.* = mem.readInt(u64, block0[i * 8 ..][0..8], .little); |
| | 527 | } |
| | 528 | } |
| 515 | | 529 | |
| 516 | initBlocks(&blocks, &h0, memory, params.p); | | |
| 517 | try processBlocks(&blocks, params.t, memory, params.p, mode, io); | 530 | try processBlocks(&blocks, params.t, memory, params.p, mode, io); |
| 518 | finalize(&blocks, memory, params.p, derived_key); | 531 | finalize(&blocks, memory, params.p, derived_key); |
| 519 | } | 532 | } |
Frank Denis <124872+jedisct1@users.noreply.github.com> 2026-09-21 03:04:38+02:00