authorgravatar for 124872+jedisct1@users.noreply.github.comFrank Denis <124872+jedisct1@users.noreply.github.com> 2026-10-06 10:48:30+02:00
committergravatar for 124872+jedisct1@users.noreply.github.comFrank Denis <124872+jedisct1@users.noreply.github.com> 2026-10-06 10:48:30+02:00
logeffdac38a82359182074174eb32e2978f315952d
treea899d6317948760954bdff97d3598542adc38bd2
parent7be72df07ee029a5e3951394f39aa69bd8d6b882
parenta3b95e1b1a5809f2fe187d7b83a0a12262df2752

Merge pull request 'crypto.hash.kangarootwelve: add a parallel update function' (#37067) from jedisct1/zig:ktparallelupdate into master

Reviewed-on: https://codeberg.org/ziglang/zig/pulls/37067

1 files changed, 88 insertions(+), 208 deletions(-)

lib/std/crypto/kangarootwelve.zig+88-208
...@@ -316,72 +316,6 @@ fn addLanesAll(...@@ -316,72 +316,6 @@ fn addLanesAll(
316 }316 }
317}317}
318318
319/// Apply Keccak-p[1600,12] to a single state (byte representation)
320fn keccakP(state: *[200]u8) void {
321 @setEvalBranchQuota(10000);
322 var lanes: [5][5]u64 = undefined;
323
324 // Load state into lanes
325 inline for (0..5) |x| {
326 inline for (0..5) |y| {
327 lanes[x][y] = load64(state[8 * (x + 5 * y) ..]);
328 }
329 }
330
331 // Apply 12 rounds
332 var round: usize = 0;
333 while (round < 12) : (round += 2) {
334 inline for (0..2) |i| {
335 // θ
336 var C: [5]u64 = undefined;
337 inline for (0..5) |x| {
338 C[x] = lanes[x][0] ^ lanes[x][1] ^ lanes[x][2] ^ lanes[x][3] ^ lanes[x][4];
339 }
340 var D: [5]u64 = undefined;
341 inline for (0..5) |x| {
342 D[x] = C[(x + 4) % 5] ^ std.math.rotl(u64, C[(x + 1) % 5], 1);
343 }
344 inline for (0..5) |x| {
345 inline for (0..5) |y| {
346 lanes[x][y] ^= D[x];
347 }
348 }
349
350 // ρ and π
351 var current = lanes[1][0];
352 var px: usize = 1;
353 var py: usize = 0;
354 inline for (0..24) |t| {
355 const temp = lanes[py][(2 * px + 3 * py) % 5];
356 const rot_amount = ((t + 1) * (t + 2) / 2) % 64;
357 lanes[py][(2 * px + 3 * py) % 5] = std.math.rotl(u64, current, @as(u6, @intCast(rot_amount)));
358 current = temp;
359 const temp_x = py;
360 py = (2 * px + 3 * py) % 5;
361 px = temp_x;
362 }
363
364 // χ
365 inline for (0..5) |y| {
366 const T = [5]u64{ lanes[0][y], lanes[1][y], lanes[2][y], lanes[3][y], lanes[4][y] };
367 inline for (0..5) |x| {
368 lanes[x][y] = T[x] ^ (~T[(x + 1) % 5] & T[(x + 2) % 5]);
369 }
370 }
371
372 // ι
373 lanes[0][0] ^= RC[round + i];
374 }
375 }
376
377 // Store lanes back to state
378 inline for (0..5) |x| {
379 inline for (0..5) |y| {
380 store64(lanes[x][y], state[8 * (x + 5 * y) ..]);
381 }
382 }
383}
384
385/// Apply Keccak-p[1600,12] to a single state (u64 lane representation)319/// Apply Keccak-p[1600,12] to a single state (u64 lane representation)
386fn keccakPLanes(lanes: *[25]u64) void {320fn keccakPLanes(lanes: *[25]u64) void {
387 @setEvalBranchQuota(10000);321 @setEvalBranchQuota(10000);
...@@ -439,38 +373,12 @@ fn turboShakeMultiSliceToBuffer(...@@ -439,38 +373,12 @@ fn turboShakeMultiSliceToBuffer(
439 separation_byte: u8,373 separation_byte: u8,
440 output: []u8,374 output: []u8,
441) void {375) void {
442 var state: [200]u8 = @splat(0);376 const StateType = if (rate == TurboSHAKE128State.block_length) TurboSHAKE128State else TurboSHAKE256State;
443 var state_pos: usize = 0;377 comptime assert(StateType.block_length == rate);
444
445 // Absorb all bytes from the multi-slice view
446 const total = view.totalLen();
447 var pos: usize = 0;
448 while (pos < total) {
449 state[state_pos] ^= view.getByte(pos);
450 state_pos += 1;
451 pos += 1;
452
453 if (state_pos == rate) {
454 keccakP(&state);
455 state_pos = 0;
456 }
457 }
458378
459 // Add separation byte and padding379 var state = StateType.init(.{ .delim = separation_byte });
460 state[state_pos] ^= separation_byte;380 for (view.slices) |slice| state.update(slice);
461 state[rate - 1] ^= 0x80;381 state.final(output);
462 keccakP(&state);
463
464 // Squeeze
465 var out_offset: usize = 0;
466 while (out_offset < output.len) {
467 const chunk = @min(rate, output.len - out_offset);
468 @memcpy(output[out_offset..][0..chunk], state[0..chunk]);
469 out_offset += chunk;
470 if (out_offset < output.len) {
471 keccakP(&state);
472 }
473 }
474}382}
475383
476/// Generic allocating TurboSHAKE384/// Generic allocating TurboSHAKE
...@@ -752,56 +660,29 @@ fn SelectLeafContext(comptime Variant: type) type {...@@ -752,56 +660,29 @@ fn SelectLeafContext(comptime Variant: type) type {
752 const Result = BatchResult(Variant);660 const Result = BatchResult(Variant);
753661
754 return struct {662 return struct {
755 view: *const MultiSliceView,663 leaves: []const u8,
756 batch_idx: usize,664 batch_idx: usize,
757 start_offset: usize,
758 num_leaves: usize,
759665
760 fn process(ctx: @This()) Result {666 fn process(ctx: @This()) Result {
667 const num_leaves = ctx.leaves.len / chunk_size;
761 var result: Result = .{668 var result: Result = .{
762 .batch_idx = ctx.batch_idx,669 .batch_idx = ctx.batch_idx,
763 .cv_len = ctx.num_leaves * cv_size,670 .cv_len = num_leaves * cv_size,
764 .cvs = undefined,671 .cvs = undefined,
765 };672 };
766673
767 var leaf_buffer: [bytes_per_batch]u8 align(cache_line_size) = undefined;674 var i: usize = 0;
768 var leaves_processed: usize = 0;675 while (i + optimal_vector_len <= num_leaves) : (i += optimal_vector_len) {
769 var byte_offset = ctx.start_offset;676 processLeaves(
770 var cv_offset: usize = 0;677 Variant,
771 const simd_batch_bytes = optimal_vector_len * chunk_size;678 optimal_vector_len,
772 while (leaves_processed + optimal_vector_len <= ctx.num_leaves) {679 ctx.leaves[i * chunk_size ..][0 .. optimal_vector_len * chunk_size],
773 if (ctx.view.tryGetSlice(byte_offset, byte_offset + simd_batch_bytes)) |leaf_data| {680 result.cvs[i * cv_size ..][0 .. optimal_vector_len * cv_size],
774 var leaf_cvs: [optimal_vector_len * Variant.cv_size]u8 = undefined;681 );
775 processLeaves(Variant, optimal_vector_len, leaf_data, &leaf_cvs);
776 @memcpy(result.cvs[cv_offset..][0..leaf_cvs.len], &leaf_cvs);
777 } else {
778 ctx.view.copyRange(byte_offset, byte_offset + simd_batch_bytes, leaf_buffer[0..simd_batch_bytes]);
779 var leaf_cvs: [optimal_vector_len * Variant.cv_size]u8 = undefined;
780 processLeaves(Variant, optimal_vector_len, leaf_buffer[0..simd_batch_bytes], &leaf_cvs);
781 @memcpy(result.cvs[cv_offset..][0..leaf_cvs.len], &leaf_cvs);
782 }
783 leaves_processed += optimal_vector_len;
784 byte_offset += optimal_vector_len * chunk_size;
785 cv_offset += optimal_vector_len * cv_size;
786 }682 }
787683 while (i < num_leaves) : (i += 1) {
788 while (leaves_processed < ctx.num_leaves) {684 const leaf = MultiSliceView.init(ctx.leaves[i * chunk_size ..][0..chunk_size], &.{}, &.{});
789 const leaf_end = byte_offset + chunk_size;685 Variant.turboShakeToBuffer(&leaf, 0x0B, result.cvs[i * cv_size ..][0..cv_size]);
790 var cv_buffer: [64]u8 = undefined;
791
792 if (ctx.view.tryGetSlice(byte_offset, leaf_end)) |leaf_data| {
793 const cv_slice = MultiSliceView.init(leaf_data, &[_]u8{}, &[_]u8{});
794 Variant.turboShakeToBuffer(&cv_slice, 0x0B, cv_buffer[0..cv_size]);
795 } else {
796 ctx.view.copyRange(byte_offset, leaf_end, leaf_buffer[0..chunk_size]);
797 const cv_slice = MultiSliceView.init(leaf_buffer[0..chunk_size], &[_]u8{}, &[_]u8{});
798 Variant.turboShakeToBuffer(&cv_slice, 0x0B, cv_buffer[0..cv_size]);
799 }
800 @memcpy(result.cvs[cv_offset..][0..cv_size], cv_buffer[0..cv_size]);
801
802 leaves_processed += 1;
803 byte_offset += chunk_size;
804 cv_offset += cv_size;
805 }686 }
806687
807 return result;688 return result;
...@@ -834,38 +715,20 @@ fn FinalLeafContext(comptime Variant: type) type {...@@ -834,38 +715,20 @@ fn FinalLeafContext(comptime Variant: type) type {
834 };715 };
835}716}
836717
837fn ktMultiThreaded(718/// Hashes complete leaves on multiple threads and absorbs chaining values in order.
719fn absorbLeavesParallel(
838 comptime Variant: type,720 comptime Variant: type,
839 allocator: Allocator,721 allocator: Allocator,
840 io: Io,722 io: Io,
841 view: *const MultiSliceView,723 final_state: *Variant.StateType,
842 total_len: usize,724 leaves: []const u8,
843 output: []u8,725) error{ OutOfMemory, Canceled }!void {
844) !void {
845 comptime assert(bytes_per_batch % (optimal_vector_len * chunk_size) == 0);726 comptime assert(bytes_per_batch % (optimal_vector_len * chunk_size) == 0);
727 assert(leaves.len % chunk_size == 0);
846728
847 const cv_size = Variant.cv_size;729 const cv_size = Variant.cv_size;
848 const StateType = Variant.StateType;
849 const leaves_per_batch = bytes_per_batch / chunk_size;730 const leaves_per_batch = bytes_per_batch / chunk_size;
850 const remaining_bytes = total_len - chunk_size;731 const full_leaves = leaves.len / chunk_size;
851 const total_leaves = std.math.divCeil(usize, remaining_bytes, chunk_size) catch unreachable;
852
853 var final_state = StateType.init(.{});
854
855 var first_chunk_buffer: [chunk_size]u8 = undefined;
856 if (view.tryGetSlice(0, chunk_size)) |first_chunk| {
857 final_state.update(first_chunk);
858 } else {
859 view.copyRange(0, chunk_size, &first_chunk_buffer);
860 final_state.update(&first_chunk_buffer);
861 }
862
863 const padding = [_]u8{ 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 };
864 final_state.update(&padding);
865
866 const full_leaves = remaining_bytes / chunk_size;
867 const has_partial_leaf = (remaining_bytes % chunk_size) != 0;
868 const partial_leaf_size = if (has_partial_leaf) remaining_bytes % chunk_size else 0;
869732
870 if (full_leaves > 0) {733 if (full_leaves > 0) {
871 const total_batches = std.math.divCeil(usize, full_leaves, leaves_per_batch) catch unreachable;734 const total_batches = std.math.divCeil(usize, full_leaves, leaves_per_batch) catch unreachable;
...@@ -893,14 +756,11 @@ fn ktMultiThreaded(...@@ -893,14 +756,11 @@ fn ktMultiThreaded(
893 while (batches_spawned < total_batches and batches_spawned - next_to_process < max_concurrent) {756 while (batches_spawned < total_batches and batches_spawned - next_to_process < max_concurrent) {
894 const batch_start_leaf = batches_spawned * leaves_per_batch;757 const batch_start_leaf = batches_spawned * leaves_per_batch;
895 const batch_leaves = @min(leaves_per_batch, full_leaves - batch_start_leaf);758 const batch_leaves = @min(leaves_per_batch, full_leaves - batch_start_leaf);
896 const start_offset = chunk_size + batch_start_leaf * chunk_size;
897759
898 select_outstanding += 1;760 select_outstanding += 1;
899 select.async(.batch, SelectLeafContext(Variant).process, .{SelectLeafContext(Variant){761 select.async(.batch, SelectLeafContext(Variant).process, .{SelectLeafContext(Variant){
900 .view = view,762 .leaves = leaves[batch_start_leaf * chunk_size ..][0 .. batch_leaves * chunk_size],
901 .batch_idx = batches_spawned,763 .batch_idx = batches_spawned,
902 .start_offset = start_offset,
903 .num_leaves = batch_leaves,
904 }});764 }});
905 batches_spawned += 1;765 batches_spawned += 1;
906 }766 }
...@@ -932,29 +792,6 @@ fn ktMultiThreaded(...@@ -932,29 +792,6 @@ fn ktMultiThreaded(
932792
933 assert(select_outstanding == 0);793 assert(select_outstanding == 0);
934 }794 }
935
936 if (has_partial_leaf) {
937 var cv_buffer: [64]u8 = undefined;
938 var leaf_buffer: [chunk_size]u8 = undefined;
939
940 const start_offset = chunk_size + full_leaves * chunk_size;
941 if (view.tryGetSlice(start_offset, start_offset + partial_leaf_size)) |leaf_data| {
942 const cv_slice = MultiSliceView.init(leaf_data, &[_]u8{}, &[_]u8{});
943 Variant.turboShakeToBuffer(&cv_slice, 0x0B, cv_buffer[0..cv_size]);
944 } else {
945 view.copyRange(start_offset, start_offset + partial_leaf_size, leaf_buffer[0..partial_leaf_size]);
946 const cv_slice = MultiSliceView.init(leaf_buffer[0..partial_leaf_size], &[_]u8{}, &[_]u8{});
947 Variant.turboShakeToBuffer(&cv_slice, 0x0B, cv_buffer[0..cv_size]);
948 }
949 final_state.update(cv_buffer[0..cv_size]);
950 }
951
952 const n_enc = rightEncode(total_leaves);
953 final_state.update(n_enc.slice());
954 const terminator = [_]u8{ 0xFF, 0xFF };
955 final_state.update(&terminator);
956
957 final_state.final(output);
958}795}
959796
960/// Generic KangarooTwelve hash function builder.797/// Generic KangarooTwelve hash function builder.
...@@ -1129,6 +966,30 @@ fn KTHash(...@@ -1129,6 +966,30 @@ fn KTHash(
1129 }966 }
1130 }967 }
1131968
969 /// Like `update`, but uses multiple threads for large inputs.
970 ///
971 /// Inputs smaller than 2 MiB are hashed on the current thread.
972 /// The allocator is only used for temporary buffers.
973 ///
974 /// After an error, the state is partially updated and must not be used anymore.
975 pub fn updateParallel(self: *Self, data: []const u8, allocator: Allocator, io: Io) error{ OutOfMemory, Canceled }!void {
976 if (data.len < large_file_threshold) {
977 return self.update(data);
978 }
979
980 // Finish the first chunk or the current leaf, so that the rest starts on a leaf boundary.
981 const head_len = if (self.final_state == null or self.buffer_len > 0) chunk_size - self.buffer_len else 0;
982 self.update(data[0..head_len]);
983 self.flushPendingChunks();
984
985 const rest = data[head_len..];
986 const full_leaves = rest.len / chunk_size;
987 try absorbLeavesParallel(Variant, allocator, io, &self.final_state.?, rest[0 .. full_leaves * chunk_size]);
988 self.num_leaves += full_leaves;
989 self.message_len += full_leaves * chunk_size;
990 self.update(rest[full_leaves * chunk_size ..]);
991 }
992
1132 /// Finalize the hash and produce output.993 /// Finalize the hash and produce output.
1133 ///994 ///
1134 /// Unlike traditional hash functions, the output can be of any length.995 /// Unlike traditional hash functions, the output can be of any length.
...@@ -1256,27 +1117,15 @@ fn KTHash(...@@ -1256,27 +1117,15 @@ fn KTHash(
1256 /// Hash with automatic parallelization for large inputs (>2MB).1117 /// Hash with automatic parallelization for large inputs (>2MB).
1257 /// Automatically uses sequential processing for smaller inputs to avoid thread overhead.1118 /// Automatically uses sequential processing for smaller inputs to avoid thread overhead.
1258 /// Allocator required for temporary buffers. IO object required for thread management.1119 /// Allocator required for temporary buffers. IO object required for thread management.
1259 pub fn hashParallel(message: []const u8, out: []u8, options: Options, allocator: Allocator, io: Io) !void {1120 pub fn hashParallel(message: []const u8, out: []u8, options: Options, allocator: Allocator, io: Io) error{ OutOfMemory, Canceled }!void {
1260 const custom = options.customization orelse &[_]u8{};
1261
1262 const custom_len_enc = rightEncode(custom.len);
1263 const view = MultiSliceView.init(message, custom, custom_len_enc.slice());
1264 const total_len = view.totalLen();
1265
1266 // Single chunk case
1267 if (total_len <= chunk_size) {
1268 singleChunkFn(&view, 0x07, out);
1269 return;
1270 }
1271
1272 // Use single-threaded processing if below threshold1121 // Use single-threaded processing if below threshold
1273 if (total_len < large_file_threshold) {1122 if (message.len < large_file_threshold) {
1274 ktSingleThreaded(Variant, &view, total_len, out);1123 return hash(message, out, options);
1275 return;
1276 }1124 }
12771125
1278 // Tree mode - multi-threaded processing1126 var hasher = Self.init(options);
1279 try ktMultiThreaded(Variant, allocator, io, &view, total_len, out);1127 try hasher.updateParallel(message, allocator, io);
1128 hasher.final(out);
1280 }1129 }
1281 };1130 };
1282}1131}
...@@ -1561,6 +1410,37 @@ test "KT256 sequential and parallel produce same output with customization" {...@@ -1561,6 +1410,37 @@ test "KT256 sequential and parallel produce same output with customization" {
1561 try std.testing.expectEqualSlices(u8, &output_seq, &output_par);1410 try std.testing.expectEqualSlices(u8, &output_seq, &output_par);
1562}1411}
15631412
1413test "parallel hashing matches sequential hashing" {
1414 const allocator = std.testing.allocator;
1415 const io = std.testing.io;
1416
1417 var prng = std.Random.DefaultPrng.init(std.testing.random_seed);
1418 const message = try allocator.alloc(u8, 3 * large_file_threshold + 5 * chunk_size + 123);
1419 defer allocator.free(message);
1420 prng.random().bytes(message);
1421
1422 const update_lens = [_]usize{ 100, large_file_threshold + chunk_size - 100, 3 * chunk_size, large_file_threshold + 77 };
1423
1424 inline for (.{ KT128, KT256 }) |KT| {
1425 var expected: [64]u8 = undefined;
1426 var actual: [64]u8 = undefined;
1427 try KT.hash(message, &expected, .{ .customization = "test" });
1428
1429 try KT.hashParallel(message, &actual, .{ .customization = "test" }, allocator, io);
1430 try std.testing.expectEqualSlices(u8, &expected, &actual);
1431
1432 var hasher = KT.init(.{ .customization = "test" });
1433 var rest: []const u8 = message;
1434 for (update_lens) |len| {
1435 try hasher.updateParallel(rest[0..len], allocator, io);
1436 rest = rest[len..];
1437 }
1438 try hasher.updateParallel(rest, allocator, io);
1439 hasher.final(&actual);
1440 try std.testing.expectEqualSlices(u8, &expected, &actual);
1441 }
1442}
1443
1564/// Helper: Generate pattern data where data[i] = (i % 251)1444/// Helper: Generate pattern data where data[i] = (i % 251)
1565fn generatePattern(allocator: Allocator, len: usize) ![]u8 {1445fn generatePattern(allocator: Allocator, len: usize) ![]u8 {
1566 const data = try allocator.alloc(u8, len);1446 const data = try allocator.alloc(u8, len);