| ... | ... | @@ -316,72 +316,6 @@ fn addLanesAll( |
| 316 | 316 | } |
| 317 | 317 | } |
| 318 | 318 | |
| 319 | | /// Apply Keccak-p[1600,12] to a single state (byte representation) |
| 320 | | fn keccakP(state: *[200]u8) void { |
| 321 | | @setEvalBranchQuota(10000); |
| 322 | | var lanes: [5][5]u64 = undefined; |
| 323 | | |
| 324 | | // Load state into lanes |
| 325 | | inline for (0..5) |x| { |
| 326 | | inline for (0..5) |y| { |
| 327 | | lanes[x][y] = load64(state[8 * (x + 5 * y) ..]); |
| 328 | | } |
| 329 | | } |
| 330 | | |
| 331 | | // Apply 12 rounds |
| 332 | | var round: usize = 0; |
| 333 | | while (round < 12) : (round += 2) { |
| 334 | | inline for (0..2) |i| { |
| 335 | | // θ |
| 336 | | var C: [5]u64 = undefined; |
| 337 | | inline for (0..5) |x| { |
| 338 | | C[x] = lanes[x][0] ^ lanes[x][1] ^ lanes[x][2] ^ lanes[x][3] ^ lanes[x][4]; |
| 339 | | } |
| 340 | | var D: [5]u64 = undefined; |
| 341 | | inline for (0..5) |x| { |
| 342 | | D[x] = C[(x + 4) % 5] ^ std.math.rotl(u64, C[(x + 1) % 5], 1); |
| 343 | | } |
| 344 | | inline for (0..5) |x| { |
| 345 | | inline for (0..5) |y| { |
| 346 | | lanes[x][y] ^= D[x]; |
| 347 | | } |
| 348 | | } |
| 349 | | |
| 350 | | // ρ and π |
| 351 | | var current = lanes[1][0]; |
| 352 | | var px: usize = 1; |
| 353 | | var py: usize = 0; |
| 354 | | inline for (0..24) |t| { |
| 355 | | const temp = lanes[py][(2 * px + 3 * py) % 5]; |
| 356 | | const rot_amount = ((t + 1) * (t + 2) / 2) % 64; |
| 357 | | lanes[py][(2 * px + 3 * py) % 5] = std.math.rotl(u64, current, @as(u6, @intCast(rot_amount))); |
| 358 | | current = temp; |
| 359 | | const temp_x = py; |
| 360 | | py = (2 * px + 3 * py) % 5; |
| 361 | | px = temp_x; |
| 362 | | } |
| 363 | | |
| 364 | | // χ |
| 365 | | inline for (0..5) |y| { |
| 366 | | const T = [5]u64{ lanes[0][y], lanes[1][y], lanes[2][y], lanes[3][y], lanes[4][y] }; |
| 367 | | inline for (0..5) |x| { |
| 368 | | lanes[x][y] = T[x] ^ (~T[(x + 1) % 5] & T[(x + 2) % 5]); |
| 369 | | } |
| 370 | | } |
| 371 | | |
| 372 | | // ι |
| 373 | | lanes[0][0] ^= RC[round + i]; |
| 374 | | } |
| 375 | | } |
| 376 | | |
| 377 | | // Store lanes back to state |
| 378 | | inline for (0..5) |x| { |
| 379 | | inline for (0..5) |y| { |
| 380 | | store64(lanes[x][y], state[8 * (x + 5 * y) ..]); |
| 381 | | } |
| 382 | | } |
| 383 | | } |
| 384 | | |
| 385 | 319 | /// Apply Keccak-p[1600,12] to a single state (u64 lane representation) |
| 386 | 320 | fn keccakPLanes(lanes: *[25]u64) void { |
| 387 | 321 | @setEvalBranchQuota(10000); |
| ... | ... | @@ -439,38 +373,12 @@ fn turboShakeMultiSliceToBuffer( |
| 439 | 373 | separation_byte: u8, |
| 440 | 374 | output: []u8, |
| 441 | 375 | ) void { |
| 442 | | var state: [200]u8 = @splat(0); |
| 443 | | var state_pos: usize = 0; |
| 444 | | |
| 445 | | // Absorb all bytes from the multi-slice view |
| 446 | | const total = view.totalLen(); |
| 447 | | var pos: usize = 0; |
| 448 | | while (pos < total) { |
| 449 | | state[state_pos] ^= view.getByte(pos); |
| 450 | | state_pos += 1; |
| 451 | | pos += 1; |
| 452 | | |
| 453 | | if (state_pos == rate) { |
| 454 | | keccakP(&state); |
| 455 | | state_pos = 0; |
| 456 | | } |
| 457 | | } |
| 376 | const StateType = if (rate == TurboSHAKE128State.block_length) TurboSHAKE128State else TurboSHAKE256State; |
| 377 | comptime assert(StateType.block_length == rate); |
| 458 | 378 | |
| 459 | | // Add separation byte and padding |
| 460 | | state[state_pos] ^= separation_byte; |
| 461 | | state[rate - 1] ^= 0x80; |
| 462 | | keccakP(&state); |
| 463 | | |
| 464 | | // Squeeze |
| 465 | | var out_offset: usize = 0; |
| 466 | | while (out_offset < output.len) { |
| 467 | | const chunk = @min(rate, output.len - out_offset); |
| 468 | | @memcpy(output[out_offset..][0..chunk], state[0..chunk]); |
| 469 | | out_offset += chunk; |
| 470 | | if (out_offset < output.len) { |
| 471 | | keccakP(&state); |
| 472 | | } |
| 473 | | } |
| 379 | var state = StateType.init(.{ .delim = separation_byte }); |
| 380 | for (view.slices) |slice| state.update(slice); |
| 381 | state.final(output); |
| 474 | 382 | } |
| 475 | 383 | |
| 476 | 384 | /// Generic allocating TurboSHAKE |
| ... | ... | @@ -752,56 +660,29 @@ fn SelectLeafContext(comptime Variant: type) type { |
| 752 | 660 | const Result = BatchResult(Variant); |
| 753 | 661 | |
| 754 | 662 | return struct { |
| 755 | | view: *const MultiSliceView, |
| 663 | leaves: []const u8, |
| 756 | 664 | batch_idx: usize, |
| 757 | | start_offset: usize, |
| 758 | | num_leaves: usize, |
| 759 | 665 | |
| 760 | 666 | fn process(ctx: @This()) Result { |
| 667 | const num_leaves = ctx.leaves.len / chunk_size; |
| 761 | 668 | var result: Result = .{ |
| 762 | 669 | .batch_idx = ctx.batch_idx, |
| 763 | | .cv_len = ctx.num_leaves * cv_size, |
| 670 | .cv_len = num_leaves * cv_size, |
| 764 | 671 | .cvs = undefined, |
| 765 | 672 | }; |
| 766 | 673 | |
| 767 | | var leaf_buffer: [bytes_per_batch]u8 align(cache_line_size) = undefined; |
| 768 | | var leaves_processed: usize = 0; |
| 769 | | var byte_offset = ctx.start_offset; |
| 770 | | var cv_offset: usize = 0; |
| 771 | | const simd_batch_bytes = optimal_vector_len * chunk_size; |
| 772 | | while (leaves_processed + optimal_vector_len <= ctx.num_leaves) { |
| 773 | | if (ctx.view.tryGetSlice(byte_offset, byte_offset + simd_batch_bytes)) |leaf_data| { |
| 774 | | var leaf_cvs: [optimal_vector_len * Variant.cv_size]u8 = undefined; |
| 775 | | processLeaves(Variant, optimal_vector_len, leaf_data, &leaf_cvs); |
| 776 | | @memcpy(result.cvs[cv_offset..][0..leaf_cvs.len], &leaf_cvs); |
| 777 | | } else { |
| 778 | | ctx.view.copyRange(byte_offset, byte_offset + simd_batch_bytes, leaf_buffer[0..simd_batch_bytes]); |
| 779 | | var leaf_cvs: [optimal_vector_len * Variant.cv_size]u8 = undefined; |
| 780 | | processLeaves(Variant, optimal_vector_len, leaf_buffer[0..simd_batch_bytes], &leaf_cvs); |
| 781 | | @memcpy(result.cvs[cv_offset..][0..leaf_cvs.len], &leaf_cvs); |
| 782 | | } |
| 783 | | leaves_processed += optimal_vector_len; |
| 784 | | byte_offset += optimal_vector_len * chunk_size; |
| 785 | | cv_offset += optimal_vector_len * cv_size; |
| 674 | var i: usize = 0; |
| 675 | while (i + optimal_vector_len <= num_leaves) : (i += optimal_vector_len) { |
| 676 | processLeaves( |
| 677 | Variant, |
| 678 | optimal_vector_len, |
| 679 | ctx.leaves[i * chunk_size ..][0 .. optimal_vector_len * chunk_size], |
| 680 | result.cvs[i * cv_size ..][0 .. optimal_vector_len * cv_size], |
| 681 | ); |
| 786 | 682 | } |
| 787 | | |
| 788 | | while (leaves_processed < ctx.num_leaves) { |
| 789 | | const leaf_end = byte_offset + chunk_size; |
| 790 | | var cv_buffer: [64]u8 = undefined; |
| 791 | | |
| 792 | | if (ctx.view.tryGetSlice(byte_offset, leaf_end)) |leaf_data| { |
| 793 | | const cv_slice = MultiSliceView.init(leaf_data, &[_]u8{}, &[_]u8{}); |
| 794 | | Variant.turboShakeToBuffer(&cv_slice, 0x0B, cv_buffer[0..cv_size]); |
| 795 | | } else { |
| 796 | | ctx.view.copyRange(byte_offset, leaf_end, leaf_buffer[0..chunk_size]); |
| 797 | | const cv_slice = MultiSliceView.init(leaf_buffer[0..chunk_size], &[_]u8{}, &[_]u8{}); |
| 798 | | Variant.turboShakeToBuffer(&cv_slice, 0x0B, cv_buffer[0..cv_size]); |
| 799 | | } |
| 800 | | @memcpy(result.cvs[cv_offset..][0..cv_size], cv_buffer[0..cv_size]); |
| 801 | | |
| 802 | | leaves_processed += 1; |
| 803 | | byte_offset += chunk_size; |
| 804 | | cv_offset += cv_size; |
| 683 | while (i < num_leaves) : (i += 1) { |
| 684 | const leaf = MultiSliceView.init(ctx.leaves[i * chunk_size ..][0..chunk_size], &.{}, &.{}); |
| 685 | Variant.turboShakeToBuffer(&leaf, 0x0B, result.cvs[i * cv_size ..][0..cv_size]); |
| 805 | 686 | } |
| 806 | 687 | |
| 807 | 688 | return result; |
| ... | ... | @@ -834,38 +715,20 @@ fn FinalLeafContext(comptime Variant: type) type { |
| 834 | 715 | }; |
| 835 | 716 | } |
| 836 | 717 | |
| 837 | | fn ktMultiThreaded( |
| 718 | /// Hashes complete leaves on multiple threads and absorbs chaining values in order. |
| 719 | fn absorbLeavesParallel( |
| 838 | 720 | comptime Variant: type, |
| 839 | 721 | allocator: Allocator, |
| 840 | 722 | io: Io, |
| 841 | | view: *const MultiSliceView, |
| 842 | | total_len: usize, |
| 843 | | output: []u8, |
| 844 | | ) !void { |
| 723 | final_state: *Variant.StateType, |
| 724 | leaves: []const u8, |
| 725 | ) error{ OutOfMemory, Canceled }!void { |
| 845 | 726 | comptime assert(bytes_per_batch % (optimal_vector_len * chunk_size) == 0); |
| 727 | assert(leaves.len % chunk_size == 0); |
| 846 | 728 | |
| 847 | 729 | const cv_size = Variant.cv_size; |
| 848 | | const StateType = Variant.StateType; |
| 849 | 730 | const leaves_per_batch = bytes_per_batch / chunk_size; |
| 850 | | const remaining_bytes = total_len - chunk_size; |
| 851 | | const total_leaves = std.math.divCeil(usize, remaining_bytes, chunk_size) catch unreachable; |
| 852 | | |
| 853 | | var final_state = StateType.init(.{}); |
| 854 | | |
| 855 | | var first_chunk_buffer: [chunk_size]u8 = undefined; |
| 856 | | if (view.tryGetSlice(0, chunk_size)) |first_chunk| { |
| 857 | | final_state.update(first_chunk); |
| 858 | | } else { |
| 859 | | view.copyRange(0, chunk_size, &first_chunk_buffer); |
| 860 | | final_state.update(&first_chunk_buffer); |
| 861 | | } |
| 862 | | |
| 863 | | const padding = [_]u8{ 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }; |
| 864 | | final_state.update(&padding); |
| 865 | | |
| 866 | | const full_leaves = remaining_bytes / chunk_size; |
| 867 | | const has_partial_leaf = (remaining_bytes % chunk_size) != 0; |
| 868 | | const partial_leaf_size = if (has_partial_leaf) remaining_bytes % chunk_size else 0; |
| 731 | const full_leaves = leaves.len / chunk_size; |
| 869 | 732 | |
| 870 | 733 | if (full_leaves > 0) { |
| 871 | 734 | const total_batches = std.math.divCeil(usize, full_leaves, leaves_per_batch) catch unreachable; |
| ... | ... | @@ -893,14 +756,11 @@ fn ktMultiThreaded( |
| 893 | 756 | while (batches_spawned < total_batches and batches_spawned - next_to_process < max_concurrent) { |
| 894 | 757 | const batch_start_leaf = batches_spawned * leaves_per_batch; |
| 895 | 758 | const batch_leaves = @min(leaves_per_batch, full_leaves - batch_start_leaf); |
| 896 | | const start_offset = chunk_size + batch_start_leaf * chunk_size; |
| 897 | 759 | |
| 898 | 760 | select_outstanding += 1; |
| 899 | 761 | select.async(.batch, SelectLeafContext(Variant).process, .{SelectLeafContext(Variant){ |
| 900 | | .view = view, |
| 762 | .leaves = leaves[batch_start_leaf * chunk_size ..][0 .. batch_leaves * chunk_size], |
| 901 | 763 | .batch_idx = batches_spawned, |
| 902 | | .start_offset = start_offset, |
| 903 | | .num_leaves = batch_leaves, |
| 904 | 764 | }}); |
| 905 | 765 | batches_spawned += 1; |
| 906 | 766 | } |
| ... | ... | @@ -932,29 +792,6 @@ fn ktMultiThreaded( |
| 932 | 792 | |
| 933 | 793 | assert(select_outstanding == 0); |
| 934 | 794 | } |
| 935 | | |
| 936 | | if (has_partial_leaf) { |
| 937 | | var cv_buffer: [64]u8 = undefined; |
| 938 | | var leaf_buffer: [chunk_size]u8 = undefined; |
| 939 | | |
| 940 | | const start_offset = chunk_size + full_leaves * chunk_size; |
| 941 | | if (view.tryGetSlice(start_offset, start_offset + partial_leaf_size)) |leaf_data| { |
| 942 | | const cv_slice = MultiSliceView.init(leaf_data, &[_]u8{}, &[_]u8{}); |
| 943 | | Variant.turboShakeToBuffer(&cv_slice, 0x0B, cv_buffer[0..cv_size]); |
| 944 | | } else { |
| 945 | | view.copyRange(start_offset, start_offset + partial_leaf_size, leaf_buffer[0..partial_leaf_size]); |
| 946 | | const cv_slice = MultiSliceView.init(leaf_buffer[0..partial_leaf_size], &[_]u8{}, &[_]u8{}); |
| 947 | | Variant.turboShakeToBuffer(&cv_slice, 0x0B, cv_buffer[0..cv_size]); |
| 948 | | } |
| 949 | | final_state.update(cv_buffer[0..cv_size]); |
| 950 | | } |
| 951 | | |
| 952 | | const n_enc = rightEncode(total_leaves); |
| 953 | | final_state.update(n_enc.slice()); |
| 954 | | const terminator = [_]u8{ 0xFF, 0xFF }; |
| 955 | | final_state.update(&terminator); |
| 956 | | |
| 957 | | final_state.final(output); |
| 958 | 795 | } |
| 959 | 796 | |
| 960 | 797 | /// Generic KangarooTwelve hash function builder. |
| ... | ... | @@ -1129,6 +966,30 @@ fn KTHash( |
| 1129 | 966 | } |
| 1130 | 967 | } |
| 1131 | 968 | |
| 969 | /// Like `update`, but uses multiple threads for large inputs. |
| 970 | /// |
| 971 | /// Inputs smaller than 2 MiB are hashed on the current thread. |
| 972 | /// The allocator is only used for temporary buffers. |
| 973 | /// |
| 974 | /// After an error, the state is partially updated and must not be used anymore. |
| 975 | pub fn updateParallel(self: *Self, data: []const u8, allocator: Allocator, io: Io) error{ OutOfMemory, Canceled }!void { |
| 976 | if (data.len < large_file_threshold) { |
| 977 | return self.update(data); |
| 978 | } |
| 979 | |
| 980 | // Finish the first chunk or the current leaf, so that the rest starts on a leaf boundary. |
| 981 | const head_len = if (self.final_state == null or self.buffer_len > 0) chunk_size - self.buffer_len else 0; |
| 982 | self.update(data[0..head_len]); |
| 983 | self.flushPendingChunks(); |
| 984 | |
| 985 | const rest = data[head_len..]; |
| 986 | const full_leaves = rest.len / chunk_size; |
| 987 | try absorbLeavesParallel(Variant, allocator, io, &self.final_state.?, rest[0 .. full_leaves * chunk_size]); |
| 988 | self.num_leaves += full_leaves; |
| 989 | self.message_len += full_leaves * chunk_size; |
| 990 | self.update(rest[full_leaves * chunk_size ..]); |
| 991 | } |
| 992 | |
| 1132 | 993 | /// Finalize the hash and produce output. |
| 1133 | 994 | /// |
| 1134 | 995 | /// Unlike traditional hash functions, the output can be of any length. |
| ... | ... | @@ -1256,27 +1117,15 @@ fn KTHash( |
| 1256 | 1117 | /// Hash with automatic parallelization for large inputs (>2MB). |
| 1257 | 1118 | /// Automatically uses sequential processing for smaller inputs to avoid thread overhead. |
| 1258 | 1119 | /// Allocator required for temporary buffers. IO object required for thread management. |
| 1259 | | pub fn hashParallel(message: []const u8, out: []u8, options: Options, allocator: Allocator, io: Io) !void { |
| 1260 | | const custom = options.customization orelse &[_]u8{}; |
| 1261 | | |
| 1262 | | const custom_len_enc = rightEncode(custom.len); |
| 1263 | | const view = MultiSliceView.init(message, custom, custom_len_enc.slice()); |
| 1264 | | const total_len = view.totalLen(); |
| 1265 | | |
| 1266 | | // Single chunk case |
| 1267 | | if (total_len <= chunk_size) { |
| 1268 | | singleChunkFn(&view, 0x07, out); |
| 1269 | | return; |
| 1270 | | } |
| 1271 | | |
| 1120 | pub fn hashParallel(message: []const u8, out: []u8, options: Options, allocator: Allocator, io: Io) error{ OutOfMemory, Canceled }!void { |
| 1272 | 1121 | // Use single-threaded processing if below threshold |
| 1273 | | if (total_len < large_file_threshold) { |
| 1274 | | ktSingleThreaded(Variant, &view, total_len, out); |
| 1275 | | return; |
| 1122 | if (message.len < large_file_threshold) { |
| 1123 | return hash(message, out, options); |
| 1276 | 1124 | } |
| 1277 | 1125 | |
| 1278 | | // Tree mode - multi-threaded processing |
| 1279 | | try ktMultiThreaded(Variant, allocator, io, &view, total_len, out); |
| 1126 | var hasher = Self.init(options); |
| 1127 | try hasher.updateParallel(message, allocator, io); |
| 1128 | hasher.final(out); |
| 1280 | 1129 | } |
| 1281 | 1130 | }; |
| 1282 | 1131 | } |
| ... | ... | @@ -1561,6 +1410,37 @@ test "KT256 sequential and parallel produce same output with customization" { |
| 1561 | 1410 | try std.testing.expectEqualSlices(u8, &output_seq, &output_par); |
| 1562 | 1411 | } |
| 1563 | 1412 | |
| 1413 | test "parallel hashing matches sequential hashing" { |
| 1414 | const allocator = std.testing.allocator; |
| 1415 | const io = std.testing.io; |
| 1416 | |
| 1417 | var prng = std.Random.DefaultPrng.init(std.testing.random_seed); |
| 1418 | const message = try allocator.alloc(u8, 3 * large_file_threshold + 5 * chunk_size + 123); |
| 1419 | defer allocator.free(message); |
| 1420 | prng.random().bytes(message); |
| 1421 | |
| 1422 | const update_lens = [_]usize{ 100, large_file_threshold + chunk_size - 100, 3 * chunk_size, large_file_threshold + 77 }; |
| 1423 | |
| 1424 | inline for (.{ KT128, KT256 }) |KT| { |
| 1425 | var expected: [64]u8 = undefined; |
| 1426 | var actual: [64]u8 = undefined; |
| 1427 | try KT.hash(message, &expected, .{ .customization = "test" }); |
| 1428 | |
| 1429 | try KT.hashParallel(message, &actual, .{ .customization = "test" }, allocator, io); |
| 1430 | try std.testing.expectEqualSlices(u8, &expected, &actual); |
| 1431 | |
| 1432 | var hasher = KT.init(.{ .customization = "test" }); |
| 1433 | var rest: []const u8 = message; |
| 1434 | for (update_lens) |len| { |
| 1435 | try hasher.updateParallel(rest[0..len], allocator, io); |
| 1436 | rest = rest[len..]; |
| 1437 | } |
| 1438 | try hasher.updateParallel(rest, allocator, io); |
| 1439 | hasher.final(&actual); |
| 1440 | try std.testing.expectEqualSlices(u8, &expected, &actual); |
| 1441 | } |
| 1442 | } |
| 1443 | |
| 1564 | 1444 | /// Helper: Generate pattern data where data[i] = (i % 251) |
| 1565 | 1445 | fn generatePattern(allocator: Allocator, len: usize) ![]u8 { |
| 1566 | 1446 | const data = try allocator.alloc(u8, len); |
Frank Denis <124872+jedisct1@users.noreply.github.com> 2026-10-06 10:48:30+02:00