authorgravatar for 124872+jedisct1@users.noreply.github.comFrank Denis <124872+jedisct1@users.noreply.github.com> 2025-10-15 14:03:56+02:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2025-10-15 14:03:56+02:00
log6669885aa2a33228baa29daba3b14985158d866f
tree857aa08a89d005ae9944808ebf1fc8b93db1594d
parent70c21fdbab064ca060e5f111010836845ca35930
signaturebadge-check Signed by PGP key B5690EEEBB952194

Faster BLAKE3 implementation (#25574)

This is a rewrite of the BLAKE3 implementation, with vectorization. On Apple Silicon, the new implementation is about twice as fast as the previous one. With AVX2, it is more than 4 times faster. With AVX512, it is more than 7.5x faster than the previous implementation (from 678 MB/s to 5086 MB/s).

1 files changed, 880 insertions(+), 381 deletions(-)

lib/std/crypto/blake3.zig+880-381
...@@ -1,391 +1,833 @@...@@ -1,391 +1,833 @@
1// Translated from BLAKE3 reference implementation.1const std = @import("std");
2// Source: https://github.com/BLAKE3-team/BLAKE3
3
4const std = @import("../std.zig");
5const builtin = @import("builtin");2const builtin = @import("builtin");
6const fmt = std.fmt;3const fmt = std.fmt;
7const math = std.math;
8const mem = std.mem;4const mem = std.mem;
9const testing = std.testing;
105
11const ChunkIterator = struct {6const Vec4 = @Vector(4, u32);
12 slice: []u8,7const Vec8 = @Vector(8, u32);
13 chunk_len: usize,8const Vec16 = @Vector(16, u32);
149
15 fn init(slice: []u8, chunk_len: usize) ChunkIterator {10const chunk_length = 1024;
16 return ChunkIterator{11const max_depth = 54;
17 .slice = slice,12
18 .chunk_len = chunk_len,13pub const simd_degree = std.simd.suggestVectorLength(u32) orelse 1;
19 };14pub const max_simd_degree = simd_degree;
15const max_simd_degree_or_2 = if (max_simd_degree > 2) max_simd_degree else 2;
16
17const iv: [8]u32 = .{
18 0x6A09E667, 0xBB67AE85, 0x3C6EF372, 0xA54FF53A,
19 0x510E527F, 0x9B05688C, 0x1F83D9AB, 0x5BE0CD19,
20};
21
22const msg_schedule: [7][16]u8 = .{
23 .{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 },
24 .{ 2, 6, 3, 10, 7, 0, 4, 13, 1, 11, 12, 5, 9, 14, 15, 8 },
25 .{ 3, 4, 10, 12, 13, 2, 7, 14, 6, 5, 9, 0, 11, 15, 8, 1 },
26 .{ 10, 7, 12, 9, 14, 3, 13, 15, 4, 0, 11, 2, 5, 8, 1, 6 },
27 .{ 12, 13, 9, 11, 15, 10, 14, 8, 7, 2, 5, 3, 0, 1, 6, 4 },
28 .{ 9, 14, 11, 5, 8, 12, 15, 1, 13, 3, 0, 10, 2, 6, 4, 7 },
29 .{ 11, 15, 5, 0, 1, 9, 8, 6, 14, 10, 2, 12, 3, 4, 7, 13 },
30};
31
32const Flags = packed struct(u8) {
33 chunk_start: bool = false,
34 chunk_end: bool = false,
35 parent: bool = false,
36 root: bool = false,
37 keyed_hash: bool = false,
38 derive_key_context: bool = false,
39 derive_key_material: bool = false,
40 reserved: bool = false,
41
42 fn toInt(self: Flags) u8 {
43 return @bitCast(self);
20 }44 }
2145
22 fn next(self: *ChunkIterator) ?[]u8 {46 fn with(self: Flags, other: Flags) Flags {
23 const next_chunk = self.slice[0..@min(self.chunk_len, self.slice.len)];47 return @bitCast(self.toInt() | other.toInt());
24 self.slice = self.slice[next_chunk.len..];
25 return if (next_chunk.len > 0) next_chunk else null;
26 }48 }
27};49};
2850
29const OUT_LEN: usize = 32;51const rotr = std.math.rotr;
30const KEY_LEN: usize = 32;
31const BLOCK_LEN: usize = 64;
32const CHUNK_LEN: usize = 1024;
3352
34const IV = [8]u32{53inline fn rotr32(w: u32, c: u5) u32 {
35 0x6A09E667, 0xBB67AE85, 0x3C6EF372, 0xA54FF53A, 0x510E527F, 0x9B05688C, 0x1F83D9AB, 0x5BE0CD19,54 return rotr(u32, w, c);
36};55}
3756
38const MSG_SCHEDULE = [7][16]u8{57inline fn load32(bytes: []const u8) u32 {
39 [_]u8{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 },58 return mem.readInt(u32, bytes[0..4], .little);
40 [_]u8{ 2, 6, 3, 10, 7, 0, 4, 13, 1, 11, 12, 5, 9, 14, 15, 8 },59}
41 [_]u8{ 3, 4, 10, 12, 13, 2, 7, 14, 6, 5, 9, 0, 11, 15, 8, 1 },
42 [_]u8{ 10, 7, 12, 9, 14, 3, 13, 15, 4, 0, 11, 2, 5, 8, 1, 6 },
43 [_]u8{ 12, 13, 9, 11, 15, 10, 14, 8, 7, 2, 5, 3, 0, 1, 6, 4 },
44 [_]u8{ 9, 14, 11, 5, 8, 12, 15, 1, 13, 3, 0, 10, 2, 6, 4, 7 },
45 [_]u8{ 11, 15, 5, 0, 1, 9, 8, 6, 14, 10, 2, 12, 3, 4, 7, 13 },
46};
4760
48// These are the internal flags that we use to domain separate root/non-root,61inline fn store32(bytes: []u8, w: u32) void {
49// chunk/parent, and chunk beginning/middle/end. These get set at the high end62 mem.writeInt(u32, bytes[0..4], w, .little);
50// of the block flags word in the compression function, so their values start63}
51// high and go down.
52const CHUNK_START: u8 = 1 << 0;
53const CHUNK_END: u8 = 1 << 1;
54const PARENT: u8 = 1 << 2;
55const ROOT: u8 = 1 << 3;
56const KEYED_HASH: u8 = 1 << 4;
57const DERIVE_KEY_CONTEXT: u8 = 1 << 5;
58const DERIVE_KEY_MATERIAL: u8 = 1 << 6;
59
60const CompressVectorized = struct {
61 const Lane = @Vector(4, u32);
62 const Rows = [4]Lane;
63
64 fn g(comptime even: bool, rows: *Rows, m: Lane) void {
65 rows[0] +%= rows[1] +% m;
66 rows[3] ^= rows[0];
67 rows[3] = math.rotr(Lane, rows[3], if (even) 8 else 16);
68 rows[2] +%= rows[3];
69 rows[1] ^= rows[2];
70 rows[1] = math.rotr(Lane, rows[1], if (even) 7 else 12);
71 }
72
73 fn diagonalize(rows: *Rows) void {
74 rows[0] = @shuffle(u32, rows[0], undefined, [_]i32{ 3, 0, 1, 2 });
75 rows[3] = @shuffle(u32, rows[3], undefined, [_]i32{ 2, 3, 0, 1 });
76 rows[2] = @shuffle(u32, rows[2], undefined, [_]i32{ 1, 2, 3, 0 });
77 }
78
79 fn undiagonalize(rows: *Rows) void {
80 rows[0] = @shuffle(u32, rows[0], undefined, [_]i32{ 1, 2, 3, 0 });
81 rows[3] = @shuffle(u32, rows[3], undefined, [_]i32{ 2, 3, 0, 1 });
82 rows[2] = @shuffle(u32, rows[2], undefined, [_]i32{ 3, 0, 1, 2 });
83 }
84
85 fn compress(
86 chaining_value: [8]u32,
87 block_words: [16]u32,
88 block_len: u32,
89 counter: u64,
90 flags: u8,
91 ) [16]u32 {
92 const md = Lane{ @as(u32, @truncate(counter)), @as(u32, @truncate(counter >> 32)), block_len, @as(u32, flags) };
93 var rows = Rows{ chaining_value[0..4].*, chaining_value[4..8].*, IV[0..4].*, md };
94
95 var m = Rows{ block_words[0..4].*, block_words[4..8].*, block_words[8..12].*, block_words[12..16].* };
96 var t0 = @shuffle(u32, m[0], m[1], [_]i32{ 0, 2, (-1 - 0), (-1 - 2) });
97 g(false, &rows, t0);
98 var t1 = @shuffle(u32, m[0], m[1], [_]i32{ 1, 3, (-1 - 1), (-1 - 3) });
99 g(true, &rows, t1);
100 diagonalize(&rows);
101 var t2 = @shuffle(u32, m[2], m[3], [_]i32{ 0, 2, (-1 - 0), (-1 - 2) });
102 t2 = @shuffle(u32, t2, undefined, [_]i32{ 3, 0, 1, 2 });
103 g(false, &rows, t2);
104 var t3 = @shuffle(u32, m[2], m[3], [_]i32{ 1, 3, (-1 - 1), (-1 - 3) });
105 t3 = @shuffle(u32, t3, undefined, [_]i32{ 3, 0, 1, 2 });
106 g(true, &rows, t3);
107 undiagonalize(&rows);
108 m = Rows{ t0, t1, t2, t3 };
109
110 var i: usize = 0;
111 while (i < 6) : (i += 1) {
112 t0 = @shuffle(u32, m[0], m[1], [_]i32{ 2, 1, (-1 - 1), (-1 - 3) });
113 t0 = @shuffle(u32, t0, undefined, [_]i32{ 1, 2, 3, 0 });
114 g(false, &rows, t0);
115 t1 = @shuffle(u32, m[2], m[3], [_]i32{ 2, 2, (-1 - 3), (-1 - 3) });
116 var tt = @shuffle(u32, m[0], undefined, [_]i32{ 3, 3, 0, 0 });
117 t1 = @shuffle(u32, tt, t1, [_]i32{ 0, (-1 - 1), 2, (-1 - 3) });
118 g(true, &rows, t1);
119 diagonalize(&rows);
120 t2 = @shuffle(u32, m[3], m[1], [_]i32{ 0, 1, (-1 - 0), (-1 - 1) });
121 tt = @shuffle(u32, t2, m[2], [_]i32{ 0, 1, 2, (-1 - 3) });
122 t2 = @shuffle(u32, tt, undefined, [_]i32{ 0, 2, 3, 1 });
123 g(false, &rows, t2);
124 t3 = @shuffle(u32, m[1], m[3], [_]i32{ 2, (-1 - 2), 3, (-1 - 3) });
125 tt = @shuffle(u32, m[2], t3, [_]i32{ 0, (-1 - 0), 1, (-1 - 1) });
126 t3 = @shuffle(u32, tt, undefined, [_]i32{ 2, 3, 1, 0 });
127 g(true, &rows, t3);
128 undiagonalize(&rows);
129 m = Rows{ t0, t1, t2, t3 };
130 }
13164
132 rows[0] ^= rows[2];65fn loadKeyWords(key: [Blake3.key_length]u8) [8]u32 {
133 rows[1] ^= rows[3];66 var key_words: [8]u32 = undefined;
134 rows[2] ^= @Vector(4, u32){ chaining_value[0], chaining_value[1], chaining_value[2], chaining_value[3] };67 for (0..8) |i| {
135 rows[3] ^= @Vector(4, u32){ chaining_value[4], chaining_value[5], chaining_value[6], chaining_value[7] };68 key_words[i] = load32(key[i * 4 ..][0..4]);
69 }
70 return key_words;
71}
13672
137 return @as([16]u32, @bitCast(rows));73fn storeCvWords(cv_words: [8]u32) [Blake3.digest_length]u8 {
74 var bytes: [Blake3.digest_length]u8 = undefined;
75 for (0..8) |i| {
76 store32(bytes[i * 4 ..][0..4], cv_words[i]);
138 }77 }
139};78 return bytes;
79}
14080
141const CompressGeneric = struct {81fn loadCvWords(bytes: [Blake3.digest_length]u8) [8]u32 {
142 fn g(state: *[16]u32, comptime a: usize, comptime b: usize, comptime c: usize, comptime d: usize, mx: u32, my: u32) void {82 var cv_words: [8]u32 = undefined;
143 state[a] +%= state[b] +% mx;83 for (0..8) |i| {
144 state[d] = math.rotr(u32, state[d] ^ state[a], 16);84 cv_words[i] = load32(bytes[i * 4 ..][0..4]);
145 state[c] +%= state[d];85 }
146 state[b] = math.rotr(u32, state[b] ^ state[c], 12);86 return cv_words;
147 state[a] +%= state[b] +% my;87}
148 state[d] = math.rotr(u32, state[d] ^ state[a], 8);88
149 state[c] +%= state[d];89inline fn counterLow(counter: u64) u32 {
150 state[b] = math.rotr(u32, state[b] ^ state[c], 7);90 return @truncate(counter);
151 }91}
15292
153 fn round(state: *[16]u32, msg: [16]u32, schedule: [16]u8) void {93inline fn counterHigh(counter: u64) u32 {
154 // Mix the columns.94 return @truncate(counter >> 32);
155 g(state, 0, 4, 8, 12, msg[schedule[0]], msg[schedule[1]]);95}
156 g(state, 1, 5, 9, 13, msg[schedule[2]], msg[schedule[3]]);96
157 g(state, 2, 6, 10, 14, msg[schedule[4]], msg[schedule[5]]);97fn highestOne(x: u64) u6 {
158 g(state, 3, 7, 11, 15, msg[schedule[6]], msg[schedule[7]]);98 if (x == 0) return 0;
15999 return @intCast(63 - @clz(x));
160 // Mix the diagonals.100}
161 g(state, 0, 5, 10, 15, msg[schedule[8]], msg[schedule[9]]);101
162 g(state, 1, 6, 11, 12, msg[schedule[10]], msg[schedule[11]]);102fn roundDownToPowerOf2(x: u64) u64 {
163 g(state, 2, 7, 8, 13, msg[schedule[12]], msg[schedule[13]]);103 return @as(u64, 1) << highestOne(x | 1);
164 g(state, 3, 4, 9, 14, msg[schedule[14]], msg[schedule[15]]);104}
165 }105
166106inline fn g(state: *[16]u32, a: usize, b: usize, c: usize, d: usize, x: u32, y: u32) void {
167 fn compress(107 state[a] +%= state[b] +% x;
168 chaining_value: [8]u32,108 state[d] = rotr32(state[d] ^ state[a], 16);
169 block_words: [16]u32,109 state[c] +%= state[d];
170 block_len: u32,110 state[b] = rotr32(state[b] ^ state[c], 12);
171 counter: u64,111 state[a] +%= state[b] +% y;
172 flags: u8,112 state[d] = rotr32(state[d] ^ state[a], 8);
173 ) [16]u32 {113 state[c] +%= state[d];
174 var state = [16]u32{114 state[b] = rotr32(state[b] ^ state[c], 7);
175 chaining_value[0],115}
176 chaining_value[1],116
177 chaining_value[2],117inline fn roundFn(state: *[16]u32, msg: *const [16]u32, round: usize) void {
178 chaining_value[3],118 const schedule = &msg_schedule[round];
179 chaining_value[4],119
180 chaining_value[5],120 g(state, 0, 4, 8, 12, msg[schedule[0]], msg[schedule[1]]);
181 chaining_value[6],121 g(state, 1, 5, 9, 13, msg[schedule[2]], msg[schedule[3]]);
182 chaining_value[7],122 g(state, 2, 6, 10, 14, msg[schedule[4]], msg[schedule[5]]);
183 IV[0],123 g(state, 3, 7, 11, 15, msg[schedule[6]], msg[schedule[7]]);
184 IV[1],124
185 IV[2],125 g(state, 0, 5, 10, 15, msg[schedule[8]], msg[schedule[9]]);
186 IV[3],126 g(state, 1, 6, 11, 12, msg[schedule[10]], msg[schedule[11]]);
187 @as(u32, @truncate(counter)),127 g(state, 2, 7, 8, 13, msg[schedule[12]], msg[schedule[13]]);
188 @as(u32, @truncate(counter >> 32)),128 g(state, 3, 4, 9, 14, msg[schedule[14]], msg[schedule[15]]);
189 block_len,129}
190 flags,130
191 };131fn compressPre(state: *[16]u32, cv: *const [8]u32, block: []const u8, block_len: u8, counter: u64, flags: Flags) void {
192 for (MSG_SCHEDULE) |schedule| {132 var block_words: [16]u32 = undefined;
193 round(&state, block_words, schedule);133 for (0..16) |i| {
134 block_words[i] = load32(block[i * 4 ..][0..4]);
135 }
136
137 for (0..8) |i| {
138 state[i] = cv[i];
139 }
140 for (0..4) |i| {
141 state[i + 8] = iv[i];
142 }
143 state[12] = counterLow(counter);
144 state[13] = counterHigh(counter);
145 state[14] = @as(u32, block_len);
146 state[15] = @as(u32, flags.toInt());
147
148 for (0..7) |round| {
149 roundFn(state, &block_words, round);
150 }
151}
152
153fn compressInPlace(cv: *[8]u32, block: []const u8, block_len: u8, counter: u64, flags: Flags) void {
154 var state: [16]u32 = undefined;
155 compressPre(&state, cv, block, block_len, counter, flags);
156 for (0..8) |i| {
157 cv[i] = state[i] ^ state[i + 8];
158 }
159}
160
161fn compressXof(cv: *const [8]u32, block: []const u8, block_len: u8, counter: u64, flags: Flags, out: *[64]u8) void {
162 var state: [16]u32 = undefined;
163 compressPre(&state, cv, block, block_len, counter, flags);
164
165 for (0..8) |i| {
166 store32(out[i * 4 ..][0..4], state[i] ^ state[i + 8]);
167 }
168 for (0..8) |i| {
169 store32(out[(i + 8) * 4 ..][0..4], state[i + 8] ^ cv[i]);
170 }
171}
172
173fn hashOne(input: []const u8, blocks: usize, key: [8]u32, counter: u64, flags: Flags, flags_start: Flags, flags_end: Flags) [Blake3.digest_length]u8 {
174 var cv = key;
175 var block_flags = flags.with(flags_start);
176 var inp = input;
177 var remaining_blocks = blocks;
178
179 while (remaining_blocks > 0) {
180 if (remaining_blocks == 1) {
181 block_flags = block_flags.with(flags_end);
194 }182 }
195 for (chaining_value, 0..) |_, i| {183 compressInPlace(&cv, inp[0..Blake3.block_length], Blake3.block_length, counter, block_flags);
196 state[i] ^= state[i + 8];184 inp = inp[Blake3.block_length..];
197 state[i + 8] ^= chaining_value[i];185 remaining_blocks -= 1;
186 block_flags = flags;
187 }
188
189 return storeCvWords(cv);
190}
191
192fn hashManyPortable(inputs: [][*]const u8, num_inputs: usize, blocks: usize, key: [8]u32, counter_arg: u64, increment_counter: bool, flags: Flags, flags_start: Flags, flags_end: Flags, out: []u8) void {
193 var counter = counter_arg;
194 for (0..num_inputs) |i| {
195 const input = inputs[i][0 .. blocks * Blake3.block_length];
196 const result = hashOne(input, blocks, key, counter, flags, flags_start, flags_end);
197 @memcpy(out[i * Blake3.digest_length ..][0..Blake3.digest_length], &result);
198 if (increment_counter) {
199 counter += 1;
198 }200 }
199 return state;
200 }201 }
201};202}
202203
203const compress = if (builtin.cpu.arch == .x86_64)204fn transposeNxN(comptime Vec: type, comptime n: comptime_int, vecs: *[n]Vec) void {
204 CompressVectorized.compress205 const temp: [n]Vec = vecs.*;
205else
206 CompressGeneric.compress;
207206
208fn first8Words(words: [16]u32) [8]u32 {207 inline for (0..n) |i| {
209 return @as(*const [8]u32, @ptrCast(&words)).*;208 inline for (0..n) |j| {
209 vecs[i][j] = temp[j][i];
210 }
211 }
210}212}
211213
212fn wordsFromLittleEndianBytes(comptime count: usize, bytes: [count * 4]u8) [count]u32 {214fn transposeMsg(comptime Vec: type, comptime n: comptime_int, inputs: [n][*]const u8, block_offset: usize, out: *[16]Vec) void {
213 var words: [count]u32 = undefined;215 const info = @typeInfo(Vec);
214 for (&words, 0..) |*word, i| {216 if (info != .vector) @compileError("transposeMsg requires a vector type");
215 word.* = mem.readInt(u32, bytes[4 * i ..][0..4], .little);217 if (info.vector.len != n) @compileError("vector width must match N");
218
219 var temp: [n][16]u32 = undefined;
220
221 for (0..n) |i| {
222 const block = inputs[i] + block_offset;
223 for (0..16) |j| {
224 temp[i][j] = load32(block[j * 4 ..][0..4]);
225 }
226 }
227
228 for (0..16) |j| {
229 var result: Vec = undefined;
230 inline for (0..n) |i| {
231 result[i] = temp[i][j];
232 }
233 out[j] = result;
216 }234 }
217 return words;
218}235}
219236
220// Each chunk or parent node can produce either an 8-word chaining value or, by237fn roundFnVec(comptime Vec: type, v: *[16]Vec, m: *const [16]Vec, r: usize) void {
221// setting the ROOT flag, any number of final output bytes. The Output struct238 const schedule = &msg_schedule[r];
222// captures the state just prior to choosing between those two possibilities.239
223const Output = struct {240 // Column round - first half
224 input_chaining_value: [8]u32 align(16),241 inline for (0..4) |i| {
225 block_words: [16]u32 align(16),242 v[i] +%= m[schedule[i * 2]];
226 block_len: u32,243 }
244 inline for (0..4) |i| {
245 v[i] +%= v[i + 4];
246 }
247 inline for (0..4) |i| {
248 v[i + 12] ^= v[i];
249 }
250 inline for (0..4) |i| {
251 v[i + 12] = rotr(Vec, v[i + 12], 16);
252 }
253 inline for (0..4) |i| {
254 v[i + 8] +%= v[i + 12];
255 }
256 inline for (0..4) |i| {
257 v[i + 4] ^= v[i + 8];
258 }
259 inline for (0..4) |i| {
260 v[i + 4] = rotr(Vec, v[i + 4], 12);
261 }
262
263 // Column round - second half
264 inline for (0..4) |i| {
265 v[i] +%= m[schedule[i * 2 + 1]];
266 }
267 inline for (0..4) |i| {
268 v[i] +%= v[i + 4];
269 }
270 inline for (0..4) |i| {
271 v[i + 12] ^= v[i];
272 }
273 inline for (0..4) |i| {
274 v[i + 12] = rotr(Vec, v[i + 12], 8);
275 }
276 inline for (0..4) |i| {
277 v[i + 8] +%= v[i + 12];
278 }
279 inline for (0..4) |i| {
280 v[i + 4] ^= v[i + 8];
281 }
282 inline for (0..4) |i| {
283 v[i + 4] = rotr(Vec, v[i + 4], 7);
284 }
285
286 // Diagonal round - first half
287 inline for (0..4) |i| {
288 v[i] +%= m[schedule[i * 2 + 8]];
289 }
290 const b_indices = [4]u8{ 5, 6, 7, 4 };
291 inline for (0..4) |i| {
292 v[i] +%= v[b_indices[i]];
293 }
294 const d_indices = [4]u8{ 15, 12, 13, 14 };
295 inline for (0..4) |i| {
296 v[d_indices[i]] ^= v[i];
297 }
298 inline for (0..4) |i| {
299 v[d_indices[i]] = rotr(Vec, v[d_indices[i]], 16);
300 }
301 const c_indices = [4]u8{ 10, 11, 8, 9 };
302 inline for (0..4) |i| {
303 v[c_indices[i]] +%= v[d_indices[i]];
304 }
305 inline for (0..4) |i| {
306 v[b_indices[i]] ^= v[c_indices[i]];
307 }
308 inline for (0..4) |i| {
309 v[b_indices[i]] = rotr(Vec, v[b_indices[i]], 12);
310 }
311
312 // Diagonal round - second half
313 inline for (0..4) |i| {
314 v[i] +%= m[schedule[i * 2 + 9]];
315 }
316 inline for (0..4) |i| {
317 v[i] +%= v[b_indices[i]];
318 }
319 inline for (0..4) |i| {
320 v[d_indices[i]] ^= v[i];
321 }
322 inline for (0..4) |i| {
323 v[d_indices[i]] = rotr(Vec, v[d_indices[i]], 8);
324 }
325 inline for (0..4) |i| {
326 v[c_indices[i]] +%= v[d_indices[i]];
327 }
328 inline for (0..4) |i| {
329 v[b_indices[i]] ^= v[c_indices[i]];
330 }
331 inline for (0..4) |i| {
332 v[b_indices[i]] = rotr(Vec, v[b_indices[i]], 7);
333 }
334}
335
336fn hashVec(
337 comptime Vec: type,
338 comptime n: comptime_int,
339 inputs: [n][*]const u8,
340 blocks: usize,
341 key: [8]u32,
227 counter: u64,342 counter: u64,
228 flags: u8,343 increment_counter: bool,
344 flags: Flags,
345 flags_start: Flags,
346 flags_end: Flags,
347 out: *[n * Blake3.digest_length]u8,
348) void {
349 var h_vecs: [8]Vec = undefined;
350 for (0..8) |i| {
351 h_vecs[i] = @splat(key[i]);
352 }
229353
230 fn chainingValue(self: *const Output) [8]u32 {354 const counter_low_vec = if (increment_counter) blk: {
231 return first8Words(compress(355 var result: Vec = undefined;
232 self.input_chaining_value,356 inline for (0..n) |i| {
233 self.block_words,357 result[i] = counterLow(counter + i);
234 self.block_len,358 }
235 self.counter,359 break :blk result;
236 self.flags,360 } else @as(Vec, @splat(counterLow(counter)));
237 ));361
238 }362 const counter_high_vec = if (increment_counter) blk: {
239363 var result: Vec = undefined;
240 fn rootOutputBytes(self: *const Output, output: []u8) void {364 inline for (0..n) |i| {
241 var out_block_it = ChunkIterator.init(output, 2 * OUT_LEN);365 result[i] = counterHigh(counter + i);
242 var output_block_counter: usize = 0;366 }
243 while (out_block_it.next()) |out_block| {367 break :blk result;
244 const words = compress(368 } else @as(Vec, @splat(counterHigh(counter)));
245 self.input_chaining_value,369
246 self.block_words,370 var block_flags = flags.with(flags_start);
247 self.block_len,371
248 output_block_counter,372 for (0..blocks) |block| {
249 self.flags | ROOT,373 if (block + 1 == blocks) {
250 );374 block_flags = block_flags.with(flags_end);
251 var out_word_it = ChunkIterator.init(out_block, 4);375 }
252 var word_counter: usize = 0;376
253 while (out_word_it.next()) |out_word| {377 const block_len_vec: Vec = @splat(Blake3.block_length);
254 var word_bytes: [4]u8 = undefined;378 const block_flags_vec: Vec = @splat(@as(u32, block_flags.toInt()));
255 mem.writeInt(u32, &word_bytes, words[word_counter], .little);379
256 @memcpy(out_word, word_bytes[0..out_word.len]);380 var msg_vecs: [16]Vec = undefined;
257 word_counter += 1;381 transposeMsg(Vec, n, inputs, block * Blake3.block_length, &msg_vecs);
382
383 var v: [16]Vec = .{
384 h_vecs[0], h_vecs[1], h_vecs[2], h_vecs[3],
385 h_vecs[4], h_vecs[5], h_vecs[6], h_vecs[7],
386 @splat(iv[0]), @splat(iv[1]), @splat(iv[2]), @splat(iv[3]),
387 counter_low_vec, counter_high_vec, block_len_vec, block_flags_vec,
388 };
389
390 inline for (0..7) |r| {
391 roundFnVec(Vec, &v, &msg_vecs, r);
392 }
393
394 inline for (0..8) |i| {
395 h_vecs[i] = v[i] ^ v[i + 8];
396 }
397
398 block_flags = flags;
399 }
400
401 // Output serialization - different strategies for different widths
402 switch (n) {
403 4 => {
404 // Special interleaved pattern for Vec4
405 var out_vecs = [4]Vec{ h_vecs[0], h_vecs[1], h_vecs[2], h_vecs[3] };
406 transposeNxN(Vec, 4, &out_vecs);
407 inline for (0..4) |i| {
408 mem.writeInt(u32, out[0 * 16 + i * 4 ..][0..4], out_vecs[0][i], .little);
258 }409 }
259 output_block_counter += 1;410 inline for (0..4) |i| {
411 mem.writeInt(u32, out[2 * 16 + i * 4 ..][0..4], out_vecs[1][i], .little);
412 }
413 inline for (0..4) |i| {
414 mem.writeInt(u32, out[4 * 16 + i * 4 ..][0..4], out_vecs[2][i], .little);
415 }
416 inline for (0..4) |i| {
417 mem.writeInt(u32, out[6 * 16 + i * 4 ..][0..4], out_vecs[3][i], .little);
418 }
419
420 out_vecs = [4]Vec{ h_vecs[4], h_vecs[5], h_vecs[6], h_vecs[7] };
421 transposeNxN(Vec, 4, &out_vecs);
422 inline for (0..4) |i| {
423 mem.writeInt(u32, out[1 * 16 + i * 4 ..][0..4], out_vecs[0][i], .little);
424 }
425 inline for (0..4) |i| {
426 mem.writeInt(u32, out[3 * 16 + i * 4 ..][0..4], out_vecs[1][i], .little);
427 }
428 inline for (0..4) |i| {
429 mem.writeInt(u32, out[5 * 16 + i * 4 ..][0..4], out_vecs[2][i], .little);
430 }
431 inline for (0..4) |i| {
432 mem.writeInt(u32, out[7 * 16 + i * 4 ..][0..4], out_vecs[3][i], .little);
433 }
434 },
435 8 => {
436 // Linear pattern with transpose for Vec8
437 var out_vecs = [8]Vec{ h_vecs[0], h_vecs[1], h_vecs[2], h_vecs[3], h_vecs[4], h_vecs[5], h_vecs[6], h_vecs[7] };
438 transposeNxN(Vec, 8, &out_vecs);
439 inline for (0..8) |i| {
440 mem.writeInt(u32, out[0 * 32 + i * 4 ..][0..4], out_vecs[0][i], .little);
441 }
442 inline for (0..8) |i| {
443 mem.writeInt(u32, out[1 * 32 + i * 4 ..][0..4], out_vecs[1][i], .little);
444 }
445 inline for (0..8) |i| {
446 mem.writeInt(u32, out[2 * 32 + i * 4 ..][0..4], out_vecs[2][i], .little);
447 }
448 inline for (0..8) |i| {
449 mem.writeInt(u32, out[3 * 32 + i * 4 ..][0..4], out_vecs[3][i], .little);
450 }
451 inline for (0..8) |i| {
452 mem.writeInt(u32, out[4 * 32 + i * 4 ..][0..4], out_vecs[4][i], .little);
453 }
454 inline for (0..8) |i| {
455 mem.writeInt(u32, out[5 * 32 + i * 4 ..][0..4], out_vecs[5][i], .little);
456 }
457 inline for (0..8) |i| {
458 mem.writeInt(u32, out[6 * 32 + i * 4 ..][0..4], out_vecs[6][i], .little);
459 }
460 inline for (0..8) |i| {
461 mem.writeInt(u32, out[7 * 32 + i * 4 ..][0..4], out_vecs[7][i], .little);
462 }
463 },
464 16 => {
465 // Direct lane-by-lane output for Vec16 (no transpose)
466 inline for (0..16) |lane| {
467 const hash_offset = lane * Blake3.digest_length;
468 inline for (0..8) |word_idx| {
469 const word = h_vecs[word_idx][lane];
470 out[hash_offset + word_idx * 4 + 0] = @truncate(word);
471 out[hash_offset + word_idx * 4 + 1] = @truncate(word >> 8);
472 out[hash_offset + word_idx * 4 + 2] = @truncate(word >> 16);
473 out[hash_offset + word_idx * 4 + 3] = @truncate(word >> 24);
474 }
475 }
476 },
477 else => @compileError("Unsupported SIMD width"),
478 }
479}
480
481fn hashManySimd(
482 inputs: [][*]const u8,
483 num_inputs: usize,
484 blocks: usize,
485 key: [8]u32,
486 counter: u64,
487 increment_counter: bool,
488 flags: Flags,
489 flags_start: Flags,
490 flags_end: Flags,
491 out: []u8,
492) void {
493 var remaining = num_inputs;
494 var inp = inputs.ptr;
495 var out_ptr = out.ptr;
496 var cnt = counter;
497
498 const simd_deg = comptime simd_degree;
499
500 if (comptime simd_deg >= 16) {
501 while (remaining >= 16) {
502 const sixteen_inputs = [16][*]const u8{
503 inp[0], inp[1], inp[2], inp[3],
504 inp[4], inp[5], inp[6], inp[7],
505 inp[8], inp[9], inp[10], inp[11],
506 inp[12], inp[13], inp[14], inp[15],
507 };
508
509 var simd_out: [16 * Blake3.digest_length]u8 = undefined;
510 hashVec(Vec16, 16, sixteen_inputs, blocks, key, cnt, increment_counter, flags, flags_start, flags_end, &simd_out);
511
512 @memcpy(out_ptr[0 .. 16 * Blake3.digest_length], &simd_out);
513
514 if (increment_counter) cnt += 16;
515 inp += 16;
516 remaining -= 16;
517 out_ptr += 16 * Blake3.digest_length;
260 }518 }
261 }519 }
262};520
521 if (comptime simd_deg >= 8) {
522 while (remaining >= 8) {
523 const eight_inputs = [8][*]const u8{
524 inp[0], inp[1], inp[2], inp[3],
525 inp[4], inp[5], inp[6], inp[7],
526 };
527
528 var simd_out: [8 * Blake3.digest_length]u8 = undefined;
529 hashVec(Vec8, 8, eight_inputs, blocks, key, cnt, increment_counter, flags, flags_start, flags_end, &simd_out);
530
531 @memcpy(out_ptr[0 .. 8 * Blake3.digest_length], &simd_out);
532
533 if (increment_counter) cnt += 8;
534 inp += 8;
535 remaining -= 8;
536 out_ptr += 8 * Blake3.digest_length;
537 }
538 }
539
540 if (comptime simd_deg >= 4) {
541 while (remaining >= 4) {
542 const four_inputs = [4][*]const u8{
543 inp[0],
544 inp[1],
545 inp[2],
546 inp[3],
547 };
548
549 var simd_out: [4 * Blake3.digest_length]u8 = undefined;
550 hashVec(Vec4, 4, four_inputs, blocks, key, cnt, increment_counter, flags, flags_start, flags_end, &simd_out);
551
552 @memcpy(out_ptr[0 .. 4 * Blake3.digest_length], &simd_out);
553
554 if (increment_counter) cnt += 4;
555 inp += 4;
556 remaining -= 4;
557 out_ptr += 4 * Blake3.digest_length;
558 }
559 }
560
561 if (remaining > 0) {
562 hashManyPortable(inp[0..remaining], remaining, blocks, key, cnt, increment_counter, flags, flags_start, flags_end, out_ptr[0 .. remaining * Blake3.digest_length]);
563 }
564}
565
566fn hashMany(inputs: [][*]const u8, num_inputs: usize, blocks: usize, key: [8]u32, counter: u64, increment_counter: bool, flags: Flags, flags_start: Flags, flags_end: Flags, out: []u8) void {
567 if (comptime max_simd_degree >= 4) {
568 hashManySimd(inputs, num_inputs, blocks, key, counter, increment_counter, flags, flags_start, flags_end, out);
569 } else {
570 hashManyPortable(inputs, num_inputs, blocks, key, counter, increment_counter, flags, flags_start, flags_end, out);
571 }
572}
573
574fn compressChunksParallel(input: []const u8, key: [8]u32, chunk_counter: u64, flags: Flags, out: []u8) usize {
575 var chunks_array: [max_simd_degree][*]const u8 = undefined;
576 var input_position: usize = 0;
577 var chunks_array_len: usize = 0;
578
579 while (input.len - input_position >= chunk_length) {
580 chunks_array[chunks_array_len] = input[input_position..].ptr;
581 input_position += chunk_length;
582 chunks_array_len += 1;
583 }
584
585 hashMany(chunks_array[0..chunks_array_len], chunks_array_len, chunk_length / Blake3.block_length, key, chunk_counter, true, flags, .{ .chunk_start = true }, .{ .chunk_end = true }, out);
586
587 if (input.len > input_position) {
588 const counter = chunk_counter + @as(u64, chunks_array_len);
589 var chunk_state = ChunkState.init(key, flags);
590 chunk_state.chunk_counter = counter;
591 chunk_state.update(input[input_position..]);
592 const output = chunk_state.output();
593 const cv = output.chainingValue();
594 const cv_bytes = storeCvWords(cv);
595 @memcpy(out[chunks_array_len * Blake3.digest_length ..][0..Blake3.digest_length], &cv_bytes);
596 return chunks_array_len + 1;
597 } else {
598 return chunks_array_len;
599 }
600}
601
602fn compressParentsParallel(child_chaining_values: []const u8, num_chaining_values: usize, key: [8]u32, flags: Flags, out: []u8) usize {
603 var parents_array: [max_simd_degree_or_2][*]const u8 = undefined;
604 var parents_array_len: usize = 0;
605
606 while (num_chaining_values - (2 * parents_array_len) >= 2) {
607 parents_array[parents_array_len] = child_chaining_values[2 * parents_array_len * Blake3.digest_length ..].ptr;
608 parents_array_len += 1;
609 }
610
611 hashMany(parents_array[0..parents_array_len], parents_array_len, 1, key, 0, false, flags.with(.{ .parent = true }), .{}, .{}, out);
612
613 if (num_chaining_values > 2 * parents_array_len) {
614 @memcpy(out[parents_array_len * Blake3.digest_length ..][0..Blake3.digest_length], child_chaining_values[2 * parents_array_len * Blake3.digest_length ..][0..Blake3.digest_length]);
615 return parents_array_len + 1;
616 } else {
617 return parents_array_len;
618 }
619}
620
621fn compressSubtreeWide(input: []const u8, key: [8]u32, chunk_counter: u64, flags: Flags, out: []u8) usize {
622 if (input.len <= max_simd_degree * chunk_length) {
623 return compressChunksParallel(input, key, chunk_counter, flags, out);
624 }
625
626 const left_input_len = leftSubtreeLen(input.len);
627 const right_input = input[left_input_len..];
628 const right_chunk_counter = chunk_counter + @as(u64, left_input_len / chunk_length);
629
630 var cv_array: [2 * max_simd_degree_or_2 * Blake3.digest_length]u8 = undefined;
631 var degree: usize = max_simd_degree;
632 if (left_input_len > chunk_length and degree == 1) {
633 degree = 2;
634 }
635 const right_cvs = cv_array[degree * Blake3.digest_length ..];
636
637 const left_n = compressSubtreeWide(input[0..left_input_len], key, chunk_counter, flags, cv_array[0..]);
638 const right_n = compressSubtreeWide(right_input, key, right_chunk_counter, flags, right_cvs);
639
640 if (left_n == 1) {
641 @memcpy(out[0 .. 2 * Blake3.digest_length], cv_array[0 .. 2 * Blake3.digest_length]);
642 return 2;
643 }
644
645 const num_chaining_values = left_n + right_n;
646 return compressParentsParallel(&cv_array, num_chaining_values, key, flags, out);
647}
648
649fn compressSubtreeToParentNode(input: []const u8, key: [8]u32, chunk_counter: u64, flags: Flags, out: *[2 * Blake3.digest_length]u8) void {
650 var cv_array: [max_simd_degree_or_2 * Blake3.digest_length]u8 = undefined;
651 var num_cvs = compressSubtreeWide(input, key, chunk_counter, flags, &cv_array);
652
653 if (max_simd_degree_or_2 > 2) {
654 var out_array: [max_simd_degree_or_2 * Blake3.digest_length / 2]u8 = undefined;
655 while (num_cvs > 2) {
656 num_cvs = compressParentsParallel(&cv_array, num_cvs, key, flags, &out_array);
657 @memcpy(cv_array[0 .. num_cvs * Blake3.digest_length], out_array[0 .. num_cvs * Blake3.digest_length]);
658 }
659 }
660
661 @memcpy(out, cv_array[0 .. 2 * Blake3.digest_length]);
662}
663
664fn leftSubtreeLen(input_len: usize) usize {
665 const full_chunks = (input_len - 1) / chunk_length;
666 return @intCast(roundDownToPowerOf2(full_chunks) * chunk_length);
667}
668
669fn parentOutput(parent_block: []const u8, key: [8]u32, flags: Flags) Output {
670 var block: [Blake3.block_length]u8 = undefined;
671 @memcpy(&block, parent_block[0..Blake3.block_length]);
672 return Output{
673 .input_cv = key,
674 .block = block,
675 .block_len = Blake3.block_length,
676 .counter = 0,
677 .flags = flags.with(.{ .parent = true }),
678 };
679}
680
681fn parentOutputFromCvs(left_cv: [8]u32, right_cv: [8]u32, key: [8]u32, flags: Flags) Output {
682 var block: [Blake3.block_length]u8 align(16) = undefined;
683 for (0..8) |i| {
684 store32(block[i * 4 ..][0..4], left_cv[i]);
685 store32(block[(i + 8) * 4 ..][0..4], right_cv[i]);
686 }
687 return Output{
688 .input_cv = key,
689 .block = block,
690 .block_len = Blake3.block_length,
691 .counter = 0,
692 .flags = flags.with(.{ .parent = true }),
693 };
694}
263695
264const ChunkState = struct {696const ChunkState = struct {
265 chaining_value: [8]u32 align(16),697 cv: [8]u32 align(16),
266 chunk_counter: u64,698 chunk_counter: u64,
267 block: [BLOCK_LEN]u8 align(16) = [_]u8{0} ** BLOCK_LEN,699 buf: [Blake3.block_length]u8 align(16),
268 block_len: u8 = 0,700 buf_len: u8,
269 blocks_compressed: u8 = 0,701 blocks_compressed: u8,
270 flags: u8,702 flags: Flags,
271703
272 fn init(key: [8]u32, chunk_counter: u64, flags: u8) ChunkState {704 fn init(key: [8]u32, flags: Flags) ChunkState {
273 return ChunkState{705 return ChunkState{
274 .chaining_value = key,706 .cv = key,
275 .chunk_counter = chunk_counter,707 .chunk_counter = 0,
708 .buf = [_]u8{0} ** Blake3.block_length,
709 .buf_len = 0,
710 .blocks_compressed = 0,
276 .flags = flags,711 .flags = flags,
277 };712 };
278 }713 }
279714
715 fn reset(self: *ChunkState, key: [8]u32, chunk_counter: u64) void {
716 self.cv = key;
717 self.chunk_counter = chunk_counter;
718 self.blocks_compressed = 0;
719 self.buf = [_]u8{0} ** Blake3.block_length;
720 self.buf_len = 0;
721 }
722
280 fn len(self: *const ChunkState) usize {723 fn len(self: *const ChunkState) usize {
281 return BLOCK_LEN * @as(usize, self.blocks_compressed) + @as(usize, self.block_len);724 return (Blake3.block_length * @as(usize, self.blocks_compressed)) + @as(usize, self.buf_len);
282 }725 }
283726
284 fn fillBlockBuf(self: *ChunkState, input: []const u8) []const u8 {727 fn fillBuf(self: *ChunkState, input: []const u8) usize {
285 const want = BLOCK_LEN - self.block_len;728 const take = @min(Blake3.block_length - @as(usize, self.buf_len), input.len);
286 const take = @min(want, input.len);729 @memcpy(self.buf[self.buf_len..][0..take], input[0..take]);
287 @memcpy(self.block[self.block_len..][0..take], input[0..take]);730 self.buf_len += @intCast(take);
288 self.block_len += @as(u8, @truncate(take));731 return take;
289 return input[take..];732 }
290 }733
291734 fn maybeStartFlag(self: *const ChunkState) Flags {
292 fn startFlag(self: *const ChunkState) u8 {735 return if (self.blocks_compressed == 0) .{ .chunk_start = true } else .{};
293 return if (self.blocks_compressed == 0) CHUNK_START else 0;736 }
294 }737
295738 fn update(self: *ChunkState, input: []const u8) void {
296 fn update(self: *ChunkState, input_slice: []const u8) void {739 var inp = input;
297 var input = input_slice;740
298 while (input.len > 0) {741 while (inp.len > 0) {
299 // If the block buffer is full, compress it and clear it. More742 if (self.buf_len == Blake3.block_length) {
300 // input is coming, so this compression is not CHUNK_END.743 compressInPlace(&self.cv, &self.buf, Blake3.block_length, self.chunk_counter, self.flags.with(self.maybeStartFlag()));
301 if (self.block_len == BLOCK_LEN) {
302 const block_words = wordsFromLittleEndianBytes(16, self.block);
303 self.chaining_value = first8Words(compress(
304 self.chaining_value,
305 block_words,
306 BLOCK_LEN,
307 self.chunk_counter,
308 self.flags | self.startFlag(),
309 ));
310 self.blocks_compressed += 1;744 self.blocks_compressed += 1;
311 self.block = [_]u8{0} ** BLOCK_LEN;745 self.buf = [_]u8{0} ** Blake3.block_length;
312 self.block_len = 0;746 self.buf_len = 0;
313 }747 }
314748
315 // Copy input bytes into the block buffer.749 const take = self.fillBuf(inp);
316 input = self.fillBlockBuf(input);750 inp = inp[take..];
317 }751 }
318 }752 }
319753
320 fn output(self: *const ChunkState) Output {754 fn output(self: *const ChunkState) Output {
321 const block_words = wordsFromLittleEndianBytes(16, self.block);755 const block_flags = self.flags.with(self.maybeStartFlag()).with(.{ .chunk_end = true });
322 return Output{756 return Output{
323 .input_chaining_value = self.chaining_value,757 .input_cv = self.cv,
324 .block_words = block_words,758 .block = self.buf,
325 .block_len = self.block_len,759 .block_len = self.buf_len,
326 .counter = self.chunk_counter,760 .counter = self.chunk_counter,
327 .flags = self.flags | self.startFlag() | CHUNK_END,761 .flags = block_flags,
328 };762 };
329 }763 }
330};764};
331765
332fn parentOutput(766const Output = struct {
333 left_child_cv: [8]u32,767 input_cv: [8]u32 align(16),
334 right_child_cv: [8]u32,768 block: [Blake3.block_length]u8 align(16),
335 key: [8]u32,769 block_len: u8,
336 flags: u8,770 counter: u64,
337) Output {771 flags: Flags,
338 var block_words: [16]u32 align(16) = undefined;
339 block_words[0..8].* = left_child_cv;
340 block_words[8..].* = right_child_cv;
341 return Output{
342 .input_chaining_value = key,
343 .block_words = block_words,
344 .block_len = BLOCK_LEN, // Always BLOCK_LEN (64) for parent nodes.
345 .counter = 0, // Always 0 for parent nodes.
346 .flags = PARENT | flags,
347 };
348}
349772
350fn parentCv(773 fn chainingValue(self: *const Output) [8]u32 {
351 left_child_cv: [8]u32,774 var cv_words = self.input_cv;
352 right_child_cv: [8]u32,775 compressInPlace(&cv_words, &self.block, self.block_len, self.counter, self.flags);
353 key: [8]u32,776 return cv_words;
354 flags: u8,777 }
355) [8]u32 {778
356 return parentOutput(left_child_cv, right_child_cv, key, flags).chainingValue();779 fn rootBytes(self: *const Output, seek: u64, out: []u8) void {
357}780 if (out.len == 0) return;
781
782 var output_block_counter = seek / 64;
783 const offset_within_block = @as(usize, @intCast(seek % 64));
784 var out_remaining = out;
785
786 if (offset_within_block > 0) {
787 var wide_buf: [64]u8 = undefined;
788 compressXof(&self.input_cv, &self.block, self.block_len, output_block_counter, self.flags.with(.{ .root = true }), &wide_buf);
789 const available_bytes = 64 - offset_within_block;
790 const bytes = @min(out_remaining.len, available_bytes);
791 @memcpy(out_remaining[0..bytes], wide_buf[offset_within_block..][0..bytes]);
792 out_remaining = out_remaining[bytes..];
793 output_block_counter += 1;
794 }
358795
359/// An incremental hasher that can accept any number of writes.796 while (out_remaining.len >= 64) {
797 compressXof(&self.input_cv, &self.block, self.block_len, output_block_counter, self.flags.with(.{ .root = true }), out_remaining[0..64]);
798 out_remaining = out_remaining[64..];
799 output_block_counter += 1;
800 }
801
802 if (out_remaining.len > 0) {
803 var wide_buf: [64]u8 = undefined;
804 compressXof(&self.input_cv, &self.block, self.block_len, output_block_counter, self.flags.with(.{ .root = true }), &wide_buf);
805 @memcpy(out_remaining, wide_buf[0..out_remaining.len]);
806 }
807 }
808};
809
810/// BLAKE3 is a cryptographic hash function that produces a 256-bit digest by default but also supports extendable output.
360pub const Blake3 = struct {811pub const Blake3 = struct {
812 pub const block_length = 64;
813 pub const digest_length = 32;
814 pub const key_length = 32;
815
361 pub const Options = struct { key: ?[digest_length]u8 = null };816 pub const Options = struct { key: ?[digest_length]u8 = null };
362 pub const KdfOptions = struct {};817 pub const KdfOptions = struct {};
363818
364 chunk_state: ChunkState,
365 key: [8]u32,819 key: [8]u32,
366 cv_stack: [54][8]u32 = undefined, // Space for 54 subtree chaining values:820 chunk: ChunkState,
367 cv_stack_len: u8 = 0, // 2^54 * CHUNK_LEN = 2^64821 cv_stack_len: u8,
368 flags: u8,822 cv_stack: [max_depth + 1][8]u32,
369
370 pub const block_length = BLOCK_LEN;
371 pub const digest_length = OUT_LEN;
372 pub const key_length = KEY_LEN;
373
374 fn init_internal(key: [8]u32, flags: u8) Blake3 {
375 return Blake3{
376 .chunk_state = ChunkState.init(key, 0, flags),
377 .key = key,
378 .flags = flags,
379 };
380 }
381823
382 /// Construct a new `Blake3` for the hash function, with an optional key824 /// Construct a new `Blake3` for the hash function, with an optional key
383 pub fn init(options: Options) Blake3 {825 pub fn init(options: Options) Blake3 {
384 if (options.key) |key| {826 if (options.key) |key| {
385 const key_words = wordsFromLittleEndianBytes(8, key);827 const key_words = loadKeyWords(key);
386 return Blake3.init_internal(key_words, KEYED_HASH);828 return init_internal(key_words, .{ .keyed_hash = true });
387 } else {829 } else {
388 return Blake3.init_internal(IV, 0);830 return init_internal(iv, .{});
389 }831 }
390 }832 }
391833
...@@ -393,12 +835,12 @@ pub const Blake3 = struct {...@@ -393,12 +835,12 @@ pub const Blake3 = struct {
393 /// string should be hardcoded, globally unique, and application-specific.835 /// string should be hardcoded, globally unique, and application-specific.
394 pub fn initKdf(context: []const u8, options: KdfOptions) Blake3 {836 pub fn initKdf(context: []const u8, options: KdfOptions) Blake3 {
395 _ = options;837 _ = options;
396 var context_hasher = Blake3.init_internal(IV, DERIVE_KEY_CONTEXT);838 var context_hasher = init_internal(iv, .{ .derive_key_context = true });
397 context_hasher.update(context);839 context_hasher.update(context);
398 var context_key: [KEY_LEN]u8 = undefined;840 var context_key: [key_length]u8 = undefined;
399 context_hasher.final(context_key[0..]);841 context_hasher.final(&context_key);
400 const context_key_words = wordsFromLittleEndianBytes(8, context_key);842 const context_key_words = loadKeyWords(context_key);
401 return Blake3.init_internal(context_key_words, DERIVE_KEY_MATERIAL);843 return init_internal(context_key_words, .{ .derive_key_material = true });
402 }844 }
403845
404 pub fn hash(b: []const u8, out: []u8, options: Options) void {846 pub fn hash(b: []const u8, out: []u8, options: Options) void {
...@@ -407,78 +849,135 @@ pub const Blake3 = struct {...@@ -407,78 +849,135 @@ pub const Blake3 = struct {
407 d.final(out);849 d.final(out);
408 }850 }
409851
410 fn pushCv(self: *Blake3, cv: [8]u32) void {852 fn init_internal(key: [8]u32, flags: Flags) Blake3 {
411 self.cv_stack[self.cv_stack_len] = cv;853 return Blake3{
412 self.cv_stack_len += 1;854 .key = key,
855 .chunk = ChunkState.init(key, flags),
856 .cv_stack_len = 0,
857 .cv_stack = undefined,
858 };
413 }859 }
414860
415 fn popCv(self: *Blake3) [8]u32 {861 fn mergeCvStack(self: *Blake3, total_len: u64) void {
416 self.cv_stack_len -= 1;862 const post_merge_stack_len = @as(u8, @intCast(@popCount(total_len)));
417 return self.cv_stack[self.cv_stack_len];863 while (self.cv_stack_len > post_merge_stack_len) {
418 }864 const left_cv = self.cv_stack[self.cv_stack_len - 2];
419865 const right_cv = self.cv_stack[self.cv_stack_len - 1];
420 // Section 5.1.2 of the BLAKE3 spec explains this algorithm in more detail.866 const output = parentOutputFromCvs(left_cv, right_cv, self.key, self.chunk.flags);
421 fn addChunkChainingValue(self: *Blake3, first_cv: [8]u32, total_chunks: u64) void {867 const cv = output.chainingValue();
422 // This chunk might complete some subtrees. For each completed subtree,868 self.cv_stack[self.cv_stack_len - 2] = cv;
423 // its left child will be the current top entry in the CV stack, and869 self.cv_stack_len -= 1;
424 // its right child will be the current value of `new_cv`. Pop each left
425 // child off the stack, merge it with `new_cv`, and overwrite `new_cv`
426 // with the result. After all these merges, push the final value of
427 // `new_cv` onto the stack. The number of completed subtrees is given
428 // by the number of trailing 0-bits in the new total number of chunks.
429 var new_cv = first_cv;
430 var chunk_counter = total_chunks;
431 while (chunk_counter & 1 == 0) {
432 new_cv = parentCv(self.popCv(), new_cv, self.key, self.flags);
433 chunk_counter >>= 1;
434 }870 }
435 self.pushCv(new_cv);871 }
872
873 fn pushCv(self: *Blake3, new_cv: [8]u32, chunk_counter: u64) void {
874 self.mergeCvStack(chunk_counter);
875 self.cv_stack[self.cv_stack_len] = new_cv;
876 self.cv_stack_len += 1;
436 }877 }
437878
438 /// Add input to the hash state. This can be called any number of times.879 /// Add input to the hash state. This can be called any number of times.
439 pub fn update(self: *Blake3, input_slice: []const u8) void {880 pub fn update(self: *Blake3, input: []const u8) void {
440 var input = input_slice;881 if (input.len == 0) return;
441 while (input.len > 0) {882
442 // If the current chunk is complete, finalize it and reset the883 var inp = input;
443 // chunk state. More input is coming, so this chunk is not ROOT.884
444 if (self.chunk_state.len() == CHUNK_LEN) {885 if (self.chunk.len() > 0) {
445 const chunk_cv = self.chunk_state.output().chainingValue();886 const take = @min(chunk_length - self.chunk.len(), inp.len);
446 const total_chunks = self.chunk_state.chunk_counter + 1;887 self.chunk.update(inp[0..take]);
447 self.addChunkChainingValue(chunk_cv, total_chunks);888 inp = inp[take..];
448 self.chunk_state = ChunkState.init(self.key, total_chunks, self.flags);889 if (inp.len > 0) {
890 const output = self.chunk.output();
891 const chunk_cv = output.chainingValue();
892 self.pushCv(chunk_cv, self.chunk.chunk_counter);
893 self.chunk.reset(self.key, self.chunk.chunk_counter + 1);
894 } else {
895 return;
449 }896 }
897 }
898
899 while (inp.len > chunk_length) {
900 var subtree_len = roundDownToPowerOf2(inp.len);
901 const count_so_far = self.chunk.chunk_counter * chunk_length;
450902
451 // Compress input bytes into the current chunk state.903 while ((subtree_len - 1) & count_so_far != 0) {
452 const want = CHUNK_LEN - self.chunk_state.len();904 subtree_len /= 2;
453 const take = @min(want, input.len);905 }
454 self.chunk_state.update(input[0..take]);906
455 input = input[take..];907 const subtree_chunks = subtree_len / chunk_length;
908 if (subtree_len <= chunk_length) {
909 var chunk_state = ChunkState.init(self.key, self.chunk.flags);
910 chunk_state.chunk_counter = self.chunk.chunk_counter;
911 chunk_state.update(inp[0..@intCast(subtree_len)]);
912 const output = chunk_state.output();
913 const cv = output.chainingValue();
914 self.pushCv(cv, chunk_state.chunk_counter);
915 } else {
916 var cv_pair: [2 * digest_length]u8 = undefined;
917 compressSubtreeToParentNode(inp[0..@intCast(subtree_len)], self.key, self.chunk.chunk_counter, self.chunk.flags, &cv_pair);
918 const left_cv = loadCvWords(cv_pair[0..digest_length].*);
919 const right_cv = loadCvWords(cv_pair[digest_length..][0..digest_length].*);
920 self.pushCv(left_cv, self.chunk.chunk_counter);
921 self.pushCv(right_cv, self.chunk.chunk_counter + (subtree_chunks / 2));
922 }
923 self.chunk.chunk_counter += subtree_chunks;
924 inp = inp[@intCast(subtree_len)..];
925 }
926
927 if (inp.len > 0) {
928 self.chunk.update(inp);
929 self.mergeCvStack(self.chunk.chunk_counter);
456 }930 }
457 }931 }
458932
459 /// Finalize the hash and write any number of output bytes.933 /// Finalize the hash and write any number of output bytes.
460 pub fn final(self: *const Blake3, out_slice: []u8) void {934 pub fn final(self: *const Blake3, out: []u8) void {
461 // Starting with the Output from the current chunk, compute all the935 self.finalizeSeek(0, out);
462 // parent chaining values along the right edge of the tree, until we936 }
463 // have the root Output.937
464 var output = self.chunk_state.output();938 /// Finalize the hash and write any number of output bytes, starting at a given seek position.
465 var parent_nodes_remaining: usize = self.cv_stack_len;939 /// This is an XOF (extendable-output function) extension.
466 while (parent_nodes_remaining > 0) {940 pub fn finalizeSeek(self: *const Blake3, seek: u64, out: []u8) void {
467 parent_nodes_remaining -= 1;941 if (out.len == 0) return;
468 output = parentOutput(942
469 self.cv_stack[parent_nodes_remaining],943 if (self.cv_stack_len == 0) {
470 output.chainingValue(),944 const output = self.chunk.output();
471 self.key,945 output.rootBytes(seek, out);
472 self.flags,946 return;
473 );
474 }947 }
475 output.rootOutputBytes(out_slice);948
949 var output: Output = undefined;
950 var cvs_remaining: usize = undefined;
951
952 if (self.chunk.len() > 0) {
953 cvs_remaining = self.cv_stack_len;
954 output = self.chunk.output();
955 } else {
956 cvs_remaining = self.cv_stack_len - 2;
957 const left_cv = self.cv_stack[cvs_remaining];
958 const right_cv = self.cv_stack[cvs_remaining + 1];
959 output = parentOutputFromCvs(left_cv, right_cv, self.key, self.chunk.flags);
960 }
961
962 while (cvs_remaining > 0) {
963 cvs_remaining -= 1;
964 const left_cv = self.cv_stack[cvs_remaining];
965 const right_cv = output.chainingValue();
966 output = parentOutputFromCvs(left_cv, right_cv, self.key, self.chunk.flags);
967 }
968
969 output.rootBytes(seek, out);
970 }
971
972 pub fn reset(self: *Blake3) void {
973 self.chunk.reset(self.key, 0);
974 self.cv_stack_len = 0;
476 }975 }
477};976};
478977
479// Use named type declarations to workaround crash with anonymous structs (issue #4373).978// Use named type declarations to workaround crash with anonymous structs (issue #4373).
480const ReferenceTest = struct {979const ReferenceTest = struct {
481 key: *const [KEY_LEN]u8,980 key: *const [Blake3.key_length]u8,
482 context_string: []const u8,981 context_string: []const u8,
483 cases: []const ReferenceTestCase,982 cases: []const ReferenceTestCase,
484};983};
...@@ -663,7 +1162,7 @@ fn testBlake3(hasher: *Blake3, input_len: usize, expected_hex: [262]u8) !void {...@@ -663,7 +1162,7 @@ fn testBlake3(hasher: *Blake3, input_len: usize, expected_hex: [262]u8) !void {
663 // Compare to expected value1162 // Compare to expected value
664 var expected_bytes: [expected_hex.len / 2]u8 = undefined;1163 var expected_bytes: [expected_hex.len / 2]u8 = undefined;
665 _ = fmt.hexToBytes(expected_bytes[0..], expected_hex[0..]) catch unreachable;1164 _ = fmt.hexToBytes(expected_bytes[0..], expected_hex[0..]) catch unreachable;
666 try testing.expectEqual(actual_bytes, expected_bytes);1165 try std.testing.expectEqual(actual_bytes, expected_bytes);
6671166
668 // Restore initial state1167 // Restore initial state
669 hasher.* = initial_state;1168 hasher.* = initial_state;