authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-10-14 00:52:36-04:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2020-10-14 00:52:36-04:00
log3811602ad72ed1823e06d2e6d2cb646bc81ca5f9
tree89e49b65a7dc498e193906656c4837ff7044424b
parent0570df69b178c99f4d9d679a57d966f507dda484
parent9f109ba0ebb4167a18a80d8ff0c173d574fc4577
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #6643 from jedisct1/chacha-vec

std/crypto: add a vectorized ChaCha20 implementation

1 files changed, 277 insertions(+), 118 deletions(-)

lib/std/crypto/chacha20.zig+277-118
......@@ -10,120 +10,300 @@ const mem = std.mem;
1010const assert = std.debug.assert;
1111const testing = std.testing;
1212const maxInt = std.math.maxInt;
13const Vector = std.meta.Vector;
1314const Poly1305 = std.crypto.onetimeauth.Poly1305;
1415
15const QuarterRound = struct {
16 a: usize,
17 b: usize,
18 c: usize,
19 d: usize,
20};
16// Vectorized implementation of the core function
17const ChaCha20VecImpl = struct {
18 const Lane = Vector(4, u32);
19 const BlockVec = [4]Lane;
20
21 fn initContext(key: [8]u32, d: [4]u32) BlockVec {
22 const c = "expand 32-byte k";
23 const constant_le = comptime Lane{
24 mem.readIntLittle(u32, c[0..4]),
25 mem.readIntLittle(u32, c[4..8]),
26 mem.readIntLittle(u32, c[8..12]),
27 mem.readIntLittle(u32, c[12..16]),
28 };
29 return BlockVec{
30 constant_le,
31 Lane{ key[0], key[1], key[2], key[3] },
32 Lane{ key[4], key[5], key[6], key[7] },
33 Lane{ d[0], d[1], d[2], d[3] },
34 };
35 }
2136
22fn Rp(a: usize, b: usize, c: usize, d: usize) QuarterRound {
23 return QuarterRound{
24 .a = a,
25 .b = b,
26 .c = c,
27 .d = d,
28 };
29}
37 inline fn rot(x: Lane, comptime n: comptime_int) Lane {
38 return (x << @splat(4, @as(u5, n))) | (x >> @splat(4, @as(u5, 32 - n)));
39 }
3040
31fn initContext(key: [8]u32, d: [4]u32) [16]u32 {
32 var ctx: [16]u32 = undefined;
33 const c = "expand 32-byte k";
34 const constant_le = comptime [_]u32{
35 mem.readIntLittle(u32, c[0..4]),
36 mem.readIntLittle(u32, c[4..8]),
37 mem.readIntLittle(u32, c[8..12]),
38 mem.readIntLittle(u32, c[12..16]),
39 };
40 mem.copy(u32, ctx[0..], constant_le[0..4]);
41 mem.copy(u32, ctx[4..12], key[0..8]);
42 mem.copy(u32, ctx[12..16], d[0..4]);
41 inline fn chacha20Core(x: *BlockVec, input: BlockVec) void {
42 x.* = input;
43
44 var r: usize = 0;
45 while (r < 20) : (r += 2) {
46 x[0] +%= x[1];
47 x[3] ^= x[0];
48 x[3] = rot(x[3], 16);
49
50 x[2] +%= x[3];
51 x[1] ^= x[2];
52 x[1] = rot(x[1], 12);
53
54 x[0] +%= x[1];
55 x[3] ^= x[0];
56 x[0] = @shuffle(u32, x[0], undefined, [_]i32{ 3, 0, 1, 2 });
57 x[3] = rot(x[3], 8);
58
59 x[2] +%= x[3];
60 x[3] = @shuffle(u32, x[3], undefined, [_]i32{ 2, 3, 0, 1 });
61 x[1] ^= x[2];
62 x[2] = @shuffle(u32, x[2], undefined, [_]i32{ 1, 2, 3, 0 });
63 x[1] = rot(x[1], 7);
64
65 x[0] +%= x[1];
66 x[3] ^= x[0];
67 x[3] = rot(x[3], 16);
68
69 x[2] +%= x[3];
70 x[1] ^= x[2];
71 x[1] = rot(x[1], 12);
72
73 x[0] +%= x[1];
74 x[3] ^= x[0];
75 x[0] = @shuffle(u32, x[0], undefined, [_]i32{ 1, 2, 3, 0 });
76 x[3] = rot(x[3], 8);
77
78 x[2] +%= x[3];
79 x[3] = @shuffle(u32, x[3], undefined, [_]i32{ 2, 3, 0, 1 });
80 x[1] ^= x[2];
81 x[2] = @shuffle(u32, x[2], undefined, [_]i32{ 3, 0, 1, 2 });
82 x[1] = rot(x[1], 7);
83 }
84 }
4385
44 return ctx;
45}
86 inline fn hashToBytes(out: *[64]u8, x: BlockVec) void {
87 var i: usize = 0;
88 while (i < 4) : (i += 1) {
89 mem.writeIntLittle(u32, out[16 * i + 0 ..][0..4], x[i][0]);
90 mem.writeIntLittle(u32, out[16 * i + 4 ..][0..4], x[i][1]);
91 mem.writeIntLittle(u32, out[16 * i + 8 ..][0..4], x[i][2]);
92 mem.writeIntLittle(u32, out[16 * i + 12 ..][0..4], x[i][3]);
93 }
94 }
4695
47// The chacha family of ciphers are based on the salsa family.
48inline fn chacha20Core(x: []u32, input: [16]u32) void {
49 for (x) |_, i|
50 x[i] = input[i];
51
52 const rounds = comptime [_]QuarterRound{
53 Rp(0, 4, 8, 12),
54 Rp(1, 5, 9, 13),
55 Rp(2, 6, 10, 14),
56 Rp(3, 7, 11, 15),
57 Rp(0, 5, 10, 15),
58 Rp(1, 6, 11, 12),
59 Rp(2, 7, 8, 13),
60 Rp(3, 4, 9, 14),
61 };
96 inline fn contextFeedback(x: *BlockVec, ctx: BlockVec) void {
97 x[0] +%= ctx[0];
98 x[1] +%= ctx[1];
99 x[2] +%= ctx[2];
100 x[3] +%= ctx[3];
101 }
62102
63 comptime var j: usize = 0;
64 inline while (j < 20) : (j += 2) {
65 // two-round cycles
66 inline for (rounds) |r| {
67 x[r.a] +%= x[r.b];
68 x[r.d] = std.math.rotl(u32, x[r.d] ^ x[r.a], @as(u32, 16));
69 x[r.c] +%= x[r.d];
70 x[r.b] = std.math.rotl(u32, x[r.b] ^ x[r.c], @as(u32, 12));
71 x[r.a] +%= x[r.b];
72 x[r.d] = std.math.rotl(u32, x[r.d] ^ x[r.a], @as(u32, 8));
73 x[r.c] +%= x[r.d];
74 x[r.b] = std.math.rotl(u32, x[r.b] ^ x[r.c], @as(u32, 7));
103 fn chaCha20Internal(out: []u8, in: []const u8, key: [8]u32, counter: [4]u32) void {
104 var ctx = initContext(key, counter);
105 var x: BlockVec = undefined;
106 var buf: [64]u8 = undefined;
107 var i: usize = 0;
108 while (i + 64 <= in.len) : (i += 64) {
109 chacha20Core(x[0..], ctx);
110 contextFeedback(&x, ctx);
111 hashToBytes(buf[0..], x);
112
113 var xout = out[i..];
114 const xin = in[i..];
115 var j: usize = 0;
116 while (j < 64) : (j += 1) {
117 xout[j] = xin[j];
118 }
119 j = 0;
120 while (j < 64) : (j += 1) {
121 xout[j] ^= buf[j];
122 }
123 ctx[3][0] += 1;
124 }
125 if (i < in.len) {
126 chacha20Core(x[0..], ctx);
127 contextFeedback(&x, ctx);
128 hashToBytes(buf[0..], x);
129
130 var xout = out[i..];
131 const xin = in[i..];
132 var j: usize = 0;
133 while (j < in.len % 64) : (j += 1) {
134 xout[j] = xin[j] ^ buf[j];
135 }
75136 }
76137 }
77}
78138
79fn hashToBytes(out: []u8, x: [16]u32) void {
80 for (x) |_, i| {
81 mem.writeIntLittle(u32, out[4 * i ..][0..4], x[i]);
139 fn hchacha20(input: [16]u8, key: [32]u8) [32]u8 {
140 var c: [4]u32 = undefined;
141 for (c) |_, i| {
142 c[i] = mem.readIntLittle(u32, input[4 * i ..][0..4]);
143 }
144 const ctx = initContext(keyToWords(key), c);
145 var x: BlockVec = undefined;
146 chacha20Core(x[0..], ctx);
147 var out: [32]u8 = undefined;
148 mem.writeIntLittle(u32, out[0..4], x[0][0]);
149 mem.writeIntLittle(u32, out[4..8], x[0][1]);
150 mem.writeIntLittle(u32, out[8..12], x[0][2]);
151 mem.writeIntLittle(u32, out[12..16], x[0][3]);
152 mem.writeIntLittle(u32, out[16..20], x[3][0]);
153 mem.writeIntLittle(u32, out[20..24], x[3][1]);
154 mem.writeIntLittle(u32, out[24..28], x[3][2]);
155 mem.writeIntLittle(u32, out[28..32], x[3][3]);
156 return out;
82157 }
83}
158};
84159
85fn chaCha20_internal(out: []u8, in: []const u8, key: [8]u32, counter: [4]u32) void {
86 var ctx = initContext(key, counter);
87 var remaining: usize = if (in.len > out.len) in.len else out.len;
88 var cursor: usize = 0;
160// Non-vectorized implementation of the core function
161const ChaCha20NonVecImpl = struct {
162 const BlockVec = [16]u32;
163
164 fn initContext(key: [8]u32, d: [4]u32) BlockVec {
165 const c = "expand 32-byte k";
166 const constant_le = comptime [4]u32{
167 mem.readIntLittle(u32, c[0..4]),
168 mem.readIntLittle(u32, c[4..8]),
169 mem.readIntLittle(u32, c[8..12]),
170 mem.readIntLittle(u32, c[12..16]),
171 };
172 return BlockVec{
173 constant_le[0], constant_le[1], constant_le[2], constant_le[3],
174 key[0], key[1], key[2], key[3],
175 key[4], key[5], key[6], key[7],
176 d[0], d[1], d[2], d[3],
177 };
178 }
89179
90 while (true) {
91 var x: [16]u32 = undefined;
92 var buf: [64]u8 = undefined;
93 chacha20Core(x[0..], ctx);
94 for (x) |_, i| {
95 x[i] +%= ctx[i];
180 const QuarterRound = struct {
181 a: usize,
182 b: usize,
183 c: usize,
184 d: usize,
185 };
186
187 fn Rp(a: usize, b: usize, c: usize, d: usize) QuarterRound {
188 return QuarterRound{
189 .a = a,
190 .b = b,
191 .c = c,
192 .d = d,
193 };
194 }
195
196 inline fn chacha20Core(x: *BlockVec, input: BlockVec) void {
197 x.* = input;
198
199 const rounds = comptime [_]QuarterRound{
200 Rp(0, 4, 8, 12),
201 Rp(1, 5, 9, 13),
202 Rp(2, 6, 10, 14),
203 Rp(3, 7, 11, 15),
204 Rp(0, 5, 10, 15),
205 Rp(1, 6, 11, 12),
206 Rp(2, 7, 8, 13),
207 Rp(3, 4, 9, 14),
208 };
209
210 comptime var j: usize = 0;
211 inline while (j < 20) : (j += 2) {
212 inline for (rounds) |r| {
213 x[r.a] +%= x[r.b];
214 x[r.d] = std.math.rotl(u32, x[r.d] ^ x[r.a], @as(u32, 16));
215 x[r.c] +%= x[r.d];
216 x[r.b] = std.math.rotl(u32, x[r.b] ^ x[r.c], @as(u32, 12));
217 x[r.a] +%= x[r.b];
218 x[r.d] = std.math.rotl(u32, x[r.d] ^ x[r.a], @as(u32, 8));
219 x[r.c] +%= x[r.d];
220 x[r.b] = std.math.rotl(u32, x[r.b] ^ x[r.c], @as(u32, 7));
221 }
96222 }
97 hashToBytes(buf[0..], x);
98 if (remaining < 64) {
99 var i: usize = 0;
100 while (i < remaining) : (i += 1)
101 out[cursor + i] = in[cursor + i] ^ buf[i];
102 return;
223 }
224
225 inline fn hashToBytes(out: *[64]u8, x: BlockVec) void {
226 var i: usize = 0;
227 while (i < 4) : (i += 1) {
228 mem.writeIntLittle(u32, out[16 * i + 0 ..][0..4], x[i * 4 + 0]);
229 mem.writeIntLittle(u32, out[16 * i + 4 ..][0..4], x[i * 4 + 1]);
230 mem.writeIntLittle(u32, out[16 * i + 8 ..][0..4], x[i * 4 + 2]);
231 mem.writeIntLittle(u32, out[16 * i + 12 ..][0..4], x[i * 4 + 3]);
103232 }
233 }
104234
235 inline fn contextFeedback(x: *BlockVec, ctx: BlockVec) void {
105236 var i: usize = 0;
106 while (i < 64) : (i += 1)
107 out[cursor + i] = in[cursor + i] ^ buf[i];
237 while (i < 16) : (i += 1) {
238 x[i] +%= ctx[i];
239 }
240 }
108241
109 cursor += 64;
110 remaining -= 64;
242 fn chaCha20Internal(out: []u8, in: []const u8, key: [8]u32, counter: [4]u32) void {
243 var ctx = initContext(key, counter);
244 var x: BlockVec = undefined;
245 var buf: [64]u8 = undefined;
246 var i: usize = 0;
247 while (i + 64 <= in.len) : (i += 64) {
248 chacha20Core(x[0..], ctx);
249 contextFeedback(&x, ctx);
250 hashToBytes(buf[0..], x);
251
252 var xout = out[i..];
253 const xin = in[i..];
254 var j: usize = 0;
255 while (j < 64) : (j += 1) {
256 xout[j] = xin[j];
257 }
258 j = 0;
259 while (j < 64) : (j += 1) {
260 xout[j] ^= buf[j];
261 }
262 ctx[12] += 1;
263 }
264 if (i < in.len) {
265 chacha20Core(x[0..], ctx);
266 contextFeedback(&x, ctx);
267 hashToBytes(buf[0..], x);
268
269 var xout = out[i..];
270 const xin = in[i..];
271 var j: usize = 0;
272 while (j < in.len % 64) : (j += 1) {
273 xout[j] = xin[j] ^ buf[j];
274 }
275 }
276 }
111277
112 ctx[12] += 1;
278 fn hchacha20(input: [16]u8, key: [32]u8) [32]u8 {
279 var c: [4]u32 = undefined;
280 for (c) |_, i| {
281 c[i] = mem.readIntLittle(u32, input[4 * i ..][0..4]);
282 }
283 const ctx = initContext(keyToWords(key), c);
284 var x: BlockVec = undefined;
285 chacha20Core(x[0..], ctx);
286 var out: [32]u8 = undefined;
287 mem.writeIntLittle(u32, out[0..4], x[0]);
288 mem.writeIntLittle(u32, out[4..8], x[1]);
289 mem.writeIntLittle(u32, out[8..12], x[2]);
290 mem.writeIntLittle(u32, out[12..16], x[3]);
291 mem.writeIntLittle(u32, out[16..20], x[12]);
292 mem.writeIntLittle(u32, out[20..24], x[13]);
293 mem.writeIntLittle(u32, out[24..28], x[14]);
294 mem.writeIntLittle(u32, out[28..32], x[15]);
295 return out;
113296 }
114}
297};
298
299const ChaCha20Impl = if (std.Target.current.cpu.arch == .x86_64) ChaCha20VecImpl else ChaCha20NonVecImpl;
115300
116301fn keyToWords(key: [32]u8) [8]u32 {
117302 var k: [8]u32 = undefined;
118 k[0] = mem.readIntLittle(u32, key[0..4]);
119 k[1] = mem.readIntLittle(u32, key[4..8]);
120 k[2] = mem.readIntLittle(u32, key[8..12]);
121 k[3] = mem.readIntLittle(u32, key[12..16]);
122 k[4] = mem.readIntLittle(u32, key[16..20]);
123 k[5] = mem.readIntLittle(u32, key[20..24]);
124 k[6] = mem.readIntLittle(u32, key[24..28]);
125 k[7] = mem.readIntLittle(u32, key[28..32]);
126
303 var i: usize = 0;
304 while (i < 8) : (i += 1) {
305 k[i] = mem.readIntLittle(u32, key[i * 4 ..][0..4]);
306 }
127307 return k;
128308}
129309
......@@ -145,7 +325,7 @@ pub const ChaCha20IETF = struct {
145325 c[1] = mem.readIntLittle(u32, nonce[0..4]);
146326 c[2] = mem.readIntLittle(u32, nonce[4..8]);
147327 c[3] = mem.readIntLittle(u32, nonce[8..12]);
148 chaCha20_internal(out, in, keyToWords(key), c);
328 ChaCha20Impl.chaCha20Internal(out, in, keyToWords(key), c);
149329 }
150330};
151331
......@@ -171,7 +351,7 @@ pub const ChaCha20With64BitNonce = struct {
171351
172352 // first partial big block
173353 if (((@intCast(u64, maxInt(u32) - @truncate(u32, counter)) + 1) << 6) < in.len) {
174 chaCha20_internal(out[cursor..big_block], in[cursor..big_block], k, c);
354 ChaCha20Impl.chaCha20Internal(out[cursor..big_block], in[cursor..big_block], k, c);
175355 cursor = big_block - cursor;
176356 c[1] += 1;
177357 if (comptime @sizeOf(usize) > 4) {
......@@ -179,14 +359,14 @@ pub const ChaCha20With64BitNonce = struct {
179359 var remaining_blocks: u32 = @intCast(u32, (in.len / big_block));
180360 var i: u32 = 0;
181361 while (remaining_blocks > 0) : (remaining_blocks -= 1) {
182 chaCha20_internal(out[cursor .. cursor + big_block], in[cursor .. cursor + big_block], k, c);
183 c[1] += 1; // upper 32-bit of counter, generic chaCha20_internal() doesn't know about this.
362 ChaCha20Impl.chaCha20Internal(out[cursor .. cursor + big_block], in[cursor .. cursor + big_block], k, c);
363 c[1] += 1; // upper 32-bit of counter, generic chaCha20Internal() doesn't know about this.
184364 cursor += big_block;
185365 }
186366 }
187367 }
188368
189 chaCha20_internal(out[cursor..], in[cursor..], k, c);
369 ChaCha20Impl.chaCha20Internal(out[cursor..], in[cursor..], k, c);
190370 }
191371};
192372
......@@ -533,33 +713,12 @@ fn chacha20poly1305Open(dst: []u8, ciphertextAndTag: []const u8, data: []const u
533713 return try chacha20poly1305OpenDetached(dst, ciphertextAndTag[0..ciphertextLen], ciphertextAndTag[ciphertextLen..][0..chacha20poly1305_tag_size], data, key, nonce);
534714}
535715
536fn hchacha20(input: [16]u8, key: [32]u8) [32]u8 {
537 var c: [4]u32 = undefined;
538 for (c) |_, i| {
539 c[i] = mem.readIntLittle(u32, input[4 * i ..][0..4]);
540 }
541 const ctx = initContext(keyToWords(key), c);
542 var x: [16]u32 = undefined;
543 chacha20Core(x[0..], ctx);
544 var out: [32]u8 = undefined;
545 mem.writeIntLittle(u32, out[0..4], x[0]);
546 mem.writeIntLittle(u32, out[4..8], x[1]);
547 mem.writeIntLittle(u32, out[8..12], x[2]);
548 mem.writeIntLittle(u32, out[12..16], x[3]);
549 mem.writeIntLittle(u32, out[16..20], x[12]);
550 mem.writeIntLittle(u32, out[20..24], x[13]);
551 mem.writeIntLittle(u32, out[24..28], x[14]);
552 mem.writeIntLittle(u32, out[28..32], x[15]);
553
554 return out;
555}
556
557716fn extend(key: [32]u8, nonce: [24]u8) struct { key: [32]u8, nonce: [12]u8 } {
558717 var subnonce: [12]u8 = undefined;
559718 mem.set(u8, subnonce[0..4], 0);
560719 mem.copy(u8, subnonce[4..], nonce[16..24]);
561720 return .{
562 .key = hchacha20(nonce[0..16].*, key),
721 .key = ChaCha20Impl.hchacha20(nonce[0..16].*, key),
563722 .nonce = subnonce,
564723 };
565724}