authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-08-21 16:52:33-04:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2020-08-21 16:52:33-04:00
logdad7af0b37f352c8c390438877ef1552a316ffde
tree51fe8aab66dddf8710136b7f7812aea5fe39c12d
parent89b6c47e042b343fba754ed8f39efa1b385093bb
parent0fe6677736fae205543c6f1c57ea864049bade73
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #6116 from jedisct1/polyfast

Make poly1305 faster

1 files changed, 164 insertions(+), 191 deletions(-)

lib/std/crypto/poly1305.zig+164-191
......@@ -3,224 +3,197 @@
33// This file is part of [zig](https://ziglang.org/), which is MIT licensed.
44// The MIT license requires this copyright notice to be included in all copies
55// and substantial portions of the software.
6// Translated from monocypher which is licensed under CC-0/BSD-3.
7//
8// https://monocypher.org/
9
10const std = @import("../std.zig");
11const builtin = std.builtin;
12
13const Endian = builtin.Endian;
14const readIntLittle = std.mem.readIntLittle;
15const writeIntLittle = std.mem.writeIntLittle;
6const std = @import("std");
7const mem = std.mem;
168
179pub const Poly1305 = struct {
18 const Self = @This();
19
10 pub const block_size: usize = 16;
2011 pub const mac_length = 16;
2112 pub const minimum_key_length = 32;
2213
2314 // constant multiplier (from the secret key)
24 r: [4]u32,
15 r: [3]u64,
2516 // accumulated hash
26 h: [5]u32,
27 // chunk of the message
28 c: [5]u32,
17 h: [3]u64 = [_]u64{ 0, 0, 0 },
2918 // random number added at the end (from the secret key)
30 pad: [4]u32,
31 // How many bytes are there in the chunk.
32 c_idx: usize,
33
34 fn secureZero(self: *Self) void {
35 std.mem.secureZero(u8, @ptrCast([*]u8, self)[0..@sizeOf(Poly1305)]);
36 }
19 pad: [2]u64,
20 // how many bytes are waiting to be processed in a partial block
21 leftover: usize = 0,
22 // partial block buffer
23 buf: [block_size]u8 align(16) = undefined,
3724
38 pub fn create(out: []u8, msg: []const u8, key: []const u8) void {
39 std.debug.assert(out.len >= mac_length);
25 pub fn init(key: []const u8) Poly1305 {
4026 std.debug.assert(key.len >= minimum_key_length);
27 const t0 = mem.readIntLittle(u64, key[0..8]);
28 const t1 = mem.readIntLittle(u64, key[8..16]);
29 return Poly1305{
30 .r = [_]u64{
31 t0 & 0xffc0fffffff,
32 ((t0 >> 44) | (t1 << 20)) & 0xfffffc0ffff,
33 ((t1 >> 24)) & 0x00ffffffc0f,
34 },
35 .pad = [_]u64{
36 mem.readIntLittle(u64, key[16..24]),
37 mem.readIntLittle(u64, key[24..32]),
38 },
39 };
40 }
4141
42 var ctx = Poly1305.init(key);
43 ctx.update(msg);
44 ctx.final(out);
42 fn blocks(st: *Poly1305, m: []const u8, last: comptime bool) void {
43 const hibit: u64 = if (last) 0 else 1 << 40;
44 const r0 = st.r[0];
45 const r1 = st.r[1];
46 const r2 = st.r[2];
47 var h0 = st.h[0];
48 var h1 = st.h[1];
49 var h2 = st.h[2];
50 const s1 = r1 * (5 << 2);
51 const s2 = r2 * (5 << 2);
52 var i: usize = 0;
53 while (i + block_size <= m.len) : (i += block_size) {
54 // h += m[i]
55 const t0 = mem.readIntLittle(u64, m[i..][0..8]);
56 const t1 = mem.readIntLittle(u64, m[i + 8 ..][0..8]);
57 h0 += @truncate(u44, t0);
58 h1 += @truncate(u44, (t0 >> 44) | (t1 << 20));
59 h2 += @truncate(u42, t1 >> 24) | hibit;
60
61 // h *= r
62 const d0 = @as(u128, h0) * r0 + @as(u128, h1) * s2 + @as(u128, h2) * s1;
63 var d1 = @as(u128, h0) * r1 + @as(u128, h1) * r0 + @as(u128, h2) * s2;
64 var d2 = @as(u128, h0) * r2 + @as(u128, h1) * r1 + @as(u128, h2) * r0;
65
66 // partial reduction
67 var carry = @intCast(u64, d0 >> 44);
68 h0 = @truncate(u44, d0);
69 d1 += carry;
70 carry = @intCast(u64, d1 >> 44);
71 h1 = @truncate(u44, d1);
72 d2 += carry;
73 carry = @intCast(u64, d2 >> 42);
74 h2 = @truncate(u42, d2);
75 h0 += @truncate(u64, carry) * 5;
76 carry = h0 >> 44;
77 h0 = @truncate(u44, h0);
78 h1 += carry;
79 }
80 st.h = [_]u64{ h0, h1, h2 };
4581 }
4682
47 // Initialize the MAC context.
48 // - key.len is sufficient size.
49 pub fn init(key: []const u8) Self {
50 var ctx: Poly1305 = undefined;
83 pub fn update(st: *Poly1305, m: []const u8) void {
84 var mb = m;
5185
52 // Initial hash is zero
53 {
54 var i: usize = 0;
55 while (i < 5) : (i += 1) {
56 ctx.h[i] = 0;
86 // handle leftover
87 if (st.leftover > 0) {
88 const want = std.math.min(block_size - st.leftover, mb.len);
89 const mc = mb[0..want];
90 for (mc) |x, i| {
91 st.buf[st.leftover + i] = x;
5792 }
58 }
59 // add 2^130 to every input block
60 ctx.c[4] = 1;
61 polyClearC(&ctx);
62
63 // load r and pad (r has some of its bits cleared)
64 {
65 var i: usize = 0;
66 while (i < 1) : (i += 1) {
67 ctx.r[0] = readIntLittle(u32, key[0..4]) & 0x0fffffff;
93 mb = mb[want..];
94 st.leftover += want;
95 if (st.leftover > block_size) {
96 return;
6897 }
98 st.blocks(&st.buf, false);
99 st.leftover = 0;
69100 }
70 {
71 var i: usize = 1;
72 while (i < 4) : (i += 1) {
73 ctx.r[i] = readIntLittle(u32, key[i * 4 ..][0..4]) & 0x0ffffffc;
74 }
75 }
76 {
77 var i: usize = 0;
78 while (i < 4) : (i += 1) {
79 ctx.pad[i] = readIntLittle(u32, key[i * 4 + 16 ..][0..4]);
80 }
81 }
82
83 return ctx;
84 }
85
86 // h = (h + c) * r
87 // preconditions:
88 // ctx->h <= 4_ffffffff_ffffffff_ffffffff_ffffffff
89 // ctx->c <= 1_ffffffff_ffffffff_ffffffff_ffffffff
90 // ctx->r <= 0ffffffc_0ffffffc_0ffffffc_0fffffff
91 // Postcondition:
92 // ctx->h <= 4_ffffffff_ffffffff_ffffffff_ffffffff
93 fn polyBlock(ctx: *Self) void {
94 // s = h + c, without carry propagation
95 const s0 = @as(u64, ctx.h[0]) + ctx.c[0]; // s0 <= 1_fffffffe
96 const s1 = @as(u64, ctx.h[1]) + ctx.c[1]; // s1 <= 1_fffffffe
97 const s2 = @as(u64, ctx.h[2]) + ctx.c[2]; // s2 <= 1_fffffffe
98 const s3 = @as(u64, ctx.h[3]) + ctx.c[3]; // s3 <= 1_fffffffe
99 const s4 = @as(u64, ctx.h[4]) + ctx.c[4]; // s4 <= 5
100
101 // Local all the things!
102 const r0 = ctx.r[0]; // r0 <= 0fffffff
103 const r1 = ctx.r[1]; // r1 <= 0ffffffc
104 const r2 = ctx.r[2]; // r2 <= 0ffffffc
105 const r3 = ctx.r[3]; // r3 <= 0ffffffc
106 const rr0 = (r0 >> 2) * 5; // rr0 <= 13fffffb // lose 2 bits...
107 const rr1 = (r1 >> 2) + r1; // rr1 <= 13fffffb // rr1 == (r1 >> 2) * 5
108 const rr2 = (r2 >> 2) + r2; // rr2 <= 13fffffb // rr1 == (r2 >> 2) * 5
109 const rr3 = (r3 >> 2) + r3; // rr3 <= 13fffffb // rr1 == (r3 >> 2) * 5
110
111 // (h + c) * r, without carry propagation
112 const x0 = s0 * r0 + s1 * rr3 + s2 * rr2 + s3 * rr1 + s4 * rr0; //<=97ffffe007fffff8
113 const x1 = s0 * r1 + s1 * r0 + s2 * rr3 + s3 * rr2 + s4 * rr1; //<=8fffffe20ffffff6
114 const x2 = s0 * r2 + s1 * r1 + s2 * r0 + s3 * rr3 + s4 * rr2; //<=87ffffe417fffff4
115 const x3 = s0 * r3 + s1 * r2 + s2 * r1 + s3 * r0 + s4 * rr3; //<=7fffffe61ffffff2
116 const x4 = s4 * (r0 & 3); // ...recover 2 bits //<= f
117
118 // partial reduction modulo 2^130 - 5
119 const _u5 = @truncate(u32, x4 + (x3 >> 32)); // u5 <= 7ffffff5
120 const _u0 = (_u5 >> 2) * 5 + (x0 & 0xffffffff);
121 const _u1 = (_u0 >> 32) + (x1 & 0xffffffff) + (x0 >> 32);
122 const _u2 = (_u1 >> 32) + (x2 & 0xffffffff) + (x1 >> 32);
123 const _u3 = (_u2 >> 32) + (x3 & 0xffffffff) + (x2 >> 32);
124 const _u4 = (_u3 >> 32) + (_u5 & 3);
125
126 // Update the hash
127 ctx.h[0] = @truncate(u32, _u0); // u0 <= 1_9ffffff0
128 ctx.h[1] = @truncate(u32, _u1); // u1 <= 1_97ffffe0
129 ctx.h[2] = @truncate(u32, _u2); // u2 <= 1_8fffffe2
130 ctx.h[3] = @truncate(u32, _u3); // u3 <= 1_87ffffe4
131 ctx.h[4] = @truncate(u32, _u4); // u4 <= 4
132 }
133
134 // (re-)initializes the input counter and input buffer
135 fn polyClearC(ctx: *Self) void {
136 ctx.c[0] = 0;
137 ctx.c[1] = 0;
138 ctx.c[2] = 0;
139 ctx.c[3] = 0;
140 ctx.c_idx = 0;
141 }
142101
143 fn polyTakeInput(ctx: *Self, input: u8) void {
144 const word = ctx.c_idx >> 2;
145 const byte = ctx.c_idx & 3;
146 ctx.c[word] |= std.math.shl(u32, input, byte * 8);
147 ctx.c_idx += 1;
148 }
102 // process full blocks
103 if (mb.len >= block_size) {
104 const want = mb.len & ~(block_size - 1);
105 st.blocks(mb[0..want], false);
106 mb = mb[want..];
107 }
149108
150 fn polyUpdate(ctx: *Self, msg: []const u8) void {
151 for (msg) |b| {
152 polyTakeInput(ctx, b);
153 if (ctx.c_idx == 16) {
154 polyBlock(ctx);
155 polyClearC(ctx);
109 // store leftover
110 if (mb.len > 0) {
111 for (mb) |x, i| {
112 st.buf[st.leftover + i] = x;
156113 }
114 st.leftover += mb.len;
157115 }
158116 }
159117
160 fn alignTo(x: usize, block_size: usize) usize {
161 return ((~x) +% 1) & (block_size - 1);
162 }
163
164 // Feed data into the MAC context.
165 pub fn update(ctx: *Self, msg: []const u8) void {
166 // Align ourselves with block boundaries
167 const alignm = std.math.min(alignTo(ctx.c_idx, 16), msg.len);
168 polyUpdate(ctx, msg[0..alignm]);
169
170 var nmsg = msg[alignm..];
171
172 // Process the msg block by block
173 const nb_blocks = nmsg.len >> 4;
174 var i: usize = 0;
175 while (i < nb_blocks) : (i += 1) {
176 ctx.c[0] = readIntLittle(u32, nmsg[0..4]);
177 ctx.c[1] = readIntLittle(u32, nmsg[4..8]);
178 ctx.c[2] = readIntLittle(u32, nmsg[8..12]);
179 ctx.c[3] = readIntLittle(u32, nmsg[12..16]);
180 polyBlock(ctx);
181 nmsg = nmsg[16..];
182 }
183 if (nb_blocks > 0) {
184 polyClearC(ctx);
118 pub fn final(st: *Poly1305, out: []u8) void {
119 std.debug.assert(out.len >= mac_length);
120 if (st.leftover > 0) {
121 var i = st.leftover;
122 st.buf[i] = 1;
123 i += 1;
124 while (i < block_size) : (i += 1) {
125 st.buf[i] = 0;
126 }
127 st.blocks(&st.buf, true);
185128 }
186
187 // remaining bytes
188 polyUpdate(ctx, nmsg[0..]);
129 // fully carry h
130 var carry = st.h[1] >> 44;
131 st.h[1] = @truncate(u44, st.h[1]);
132 st.h[2] += carry;
133 carry = st.h[2] >> 42;
134 st.h[2] = @truncate(u42, st.h[2]);
135 st.h[0] += carry * 5;
136 carry = st.h[0] >> 44;
137 st.h[0] = @truncate(u44, st.h[0]);
138 st.h[1] += carry;
139 carry = st.h[1] >> 44;
140 st.h[1] = @truncate(u44, st.h[1]);
141 st.h[2] += carry;
142 carry = st.h[2] >> 42;
143 st.h[2] = @truncate(u42, st.h[2]);
144 st.h[0] += carry * 5;
145 carry = st.h[0] >> 44;
146 st.h[0] = @truncate(u44, st.h[0]);
147 st.h[1] += carry;
148
149 // compute h + -p
150 var g0 = st.h[0] + 5;
151 carry = g0 >> 44;
152 g0 = @truncate(u44, g0);
153 var g1 = st.h[1] + carry;
154 carry = g1 >> 44;
155 g1 = @truncate(u44, g1);
156 var g2 = st.h[2] + carry -% (1 << 42);
157
158 // (hopefully) constant-time select h if h < p, or h + -p if h >= p
159 const mask = (g2 >> 63) -% 1;
160 g0 &= mask;
161 g1 &= mask;
162 g2 &= mask;
163 const nmask = ~mask;
164 st.h[0] = (st.h[0] & nmask) | g0;
165 st.h[1] = (st.h[1] & nmask) | g1;
166 st.h[2] = (st.h[2] & nmask) | g2;
167
168 // h = (h + pad)
169 const t0 = st.pad[0];
170 const t1 = st.pad[1];
171 st.h[0] += @truncate(u44, t0);
172 carry = st.h[0] >> 44;
173 st.h[0] = @truncate(u44, st.h[0]);
174 st.h[1] += @truncate(u44, (t0 >> 44) | (t1 << 20)) + carry;
175 carry = st.h[1] >> 44;
176 st.h[1] = @truncate(u44, st.h[1]);
177 st.h[2] += @truncate(u42, t1 >> 24) + carry;
178 st.h[2] = @truncate(u42, st.h[2]);
179
180 // mac = h % (2^128)
181 st.h[0] |= st.h[1] << 44;
182 st.h[1] = (st.h[1] >> 20) | (st.h[2] << 24);
183
184 mem.writeIntLittle(u64, out[0..8], st.h[0]);
185 mem.writeIntLittle(u64, out[8..16], st.h[1]);
186
187 std.mem.secureZero(u8, @ptrCast([*]u8, st)[0..@sizeOf(Poly1305)]);
189188 }
190189
191 // Finalize the MAC and output into buffer provided by caller.
192 pub fn final(ctx: *Self, out: []u8) void {
193 // Process the last block (if any)
194 if (ctx.c_idx != 0) {
195 // move the final 1 according to remaining input length
196 // (We may add less than 2^130 to the last input block)
197 ctx.c[4] = 0;
198 polyTakeInput(ctx, 1);
199 // one last hash update
200 polyBlock(ctx);
201 }
190 pub fn create(out: []u8, msg: []const u8, key: []const u8) void {
191 std.debug.assert(out.len >= mac_length);
192 std.debug.assert(key.len >= minimum_key_length);
202193
203 // check if we should subtract 2^130-5 by performing the
204 // corresponding carry propagation.
205 const _u0 = @as(u64, 5) + ctx.h[0]; // <= 1_00000004
206 const _u1 = (_u0 >> 32) + ctx.h[1]; // <= 1_00000000
207 const _u2 = (_u1 >> 32) + ctx.h[2]; // <= 1_00000000
208 const _u3 = (_u2 >> 32) + ctx.h[3]; // <= 1_00000000
209 const _u4 = (_u3 >> 32) + ctx.h[4]; // <= 5
210 // u4 indicates how many times we should subtract 2^130-5 (0 or 1)
211
212 // h + pad, minus 2^130-5 if u4 exceeds 3
213 const uu0 = (_u4 >> 2) * 5 + ctx.h[0] + ctx.pad[0]; // <= 2_00000003
214 const uu1 = (uu0 >> 32) + ctx.h[1] + ctx.pad[1]; // <= 2_00000000
215 const uu2 = (uu1 >> 32) + ctx.h[2] + ctx.pad[2]; // <= 2_00000000
216 const uu3 = (uu2 >> 32) + ctx.h[3] + ctx.pad[3]; // <= 2_00000000
217
218 writeIntLittle(u32, out[0..4], @truncate(u32, uu0));
219 writeIntLittle(u32, out[4..8], @truncate(u32, uu1));
220 writeIntLittle(u32, out[8..12], @truncate(u32, uu2));
221 writeIntLittle(u32, out[12..16], @truncate(u32, uu3));
222
223 ctx.secureZero();
194 var st = Poly1305.init(key);
195 st.update(msg);
196 st.final(out);
224197 }
225198};
226199