authorgravatar for 124872+jedisct1@users.noreply.github.comFrank Denis <124872+jedisct1@users.noreply.github.com> 2020-10-01 01:34:23+02:00
committergravatar for 124872+jedisct1@users.noreply.github.comFrank Denis <124872+jedisct1@users.noreply.github.com> 2020-10-01 02:05:11+02:00
log97fd0974b9372f65c661908fb1ffc01d68255dae
tree9e4d7d96dd7a4bf8d6434a59d3822bf5ae79ca1b
parent8161de7fa44393c08cf96c9f9aa5379f98925472

ghash: add pclmul support on x86_64

1 files changed, 34 insertions(+), 20 deletions(-)

lib/std/crypto/ghash.zig+34-20
...@@ -94,7 +94,18 @@ pub const Ghash = struct {...@@ -94,7 +94,18 @@ pub const Ghash = struct {
94 }94 }
95 }95 }
9696
97 fn bmul(x: u64, y: u64) u64 {97 inline fn clmul_pclmul(x: u64, y: u64) u64 {
98 const Vector = std.meta.Vector;
99 const product = asm (
100 \\ vpclmulqdq $0x00, %[x], %[y], %[out]
101 : [out] "=x" (-> Vector(2, u64))
102 : [x] "x" (@bitCast(Vector(2, u64), @as(u128, x))),
103 [y] "x" (@bitCast(Vector(2, u64), @as(u128, y)))
104 );
105 return product[0];
106 }
107
108 fn clmul_soft(x: u64, y: u64) u64 {
98 const x0 = x & 0x1111111111111111;109 const x0 = x & 0x1111111111111111;
99 const x1 = x & 0x2222222222222222;110 const x1 = x & 0x2222222222222222;
100 const x2 = x & 0x4444444444444444;111 const x2 = x & 0x4444444444444444;
...@@ -111,10 +122,13 @@ pub const Ghash = struct {...@@ -111,10 +122,13 @@ pub const Ghash = struct {
111 z1 &= 0x2222222222222222;122 z1 &= 0x2222222222222222;
112 z2 &= 0x4444444444444444;123 z2 &= 0x4444444444444444;
113 z3 &= 0x8888888888888888;124 z3 &= 0x8888888888888888;
114
115 return z0 | z1 | z2 | z3;125 return z0 | z1 | z2 | z3;
116 }126 }
117127
128 const has_pclmul = comptime std.Target.x86.featureSetHas(std.Target.current.cpu.features, .pclmul);
129 const has_avx = comptime std.Target.x86.featureSetHas(std.Target.current.cpu.features, .avx);
130 const clmul = if (std.Target.current.cpu.arch == .x86_64 and has_pclmul and has_avx) clmul_pclmul else clmul_soft;
131
118 fn blocks(st: *Ghash, msg: []const u8) void {132 fn blocks(st: *Ghash, msg: []const u8) void {
119 assert(msg.len % 16 == 0); // GHASH blocks() expects full blocks133 assert(msg.len % 16 == 0); // GHASH blocks() expects full blocks
120 var y1 = st.y1;134 var y1 = st.y1;
...@@ -134,12 +148,12 @@ pub const Ghash = struct {...@@ -134,12 +148,12 @@ pub const Ghash = struct {
134 const y2 = y0 ^ y1;148 const y2 = y0 ^ y1;
135 const y2r = y0r ^ y1r;149 const y2r = y0r ^ y1r;
136150
137 var z0 = bmul(y0, st.hh0);151 var z0 = clmul(y0, st.hh0);
138 var z1 = bmul(y1, st.hh1);152 var z1 = clmul(y1, st.hh1);
139 var z2 = bmul(y2, st.hh2) ^ z0 ^ z1;153 var z2 = clmul(y2, st.hh2) ^ z0 ^ z1;
140 var z0h = bmul(y0r, st.hh0r);154 var z0h = clmul(y0r, st.hh0r);
141 var z1h = bmul(y1r, st.hh1r);155 var z1h = clmul(y1r, st.hh1r);
142 var z2h = bmul(y2r, st.hh2r) ^ z0h ^ z1h;156 var z2h = clmul(y2r, st.hh2r) ^ z0h ^ z1h;
143157
144 // B1 * H unreduced158 // B1 * H unreduced
145 const sy1 = mem.readIntBig(u64, msg[i..][16..24]);159 const sy1 = mem.readIntBig(u64, msg[i..][16..24]);
...@@ -150,12 +164,12 @@ pub const Ghash = struct {...@@ -150,12 +164,12 @@ pub const Ghash = struct {
150 const sy2 = sy0 ^ sy1;164 const sy2 = sy0 ^ sy1;
151 const sy2r = sy0r ^ sy1r;165 const sy2r = sy0r ^ sy1r;
152166
153 const sz0 = bmul(sy0, st.h0);167 const sz0 = clmul(sy0, st.h0);
154 const sz1 = bmul(sy1, st.h1);168 const sz1 = clmul(sy1, st.h1);
155 const sz2 = bmul(sy2, st.h2) ^ sz0 ^ sz1;169 const sz2 = clmul(sy2, st.h2) ^ sz0 ^ sz1;
156 const sz0h = bmul(sy0r, st.h0r);170 const sz0h = clmul(sy0r, st.h0r);
157 const sz1h = bmul(sy1r, st.h1r);171 const sz1h = clmul(sy1r, st.h1r);
158 const sz2h = bmul(sy2r, st.h2r) ^ sz0h ^ sz1h;172 const sz2h = clmul(sy2r, st.h2r) ^ sz0h ^ sz1h;
159173
160 // ((B0 * H^2) + B1 * H) (mod M)174 // ((B0 * H^2) + B1 * H) (mod M)
161 z0 ^= sz0;175 z0 ^= sz0;
...@@ -195,12 +209,12 @@ pub const Ghash = struct {...@@ -195,12 +209,12 @@ pub const Ghash = struct {
195 const y2 = y0 ^ y1;209 const y2 = y0 ^ y1;
196 const y2r = y0r ^ y1r;210 const y2r = y0r ^ y1r;
197211
198 const z0 = bmul(y0, st.h0);212 const z0 = clmul(y0, st.h0);
199 const z1 = bmul(y1, st.h1);213 const z1 = clmul(y1, st.h1);
200 var z2 = bmul(y2, st.h2) ^ z0 ^ z1;214 var z2 = clmul(y2, st.h2) ^ z0 ^ z1;
201 var z0h = bmul(y0r, st.h0r);215 var z0h = clmul(y0r, st.h0r);
202 var z1h = bmul(y1r, st.h1r);216 var z1h = clmul(y1r, st.h1r);
203 var z2h = bmul(y2r, st.h2r) ^ z0h ^ z1h;217 var z2h = clmul(y2r, st.h2r) ^ z0h ^ z1h;
204 z0h = @bitReverse(u64, z0h) >> 1;218 z0h = @bitReverse(u64, z0h) >> 1;
205 z1h = @bitReverse(u64, z1h) >> 1;219 z1h = @bitReverse(u64, z1h) >> 1;
206 z2h = @bitReverse(u64, z2h) >> 1;220 z2h = @bitReverse(u64, z2h) >> 1;