| ... | @@ -192,85 +192,89 @@ fn Sha2x32(comptime params: Sha2Params32) type { | ... | @@ -192,85 +192,89 @@ fn Sha2x32(comptime params: Sha2Params32) type { |
| 192 | s[i] |= @as(u32, b[i * 4 + 3]) << 0; | 192 | s[i] |= @as(u32, b[i * 4 + 3]) << 0; |
| 193 | } | 193 | } |
| 194 | | 194 | |
| 195 | if (builtin.cpu.arch == .aarch64 and builtin.cpu.features.isEnabled(@enumToInt(std.Target.aarch64.Feature.sha2))) { | 195 | switch (builtin.cpu.arch) { |
| 196 | var x: v4u32 = d.s[0..4].*; | 196 | .aarch64 => if (comptime builtin.cpu.features.isEnabled(@enumToInt(std.Target.aarch64.Feature.sha2))) { |
| 197 | var y: v4u32 = d.s[4..8].*; | 197 | var x: v4u32 = d.s[0..4].*; |
| 198 | const s_v = @ptrCast(*[16]v4u32, &s); | 198 | var y: v4u32 = d.s[4..8].*; |
| 199 | | 199 | const s_v = @ptrCast(*[16]v4u32, &s); |
| 200 | comptime var k: u8 = 0; | 200 | |
| 201 | inline while (k < 16) : (k += 1) { | 201 | comptime var k: u8 = 0; |
| 202 | if (k > 3) { | 202 | inline while (k < 16) : (k += 1) { |
| 203 | s_v[k] = asm ( | 203 | if (k > 3) { |
| 204 | \\sha256su0.4s %[w0_3], %[w4_7] | 204 | s_v[k] = asm ( |
| 205 | \\sha256su1.4s %[w0_3], %[w8_11], %[w12_15] | 205 | \\sha256su0.4s %[w0_3], %[w4_7] |
| 206 | : [w0_3] "=w" (-> v4u32), | 206 | \\sha256su1.4s %[w0_3], %[w8_11], %[w12_15] |
| 207 | : [_] "0" (s_v[k - 4]), | 207 | : [w0_3] "=w" (-> v4u32), |
| 208 | [w4_7] "w" (s_v[k - 3]), | 208 | : [_] "0" (s_v[k - 4]), |
| 209 | [w8_11] "w" (s_v[k - 2]), | 209 | [w4_7] "w" (s_v[k - 3]), |
| 210 | [w12_15] "w" (s_v[k - 1]), | 210 | [w8_11] "w" (s_v[k - 2]), |
| | 211 | [w12_15] "w" (s_v[k - 1]), |
| | 212 | ); |
| | 213 | } |
| | 214 | |
| | 215 | const w: v4u32 = s_v[k] +% @as(v4u32, W[4 * k ..][0..4].*); |
| | 216 | asm volatile ( |
| | 217 | \\mov.4s v0, %[x] |
| | 218 | \\sha256h.4s %[x], %[y], %[w] |
| | 219 | \\sha256h2.4s %[y], v0, %[w] |
| | 220 | : [x] "=w" (x), |
| | 221 | [y] "=w" (y), |
| | 222 | : [_] "0" (x), |
| | 223 | [_] "1" (y), |
| | 224 | [w] "w" (w), |
| | 225 | : "v0" |
| 211 | ); | 226 | ); |
| 212 | } | 227 | } |
| 213 | | 228 | |
| 214 | const w: v4u32 = s_v[k] +% @as(v4u32, W[4 * k ..][0..4].*); | 229 | d.s[0..4].* = x +% @as(v4u32, d.s[0..4].*); |
| 215 | asm volatile ( | 230 | d.s[4..8].* = y +% @as(v4u32, d.s[4..8].*); |
| 216 | \\mov.4s v0, %[x] | 231 | return; |
| 217 | \\sha256h.4s %[x], %[y], %[w] | 232 | }, |
| 218 | \\sha256h2.4s %[y], v0, %[w] | 233 | .x86_64 => if (comptime builtin.cpu.features.isEnabled(@enumToInt(std.Target.x86.Feature.sha))) { |
| 219 | : [x] "=w" (x), | 234 | var x: v4u32 = [_]u32{ d.s[5], d.s[4], d.s[1], d.s[0] }; |
| 220 | [y] "=w" (y), | 235 | var y: v4u32 = [_]u32{ d.s[7], d.s[6], d.s[3], d.s[2] }; |
| 221 | : [_] "0" (x), | 236 | const s_v = @ptrCast(*[16]v4u32, &s); |
| 222 | [_] "1" (y), | 237 | |
| 223 | [w] "w" (w), | 238 | comptime var k: u8 = 0; |
| 224 | : "v0" | 239 | inline while (k < 16) : (k += 1) { |
| 225 | ); | 240 | if (k < 12) { |
| 226 | } | 241 | const r = asm ("sha256msg1 %[w4_7], %[w0_3]" |
| 227 | | 242 | : [w0_3] "=x" (-> v4u32), |
| 228 | d.s[0..4].* = x +% @as(v4u32, d.s[0..4].*); | 243 | : [_] "0" (s_v[k]), |
| 229 | d.s[4..8].* = y +% @as(v4u32, d.s[4..8].*); | 244 | [w4_7] "x" (s_v[k + 1]), |
| 230 | return; | 245 | ); |
| 231 | } else if (builtin.cpu.arch == .x86_64 and builtin.cpu.features.isEnabled(@enumToInt(std.Target.x86.Feature.sha))) { | 246 | const t = @shuffle(u32, s_v[k + 2], s_v[k + 3], [_]i32{ 1, 2, 3, -1 }); |
| 232 | var x: v4u32 = [_]u32{ d.s[5], d.s[4], d.s[1], d.s[0] }; | 247 | s_v[k + 4] = asm ("sha256msg2 %[w12_15], %[t]" |
| 233 | var y: v4u32 = [_]u32{ d.s[7], d.s[6], d.s[3], d.s[2] }; | 248 | : [t] "=x" (-> v4u32), |
| 234 | const s_v = @ptrCast(*[16]v4u32, &s); | 249 | : [_] "0" (r +% t), |
| 235 | | 250 | [w12_15] "x" (s_v[k + 3]), |
| 236 | comptime var k: u8 = 0; | 251 | ); |
| 237 | inline while (k < 16) : (k += 1) { | 252 | } |
| 238 | if (k < 12) { | 253 | |
| 239 | const r = asm ("sha256msg1 %[w4_7], %[w0_3]" | 254 | const w: v4u32 = s_v[k] +% @as(v4u32, W[4 * k ..][0..4].*); |
| 240 | : [w0_3] "=x" (-> v4u32), | 255 | asm volatile ( |
| 241 | : [_] "0" (s_v[k]), | 256 | \\sha256rnds2 %[x], %[y] |
| 242 | [w4_7] "x" (s_v[k + 1]), | 257 | \\pshufd $0xe, %%xmm0, %%xmm0 |
| 243 | ); | 258 | \\sha256rnds2 %[y], %[x] |
| 244 | const t = @shuffle(u32, s_v[k + 2], s_v[k + 3], [_]i32{ 1, 2, 3, -1 }); | 259 | : [y] "=x" (y), |
| 245 | s_v[k + 4] = asm ("sha256msg2 %[w12_15], %[t]" | 260 | [x] "=x" (x), |
| 246 | : [t] "=x" (-> v4u32), | 261 | : [_] "0" (y), |
| 247 | : [_] "0" (r +% t), | 262 | [_] "1" (x), |
| 248 | [w12_15] "x" (s_v[k + 3]), | 263 | [_] "{xmm0}" (w), |
| 249 | ); | 264 | ); |
| 250 | } | 265 | } |
| 251 | | 266 | |
| 252 | const w: v4u32 = s_v[k] +% @as(v4u32, W[4 * k ..][0..4].*); | 267 | d.s[0] +%= x[3]; |
| 253 | asm volatile ( | 268 | d.s[1] +%= x[2]; |
| 254 | \\sha256rnds2 %[x], %[y] | 269 | d.s[4] +%= x[1]; |
| 255 | \\pshufd $0xe, %%xmm0, %%xmm0 | 270 | d.s[5] +%= x[0]; |
| 256 | \\sha256rnds2 %[y], %[x] | 271 | d.s[2] +%= y[3]; |
| 257 | : [y] "=x" (y), | 272 | d.s[3] +%= y[2]; |
| 258 | [x] "=x" (x), | 273 | d.s[6] +%= y[1]; |
| 259 | : [_] "0" (y), | 274 | d.s[7] +%= y[0]; |
| 260 | [_] "1" (x), | 275 | return; |
| 261 | [_] "{xmm0}" (w), | 276 | }, |
| 262 | ); | 277 | else => {}, |
| 263 | } | | |
| 264 | | | |
| 265 | d.s[0] +%= x[3]; | | |
| 266 | d.s[1] +%= x[2]; | | |
| 267 | d.s[4] +%= x[1]; | | |
| 268 | d.s[5] +%= x[0]; | | |
| 269 | d.s[2] +%= y[3]; | | |
| 270 | d.s[3] +%= y[2]; | | |
| 271 | d.s[6] +%= y[1]; | | |
| 272 | d.s[7] +%= y[0]; | | |
| 273 | return; | | |
| 274 | } | 278 | } |
| 275 | | 279 | |
| 276 | while (i < 64) : (i += 1) { | 280 | while (i < 64) : (i += 1) { |
Cody Tapscott <topolarity@tapscott.me> 2022-10-24 00:38:10-07:00
lib