| ... | @@ -192,94 +192,96 @@ fn Sha2x32(comptime params: Sha2Params32) type { | ... | @@ -192,94 +192,96 @@ fn Sha2x32(comptime params: Sha2Params32) type { |
| 192 | s[i] = mem.readIntBig(u32, mem.asBytes(elem)); | 192 | s[i] = mem.readIntBig(u32, mem.asBytes(elem)); |
| 193 | } | 193 | } |
| 194 | | 194 | |
| 195 | switch (builtin.cpu.arch) { | 195 | if (!isComptime()) { |
| 196 | .aarch64 => if (!isComptime() and comptime builtin.cpu.features.isEnabled(@enumToInt(std.Target.aarch64.Feature.sha2))) { | 196 | switch (builtin.cpu.arch) { |
| 197 | var x: v4u32 = d.s[0..4].*; | 197 | .aarch64 => if (comptime std.Target.aarch64.featureSetHas(builtin.cpu.features, .sha2)) { |
| 198 | var y: v4u32 = d.s[4..8].*; | 198 | var x: v4u32 = d.s[0..4].*; |
| 199 | const s_v = @ptrCast(*[16]v4u32, &s); | 199 | var y: v4u32 = d.s[4..8].*; |
| 200 | | 200 | const s_v = @ptrCast(*[16]v4u32, &s); |
| 201 | comptime var k: u8 = 0; | 201 | |
| 202 | inline while (k < 16) : (k += 1) { | 202 | comptime var k: u8 = 0; |
| 203 | if (k > 3) { | 203 | inline while (k < 16) : (k += 1) { |
| 204 | s_v[k] = asm ( | 204 | if (k > 3) { |
| 205 | \\sha256su0.4s %[w0_3], %[w4_7] | 205 | s_v[k] = asm ( |
| 206 | \\sha256su1.4s %[w0_3], %[w8_11], %[w12_15] | 206 | \\sha256su0.4s %[w0_3], %[w4_7] |
| 207 | : [w0_3] "=w" (-> v4u32), | 207 | \\sha256su1.4s %[w0_3], %[w8_11], %[w12_15] |
| 208 | : [_] "0" (s_v[k - 4]), | 208 | : [w0_3] "=w" (-> v4u32), |
| 209 | [w4_7] "w" (s_v[k - 3]), | 209 | : [_] "0" (s_v[k - 4]), |
| 210 | [w8_11] "w" (s_v[k - 2]), | 210 | [w4_7] "w" (s_v[k - 3]), |
| 211 | [w12_15] "w" (s_v[k - 1]), | 211 | [w8_11] "w" (s_v[k - 2]), |
| | 212 | [w12_15] "w" (s_v[k - 1]), |
| | 213 | ); |
| | 214 | } |
| | 215 | |
| | 216 | const w: v4u32 = s_v[k] +% @as(v4u32, W[4 * k ..][0..4].*); |
| | 217 | asm volatile ( |
| | 218 | \\mov.4s v0, %[x] |
| | 219 | \\sha256h.4s %[x], %[y], %[w] |
| | 220 | \\sha256h2.4s %[y], v0, %[w] |
| | 221 | : [x] "=w" (x), |
| | 222 | [y] "=w" (y), |
| | 223 | : [_] "0" (x), |
| | 224 | [_] "1" (y), |
| | 225 | [w] "w" (w), |
| | 226 | : "v0" |
| 212 | ); | 227 | ); |
| 213 | } | 228 | } |
| 214 | | 229 | |
| 215 | const w: v4u32 = s_v[k] +% @as(v4u32, W[4 * k ..][0..4].*); | 230 | d.s[0..4].* = x +% @as(v4u32, d.s[0..4].*); |
| 216 | asm volatile ( | 231 | d.s[4..8].* = y +% @as(v4u32, d.s[4..8].*); |
| 217 | \\mov.4s v0, %[x] | 232 | return; |
| 218 | \\sha256h.4s %[x], %[y], %[w] | 233 | }, |
| 219 | \\sha256h2.4s %[y], v0, %[w] | 234 | .x86_64 => if (comptime std.Target.x86.featureSetHas(builtin.cpu.features, .sha)) { |
| 220 | : [x] "=w" (x), | 235 | var x: v4u32 = [_]u32{ d.s[5], d.s[4], d.s[1], d.s[0] }; |
| 221 | [y] "=w" (y), | 236 | var y: v4u32 = [_]u32{ d.s[7], d.s[6], d.s[3], d.s[2] }; |
| 222 | : [_] "0" (x), | 237 | const s_v = @ptrCast(*[16]v4u32, &s); |
| 223 | [_] "1" (y), | 238 | |
| 224 | [w] "w" (w), | 239 | comptime var k: u8 = 0; |
| 225 | : "v0" | 240 | inline while (k < 16) : (k += 1) { |
| 226 | ); | 241 | if (k < 12) { |
| 227 | } | 242 | var tmp = s_v[k]; |
| 228 | | 243 | s_v[k + 4] = asm ( |
| 229 | d.s[0..4].* = x +% @as(v4u32, d.s[0..4].*); | 244 | \\ sha256msg1 %[w4_7], %[tmp] |
| 230 | d.s[4..8].* = y +% @as(v4u32, d.s[4..8].*); | 245 | \\ vpalignr $0x4, %[w8_11], %[w12_15], %[result] |
| 231 | return; | 246 | \\ paddd %[tmp], %[result] |
| 232 | }, | 247 | \\ sha256msg2 %[w12_15], %[result] |
| 233 | .x86_64 => if (!isComptime() and comptime builtin.cpu.features.isEnabled(@enumToInt(std.Target.x86.Feature.sha))) { | 248 | : [tmp] "=&x" (tmp), |
| 234 | var x: v4u32 = [_]u32{ d.s[5], d.s[4], d.s[1], d.s[0] }; | 249 | [result] "=&x" (-> v4u32), |
| 235 | var y: v4u32 = [_]u32{ d.s[7], d.s[6], d.s[3], d.s[2] }; | 250 | : [_] "0" (tmp), |
| 236 | const s_v = @ptrCast(*[16]v4u32, &s); | 251 | [w4_7] "x" (s_v[k + 1]), |
| 237 | | 252 | [w8_11] "x" (s_v[k + 2]), |
| 238 | comptime var k: u8 = 0; | 253 | [w12_15] "x" (s_v[k + 3]), |
| 239 | inline while (k < 16) : (k += 1) { | 254 | ); |
| 240 | if (k < 12) { | 255 | } |
| 241 | var tmp = s_v[k]; | 256 | |
| 242 | s_v[k + 4] = asm ( | 257 | const w: v4u32 = s_v[k] +% @as(v4u32, W[4 * k ..][0..4].*); |
| 243 | \\ sha256msg1 %[w4_7], %[tmp] | 258 | y = asm ("sha256rnds2 %[x], %[y]" |
| 244 | \\ vpalignr $0x4, %[w8_11], %[w12_15], %[result] | 259 | : [y] "=x" (-> v4u32), |
| 245 | \\ paddd %[tmp], %[result] | 260 | : [_] "0" (y), |
| 246 | \\ sha256msg2 %[w12_15], %[result] | 261 | [x] "x" (x), |
| 247 | : [tmp] "=&x" (tmp), | 262 | [_] "{xmm0}" (w), |
| 248 | [result] "=&x" (-> v4u32), | 263 | ); |
| 249 | : [_] "0" (tmp), | 264 | |
| 250 | [w4_7] "x" (s_v[k + 1]), | 265 | x = asm ("sha256rnds2 %[y], %[x]" |
| 251 | [w8_11] "x" (s_v[k + 2]), | 266 | : [x] "=x" (-> v4u32), |
| 252 | [w12_15] "x" (s_v[k + 3]), | 267 | : [_] "0" (x), |
| | 268 | [y] "x" (y), |
| | 269 | [_] "{xmm0}" (@bitCast(v4u32, @bitCast(u128, w) >> 64)), |
| 253 | ); | 270 | ); |
| 254 | } | 271 | } |
| 255 | | 272 | |
| 256 | const w: v4u32 = s_v[k] +% @as(v4u32, W[4 * k ..][0..4].*); | 273 | d.s[0] +%= x[3]; |
| 257 | y = asm ("sha256rnds2 %[x], %[y]" | 274 | d.s[1] +%= x[2]; |
| 258 | : [y] "=x" (-> v4u32), | 275 | d.s[4] +%= x[1]; |
| 259 | : [_] "0" (y), | 276 | d.s[5] +%= x[0]; |
| 260 | [x] "x" (x), | 277 | d.s[2] +%= y[3]; |
| 261 | [_] "{xmm0}" (w), | 278 | d.s[3] +%= y[2]; |
| 262 | ); | 279 | d.s[6] +%= y[1]; |
| 263 | | 280 | d.s[7] +%= y[0]; |
| 264 | x = asm ("sha256rnds2 %[y], %[x]" | 281 | return; |
| 265 | : [x] "=x" (-> v4u32), | 282 | }, |
| 266 | : [_] "0" (x), | 283 | else => {}, |
| 267 | [y] "x" (y), | 284 | } |
| 268 | [_] "{xmm0}" (@bitCast(v4u32, @bitCast(u128, w) >> 64)), | | |
| 269 | ); | | |
| 270 | } | | |
| 271 | | | |
| 272 | d.s[0] +%= x[3]; | | |
| 273 | d.s[1] +%= x[2]; | | |
| 274 | d.s[4] +%= x[1]; | | |
| 275 | d.s[5] +%= x[0]; | | |
| 276 | d.s[2] +%= y[3]; | | |
| 277 | d.s[3] +%= y[2]; | | |
| 278 | d.s[6] +%= y[1]; | | |
| 279 | d.s[7] +%= y[0]; | | |
| 280 | return; | | |
| 281 | }, | | |
| 282 | else => {}, | | |
| 283 | } | 285 | } |
| 284 | | 286 | |
| 285 | var i: usize = 16; | 287 | var i: usize = 16; |
Cody Tapscott <topolarity@tapscott.me> 2022-10-28 17:17:08-07:00
lib