| ... | ... | @@ -192,94 +192,96 @@ fn Sha2x32(comptime params: Sha2Params32) type { |
| 192 | 192 | s[i] = mem.readIntBig(u32, mem.asBytes(elem)); |
| 193 | 193 | } |
| 194 | 194 | |
| 195 | | switch (builtin.cpu.arch) { |
| 196 | | .aarch64 => if (!isComptime() and comptime builtin.cpu.features.isEnabled(@enumToInt(std.Target.aarch64.Feature.sha2))) { |
| 197 | | var x: v4u32 = d.s[0..4].*; |
| 198 | | var y: v4u32 = d.s[4..8].*; |
| 199 | | const s_v = @ptrCast(*[16]v4u32, &s); |
| 200 | | |
| 201 | | comptime var k: u8 = 0; |
| 202 | | inline while (k < 16) : (k += 1) { |
| 203 | | if (k > 3) { |
| 204 | | s_v[k] = asm ( |
| 205 | | \\sha256su0.4s %[w0_3], %[w4_7] |
| 206 | | \\sha256su1.4s %[w0_3], %[w8_11], %[w12_15] |
| 207 | | : [w0_3] "=w" (-> v4u32), |
| 208 | | : [_] "0" (s_v[k - 4]), |
| 209 | | [w4_7] "w" (s_v[k - 3]), |
| 210 | | [w8_11] "w" (s_v[k - 2]), |
| 211 | | [w12_15] "w" (s_v[k - 1]), |
| 195 | if (!isComptime()) { |
| 196 | switch (builtin.cpu.arch) { |
| 197 | .aarch64 => if (comptime std.Target.aarch64.featureSetHas(builtin.cpu.features, .sha2)) { |
| 198 | var x: v4u32 = d.s[0..4].*; |
| 199 | var y: v4u32 = d.s[4..8].*; |
| 200 | const s_v = @ptrCast(*[16]v4u32, &s); |
| 201 | |
| 202 | comptime var k: u8 = 0; |
| 203 | inline while (k < 16) : (k += 1) { |
| 204 | if (k > 3) { |
| 205 | s_v[k] = asm ( |
| 206 | \\sha256su0.4s %[w0_3], %[w4_7] |
| 207 | \\sha256su1.4s %[w0_3], %[w8_11], %[w12_15] |
| 208 | : [w0_3] "=w" (-> v4u32), |
| 209 | : [_] "0" (s_v[k - 4]), |
| 210 | [w4_7] "w" (s_v[k - 3]), |
| 211 | [w8_11] "w" (s_v[k - 2]), |
| 212 | [w12_15] "w" (s_v[k - 1]), |
| 213 | ); |
| 214 | } |
| 215 | |
| 216 | const w: v4u32 = s_v[k] +% @as(v4u32, W[4 * k ..][0..4].*); |
| 217 | asm volatile ( |
| 218 | \\mov.4s v0, %[x] |
| 219 | \\sha256h.4s %[x], %[y], %[w] |
| 220 | \\sha256h2.4s %[y], v0, %[w] |
| 221 | : [x] "=w" (x), |
| 222 | [y] "=w" (y), |
| 223 | : [_] "0" (x), |
| 224 | [_] "1" (y), |
| 225 | [w] "w" (w), |
| 226 | : "v0" |
| 212 | 227 | ); |
| 213 | 228 | } |
| 214 | 229 | |
| 215 | | const w: v4u32 = s_v[k] +% @as(v4u32, W[4 * k ..][0..4].*); |
| 216 | | asm volatile ( |
| 217 | | \\mov.4s v0, %[x] |
| 218 | | \\sha256h.4s %[x], %[y], %[w] |
| 219 | | \\sha256h2.4s %[y], v0, %[w] |
| 220 | | : [x] "=w" (x), |
| 221 | | [y] "=w" (y), |
| 222 | | : [_] "0" (x), |
| 223 | | [_] "1" (y), |
| 224 | | [w] "w" (w), |
| 225 | | : "v0" |
| 226 | | ); |
| 227 | | } |
| 228 | | |
| 229 | | d.s[0..4].* = x +% @as(v4u32, d.s[0..4].*); |
| 230 | | d.s[4..8].* = y +% @as(v4u32, d.s[4..8].*); |
| 231 | | return; |
| 232 | | }, |
| 233 | | .x86_64 => if (!isComptime() and comptime builtin.cpu.features.isEnabled(@enumToInt(std.Target.x86.Feature.sha))) { |
| 234 | | var x: v4u32 = [_]u32{ d.s[5], d.s[4], d.s[1], d.s[0] }; |
| 235 | | var y: v4u32 = [_]u32{ d.s[7], d.s[6], d.s[3], d.s[2] }; |
| 236 | | const s_v = @ptrCast(*[16]v4u32, &s); |
| 237 | | |
| 238 | | comptime var k: u8 = 0; |
| 239 | | inline while (k < 16) : (k += 1) { |
| 240 | | if (k < 12) { |
| 241 | | var tmp = s_v[k]; |
| 242 | | s_v[k + 4] = asm ( |
| 243 | | \\ sha256msg1 %[w4_7], %[tmp] |
| 244 | | \\ vpalignr $0x4, %[w8_11], %[w12_15], %[result] |
| 245 | | \\ paddd %[tmp], %[result] |
| 246 | | \\ sha256msg2 %[w12_15], %[result] |
| 247 | | : [tmp] "=&x" (tmp), |
| 248 | | [result] "=&x" (-> v4u32), |
| 249 | | : [_] "0" (tmp), |
| 250 | | [w4_7] "x" (s_v[k + 1]), |
| 251 | | [w8_11] "x" (s_v[k + 2]), |
| 252 | | [w12_15] "x" (s_v[k + 3]), |
| 230 | d.s[0..4].* = x +% @as(v4u32, d.s[0..4].*); |
| 231 | d.s[4..8].* = y +% @as(v4u32, d.s[4..8].*); |
| 232 | return; |
| 233 | }, |
| 234 | .x86_64 => if (comptime std.Target.x86.featureSetHas(builtin.cpu.features, .sha)) { |
| 235 | var x: v4u32 = [_]u32{ d.s[5], d.s[4], d.s[1], d.s[0] }; |
| 236 | var y: v4u32 = [_]u32{ d.s[7], d.s[6], d.s[3], d.s[2] }; |
| 237 | const s_v = @ptrCast(*[16]v4u32, &s); |
| 238 | |
| 239 | comptime var k: u8 = 0; |
| 240 | inline while (k < 16) : (k += 1) { |
| 241 | if (k < 12) { |
| 242 | var tmp = s_v[k]; |
| 243 | s_v[k + 4] = asm ( |
| 244 | \\ sha256msg1 %[w4_7], %[tmp] |
| 245 | \\ vpalignr $0x4, %[w8_11], %[w12_15], %[result] |
| 246 | \\ paddd %[tmp], %[result] |
| 247 | \\ sha256msg2 %[w12_15], %[result] |
| 248 | : [tmp] "=&x" (tmp), |
| 249 | [result] "=&x" (-> v4u32), |
| 250 | : [_] "0" (tmp), |
| 251 | [w4_7] "x" (s_v[k + 1]), |
| 252 | [w8_11] "x" (s_v[k + 2]), |
| 253 | [w12_15] "x" (s_v[k + 3]), |
| 254 | ); |
| 255 | } |
| 256 | |
| 257 | const w: v4u32 = s_v[k] +% @as(v4u32, W[4 * k ..][0..4].*); |
| 258 | y = asm ("sha256rnds2 %[x], %[y]" |
| 259 | : [y] "=x" (-> v4u32), |
| 260 | : [_] "0" (y), |
| 261 | [x] "x" (x), |
| 262 | [_] "{xmm0}" (w), |
| 263 | ); |
| 264 | |
| 265 | x = asm ("sha256rnds2 %[y], %[x]" |
| 266 | : [x] "=x" (-> v4u32), |
| 267 | : [_] "0" (x), |
| 268 | [y] "x" (y), |
| 269 | [_] "{xmm0}" (@bitCast(v4u32, @bitCast(u128, w) >> 64)), |
| 253 | 270 | ); |
| 254 | 271 | } |
| 255 | 272 | |
| 256 | | const w: v4u32 = s_v[k] +% @as(v4u32, W[4 * k ..][0..4].*); |
| 257 | | y = asm ("sha256rnds2 %[x], %[y]" |
| 258 | | : [y] "=x" (-> v4u32), |
| 259 | | : [_] "0" (y), |
| 260 | | [x] "x" (x), |
| 261 | | [_] "{xmm0}" (w), |
| 262 | | ); |
| 263 | | |
| 264 | | x = asm ("sha256rnds2 %[y], %[x]" |
| 265 | | : [x] "=x" (-> v4u32), |
| 266 | | : [_] "0" (x), |
| 267 | | [y] "x" (y), |
| 268 | | [_] "{xmm0}" (@bitCast(v4u32, @bitCast(u128, w) >> 64)), |
| 269 | | ); |
| 270 | | } |
| 271 | | |
| 272 | | d.s[0] +%= x[3]; |
| 273 | | d.s[1] +%= x[2]; |
| 274 | | d.s[4] +%= x[1]; |
| 275 | | d.s[5] +%= x[0]; |
| 276 | | d.s[2] +%= y[3]; |
| 277 | | d.s[3] +%= y[2]; |
| 278 | | d.s[6] +%= y[1]; |
| 279 | | d.s[7] +%= y[0]; |
| 280 | | return; |
| 281 | | }, |
| 282 | | else => {}, |
| 273 | d.s[0] +%= x[3]; |
| 274 | d.s[1] +%= x[2]; |
| 275 | d.s[4] +%= x[1]; |
| 276 | d.s[5] +%= x[0]; |
| 277 | d.s[2] +%= y[3]; |
| 278 | d.s[3] +%= y[2]; |
| 279 | d.s[6] +%= y[1]; |
| 280 | d.s[7] +%= y[0]; |
| 281 | return; |
| 282 | }, |
| 283 | else => {}, |
| 284 | } |
| 283 | 285 | } |
| 284 | 286 | |
| 285 | 287 | var i: usize = 16; |
Cody Tapscott <topolarity@tapscott.me> 2022-10-28 17:17:08-07:00
lib