authorgravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2025-03-11 18:24:37-04:00
committergravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2025-03-21 21:51:08-04:00
logc5c1c8538dc047bc7df9acd685791c4ba3e20ffb
treeca21b8d0f19c2b3e2a5d73d1659b495e2e2a6b4c
parented284c1f98ae08a36a7ef0776e22a4cfb2527fee

x86_64: rewrite wrapping multiplication

14 files changed, 3197 insertions(+), 374 deletions(-)

lib/std/zig/Zir.zig+5-1
...@@ -2136,7 +2136,7 @@ pub const Inst = struct {...@@ -2136,7 +2136,7 @@ pub const Inst = struct {
2136 ref_start_index = static_len,2136 ref_start_index = static_len,
2137 _,2137 _,
21382138
2139 pub const static_len = 93;2139 pub const static_len = 97;
21402140
2141 pub fn toRef(i: Index) Inst.Ref {2141 pub fn toRef(i: Index) Inst.Ref {
2142 return @enumFromInt(@intFromEnum(Index.ref_start_index) + @intFromEnum(i));2142 return @enumFromInt(@intFromEnum(Index.ref_start_index) + @intFromEnum(i));
...@@ -2221,6 +2221,10 @@ pub const Inst = struct {...@@ -2221,6 +2221,10 @@ pub const Inst = struct {
2221 slice_const_u8_sentinel_0_type,2221 slice_const_u8_sentinel_0_type,
2222 vector_16_i8_type,2222 vector_16_i8_type,
2223 vector_32_i8_type,2223 vector_32_i8_type,
2224 vector_1_u8_type,
2225 vector_2_u8_type,
2226 vector_4_u8_type,
2227 vector_8_u8_type,
2224 vector_16_u8_type,2228 vector_16_u8_type,
2225 vector_32_u8_type,2229 vector_32_u8_type,
2226 vector_8_i16_type,2230 vector_8_i16_type,
src/Air.zig+4
...@@ -985,6 +985,10 @@ pub const Inst = struct {...@@ -985,6 +985,10 @@ pub const Inst = struct {
985 slice_const_u8_sentinel_0_type = @intFromEnum(InternPool.Index.slice_const_u8_sentinel_0_type),985 slice_const_u8_sentinel_0_type = @intFromEnum(InternPool.Index.slice_const_u8_sentinel_0_type),
986 vector_16_i8_type = @intFromEnum(InternPool.Index.vector_16_i8_type),986 vector_16_i8_type = @intFromEnum(InternPool.Index.vector_16_i8_type),
987 vector_32_i8_type = @intFromEnum(InternPool.Index.vector_32_i8_type),987 vector_32_i8_type = @intFromEnum(InternPool.Index.vector_32_i8_type),
988 vector_1_u8_type = @intFromEnum(InternPool.Index.vector_1_u8_type),
989 vector_2_u8_type = @intFromEnum(InternPool.Index.vector_2_u8_type),
990 vector_4_u8_type = @intFromEnum(InternPool.Index.vector_4_u8_type),
991 vector_8_u8_type = @intFromEnum(InternPool.Index.vector_8_u8_type),
988 vector_16_u8_type = @intFromEnum(InternPool.Index.vector_16_u8_type),992 vector_16_u8_type = @intFromEnum(InternPool.Index.vector_16_u8_type),
989 vector_32_u8_type = @intFromEnum(InternPool.Index.vector_32_u8_type),993 vector_32_u8_type = @intFromEnum(InternPool.Index.vector_32_u8_type),
990 vector_8_i16_type = @intFromEnum(InternPool.Index.vector_8_i16_type),994 vector_8_i16_type = @intFromEnum(InternPool.Index.vector_8_i16_type),
src/InternPool.zig+20
...@@ -4573,6 +4573,10 @@ pub const Index = enum(u32) {...@@ -4573,6 +4573,10 @@ pub const Index = enum(u32) {
45734573
4574 vector_16_i8_type,4574 vector_16_i8_type,
4575 vector_32_i8_type,4575 vector_32_i8_type,
4576 vector_1_u8_type,
4577 vector_2_u8_type,
4578 vector_4_u8_type,
4579 vector_8_u8_type,
4576 vector_16_u8_type,4580 vector_16_u8_type,
4577 vector_32_u8_type,4581 vector_32_u8_type,
4578 vector_8_i16_type,4582 vector_8_i16_type,
...@@ -5089,6 +5093,14 @@ pub const static_keys = [_]Key{...@@ -5089,6 +5093,14 @@ pub const static_keys = [_]Key{
5089 .{ .vector_type = .{ .len = 16, .child = .i8_type } },5093 .{ .vector_type = .{ .len = 16, .child = .i8_type } },
5090 // @Vector(32, i8)5094 // @Vector(32, i8)
5091 .{ .vector_type = .{ .len = 32, .child = .i8_type } },5095 .{ .vector_type = .{ .len = 32, .child = .i8_type } },
5096 // @Vector(1, u8)
5097 .{ .vector_type = .{ .len = 1, .child = .u8_type } },
5098 // @Vector(2, u8)
5099 .{ .vector_type = .{ .len = 2, .child = .u8_type } },
5100 // @Vector(4, u8)
5101 .{ .vector_type = .{ .len = 4, .child = .u8_type } },
5102 // @Vector(8, u8)
5103 .{ .vector_type = .{ .len = 8, .child = .u8_type } },
5092 // @Vector(16, u8)5104 // @Vector(16, u8)
5093 .{ .vector_type = .{ .len = 16, .child = .u8_type } },5105 .{ .vector_type = .{ .len = 16, .child = .u8_type } },
5094 // @Vector(32, u8)5106 // @Vector(32, u8)
...@@ -11766,6 +11778,10 @@ pub fn typeOf(ip: *const InternPool, index: Index) Index {...@@ -11766,6 +11778,10 @@ pub fn typeOf(ip: *const InternPool, index: Index) Index {
11766 .slice_const_u8_sentinel_0_type,11778 .slice_const_u8_sentinel_0_type,
11767 .vector_16_i8_type,11779 .vector_16_i8_type,
11768 .vector_32_i8_type,11780 .vector_32_i8_type,
11781 .vector_1_u8_type,
11782 .vector_2_u8_type,
11783 .vector_4_u8_type,
11784 .vector_8_u8_type,
11769 .vector_16_u8_type,11785 .vector_16_u8_type,
11770 .vector_32_u8_type,11786 .vector_32_u8_type,
11771 .vector_8_i16_type,11787 .vector_8_i16_type,
...@@ -12106,6 +12122,10 @@ pub fn zigTypeTag(ip: *const InternPool, index: Index) std.builtin.TypeId {...@@ -12106,6 +12122,10 @@ pub fn zigTypeTag(ip: *const InternPool, index: Index) std.builtin.TypeId {
1210612122
12107 .vector_16_i8_type,12123 .vector_16_i8_type,
12108 .vector_32_i8_type,12124 .vector_32_i8_type,
12125 .vector_1_u8_type,
12126 .vector_2_u8_type,
12127 .vector_4_u8_type,
12128 .vector_8_u8_type,
12109 .vector_16_u8_type,12129 .vector_16_u8_type,
12110 .vector_32_u8_type,12130 .vector_32_u8_type,
12111 .vector_8_i16_type,12131 .vector_8_i16_type,
src/Sema.zig+4
...@@ -36377,6 +36377,10 @@ pub fn typeHasOnePossibleValue(sema: *Sema, ty: Type) CompileError!?Value {...@@ -36377,6 +36377,10 @@ pub fn typeHasOnePossibleValue(sema: *Sema, ty: Type) CompileError!?Value {
36377 .slice_const_u8_sentinel_0_type,36377 .slice_const_u8_sentinel_0_type,
36378 .vector_16_i8_type,36378 .vector_16_i8_type,
36379 .vector_32_i8_type,36379 .vector_32_i8_type,
36380 .vector_1_u8_type,
36381 .vector_2_u8_type,
36382 .vector_4_u8_type,
36383 .vector_8_u8_type,
36380 .vector_16_u8_type,36384 .vector_16_u8_type,
36381 .vector_32_u8_type,36385 .vector_32_u8_type,
36382 .vector_8_i16_type,36386 .vector_8_i16_type,
src/Type.zig+4
...@@ -4201,6 +4201,10 @@ pub const slice_const_u8_sentinel_0: Type = .{ .ip_index = .slice_const_u8_senti...@@ -4201,6 +4201,10 @@ pub const slice_const_u8_sentinel_0: Type = .{ .ip_index = .slice_const_u8_senti
42014201
4202pub const vector_16_i8: Type = .{ .ip_index = .vector_16_i8_type };4202pub const vector_16_i8: Type = .{ .ip_index = .vector_16_i8_type };
4203pub const vector_32_i8: Type = .{ .ip_index = .vector_32_i8_type };4203pub const vector_32_i8: Type = .{ .ip_index = .vector_32_i8_type };
4204pub const vector_1_u8: Type = .{ .ip_index = .vector_1_u8_type };
4205pub const vector_2_u8: Type = .{ .ip_index = .vector_2_u8_type };
4206pub const vector_4_u8: Type = .{ .ip_index = .vector_4_u8_type };
4207pub const vector_8_u8: Type = .{ .ip_index = .vector_8_u8_type };
4204pub const vector_16_u8: Type = .{ .ip_index = .vector_16_u8_type };4208pub const vector_16_u8: Type = .{ .ip_index = .vector_16_u8_type };
4205pub const vector_32_u8: Type = .{ .ip_index = .vector_32_u8_type };4209pub const vector_32_u8: Type = .{ .ip_index = .vector_32_u8_type };
4206pub const vector_8_i16: Type = .{ .ip_index = .vector_8_i16_type };4210pub const vector_8_i16: Type = .{ .ip_index = .vector_8_i16_type };
src/arch/x86_64/CodeGen.zig+3082-364
...@@ -2069,7 +2069,7 @@ fn asmRegisterMemoryImmediate(...@@ -2069,7 +2069,7 @@ fn asmRegisterMemoryImmediate(
2069 } else {2069 } else {
2070 const payload = try self.addExtra(Mir.Imm32{ .imm = switch (imm) {2070 const payload = try self.addExtra(Mir.Imm32{ .imm = switch (imm) {
2071 .signed => |s| @bitCast(s),2071 .signed => |s| @bitCast(s),
2072 .unsigned => unreachable,2072 .unsigned => |u| @as(u32, @intCast(u)),
2073 .reloc => unreachable,2073 .reloc => unreachable,
2074 } });2074 } });
2075 assert(payload + 1 == try self.addExtra(Mir.Memory.encode(m)));2075 assert(payload + 1 == try self.addExtra(Mir.Memory.encode(m)));
...@@ -2418,7 +2418,7 @@ fn genBodyBlock(self: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -2418,7 +2418,7 @@ fn genBodyBlock(self: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2418}2418}
24192419
2420fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {2420fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2421 @setEvalBranchQuota(13_600);2421 @setEvalBranchQuota(13_800);
2422 const pt = cg.pt;2422 const pt = cg.pt;
2423 const zcu = pt.zcu;2423 const zcu = pt.zcu;
2424 const ip = &zcu.intern_pool;2424 const ip = &zcu.intern_pool;
...@@ -2457,9 +2457,6 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -2457,9 +2457,6 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2457 .shr, .shr_exact => try cg.airShlShrBinOp(inst),2457 .shr, .shr_exact => try cg.airShlShrBinOp(inst),
2458 .shl, .shl_exact => try cg.airShlShrBinOp(inst),2458 .shl, .shl_exact => try cg.airShlShrBinOp(inst),
24592459
2460 .mul_wrap,
2461 => |air_tag| try cg.airMulDivBinOp(inst, air_tag),
2462
2463 .add_sat => try cg.airAddSat(inst),2460 .add_sat => try cg.airAddSat(inst),
2464 .sub_sat => try cg.airSubSat(inst),2461 .sub_sat => try cg.airSubSat(inst),
2465 .mul_sat => try cg.airMulSat(inst),2462 .mul_sat => try cg.airMulSat(inst),
...@@ -5215,13 +5212,2516 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -5215,13 +5212,2516 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
5215 .each = .{ .once = &.{5212 .each = .{ .once = &.{
5216 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },5213 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
5217 .{ .@"0:", ._, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ },5214 .{ .@"0:", ._, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ },
5218 .{ ._, ._, .sub, .tmp1q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ },5215 .{ ._, ._, .sub, .tmp1q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ },
5216 .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1q, ._, ._ },
5217 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
5218 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
5219 } },
5220 }, .{
5221 .required_features = .{ .@"64bit", null, null, null },
5222 .src_constraints = .{
5223 .{ .scalar_remainder_int = .{ .of = .qword, .is = .qword } },
5224 .{ .scalar_remainder_int = .{ .of = .qword, .is = .qword } },
5225 .any,
5226 },
5227 .patterns = &.{
5228 .{ .src = .{ .to_mem, .to_mem, .none } },
5229 },
5230 .extra_temps = .{
5231 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
5232 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
5233 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
5234 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
5235 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
5236 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
5237 .unused,
5238 .unused,
5239 .unused,
5240 .unused,
5241 .unused,
5242 },
5243 .dst_temps = .{ .mem, .unused },
5244 .each = .{ .once = &.{
5245 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
5246 .{ .@"0:", ._, .lea, .tmp1p, .memia(.src0, .tmp0, .add_unaligned_size_add_elem_size), ._, ._ },
5247 .{ ._, ._, .lea, .tmp2p, .memia(.src1, .tmp0, .add_unaligned_size_add_elem_size), ._, ._ },
5248 .{ ._, ._, .lea, .tmp3p, .memia(.dst0, .tmp0, .add_unaligned_size_add_elem_size), ._, ._ },
5249 .{ ._, ._, .mov, .tmp4p, .sa(.src0, .sub_elem_size_div_8), ._, ._ },
5250 .{ ._, ._c, .cl, ._, ._, ._, ._ },
5251 .{ .@"1:", ._, .mov, .tmp5q, .leasi(.tmp1q, .@"8", .tmp4), ._, ._ },
5252 .{ ._, ._, .sbb, .tmp5q, .leasi(.tmp2q, .@"8", .tmp4), ._, ._ },
5253 .{ ._, ._, .mov, .leasi(.tmp3q, .@"8", .tmp4), .tmp5q, ._, ._ },
5254 .{ ._, ._c, .in, .tmp4p, ._, ._, ._ },
5255 .{ ._, ._nz, .j, .@"1b", ._, ._, ._ },
5256 .{ ._, ._, .add, .tmp0p, .sa(.src0, .add_elem_size), ._, ._ },
5257 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
5258 } },
5259 }, .{
5260 .src_constraints = .{
5261 .{ .scalar_remainder_int = .{ .of = .dword, .is = .dword } },
5262 .{ .scalar_remainder_int = .{ .of = .dword, .is = .dword } },
5263 .any,
5264 },
5265 .patterns = &.{
5266 .{ .src = .{ .to_mem, .to_mem, .none } },
5267 },
5268 .extra_temps = .{
5269 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
5270 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
5271 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
5272 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
5273 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
5274 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
5275 .unused,
5276 .unused,
5277 .unused,
5278 .unused,
5279 .unused,
5280 },
5281 .dst_temps = .{ .mem, .unused },
5282 .each = .{ .once = &.{
5283 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
5284 .{ .@"0:", ._, .lea, .tmp1p, .memia(.src0, .tmp0, .add_unaligned_size_add_elem_size), ._, ._ },
5285 .{ ._, ._, .lea, .tmp2p, .memia(.src1, .tmp0, .add_unaligned_size_add_elem_size), ._, ._ },
5286 .{ ._, ._, .lea, .tmp3p, .memia(.dst0, .tmp0, .add_unaligned_size_add_elem_size), ._, ._ },
5287 .{ ._, ._, .mov, .tmp4p, .sa(.src0, .sub_elem_size_div_4), ._, ._ },
5288 .{ ._, ._c, .cl, ._, ._, ._, ._ },
5289 .{ .@"1:", ._, .mov, .tmp5d, .leasi(.tmp1d, .@"4", .tmp4), ._, ._ },
5290 .{ ._, ._, .sbb, .tmp5d, .leasi(.tmp2d, .@"4", .tmp4), ._, ._ },
5291 .{ ._, ._, .mov, .leasi(.tmp3d, .@"4", .tmp4), .tmp5d, ._, ._ },
5292 .{ ._, ._c, .in, .tmp4p, ._, ._, ._ },
5293 .{ ._, ._nz, .j, .@"1b", ._, ._, ._ },
5294 .{ ._, ._, .add, .tmp0p, .sa(.src0, .add_elem_size), ._, ._ },
5295 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
5296 } },
5297 }, .{
5298 .required_features = .{ .f16c, null, null, null },
5299 .src_constraints = .{
5300 .{ .scalar_float = .{ .of = .word, .is = .word } },
5301 .{ .scalar_float = .{ .of = .word, .is = .word } },
5302 .any,
5303 },
5304 .patterns = &.{
5305 .{ .src = .{ .to_sse, .to_sse, .none } },
5306 },
5307 .extra_temps = .{
5308 .{ .type = .f32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } },
5309 .unused,
5310 .unused,
5311 .unused,
5312 .unused,
5313 .unused,
5314 .unused,
5315 .unused,
5316 .unused,
5317 .unused,
5318 .unused,
5319 },
5320 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
5321 .each = .{ .once = &.{
5322 .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ },
5323 .{ ._, .v_ps, .cvtph2, .tmp0x, .src1q, ._, ._ },
5324 .{ ._, .v_ss, .sub, .dst0x, .dst0x, .tmp0d, ._ },
5325 .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ },
5326 } },
5327 }, .{
5328 .required_features = .{ .sse, null, null, null },
5329 .src_constraints = .{
5330 .{ .scalar_float = .{ .of = .word, .is = .word } },
5331 .{ .scalar_float = .{ .of = .word, .is = .word } },
5332 .any,
5333 },
5334 .patterns = &.{
5335 .{ .src = .{ .{ .to_reg = .xmm0 }, .{ .to_reg = .xmm1 }, .none } },
5336 },
5337 .call_frame = .{ .alignment = .@"16" },
5338 .extra_temps = .{
5339 .{ .type = .usize, .kind = .{ .symbol = &.{ .name = "__subhf3" } } },
5340 .unused,
5341 .unused,
5342 .unused,
5343 .unused,
5344 .unused,
5345 .unused,
5346 .unused,
5347 .unused,
5348 .unused,
5349 .unused,
5350 },
5351 .dst_temps = .{ .{ .ref = .src0 }, .unused },
5352 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
5353 .each = .{ .once = &.{
5354 .{ ._, ._, .call, .tmp0d, ._, ._, ._ },
5355 } },
5356 }, .{
5357 .required_features = .{ .f16c, null, null, null },
5358 .src_constraints = .{
5359 .{ .scalar_float = .{ .of = .qword, .is = .word } },
5360 .{ .scalar_float = .{ .of = .qword, .is = .word } },
5361 .any,
5362 },
5363 .patterns = &.{
5364 .{ .src = .{ .mem, .mem, .none } },
5365 .{ .src = .{ .to_sse, .mem, .none } },
5366 .{ .src = .{ .mem, .to_sse, .none } },
5367 .{ .src = .{ .to_sse, .to_sse, .none } },
5368 },
5369 .extra_temps = .{
5370 .{ .type = .vector_4_f32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } },
5371 .unused,
5372 .unused,
5373 .unused,
5374 .unused,
5375 .unused,
5376 .unused,
5377 .unused,
5378 .unused,
5379 .unused,
5380 .unused,
5381 },
5382 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
5383 .each = .{ .once = &.{
5384 .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ },
5385 .{ ._, .v_ps, .cvtph2, .tmp0x, .src1q, ._, ._ },
5386 .{ ._, .v_ps, .sub, .dst0x, .dst0x, .tmp0x, ._ },
5387 .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ },
5388 } },
5389 }, .{
5390 .required_features = .{ .f16c, null, null, null },
5391 .src_constraints = .{
5392 .{ .scalar_float = .{ .of = .xword, .is = .word } },
5393 .{ .scalar_float = .{ .of = .xword, .is = .word } },
5394 .any,
5395 },
5396 .patterns = &.{
5397 .{ .src = .{ .mem, .mem, .none } },
5398 .{ .src = .{ .to_sse, .mem, .none } },
5399 .{ .src = .{ .mem, .to_sse, .none } },
5400 .{ .src = .{ .to_sse, .to_sse, .none } },
5401 },
5402 .extra_temps = .{
5403 .{ .type = .vector_8_f32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } },
5404 .unused,
5405 .unused,
5406 .unused,
5407 .unused,
5408 .unused,
5409 .unused,
5410 .unused,
5411 .unused,
5412 .unused,
5413 .unused,
5414 },
5415 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
5416 .each = .{ .once = &.{
5417 .{ ._, .v_ps, .cvtph2, .dst0y, .src0x, ._, ._ },
5418 .{ ._, .v_ps, .cvtph2, .tmp0y, .src1x, ._, ._ },
5419 .{ ._, .v_ps, .sub, .dst0y, .dst0y, .tmp0y, ._ },
5420 .{ ._, .v_, .cvtps2ph, .dst0x, .dst0y, .rm(.{}), ._ },
5421 } },
5422 }, .{
5423 .required_features = .{ .f16c, null, null, null },
5424 .src_constraints = .{
5425 .{ .multiple_scalar_float = .{ .of = .xword, .is = .word } },
5426 .{ .multiple_scalar_float = .{ .of = .xword, .is = .word } },
5427 .any,
5428 },
5429 .patterns = &.{
5430 .{ .src = .{ .to_mem, .to_mem, .none } },
5431 },
5432 .extra_temps = .{
5433 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
5434 .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } },
5435 .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } },
5436 .unused,
5437 .unused,
5438 .unused,
5439 .unused,
5440 .unused,
5441 .unused,
5442 .unused,
5443 .unused,
5444 },
5445 .dst_temps = .{ .mem, .unused },
5446 .clobbers = .{ .eflags = true },
5447 .each = .{ .once = &.{
5448 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
5449 .{ .@"0:", .v_ps, .cvtph2, .tmp1y, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ },
5450 .{ ._, .v_ps, .cvtph2, .tmp2y, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ },
5451 .{ ._, .v_ps, .sub, .tmp1y, .tmp1y, .tmp2y, ._ },
5452 .{ ._, .v_, .cvtps2ph, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1y, .rm(.{}), ._ },
5453 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
5454 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
5455 } },
5456 }, .{
5457 .required_features = .{ .avx, null, null, null },
5458 .src_constraints = .{
5459 .{ .multiple_scalar_float = .{ .of = .word, .is = .word } },
5460 .{ .multiple_scalar_float = .{ .of = .word, .is = .word } },
5461 .any,
5462 },
5463 .patterns = &.{
5464 .{ .src = .{ .to_mem, .to_mem, .none } },
5465 },
5466 .call_frame = .{ .alignment = .@"16" },
5467 .extra_temps = .{
5468 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
5469 .{ .type = .f16, .kind = .{ .reg = .xmm0 } },
5470 .{ .type = .f16, .kind = .{ .reg = .xmm1 } },
5471 .{ .type = .usize, .kind = .{ .symbol = &.{ .name = "__subhf3" } } },
5472 .unused,
5473 .unused,
5474 .unused,
5475 .unused,
5476 .unused,
5477 .unused,
5478 .unused,
5479 },
5480 .dst_temps = .{ .mem, .unused },
5481 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
5482 .each = .{ .once = &.{
5483 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
5484 .{ .@"0:", .vp_, .xor, .tmp2x, .tmp2x, .tmp2x, ._ },
5485 .{ ._, .vp_w, .insr, .tmp1x, .tmp2x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(0) },
5486 .{ ._, .vp_w, .insr, .tmp2x, .tmp2x, .memia(.src1w, .tmp0, .add_unaligned_size), .ui(0) },
5487 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
5488 .{ ._, .vp_w, .extr, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1x, .ui(0), ._ },
5489 .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ },
5490 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
5491 } },
5492 }, .{
5493 .required_features = .{ .sse4_1, null, null, null },
5494 .src_constraints = .{
5495 .{ .multiple_scalar_float = .{ .of = .word, .is = .word } },
5496 .{ .multiple_scalar_float = .{ .of = .word, .is = .word } },
5497 .any,
5498 },
5499 .patterns = &.{
5500 .{ .src = .{ .to_mem, .to_mem, .none } },
5501 },
5502 .call_frame = .{ .alignment = .@"16" },
5503 .extra_temps = .{
5504 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
5505 .{ .type = .f16, .kind = .{ .reg = .xmm0 } },
5506 .{ .type = .f16, .kind = .{ .reg = .xmm1 } },
5507 .{ .type = .usize, .kind = .{ .symbol = &.{ .name = "__subhf3" } } },
5508 .unused,
5509 .unused,
5510 .unused,
5511 .unused,
5512 .unused,
5513 .unused,
5514 .unused,
5515 },
5516 .dst_temps = .{ .mem, .unused },
5517 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
5518 .each = .{ .once = &.{
5519 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
5520 .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ },
5521 .{ ._, .p_, .xor, .tmp2x, .tmp2x, ._, ._ },
5522 .{ ._, .p_w, .insr, .tmp1x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(0), ._ },
5523 .{ ._, .p_w, .insr, .tmp2x, .memia(.src1w, .tmp0, .add_unaligned_size), .ui(0), ._ },
5524 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
5525 .{ ._, .p_w, .extr, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1x, .ui(0), ._ },
5526 .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ },
5527 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
5528 } },
5529 }, .{
5530 .required_features = .{ .sse2, null, null, null },
5531 .src_constraints = .{
5532 .{ .multiple_scalar_float = .{ .of = .word, .is = .word } },
5533 .{ .multiple_scalar_float = .{ .of = .word, .is = .word } },
5534 .any,
5535 },
5536 .patterns = &.{
5537 .{ .src = .{ .to_mem, .to_mem, .none } },
5538 },
5539 .call_frame = .{ .alignment = .@"16" },
5540 .extra_temps = .{
5541 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
5542 .{ .type = .f16, .kind = .{ .reg = .xmm0 } },
5543 .{ .type = .f16, .kind = .{ .reg = .xmm1 } },
5544 .{ .type = .usize, .kind = .{ .symbol = &.{ .name = "__subhf3" } } },
5545 .{ .type = .f16, .kind = .{ .reg = .ax } },
5546 .unused,
5547 .unused,
5548 .unused,
5549 .unused,
5550 .unused,
5551 .unused,
5552 },
5553 .dst_temps = .{ .mem, .unused },
5554 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
5555 .each = .{ .once = &.{
5556 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
5557 .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ },
5558 .{ ._, .p_, .xor, .tmp2x, .tmp2x, ._, ._ },
5559 .{ ._, .p_w, .insr, .tmp1x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(0), ._ },
5560 .{ ._, .p_w, .insr, .tmp2x, .memia(.src1w, .tmp0, .add_unaligned_size), .ui(0), ._ },
5561 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
5562 .{ ._, .p_w, .extr, .tmp4d, .tmp1x, .ui(0), ._ },
5563 .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp4w, ._, ._ },
5564 .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ },
5565 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
5566 } },
5567 }, .{
5568 .required_features = .{ .sse, null, null, null },
5569 .src_constraints = .{
5570 .{ .multiple_scalar_float = .{ .of = .word, .is = .word } },
5571 .{ .multiple_scalar_float = .{ .of = .word, .is = .word } },
5572 .any,
5573 },
5574 .patterns = &.{
5575 .{ .src = .{ .to_mem, .to_mem, .none } },
5576 },
5577 .call_frame = .{ .alignment = .@"16" },
5578 .extra_temps = .{
5579 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
5580 .{ .type = .f16, .kind = .{ .reg = .ax } },
5581 .{ .type = .f32, .kind = .mem },
5582 .{ .type = .f16, .kind = .{ .reg = .xmm0 } },
5583 .{ .type = .f16, .kind = .{ .reg = .xmm1 } },
5584 .{ .type = .usize, .kind = .{ .symbol = &.{ .name = "__subhf3" } } },
5585 .unused,
5586 .unused,
5587 .unused,
5588 .unused,
5589 .unused,
5590 },
5591 .dst_temps = .{ .mem, .unused },
5592 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
5593 .each = .{ .once = &.{
5594 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
5595 .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ },
5596 .{ ._, ._, .mov, .mem(.tmp2d), .tmp1d, ._, ._ },
5597 .{ ._, ._ss, .mov, .tmp3x, .mem(.tmp2d), ._, ._ },
5598 .{ ._, ._, .movzx, .tmp1d, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ },
5599 .{ ._, ._, .mov, .mem(.tmp2d), .tmp1d, ._, ._ },
5600 .{ ._, ._ss, .mov, .tmp4x, .mem(.tmp2d), ._, ._ },
5601 .{ ._, ._, .call, .tmp5d, ._, ._, ._ },
5602 .{ ._, ._ss, .mov, .mem(.tmp2d), .tmp3x, ._, ._ },
5603 .{ ._, ._, .mov, .tmp1d, .mem(.tmp2d), ._, ._ },
5604 .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ },
5605 .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ },
5606 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
5607 } },
5608 }, .{
5609 .required_features = .{ .avx, null, null, null },
5610 .src_constraints = .{
5611 .{ .scalar_float = .{ .of = .dword, .is = .dword } },
5612 .{ .scalar_float = .{ .of = .dword, .is = .dword } },
5613 .any,
5614 },
5615 .patterns = &.{
5616 .{ .src = .{ .to_sse, .mem, .none } },
5617 .{ .src = .{ .to_sse, .to_sse, .none } },
5618 },
5619 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
5620 .each = .{ .once = &.{
5621 .{ ._, .v_ss, .sub, .dst0x, .src0x, .src1d, ._ },
5622 } },
5623 }, .{
5624 .required_features = .{ .sse, null, null, null },
5625 .src_constraints = .{
5626 .{ .scalar_float = .{ .of = .dword, .is = .dword } },
5627 .{ .scalar_float = .{ .of = .dword, .is = .dword } },
5628 .any,
5629 },
5630 .patterns = &.{
5631 .{ .src = .{ .to_mut_sse, .mem, .none } },
5632 .{ .src = .{ .to_mut_sse, .to_sse, .none } },
5633 },
5634 .dst_temps = .{ .{ .ref = .src0 }, .unused },
5635 .each = .{ .once = &.{
5636 .{ ._, ._ss, .sub, .dst0x, .src1d, ._, ._ },
5637 } },
5638 }, .{
5639 .required_features = .{ .avx, null, null, null },
5640 .src_constraints = .{
5641 .{ .scalar_float = .{ .of = .xword, .is = .dword } },
5642 .{ .scalar_float = .{ .of = .xword, .is = .dword } },
5643 .any,
5644 },
5645 .patterns = &.{
5646 .{ .src = .{ .to_sse, .mem, .none } },
5647 .{ .src = .{ .to_sse, .to_sse, .none } },
5648 },
5649 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
5650 .each = .{ .once = &.{
5651 .{ ._, .v_ps, .sub, .dst0x, .src0x, .src1x, ._ },
5652 } },
5653 }, .{
5654 .required_features = .{ .sse, null, null, null },
5655 .src_constraints = .{
5656 .{ .scalar_float = .{ .of = .xword, .is = .dword } },
5657 .{ .scalar_float = .{ .of = .xword, .is = .dword } },
5658 .any,
5659 },
5660 .patterns = &.{
5661 .{ .src = .{ .to_mut_sse, .mem, .none } },
5662 .{ .src = .{ .to_mut_sse, .to_sse, .none } },
5663 },
5664 .dst_temps = .{ .{ .ref = .src0 }, .unused },
5665 .each = .{ .once = &.{
5666 .{ ._, ._ps, .sub, .dst0x, .src1x, ._, ._ },
5667 } },
5668 }, .{
5669 .required_features = .{ .avx, null, null, null },
5670 .src_constraints = .{
5671 .{ .scalar_float = .{ .of = .yword, .is = .dword } },
5672 .{ .scalar_float = .{ .of = .yword, .is = .dword } },
5673 .any,
5674 },
5675 .patterns = &.{
5676 .{ .src = .{ .to_sse, .mem, .none } },
5677 .{ .src = .{ .to_sse, .to_sse, .none } },
5678 },
5679 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
5680 .each = .{ .once = &.{
5681 .{ ._, .v_ps, .sub, .dst0y, .src0y, .src1y, ._ },
5682 } },
5683 }, .{
5684 .required_features = .{ .avx, null, null, null },
5685 .src_constraints = .{
5686 .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } },
5687 .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } },
5688 .any,
5689 },
5690 .patterns = &.{
5691 .{ .src = .{ .to_mem, .to_mem, .none } },
5692 },
5693 .extra_temps = .{
5694 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
5695 .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } },
5696 .unused,
5697 .unused,
5698 .unused,
5699 .unused,
5700 .unused,
5701 .unused,
5702 .unused,
5703 .unused,
5704 .unused,
5705 },
5706 .dst_temps = .{ .mem, .unused },
5707 .clobbers = .{ .eflags = true },
5708 .each = .{ .once = &.{
5709 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
5710 .{ .@"0:", .v_ps, .mova, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ },
5711 .{ ._, .v_ps, .sub, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ },
5712 .{ ._, .v_ps, .mova, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ },
5713 .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ },
5714 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
5715 } },
5716 }, .{
5717 .required_features = .{ .sse, null, null, null },
5718 .src_constraints = .{
5719 .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } },
5720 .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } },
5721 .any,
5722 },
5723 .patterns = &.{
5724 .{ .src = .{ .to_mem, .to_mem, .none } },
5725 },
5726 .extra_temps = .{
5727 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
5728 .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } },
5729 .unused,
5730 .unused,
5731 .unused,
5732 .unused,
5733 .unused,
5734 .unused,
5735 .unused,
5736 .unused,
5737 .unused,
5738 },
5739 .dst_temps = .{ .mem, .unused },
5740 .clobbers = .{ .eflags = true },
5741 .each = .{ .once = &.{
5742 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
5743 .{ .@"0:", ._ps, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ },
5744 .{ ._, ._ps, .sub, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ },
5745 .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ },
5746 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
5747 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
5748 } },
5749 }, .{
5750 .required_features = .{ .avx, null, null, null },
5751 .src_constraints = .{
5752 .{ .scalar_float = .{ .of = .qword, .is = .qword } },
5753 .{ .scalar_float = .{ .of = .qword, .is = .qword } },
5754 .any,
5755 },
5756 .patterns = &.{
5757 .{ .src = .{ .to_sse, .mem, .none } },
5758 .{ .src = .{ .to_sse, .to_sse, .none } },
5759 },
5760 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
5761 .each = .{ .once = &.{
5762 .{ ._, .v_sd, .sub, .dst0x, .src0x, .src1q, ._ },
5763 } },
5764 }, .{
5765 .required_features = .{ .sse2, null, null, null },
5766 .src_constraints = .{
5767 .{ .scalar_float = .{ .of = .qword, .is = .qword } },
5768 .{ .scalar_float = .{ .of = .qword, .is = .qword } },
5769 .any,
5770 },
5771 .patterns = &.{
5772 .{ .src = .{ .to_mut_sse, .mem, .none } },
5773 .{ .src = .{ .to_mut_sse, .to_sse, .none } },
5774 },
5775 .dst_temps = .{ .{ .ref = .src0 }, .unused },
5776 .each = .{ .once = &.{
5777 .{ ._, ._sd, .sub, .dst0x, .src1q, ._, ._ },
5778 } },
5779 }, .{
5780 .required_features = .{ .x87, null, null, null },
5781 .src_constraints = .{
5782 .{ .scalar_float = .{ .of = .qword, .is = .qword } },
5783 .{ .scalar_float = .{ .of = .qword, .is = .qword } },
5784 .any,
5785 },
5786 .patterns = &.{
5787 .{ .src = .{ .to_mem, .to_mem, .none } },
5788 },
5789 .extra_temps = .{
5790 .{ .type = .f64, .kind = .{ .reg = .st6 } },
5791 .{ .type = .f64, .kind = .{ .reg = .st7 } },
5792 .unused,
5793 .unused,
5794 .unused,
5795 .unused,
5796 .unused,
5797 .unused,
5798 .unused,
5799 .unused,
5800 .unused,
5801 },
5802 .dst_temps = .{ .mem, .unused },
5803 .each = .{ .once = &.{
5804 .{ ._, .f_, .ld, .src0q, ._, ._, ._ },
5805 .{ ._, .f_, .sub, .src1q, ._, ._, ._ },
5806 .{ ._, .f_p, .st, .dst0q, ._, ._, ._ },
5807 } },
5808 }, .{
5809 .required_features = .{ .avx, null, null, null },
5810 .src_constraints = .{
5811 .{ .scalar_float = .{ .of = .xword, .is = .qword } },
5812 .{ .scalar_float = .{ .of = .xword, .is = .qword } },
5813 .any,
5814 },
5815 .patterns = &.{
5816 .{ .src = .{ .to_sse, .mem, .none } },
5817 .{ .src = .{ .to_sse, .to_sse, .none } },
5818 },
5819 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
5820 .each = .{ .once = &.{
5821 .{ ._, .v_pd, .sub, .dst0x, .src0x, .src1x, ._ },
5822 } },
5823 }, .{
5824 .required_features = .{ .sse2, null, null, null },
5825 .src_constraints = .{
5826 .{ .scalar_float = .{ .of = .xword, .is = .qword } },
5827 .{ .scalar_float = .{ .of = .xword, .is = .qword } },
5828 .any,
5829 },
5830 .patterns = &.{
5831 .{ .src = .{ .to_mut_sse, .mem, .none } },
5832 .{ .src = .{ .to_mut_sse, .to_sse, .none } },
5833 },
5834 .dst_temps = .{ .{ .ref = .src0 }, .unused },
5835 .each = .{ .once = &.{
5836 .{ ._, ._pd, .sub, .dst0x, .src1x, ._, ._ },
5837 } },
5838 }, .{
5839 .required_features = .{ .avx, null, null, null },
5840 .src_constraints = .{
5841 .{ .scalar_float = .{ .of = .yword, .is = .qword } },
5842 .{ .scalar_float = .{ .of = .yword, .is = .qword } },
5843 .any,
5844 },
5845 .patterns = &.{
5846 .{ .src = .{ .to_sse, .mem, .none } },
5847 .{ .src = .{ .to_sse, .to_sse, .none } },
5848 },
5849 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
5850 .each = .{ .once = &.{
5851 .{ ._, .v_pd, .sub, .dst0y, .src0y, .src1y, ._ },
5852 } },
5853 }, .{
5854 .required_features = .{ .avx, null, null, null },
5855 .src_constraints = .{
5856 .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } },
5857 .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } },
5858 .any,
5859 },
5860 .patterns = &.{
5861 .{ .src = .{ .to_mem, .to_mem, .none } },
5862 },
5863 .extra_temps = .{
5864 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
5865 .{ .type = .vector_4_f64, .kind = .{ .rc = .sse } },
5866 .unused,
5867 .unused,
5868 .unused,
5869 .unused,
5870 .unused,
5871 .unused,
5872 .unused,
5873 .unused,
5874 .unused,
5875 },
5876 .dst_temps = .{ .mem, .unused },
5877 .clobbers = .{ .eflags = true },
5878 .each = .{ .once = &.{
5879 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
5880 .{ .@"0:", .v_pd, .mova, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ },
5881 .{ ._, .v_pd, .sub, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ },
5882 .{ ._, .v_pd, .mova, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ },
5883 .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ },
5884 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
5885 } },
5886 }, .{
5887 .required_features = .{ .sse2, null, null, null },
5888 .src_constraints = .{
5889 .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } },
5890 .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } },
5891 .any,
5892 },
5893 .patterns = &.{
5894 .{ .src = .{ .to_mem, .to_mem, .none } },
5895 },
5896 .extra_temps = .{
5897 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
5898 .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } },
5899 .unused,
5900 .unused,
5901 .unused,
5902 .unused,
5903 .unused,
5904 .unused,
5905 .unused,
5906 .unused,
5907 .unused,
5908 },
5909 .dst_temps = .{ .mem, .unused },
5910 .clobbers = .{ .eflags = true },
5911 .each = .{ .once = &.{
5912 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
5913 .{ .@"0:", ._pd, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ },
5914 .{ ._, ._pd, .sub, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ },
5915 .{ ._, ._pd, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ },
5916 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
5917 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
5918 } },
5919 }, .{
5920 .required_features = .{ .x87, null, null, null },
5921 .src_constraints = .{
5922 .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } },
5923 .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } },
5924 .any,
5925 },
5926 .patterns = &.{
5927 .{ .src = .{ .to_mem, .to_mem, .none } },
5928 },
5929 .extra_temps = .{
5930 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
5931 .{ .type = .f64, .kind = .{ .reg = .st6 } },
5932 .{ .type = .f64, .kind = .{ .reg = .st7 } },
5933 .unused,
5934 .unused,
5935 .unused,
5936 .unused,
5937 .unused,
5938 .unused,
5939 .unused,
5940 .unused,
5941 },
5942 .dst_temps = .{ .mem, .unused },
5943 .each = .{ .once = &.{
5944 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
5945 .{ .@"0:", .f_, .ld, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._, ._ },
5946 .{ ._, .f_, .sub, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._, ._ },
5947 .{ ._, .f_p, .st, .memia(.dst0q, .tmp0, .add_unaligned_size), ._, ._, ._ },
5948 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
5949 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
5950 } },
5951 }, .{
5952 .required_features = .{ .x87, null, null, null },
5953 .src_constraints = .{
5954 .{ .scalar_float = .{ .of = .xword, .is = .tbyte } },
5955 .{ .scalar_float = .{ .of = .xword, .is = .tbyte } },
5956 .any,
5957 },
5958 .patterns = &.{
5959 .{ .src = .{ .mem, .mem, .none } },
5960 },
5961 .extra_temps = .{
5962 .{ .type = .f80, .kind = .{ .reg = .st6 } },
5963 .{ .type = .f80, .kind = .{ .reg = .st7 } },
5964 .unused,
5965 .unused,
5966 .unused,
5967 .unused,
5968 .unused,
5969 .unused,
5970 .unused,
5971 .unused,
5972 .unused,
5973 },
5974 .dst_temps = .{ .{ .rc = .x87 }, .unused },
5975 .each = .{ .once = &.{
5976 .{ ._, .f_, .ld, .src0t, ._, ._, ._ },
5977 .{ ._, .f_, .ld, .src1t, ._, ._, ._ },
5978 .{ ._, .f_p, .sub, ._, ._, ._, ._ },
5979 .{ ._, .f_p, .st, .dst0t, ._, ._, ._ },
5980 } },
5981 }, .{
5982 .required_features = .{ .x87, null, null, null },
5983 .src_constraints = .{
5984 .{ .scalar_float = .{ .of = .xword, .is = .tbyte } },
5985 .{ .scalar_float = .{ .of = .xword, .is = .tbyte } },
5986 .any,
5987 },
5988 .patterns = &.{
5989 .{ .src = .{ .to_x87, .mem, .none }, .commute = .{ 0, 1 } },
5990 },
5991 .extra_temps = .{
5992 .{ .type = .f80, .kind = .{ .reg = .st7 } },
5993 .unused,
5994 .unused,
5995 .unused,
5996 .unused,
5997 .unused,
5998 .unused,
5999 .unused,
6000 .unused,
6001 .unused,
6002 .unused,
6003 },
6004 .dst_temps = .{ .{ .rc = .x87 }, .unused },
6005 .each = .{ .once = &.{
6006 .{ ._, .f_, .ld, .src0t, ._, ._, ._ },
6007 .{ ._, .f_, .subr, .tmp0t, .src1t, ._, ._ },
6008 .{ ._, .f_p, .st, .dst0t, ._, ._, ._ },
6009 } },
6010 }, .{
6011 .required_features = .{ .x87, null, null, null },
6012 .src_constraints = .{
6013 .{ .scalar_float = .{ .of = .xword, .is = .tbyte } },
6014 .{ .scalar_float = .{ .of = .xword, .is = .tbyte } },
6015 .any,
6016 },
6017 .patterns = &.{
6018 .{ .src = .{ .mem, .to_x87, .none } },
6019 .{ .src = .{ .to_x87, .to_x87, .none } },
6020 },
6021 .extra_temps = .{
6022 .{ .type = .f80, .kind = .{ .reg = .st7 } },
6023 .unused,
6024 .unused,
6025 .unused,
6026 .unused,
6027 .unused,
6028 .unused,
6029 .unused,
6030 .unused,
6031 .unused,
6032 .unused,
6033 },
6034 .dst_temps = .{ .{ .rc = .x87 }, .unused },
6035 .each = .{ .once = &.{
6036 .{ ._, .f_, .ld, .src0t, ._, ._, ._ },
6037 .{ ._, .f_, .sub, .tmp0t, .src1t, ._, ._ },
6038 .{ ._, .f_p, .st, .dst0t, ._, ._, ._ },
6039 } },
6040 }, .{
6041 .required_features = .{ .x87, null, null, null },
6042 .src_constraints = .{
6043 .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } },
6044 .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } },
6045 .any,
6046 },
6047 .patterns = &.{
6048 .{ .src = .{ .to_mem, .to_mem, .none } },
6049 },
6050 .extra_temps = .{
6051 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
6052 .{ .type = .f80, .kind = .{ .reg = .st6 } },
6053 .{ .type = .f80, .kind = .{ .reg = .st7 } },
6054 .unused,
6055 .unused,
6056 .unused,
6057 .unused,
6058 .unused,
6059 .unused,
6060 .unused,
6061 .unused,
6062 },
6063 .dst_temps = .{ .mem, .unused },
6064 .each = .{ .once = &.{
6065 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
6066 .{ .@"0:", .f_, .ld, .memia(.src0t, .tmp0, .add_unaligned_size), ._, ._, ._ },
6067 .{ ._, .f_, .ld, .memia(.src1t, .tmp0, .add_unaligned_size), ._, ._, ._ },
6068 .{ ._, .f_p, .sub, ._, ._, ._, ._ },
6069 .{ ._, .f_p, .st, .memia(.dst0t, .tmp0, .add_unaligned_size), ._, ._, ._ },
6070 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
6071 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
6072 } },
6073 }, .{
6074 .required_features = .{ .sse, null, null, null },
6075 .src_constraints = .{
6076 .{ .scalar_float = .{ .of = .xword, .is = .xword } },
6077 .{ .scalar_float = .{ .of = .xword, .is = .xword } },
6078 .any,
6079 },
6080 .patterns = &.{
6081 .{ .src = .{ .{ .to_reg = .xmm0 }, .{ .to_reg = .xmm1 }, .none } },
6082 },
6083 .call_frame = .{ .alignment = .@"16" },
6084 .extra_temps = .{
6085 .{ .type = .usize, .kind = .{ .symbol = &.{ .name = "__subtf3" } } },
6086 .unused,
6087 .unused,
6088 .unused,
6089 .unused,
6090 .unused,
6091 .unused,
6092 .unused,
6093 .unused,
6094 .unused,
6095 .unused,
6096 },
6097 .dst_temps = .{ .{ .ref = .src0 }, .unused },
6098 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
6099 .each = .{ .once = &.{
6100 .{ ._, ._, .call, .tmp0d, ._, ._, ._ },
6101 } },
6102 }, .{
6103 .required_features = .{ .avx, null, null, null },
6104 .src_constraints = .{
6105 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
6106 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
6107 .any,
6108 },
6109 .patterns = &.{
6110 .{ .src = .{ .to_mem, .to_mem, .none } },
6111 },
6112 .call_frame = .{ .alignment = .@"16" },
6113 .extra_temps = .{
6114 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
6115 .{ .type = .f128, .kind = .{ .reg = .xmm0 } },
6116 .{ .type = .f128, .kind = .{ .reg = .xmm1 } },
6117 .{ .type = .usize, .kind = .{ .symbol = &.{ .name = "__subtf3" } } },
6118 .unused,
6119 .unused,
6120 .unused,
6121 .unused,
6122 .unused,
6123 .unused,
6124 .unused,
6125 },
6126 .dst_temps = .{ .mem, .unused },
6127 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
6128 .each = .{ .once = &.{
6129 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
6130 .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ },
6131 .{ ._, .v_dqa, .mov, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ },
6132 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
6133 .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ },
6134 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
6135 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
6136 } },
6137 }, .{
6138 .required_features = .{ .sse2, null, null, null },
6139 .src_constraints = .{
6140 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
6141 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
6142 .any,
6143 },
6144 .patterns = &.{
6145 .{ .src = .{ .to_mem, .to_mem, .none } },
6146 },
6147 .call_frame = .{ .alignment = .@"16" },
6148 .extra_temps = .{
6149 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
6150 .{ .type = .f128, .kind = .{ .reg = .xmm0 } },
6151 .{ .type = .f128, .kind = .{ .reg = .xmm1 } },
6152 .{ .type = .usize, .kind = .{ .symbol = &.{ .name = "__subtf3" } } },
6153 .unused,
6154 .unused,
6155 .unused,
6156 .unused,
6157 .unused,
6158 .unused,
6159 .unused,
6160 },
6161 .dst_temps = .{ .mem, .unused },
6162 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
6163 .each = .{ .once = &.{
6164 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
6165 .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ },
6166 .{ ._, ._dqa, .mov, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ },
6167 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
6168 .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ },
6169 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
6170 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
6171 } },
6172 }, .{
6173 .required_features = .{ .sse, null, null, null },
6174 .src_constraints = .{
6175 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
6176 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
6177 .any,
6178 },
6179 .patterns = &.{
6180 .{ .src = .{ .to_mem, .to_mem, .none } },
6181 },
6182 .call_frame = .{ .alignment = .@"16" },
6183 .extra_temps = .{
6184 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
6185 .{ .type = .f128, .kind = .{ .reg = .xmm0 } },
6186 .{ .type = .f128, .kind = .{ .reg = .xmm1 } },
6187 .{ .type = .usize, .kind = .{ .symbol = &.{ .name = "__subtf3" } } },
6188 .unused,
6189 .unused,
6190 .unused,
6191 .unused,
6192 .unused,
6193 .unused,
6194 .unused,
6195 },
6196 .dst_temps = .{ .mem, .unused },
6197 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
6198 .each = .{ .once = &.{
6199 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
6200 .{ .@"0:", ._ps, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ },
6201 .{ ._, ._ps, .mova, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ },
6202 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
6203 .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ },
6204 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
6205 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
6206 } },
6207 } }) catch |err| switch (err) {
6208 error.SelectFailed => return cg.fail("failed to select {s} {} {} {}", .{
6209 @tagName(air_tag),
6210 cg.typeOf(bin_op.lhs).fmt(pt),
6211 ops[0].tracking(cg),
6212 ops[1].tracking(cg),
6213 }),
6214 else => |e| return e,
6215 };
6216 switch (air_tag) {
6217 else => unreachable,
6218 .sub, .sub_optimized => {},
6219 .sub_wrap => res[0].wrapInt(cg) catch |err| switch (err) {
6220 error.SelectFailed => return cg.fail("failed to select wrap {} {} {}", .{
6221 cg.typeOf(bin_op.lhs).fmt(pt),
6222 ops[0].tracking(cg),
6223 ops[1].tracking(cg),
6224 }),
6225 else => |e| return e,
6226 },
6227 }
6228 try res[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg);
6229 },
6230 .sub_safe => unreachable,
6231 .mul, .mul_optimized => |air_tag| if (use_old) try cg.airMulDivBinOp(inst, .mul) else {
6232 const bin_op = air_datas[@intFromEnum(inst)].bin_op;
6233 const ty = cg.typeOf(bin_op.lhs);
6234 var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs });
6235 var res: [1]Temp = undefined;
6236 cg.select(&res, &.{ty}, &ops, comptime &.{ .{
6237 .src_constraints = .{ .{ .signed_int = .byte }, .{ .signed_int = .byte }, .any },
6238 .patterns = &.{
6239 .{ .src = .{ .{ .to_reg = .al }, .mem, .none } },
6240 .{ .src = .{ .mem, .{ .to_reg = .al }, .none }, .commute = .{ 0, 1 } },
6241 .{ .src = .{ .{ .to_reg = .al }, .to_gpr, .none } },
6242 },
6243 .dst_temps = .{ .{ .ref = .src0 }, .unused },
6244 .clobbers = .{ .eflags = true },
6245 .each = .{ .once = &.{
6246 .{ ._, .i_, .mul, .src1b, ._, ._, ._ },
6247 } },
6248 }, .{
6249 .src_constraints = .{ .{ .unsigned_int = .byte }, .{ .unsigned_int = .byte }, .any },
6250 .patterns = &.{
6251 .{ .src = .{ .{ .to_reg = .al }, .mem, .none } },
6252 .{ .src = .{ .mem, .{ .to_reg = .al }, .none }, .commute = .{ 0, 1 } },
6253 .{ .src = .{ .{ .to_reg = .al }, .to_gpr, .none } },
6254 },
6255 .dst_temps = .{ .{ .ref = .src0 }, .unused },
6256 .clobbers = .{ .eflags = true },
6257 .each = .{ .once = &.{
6258 .{ ._, ._, .mul, .src1b, ._, ._, ._ },
6259 } },
6260 }, .{
6261 .src_constraints = .{ .{ .int = .word }, .{ .int = .word }, .any },
6262 .patterns = &.{
6263 .{ .src = .{ .mem, .imm16, .none } },
6264 .{ .src = .{ .imm16, .mem, .none }, .commute = .{ 0, 1 } },
6265 .{ .src = .{ .to_gpr, .imm16, .none } },
6266 .{ .src = .{ .imm16, .to_gpr, .none }, .commute = .{ 0, 1 } },
6267 },
6268 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }, .unused },
6269 .clobbers = .{ .eflags = true },
6270 .each = .{ .once = &.{
6271 .{ ._, .i_, .mul, .dst0w, .src0w, .src1w, ._ },
6272 } },
6273 }, .{
6274 .src_constraints = .{ .{ .int = .word }, .{ .int = .word }, .any },
6275 .patterns = &.{
6276 .{ .src = .{ .to_mut_gpr, .mem, .none } },
6277 .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } },
6278 .{ .src = .{ .to_mut_gpr, .to_gpr, .none } },
6279 },
6280 .dst_temps = .{ .{ .ref = .src0 }, .unused },
6281 .clobbers = .{ .eflags = true },
6282 .each = .{ .once = &.{
6283 .{ ._, .i_, .mul, .dst0w, .src1w, ._, ._ },
6284 } },
6285 }, .{
6286 .src_constraints = .{ .{ .int = .dword }, .{ .int = .dword }, .any },
6287 .patterns = &.{
6288 .{ .src = .{ .mem, .imm32, .none } },
6289 .{ .src = .{ .imm32, .mem, .none }, .commute = .{ 0, 1 } },
6290 .{ .src = .{ .to_gpr, .imm32, .none } },
6291 .{ .src = .{ .imm32, .to_gpr, .none }, .commute = .{ 0, 1 } },
6292 },
6293 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }, .unused },
6294 .clobbers = .{ .eflags = true },
6295 .each = .{ .once = &.{
6296 .{ ._, .i_, .mul, .dst0d, .src0d, .src1d, ._ },
6297 } },
6298 }, .{
6299 .src_constraints = .{ .{ .int = .dword }, .{ .int = .dword }, .any },
6300 .patterns = &.{
6301 .{ .src = .{ .to_mut_gpr, .mem, .none } },
6302 .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } },
6303 .{ .src = .{ .to_mut_gpr, .to_gpr, .none } },
6304 },
6305 .dst_temps = .{ .{ .ref = .src0 }, .unused },
6306 .clobbers = .{ .eflags = true },
6307 .each = .{ .once = &.{
6308 .{ ._, .i_, .mul, .dst0d, .src1d, ._, ._ },
6309 } },
6310 }, .{
6311 .required_features = .{ .@"64bit", null, null, null },
6312 .src_constraints = .{ .{ .int = .qword }, .{ .int = .qword }, .any },
6313 .patterns = &.{
6314 .{ .src = .{ .mem, .simm32, .none } },
6315 .{ .src = .{ .simm32, .mem, .none }, .commute = .{ 0, 1 } },
6316 .{ .src = .{ .to_gpr, .simm32, .none } },
6317 .{ .src = .{ .simm32, .to_gpr, .none }, .commute = .{ 0, 1 } },
6318 },
6319 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }, .unused },
6320 .clobbers = .{ .eflags = true },
6321 .each = .{ .once = &.{
6322 .{ ._, .i_, .mul, .dst0q, .src0q, .src1q, ._ },
6323 } },
6324 }, .{
6325 .required_features = .{ .@"64bit", null, null, null },
6326 .src_constraints = .{ .{ .int = .qword }, .{ .int = .qword }, .any },
6327 .patterns = &.{
6328 .{ .src = .{ .to_mut_gpr, .mem, .none } },
6329 .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } },
6330 .{ .src = .{ .to_mut_gpr, .to_gpr, .none } },
6331 },
6332 .dst_temps = .{ .{ .ref = .src0 }, .unused },
6333 .clobbers = .{ .eflags = true },
6334 .each = .{ .once = &.{
6335 .{ ._, .i_, .mul, .dst0q, .src1q, ._, ._ },
6336 } },
6337 }, .{
6338 .required_features = .{ .@"64bit", null, null, null },
6339 .src_constraints = .{ .{ .int = .xword }, .{ .int = .xword }, .any },
6340 .patterns = &.{
6341 .{ .src = .{ .to_mem, .to_mem, .none } },
6342 },
6343 .extra_temps = .{
6344 .{ .type = .u64, .kind = .{ .reg = .rax } },
6345 .{ .type = .u64, .kind = .{ .reg = .rdx } },
6346 .unused,
6347 .unused,
6348 .unused,
6349 .unused,
6350 .unused,
6351 .unused,
6352 .unused,
6353 .unused,
6354 .unused,
6355 },
6356 .dst_temps = .{ .mem, .unused },
6357 .clobbers = .{ .eflags = true },
6358 .each = .{ .once = &.{
6359 .{ ._, ._, .mov, .tmp0q, .src0q, ._, ._ },
6360 .{ ._, ._, .mul, .src1q, ._, ._, ._ },
6361 .{ ._, ._, .mov, .dst0q, .tmp0q, ._, ._ },
6362 .{ ._, ._, .mov, .tmp0q, .src0q, ._, ._ },
6363 .{ ._, .i_, .mul, .tmp0q, .memd(.src1q, 8), ._, ._ },
6364 .{ ._, ._, .add, .tmp1q, .tmp0q, ._, ._ },
6365 .{ ._, ._, .mov, .tmp0q, .src1q, ._, ._ },
6366 .{ ._, .i_, .mul, .tmp0q, .memd(.src0q, 8), ._, ._ },
6367 .{ ._, ._, .add, .tmp1q, .tmp0q, ._, ._ },
6368 .{ ._, ._, .mov, .memd(.dst0q, 8), .tmp1q, ._, ._ },
6369 } },
6370 }, .{
6371 .required_features = .{ .@"64bit", .bmi2, .adx, null },
6372 .src_constraints = .{
6373 .{ .remainder_int = .{ .of = .qword, .is = .qword } },
6374 .{ .remainder_int = .{ .of = .qword, .is = .qword } },
6375 .any,
6376 },
6377 .patterns = &.{
6378 .{ .src = .{ .to_mem, .to_mem, .none } },
6379 },
6380 .extra_temps = .{
6381 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
6382 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
6383 .{ .type = .u64, .kind = .{ .reg = .rdx } },
6384 .{ .type = .isize, .kind = .{ .reg = .rcx } },
6385 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
6386 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
6387 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
6388 .unused,
6389 .unused,
6390 .unused,
6391 .unused,
6392 },
6393 .dst_temps = .{ .mem, .unused },
6394 .clobbers = .{ .eflags = true },
6395 .each = .{ .once = &.{
6396 .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ },
6397 .{ ._, ._, .lea, .tmp1p, .mem(.src1), ._, ._ },
6398 .{ .@"0:", ._, .xor, .tmp2d, .tmp2d, ._, ._ },
6399 .{ ._, ._, .@"or", .tmp2q, .memi(.src0q, .tmp0), ._, ._ },
6400 .{ ._, ._z, .j, .@"2f", ._, ._, ._ },
6401 .{ ._, ._, .lea, .tmp3p, .leaad(.tmp0, .sub_src0_size, 8), ._, ._ },
6402 .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ },
6403 .{ .@"1:", ._x, .mul, .tmp6q, .tmp5q, .leai(.tmp1q, .tmp3), ._ },
6404 .{ ._, ._x, .adc, .tmp5q, .tmp4q, ._, ._ },
6405 .{ ._, ._, .mov, .memiad(.dst0q, .tmp3, .add_size, -8), .tmp5q, ._, ._ },
6406 .{ ._, ._rcxz, .j, .@"1f", ._, ._, ._ },
6407 .{ ._, ._x, .ado, .tmp6q, .memia(.dst0q, .tmp3, .add_size), ._, ._ },
6408 .{ ._, ._, .mov, .tmp4q, .tmp6q, ._, ._ },
6409 .{ ._, ._, .lea, .tmp3p, .lead(.tmp3, 8), ._, ._ },
6410 .{ ._, ._mp, .j, .@"1b", ._, ._, ._ },
6411 .{ .@"2:", ._, .mov, .memi(.dst0q, .tmp0), .tmp2q, ._, ._ },
6412 .{ .@"1:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ },
6413 .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ },
6414 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
6415 } },
6416 }, .{
6417 .required_features = .{ .@"64bit", .bmi2, .slow_incdec, null },
6418 .src_constraints = .{
6419 .{ .remainder_int = .{ .of = .qword, .is = .qword } },
6420 .{ .remainder_int = .{ .of = .qword, .is = .qword } },
6421 .any,
6422 },
6423 .patterns = &.{
6424 .{ .src = .{ .to_mem, .to_mem, .none } },
6425 },
6426 .extra_temps = .{
6427 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
6428 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
6429 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
6430 .{ .type = .u64, .kind = .{ .reg = .rdx } },
6431 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
6432 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
6433 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
6434 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
6435 .unused,
6436 .unused,
6437 .unused,
6438 },
6439 .dst_temps = .{ .mem, .unused },
6440 .clobbers = .{ .eflags = true },
6441 .each = .{ .once = &.{
6442 .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ },
6443 .{ ._, ._, .lea, .tmp1p, .memd(.src1, 8), ._, ._ },
6444 .{ .@"0:", ._, .lea, .tmp2p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ },
6445 .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ },
6446 .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ },
6447 .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ },
6448 .{ ._, ._, .@"or", .tmp3q, .memsi(.src0q, .@"8", .tmp0), ._, ._ },
6449 .{ ._, ._nz, .j, .@"2f", ._, ._, ._ },
6450 .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ },
6451 .{ ._, ._mp, .j, .@"3f", ._, ._, ._ },
6452 .{ .@"1:", ._, .adc, .tmp7q, .memsia(.dst0q, .@"8", .tmp2, .add_size), ._, ._ },
6453 .{ ._, ._, .adc, .tmp4b, .si(0), ._, ._ },
6454 .{ ._, ._, .mov, .tmp5q, .tmp7q, ._, ._ },
6455 .{ ._, ._l, .sh, .tmp4b, .ui(4), ._, ._ },
6456 .{ .@"2:", ._x, .mul, .tmp7q, .tmp6q, .leasi(.tmp1q, .@"8", .tmp2), ._ },
6457 .{ ._, ._, .adc, .tmp6q, .tmp5q, ._, ._ },
6458 .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp2, .add_size), .tmp6q, ._, ._ },
6459 .{ ._, ._c, .in, .tmp2p, ._, ._, ._ },
6460 .{ ._, ._nz, .j, .@"1b", ._, ._, ._ },
6461 .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ },
6462 .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ },
6463 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
6464 } },
6465 }, .{
6466 .required_features = .{ .@"64bit", .bmi2, null, null },
6467 .src_constraints = .{
6468 .{ .remainder_int = .{ .of = .qword, .is = .qword } },
6469 .{ .remainder_int = .{ .of = .qword, .is = .qword } },
6470 .any,
6471 },
6472 .patterns = &.{
6473 .{ .src = .{ .to_mem, .to_mem, .none } },
6474 },
6475 .extra_temps = .{
6476 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
6477 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
6478 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
6479 .{ .type = .u64, .kind = .{ .reg = .rdx } },
6480 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
6481 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
6482 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
6483 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
6484 .unused,
6485 .unused,
6486 .unused,
6487 },
6488 .dst_temps = .{ .mem, .unused },
6489 .clobbers = .{ .eflags = true },
6490 .each = .{ .once = &.{
6491 .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ },
6492 .{ ._, ._, .lea, .tmp1p, .memd(.src1, 8), ._, ._ },
6493 .{ .@"0:", ._, .lea, .tmp2p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ },
6494 .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ },
6495 .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ },
6496 .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ },
6497 .{ ._, ._, .@"or", .tmp3q, .memsi(.src0q, .@"8", .tmp0), ._, ._ },
6498 .{ ._, ._nz, .j, .@"2f", ._, ._, ._ },
6499 .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ },
6500 .{ ._, ._mp, .j, .@"3f", ._, ._, ._ },
6501 .{ .@"1:", ._, .adc, .tmp7q, .memsia(.dst0q, .@"8", .tmp2, .add_size), ._, ._ },
6502 .{ ._, ._, .adc, .tmp4b, .si(0), ._, ._ },
6503 .{ ._, ._, .mov, .tmp5q, .tmp7q, ._, ._ },
6504 .{ ._, ._l, .sh, .tmp4b, .ui(4), ._, ._ },
6505 .{ .@"2:", ._x, .mul, .tmp7q, .tmp6q, .leasi(.tmp1q, .@"8", .tmp2), ._ },
6506 .{ ._, ._, .adc, .tmp6q, .tmp5q, ._, ._ },
6507 .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp2, .add_size), .tmp6q, ._, ._ },
6508 .{ ._, ._c, .in, .tmp2p, ._, ._, ._ },
6509 .{ ._, ._nz, .j, .@"1b", ._, ._, ._ },
6510 .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ },
6511 .{ ._, ._c, .de, .tmp0d, ._, ._, ._ },
6512 .{ ._, ._ns, .j, .@"0b", ._, ._, ._ },
6513 } },
6514 }, .{
6515 .required_features = .{ .@"64bit", .slow_incdec, null, null },
6516 .src_constraints = .{
6517 .{ .remainder_int = .{ .of = .qword, .is = .qword } },
6518 .{ .remainder_int = .{ .of = .qword, .is = .qword } },
6519 .any,
6520 },
6521 .patterns = &.{
6522 .{ .src = .{ .to_mem, .to_mem, .none } },
6523 },
6524 .extra_temps = .{
6525 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
6526 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
6527 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
6528 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
6529 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
6530 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
6531 .{ .type = .u64, .kind = .{ .reg = .rax } },
6532 .{ .type = .u64, .kind = .{ .reg = .rdx } },
6533 .unused,
6534 .unused,
6535 .unused,
6536 },
6537 .dst_temps = .{ .mem, .unused },
6538 .clobbers = .{ .eflags = true },
6539 .each = .{ .once = &.{
6540 .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ },
6541 .{ ._, ._, .lea, .tmp1p, .memd(.src1, 8), ._, ._ },
6542 .{ .@"0:", ._, .lea, .tmp2p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ },
6543 .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ },
6544 .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ },
6545 .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ },
6546 .{ ._, ._, .@"or", .tmp3q, .memsi(.src0q, .@"8", .tmp0), ._, ._ },
6547 .{ ._, ._nz, .j, .@"2f", ._, ._, ._ },
6548 .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ },
6549 .{ ._, ._mp, .j, .@"3f", ._, ._, ._ },
6550 .{ .@"1:", ._, .adc, .tmp7q, .memsia(.dst0q, .@"8", .tmp2, .add_size), ._, ._ },
6551 .{ ._, ._, .adc, .tmp4b, .si(0), ._, ._ },
6552 .{ ._, ._, .mov, .tmp5q, .tmp7q, ._, ._ },
6553 .{ .@"2:", ._, .mov, .tmp6q, .tmp3q, ._, ._ },
6554 .{ ._, ._, .mul, .leasi(.tmp1q, .@"8", .tmp2), ._, ._, ._ },
6555 .{ ._, ._l, .sh, .tmp4b, .ui(4), ._, ._ },
6556 .{ ._, ._, .adc, .tmp6q, .tmp5q, ._, ._ },
6557 .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp2, .add_size), .tmp6q, ._, ._ },
6558 .{ ._, ._c, .in, .tmp2p, ._, ._, ._ },
6559 .{ ._, ._nz, .j, .@"1b", ._, ._, ._ },
6560 .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ },
6561 .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ },
6562 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
6563 } },
6564 }, .{
6565 .required_features = .{ .@"64bit", null, null, null },
6566 .src_constraints = .{
6567 .{ .remainder_int = .{ .of = .qword, .is = .qword } },
6568 .{ .remainder_int = .{ .of = .qword, .is = .qword } },
6569 .any,
6570 },
6571 .patterns = &.{
6572 .{ .src = .{ .to_mem, .to_mem, .none } },
6573 },
6574 .extra_temps = .{
6575 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
6576 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
6577 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
6578 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
6579 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
6580 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
6581 .{ .type = .u64, .kind = .{ .reg = .rax } },
6582 .{ .type = .u64, .kind = .{ .reg = .rdx } },
6583 .unused,
6584 .unused,
6585 .unused,
6586 },
6587 .dst_temps = .{ .mem, .unused },
6588 .clobbers = .{ .eflags = true },
6589 .each = .{ .once = &.{
6590 .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ },
6591 .{ ._, ._, .lea, .tmp1p, .memd(.src1, 8), ._, ._ },
6592 .{ .@"0:", ._, .lea, .tmp2p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ },
6593 .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ },
6594 .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ },
6595 .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ },
6596 .{ ._, ._, .@"or", .tmp3q, .memsi(.src0q, .@"8", .tmp0), ._, ._ },
6597 .{ ._, ._nz, .j, .@"2f", ._, ._, ._ },
6598 .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ },
6599 .{ ._, ._mp, .j, .@"3f", ._, ._, ._ },
6600 .{ .@"1:", ._, .adc, .tmp7q, .memsia(.dst0q, .@"8", .tmp2, .add_size), ._, ._ },
6601 .{ ._, ._, .adc, .tmp4b, .si(0), ._, ._ },
6602 .{ ._, ._, .mov, .tmp5q, .tmp7q, ._, ._ },
6603 .{ .@"2:", ._, .mov, .tmp6q, .tmp3q, ._, ._ },
6604 .{ ._, ._, .mul, .leasi(.tmp1q, .@"8", .tmp2), ._, ._, ._ },
6605 .{ ._, ._l, .sh, .tmp4b, .ui(4), ._, ._ },
6606 .{ ._, ._, .adc, .tmp6q, .tmp5q, ._, ._ },
6607 .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp2, .add_size), .tmp6q, ._, ._ },
6608 .{ ._, ._c, .in, .tmp2p, ._, ._, ._ },
6609 .{ ._, ._nz, .j, .@"1b", ._, ._, ._ },
6610 .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ },
6611 .{ ._, ._c, .de, .tmp0d, ._, ._, ._ },
6612 .{ ._, ._ns, .j, .@"0b", ._, ._, ._ },
6613 } },
6614 }, .{
6615 .required_features = .{ .avx, null, null, null },
6616 .src_constraints = .{
6617 .{ .scalar_signed_int = .{ .of = .qword, .is = .byte } },
6618 .{ .scalar_signed_int = .{ .of = .qword, .is = .byte } },
6619 .any,
6620 },
6621 .patterns = &.{
6622 .{ .src = .{ .mem, .mem, .none } },
6623 .{ .src = .{ .to_sse, .mem, .none } },
6624 .{ .src = .{ .mem, .to_sse, .none } },
6625 .{ .src = .{ .to_sse, .to_sse, .none } },
6626 },
6627 .extra_temps = .{
6628 .{ .type = .vector_8_i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } },
6629 .unused,
6630 .unused,
6631 .unused,
6632 .unused,
6633 .unused,
6634 .unused,
6635 .unused,
6636 .unused,
6637 .unused,
6638 .unused,
6639 },
6640 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
6641 .each = .{ .once = &.{
6642 .{ ._, .vp_w, .movsxb, .dst0x, .src0q, ._, ._ },
6643 .{ ._, .vp_w, .movsxb, .tmp0x, .src1q, ._, ._ },
6644 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ },
6645 .{ ._, .vp_b, .ackssw, .dst0x, .dst0x, .dst0x, ._ },
6646 } },
6647 }, .{
6648 .required_features = .{ .sse4_1, null, null, null },
6649 .src_constraints = .{
6650 .{ .scalar_signed_int = .{ .of = .qword, .is = .byte } },
6651 .{ .scalar_signed_int = .{ .of = .qword, .is = .byte } },
6652 .any,
6653 },
6654 .patterns = &.{
6655 .{ .src = .{ .mem, .mem, .none } },
6656 .{ .src = .{ .to_sse, .mem, .none } },
6657 .{ .src = .{ .mem, .to_sse, .none } },
6658 .{ .src = .{ .to_sse, .to_sse, .none } },
6659 },
6660 .extra_temps = .{
6661 .{ .type = .vector_8_i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } },
6662 .unused,
6663 .unused,
6664 .unused,
6665 .unused,
6666 .unused,
6667 .unused,
6668 .unused,
6669 .unused,
6670 .unused,
6671 .unused,
6672 },
6673 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
6674 .each = .{ .once = &.{
6675 .{ ._, .p_w, .movsxb, .dst0x, .src0q, ._, ._ },
6676 .{ ._, .p_w, .movsxb, .tmp0x, .src1q, ._, ._ },
6677 .{ ._, .p_w, .mull, .dst0x, .tmp0x, ._, ._ },
6678 .{ ._, .p_b, .ackssw, .dst0x, .dst0x, ._, ._ },
6679 } },
6680 }, .{
6681 .required_features = .{ .sse2, null, null, null },
6682 .src_constraints = .{
6683 .{ .scalar_signed_int = .{ .of = .qword, .is = .byte } },
6684 .{ .scalar_signed_int = .{ .of = .qword, .is = .byte } },
6685 .any,
6686 },
6687 .patterns = &.{
6688 .{ .src = .{ .to_mut_sse, .to_mut_sse, .none } },
6689 },
6690 .extra_temps = .{
6691 .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } },
6692 .unused,
6693 .unused,
6694 .unused,
6695 .unused,
6696 .unused,
6697 .unused,
6698 .unused,
6699 .unused,
6700 .unused,
6701 .unused,
6702 },
6703 .dst_temps = .{ .{ .ref = .src0 }, .unused },
6704 .each = .{ .once = &.{
6705 .{ ._, .p_, .xor, .tmp0x, .tmp0x, ._, ._ },
6706 .{ ._, .p_b, .cmpgt, .tmp0x, .src0x, ._, ._ },
6707 .{ ._, .p_, .unpcklbw, .dst0x, .tmp0x, ._, ._ },
6708 .{ ._, .p_, .xor, .tmp0x, .tmp0x, ._, ._ },
6709 .{ ._, .p_b, .cmpgt, .tmp0x, .src1x, ._, ._ },
6710 .{ ._, .p_, .unpcklbw, .src1x, .tmp0x, ._, ._ },
6711 .{ ._, .p_w, .mull, .dst0x, .src1x, ._, ._ },
6712 .{ ._, .p_b, .ackssw, .dst0x, .dst0x, ._, ._ },
6713 } },
6714 }, .{
6715 .required_features = .{ .avx2, null, null, null },
6716 .src_constraints = .{
6717 .{ .scalar_signed_int = .{ .of = .xword, .is = .byte } },
6718 .{ .scalar_signed_int = .{ .of = .xword, .is = .byte } },
6719 .any,
6720 },
6721 .patterns = &.{
6722 .{ .src = .{ .mem, .mem, .none } },
6723 .{ .src = .{ .to_sse, .mem, .none } },
6724 .{ .src = .{ .mem, .to_sse, .none } },
6725 .{ .src = .{ .to_sse, .to_sse, .none } },
6726 },
6727 .extra_temps = .{
6728 .{ .type = .vector_16_i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } },
6729 .unused,
6730 .unused,
6731 .unused,
6732 .unused,
6733 .unused,
6734 .unused,
6735 .unused,
6736 .unused,
6737 .unused,
6738 .unused,
6739 },
6740 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
6741 .each = .{ .once = &.{
6742 .{ ._, .vp_w, .movsxb, .dst0y, .src0x, ._, ._ },
6743 .{ ._, .vp_w, .movsxb, .tmp0y, .src1x, ._, ._ },
6744 .{ ._, .vp_w, .mull, .dst0y, .dst0y, .tmp0y, ._ },
6745 .{ ._, .vp_b, .ackssw, .dst0y, .dst0y, .dst0y, ._ },
6746 .{ ._, .v_q, .perm, .dst0y, .dst0y, .ui(0b10_00_10_00), ._ },
6747 } },
6748 }, .{
6749 .required_features = .{ .avx, null, null, null },
6750 .src_constraints = .{
6751 .{ .scalar_unsigned_int = .{ .of = .qword, .is = .byte } },
6752 .{ .scalar_unsigned_int = .{ .of = .qword, .is = .byte } },
6753 .any,
6754 },
6755 .patterns = &.{
6756 .{ .src = .{ .mem, .mem, .none } },
6757 .{ .src = .{ .to_sse, .mem, .none } },
6758 .{ .src = .{ .mem, .to_sse, .none } },
6759 .{ .src = .{ .to_sse, .to_sse, .none } },
6760 },
6761 .extra_temps = .{
6762 .{ .type = .vector_8_u16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } },
6763 .unused,
6764 .unused,
6765 .unused,
6766 .unused,
6767 .unused,
6768 .unused,
6769 .unused,
6770 .unused,
6771 .unused,
6772 .unused,
6773 },
6774 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
6775 .each = .{ .once = &.{
6776 .{ ._, .vp_w, .movzxb, .dst0x, .src0q, ._, ._ },
6777 .{ ._, .vp_w, .movzxb, .tmp0x, .src1q, ._, ._ },
6778 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ },
6779 .{ ._, .vp_b, .ackusw, .dst0x, .dst0x, .dst0x, ._ },
6780 } },
6781 }, .{
6782 .required_features = .{ .sse4_1, null, null, null },
6783 .src_constraints = .{
6784 .{ .scalar_unsigned_int = .{ .of = .qword, .is = .byte } },
6785 .{ .scalar_unsigned_int = .{ .of = .qword, .is = .byte } },
6786 .any,
6787 },
6788 .patterns = &.{
6789 .{ .src = .{ .mem, .mem, .none } },
6790 .{ .src = .{ .to_sse, .mem, .none } },
6791 .{ .src = .{ .mem, .to_sse, .none } },
6792 .{ .src = .{ .to_sse, .to_sse, .none } },
6793 },
6794 .extra_temps = .{
6795 .{ .type = .vector_8_u16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } },
6796 .unused,
6797 .unused,
6798 .unused,
6799 .unused,
6800 .unused,
6801 .unused,
6802 .unused,
6803 .unused,
6804 .unused,
6805 .unused,
6806 },
6807 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
6808 .each = .{ .once = &.{
6809 .{ ._, .p_w, .movzxb, .dst0x, .src0q, ._, ._ },
6810 .{ ._, .p_w, .movzxb, .tmp0x, .src1q, ._, ._ },
6811 .{ ._, .p_w, .mull, .dst0x, .tmp0x, ._, ._ },
6812 .{ ._, .p_b, .ackusw, .dst0x, .dst0x, ._, ._ },
6813 } },
6814 }, .{
6815 .required_features = .{ .sse2, null, null, null },
6816 .src_constraints = .{
6817 .{ .scalar_unsigned_int = .{ .of = .qword, .is = .byte } },
6818 .{ .scalar_unsigned_int = .{ .of = .qword, .is = .byte } },
6819 .any,
6820 },
6821 .patterns = &.{
6822 .{ .src = .{ .to_mut_sse, .to_mut_sse, .none } },
6823 },
6824 .extra_temps = .{
6825 .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } },
6826 .unused,
6827 .unused,
6828 .unused,
6829 .unused,
6830 .unused,
6831 .unused,
6832 .unused,
6833 .unused,
6834 .unused,
6835 .unused,
6836 },
6837 .dst_temps = .{ .{ .ref = .src0 }, .unused },
6838 .each = .{ .once = &.{
6839 .{ ._, .p_, .xor, .tmp0x, .tmp0x, ._, ._ },
6840 .{ ._, .p_, .unpcklbw, .dst0x, .tmp0x, ._, ._ },
6841 .{ ._, .p_, .unpcklbw, .src1x, .tmp0x, ._, ._ },
6842 .{ ._, .p_w, .mull, .dst0x, .src1x, ._, ._ },
6843 .{ ._, .p_b, .ackusw, .dst0x, .dst0x, ._, ._ },
6844 } },
6845 }, .{
6846 .required_features = .{ .avx2, null, null, null },
6847 .src_constraints = .{
6848 .{ .scalar_unsigned_int = .{ .of = .xword, .is = .byte } },
6849 .{ .scalar_unsigned_int = .{ .of = .xword, .is = .byte } },
6850 .any,
6851 },
6852 .patterns = &.{
6853 .{ .src = .{ .mem, .mem, .none } },
6854 .{ .src = .{ .to_sse, .mem, .none } },
6855 .{ .src = .{ .mem, .to_sse, .none } },
6856 .{ .src = .{ .to_sse, .to_sse, .none } },
6857 },
6858 .extra_temps = .{
6859 .{ .type = .vector_16_u16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } },
6860 .unused,
6861 .unused,
6862 .unused,
6863 .unused,
6864 .unused,
6865 .unused,
6866 .unused,
6867 .unused,
6868 .unused,
6869 .unused,
6870 },
6871 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
6872 .each = .{ .once = &.{
6873 .{ ._, .vp_w, .movzxb, .dst0y, .src0x, ._, ._ },
6874 .{ ._, .vp_w, .movzxb, .tmp0y, .src1x, ._, ._ },
6875 .{ ._, .vp_w, .mull, .dst0y, .dst0y, .tmp0y, ._ },
6876 .{ ._, .vp_b, .ackusw, .dst0y, .dst0y, .dst0y, ._ },
6877 .{ ._, .v_q, .perm, .dst0y, .dst0y, .ui(0b10_00_10_00), ._ },
6878 } },
6879 }, .{
6880 .required_features = .{ .avx2, null, null, null },
6881 .src_constraints = .{
6882 .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } },
6883 .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } },
6884 .any,
6885 },
6886 .patterns = &.{
6887 .{ .src = .{ .to_mem, .to_mem, .none } },
6888 },
6889 .extra_temps = .{
6890 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
6891 .{ .type = .vector_16_i16, .kind = .{ .rc = .sse } },
6892 .{ .type = .vector_16_i16, .kind = .{ .rc = .sse } },
6893 .unused,
6894 .unused,
6895 .unused,
6896 .unused,
6897 .unused,
6898 .unused,
6899 .unused,
6900 .unused,
6901 },
6902 .dst_temps = .{ .mem, .unused },
6903 .clobbers = .{ .eflags = true },
6904 .each = .{ .once = &.{
6905 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
6906 .{ .@"0:", .vp_w, .movsxb, .tmp1y, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ },
6907 .{ ._, .vp_w, .movsxb, .tmp2y, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ },
6908 .{ ._, .vp_w, .mull, .tmp1y, .tmp1y, .tmp2y, ._ },
6909 .{ ._, .vp_b, .ackssw, .tmp1y, .tmp1y, .tmp1y, ._ },
6910 .{ ._, .v_q, .perm, .tmp1y, .tmp1y, .ui(0b10_00_10_00), ._ },
6911 .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ },
6912 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
6913 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
6914 } },
6915 }, .{
6916 .required_features = .{ .avx, null, null, null },
6917 .src_constraints = .{
6918 .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .byte } },
6919 .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .byte } },
6920 .any,
6921 },
6922 .patterns = &.{
6923 .{ .src = .{ .to_mem, .to_mem, .none } },
6924 },
6925 .extra_temps = .{
6926 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
6927 .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } },
6928 .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } },
6929 .unused,
6930 .unused,
6931 .unused,
6932 .unused,
6933 .unused,
6934 .unused,
6935 .unused,
6936 .unused,
6937 },
6938 .dst_temps = .{ .mem, .unused },
6939 .clobbers = .{ .eflags = true },
6940 .each = .{ .once = &.{
6941 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
6942 .{ .@"0:", .vp_w, .movsxb, .tmp1x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ },
6943 .{ ._, .vp_w, .movsxb, .tmp2x, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ },
6944 .{ ._, .vp_w, .mull, .tmp1x, .tmp1x, .tmp2x, ._ },
6945 .{ ._, .vp_b, .ackssw, .tmp1x, .tmp1x, .tmp1x, ._ },
6946 .{ ._, .v_q, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ },
6947 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
6948 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
6949 } },
6950 }, .{
6951 .required_features = .{ .sse4_1, null, null, null },
6952 .src_constraints = .{
6953 .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .byte } },
6954 .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .byte } },
6955 .any,
6956 },
6957 .patterns = &.{
6958 .{ .src = .{ .to_mem, .to_mem, .none } },
6959 },
6960 .extra_temps = .{
6961 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
6962 .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } },
6963 .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } },
6964 .unused,
6965 .unused,
6966 .unused,
6967 .unused,
6968 .unused,
6969 .unused,
6970 .unused,
6971 .unused,
6972 },
6973 .dst_temps = .{ .mem, .unused },
6974 .clobbers = .{ .eflags = true },
6975 .each = .{ .once = &.{
6976 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
6977 .{ .@"0:", .p_w, .movsxb, .tmp1x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ },
6978 .{ ._, .p_w, .movsxb, .tmp2x, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ },
6979 .{ ._, .p_w, .mull, .tmp1x, .tmp2x, ._, ._ },
6980 .{ ._, .p_b, .ackssw, .tmp1x, .tmp1x, ._, ._ },
6981 .{ ._, ._q, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ },
6982 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
6983 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
6984 } },
6985 }, .{
6986 .required_features = .{ .sse2, null, null, null },
6987 .src_constraints = .{
6988 .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .byte } },
6989 .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .byte } },
6990 .any,
6991 },
6992 .patterns = &.{
6993 .{ .src = .{ .to_mem, .to_mem, .none } },
6994 },
6995 .extra_temps = .{
6996 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
6997 .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } },
6998 .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } },
6999 .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } },
7000 .unused,
7001 .unused,
7002 .unused,
7003 .unused,
7004 .unused,
7005 .unused,
7006 .unused,
7007 },
7008 .dst_temps = .{ .mem, .unused },
7009 .clobbers = .{ .eflags = true },
7010 .each = .{ .once = &.{
7011 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
7012 .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ },
7013 .{ ._, ._q, .mov, .tmp2x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ },
7014 .{ ._, .p_b, .cmpgt, .tmp1x, .tmp2x, ._, ._ },
7015 .{ ._, .p_, .unpcklbw, .tmp2x, .tmp1x, ._, ._ },
7016 .{ ._, .p_, .xor, .tmp1x, .tmp1x, ._, ._ },
7017 .{ ._, ._q, .mov, .tmp3x, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ },
7018 .{ ._, .p_b, .cmpgt, .tmp1x, .tmp3x, ._, ._ },
7019 .{ ._, .p_, .unpcklbw, .tmp3x, .tmp1x, ._, ._ },
7020 .{ ._, .p_w, .mull, .tmp2x, .tmp3x, ._, ._ },
7021 .{ ._, .p_b, .ackssw, .tmp2x, .tmp2x, ._, ._ },
7022 .{ ._, ._q, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp2x, ._, ._ },
7023 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
7024 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
7025 } },
7026 }, .{
7027 .required_features = .{ .slow_incdec, null, null, null },
7028 .src_constraints = .{
7029 .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } },
7030 .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } },
7031 .any,
7032 },
7033 .patterns = &.{
7034 .{ .src = .{ .to_mem, .to_mem, .none } },
7035 },
7036 .extra_temps = .{
7037 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
7038 .{ .type = .i8, .kind = .{ .reg = .al } },
7039 .unused,
7040 .unused,
7041 .unused,
7042 .unused,
7043 .unused,
7044 .unused,
7045 .unused,
7046 .unused,
7047 .unused,
7048 },
7049 .dst_temps = .{ .mem, .unused },
7050 .clobbers = .{ .eflags = true },
7051 .each = .{ .once = &.{
7052 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
7053 .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ },
7054 .{ ._, .i_, .mul, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._, ._ },
7055 .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ },
7056 .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
7057 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
7058 } },
7059 }, .{
7060 .src_constraints = .{
7061 .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } },
7062 .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } },
7063 .any,
7064 },
7065 .patterns = &.{
7066 .{ .src = .{ .to_mem, .to_mem, .none } },
7067 },
7068 .extra_temps = .{
7069 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
7070 .{ .type = .i8, .kind = .{ .reg = .al } },
7071 .unused,
7072 .unused,
7073 .unused,
7074 .unused,
7075 .unused,
7076 .unused,
7077 .unused,
7078 .unused,
7079 .unused,
7080 },
7081 .dst_temps = .{ .mem, .unused },
7082 .clobbers = .{ .eflags = true },
7083 .each = .{ .once = &.{
7084 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
7085 .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ },
7086 .{ ._, .i_, .mul, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._, ._ },
7087 .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ },
7088 .{ ._, ._c, .in, .tmp0p, ._, ._, ._ },
7089 .{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
7090 } },
7091 }, .{
7092 .required_features = .{ .avx2, null, null, null },
7093 .src_constraints = .{
7094 .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .byte } },
7095 .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .byte } },
7096 .any,
7097 },
7098 .patterns = &.{
7099 .{ .src = .{ .to_mem, .to_mem, .none } },
7100 },
7101 .extra_temps = .{
7102 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
7103 .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } },
7104 .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } },
7105 .unused,
7106 .unused,
7107 .unused,
7108 .unused,
7109 .unused,
7110 .unused,
7111 .unused,
7112 .unused,
7113 },
7114 .dst_temps = .{ .mem, .unused },
7115 .clobbers = .{ .eflags = true },
7116 .each = .{ .once = &.{
7117 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
7118 .{ .@"0:", .vp_w, .movzxb, .tmp1y, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ },
7119 .{ ._, .vp_w, .movzxb, .tmp2y, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ },
7120 .{ ._, .vp_w, .mull, .tmp1y, .tmp1y, .tmp2y, ._ },
7121 .{ ._, .vp_b, .ackusw, .tmp1y, .tmp1y, .tmp1y, ._ },
7122 .{ ._, .v_q, .perm, .tmp1y, .tmp1y, .ui(0b10_00_10_00), ._ },
7123 .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ },
7124 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
7125 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
7126 } },
7127 }, .{
7128 .required_features = .{ .avx, null, null, null },
7129 .src_constraints = .{
7130 .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .byte } },
7131 .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .byte } },
7132 .any,
7133 },
7134 .patterns = &.{
7135 .{ .src = .{ .to_mem, .to_mem, .none } },
7136 },
7137 .extra_temps = .{
7138 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
7139 .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } },
7140 .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } },
7141 .unused,
7142 .unused,
7143 .unused,
7144 .unused,
7145 .unused,
7146 .unused,
7147 .unused,
7148 .unused,
7149 },
7150 .dst_temps = .{ .mem, .unused },
7151 .clobbers = .{ .eflags = true },
7152 .each = .{ .once = &.{
7153 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
7154 .{ .@"0:", .vp_w, .movzxb, .tmp1x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ },
7155 .{ ._, .vp_w, .movzxb, .tmp2x, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ },
7156 .{ ._, .vp_w, .mull, .tmp1x, .tmp1x, .tmp2x, ._ },
7157 .{ ._, .vp_b, .ackusw, .tmp1x, .tmp1x, .tmp1x, ._ },
7158 .{ ._, .v_q, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ },
7159 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
7160 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
7161 } },
7162 }, .{
7163 .required_features = .{ .sse4_1, null, null, null },
7164 .src_constraints = .{
7165 .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .byte } },
7166 .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .byte } },
7167 .any,
7168 },
7169 .patterns = &.{
7170 .{ .src = .{ .to_mem, .to_mem, .none } },
7171 },
7172 .extra_temps = .{
7173 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
7174 .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } },
7175 .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } },
7176 .unused,
7177 .unused,
7178 .unused,
7179 .unused,
7180 .unused,
7181 .unused,
7182 .unused,
7183 .unused,
7184 },
7185 .dst_temps = .{ .mem, .unused },
7186 .clobbers = .{ .eflags = true },
7187 .each = .{ .once = &.{
7188 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
7189 .{ .@"0:", .p_w, .movzxb, .tmp1x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ },
7190 .{ ._, .p_w, .movzxb, .tmp2x, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ },
7191 .{ ._, .p_w, .mull, .tmp1x, .tmp2x, ._, ._ },
7192 .{ ._, .p_b, .ackusw, .tmp1x, .tmp1x, ._, ._ },
7193 .{ ._, ._q, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ },
7194 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
7195 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
7196 } },
7197 }, .{
7198 .required_features = .{ .sse2, null, null, null },
7199 .src_constraints = .{
7200 .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .byte } },
7201 .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .byte } },
7202 .any,
7203 },
7204 .patterns = &.{
7205 .{ .src = .{ .to_mem, .to_mem, .none } },
7206 },
7207 .extra_temps = .{
7208 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
7209 .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } },
7210 .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } },
7211 .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } },
7212 .unused,
7213 .unused,
7214 .unused,
7215 .unused,
7216 .unused,
7217 .unused,
7218 .unused,
7219 },
7220 .dst_temps = .{ .mem, .unused },
7221 .clobbers = .{ .eflags = true },
7222 .each = .{ .once = &.{
7223 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
7224 .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ },
7225 .{ ._, ._q, .mov, .tmp2x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ },
7226 .{ ._, .p_, .unpcklbw, .tmp2x, .tmp1x, ._, ._ },
7227 .{ ._, ._q, .mov, .tmp3x, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ },
7228 .{ ._, .p_, .unpcklbw, .tmp3x, .tmp1x, ._, ._ },
7229 .{ ._, .p_w, .mull, .tmp2x, .tmp3x, ._, ._ },
7230 .{ ._, .p_b, .ackusw, .tmp2x, .tmp2x, ._, ._ },
7231 .{ ._, ._q, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp2x, ._, ._ },
7232 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
7233 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
7234 } },
7235 }, .{
7236 .required_features = .{ .slow_incdec, null, null, null },
7237 .src_constraints = .{
7238 .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } },
7239 .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } },
7240 .any,
7241 },
7242 .patterns = &.{
7243 .{ .src = .{ .to_mem, .to_mem, .none } },
7244 },
7245 .extra_temps = .{
7246 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
7247 .{ .type = .u8, .kind = .{ .reg = .al } },
7248 .unused,
7249 .unused,
7250 .unused,
7251 .unused,
7252 .unused,
7253 .unused,
7254 .unused,
7255 .unused,
7256 .unused,
7257 },
7258 .dst_temps = .{ .mem, .unused },
7259 .clobbers = .{ .eflags = true },
7260 .each = .{ .once = &.{
7261 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
7262 .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ },
7263 .{ ._, ._, .mul, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._, ._ },
7264 .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ },
7265 .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
7266 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
7267 } },
7268 }, .{
7269 .src_constraints = .{
7270 .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } },
7271 .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } },
7272 .any,
7273 },
7274 .patterns = &.{
7275 .{ .src = .{ .to_mem, .to_mem, .none } },
7276 },
7277 .extra_temps = .{
7278 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
7279 .{ .type = .u8, .kind = .{ .reg = .al } },
7280 .unused,
7281 .unused,
7282 .unused,
7283 .unused,
7284 .unused,
7285 .unused,
7286 .unused,
7287 .unused,
7288 .unused,
7289 },
7290 .dst_temps = .{ .mem, .unused },
7291 .clobbers = .{ .eflags = true },
7292 .each = .{ .once = &.{
7293 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
7294 .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ },
7295 .{ ._, ._, .mul, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._, ._ },
7296 .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ },
7297 .{ ._, ._c, .in, .tmp0p, ._, ._, ._ },
7298 .{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
7299 } },
7300 }, .{
7301 .required_features = .{ .avx, null, null, null },
7302 .src_constraints = .{
7303 .{ .scalar_int = .{ .of = .xword, .is = .word } },
7304 .{ .scalar_int = .{ .of = .xword, .is = .word } },
7305 .any,
7306 },
7307 .patterns = &.{
7308 .{ .src = .{ .to_sse, .mem, .none } },
7309 .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } },
7310 .{ .src = .{ .to_sse, .to_sse, .none } },
7311 },
7312 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
7313 .each = .{ .once = &.{
7314 .{ ._, .vp_w, .mull, .dst0x, .src0x, .src1x, ._ },
7315 } },
7316 }, .{
7317 .required_features = .{ .sse2, null, null, null },
7318 .src_constraints = .{
7319 .{ .scalar_int = .{ .of = .xword, .is = .word } },
7320 .{ .scalar_int = .{ .of = .xword, .is = .word } },
7321 .any,
7322 },
7323 .patterns = &.{
7324 .{ .src = .{ .to_mut_sse, .mem, .none } },
7325 .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } },
7326 .{ .src = .{ .to_mut_sse, .to_sse, .none } },
7327 },
7328 .dst_temps = .{ .{ .ref = .src0 }, .unused },
7329 .each = .{ .once = &.{
7330 .{ ._, .p_w, .mull, .dst0x, .src1x, ._, ._ },
7331 } },
7332 }, .{
7333 .required_features = .{ .avx2, null, null, null },
7334 .src_constraints = .{
7335 .{ .scalar_int = .{ .of = .yword, .is = .word } },
7336 .{ .scalar_int = .{ .of = .yword, .is = .word } },
7337 .any,
7338 },
7339 .patterns = &.{
7340 .{ .src = .{ .to_sse, .mem, .none } },
7341 .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } },
7342 .{ .src = .{ .to_sse, .to_sse, .none } },
7343 },
7344 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
7345 .each = .{ .once = &.{
7346 .{ ._, .vp_w, .mull, .dst0y, .src0y, .src1y, ._ },
7347 } },
7348 }, .{
7349 .required_features = .{ .avx2, null, null, null },
7350 .src_constraints = .{
7351 .{ .multiple_scalar_int = .{ .of = .yword, .is = .word } },
7352 .{ .multiple_scalar_int = .{ .of = .yword, .is = .word } },
7353 .any,
7354 },
7355 .patterns = &.{
7356 .{ .src = .{ .to_mem, .to_mem, .none } },
7357 },
7358 .extra_temps = .{
7359 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
7360 .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } },
7361 .unused,
7362 .unused,
7363 .unused,
7364 .unused,
7365 .unused,
7366 .unused,
7367 .unused,
7368 .unused,
7369 .unused,
7370 },
7371 .dst_temps = .{ .mem, .unused },
7372 .clobbers = .{ .eflags = true },
7373 .each = .{ .once = &.{
7374 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
7375 .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ },
7376 .{ ._, .vp_w, .mull, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ },
7377 .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ },
7378 .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ },
7379 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
7380 } },
7381 }, .{
7382 .required_features = .{ .avx, null, null, null },
7383 .src_constraints = .{
7384 .{ .multiple_scalar_int = .{ .of = .xword, .is = .word } },
7385 .{ .multiple_scalar_int = .{ .of = .xword, .is = .word } },
7386 .any,
7387 },
7388 .patterns = &.{
7389 .{ .src = .{ .to_mem, .to_mem, .none } },
7390 },
7391 .extra_temps = .{
7392 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
7393 .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } },
7394 .unused,
7395 .unused,
7396 .unused,
7397 .unused,
7398 .unused,
7399 .unused,
7400 .unused,
7401 .unused,
7402 .unused,
7403 },
7404 .dst_temps = .{ .mem, .unused },
7405 .clobbers = .{ .eflags = true },
7406 .each = .{ .once = &.{
7407 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
7408 .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ },
7409 .{ ._, .vp_w, .mull, .tmp1x, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._ },
7410 .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ },
7411 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
7412 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
7413 } },
7414 }, .{
7415 .required_features = .{ .sse2, null, null, null },
7416 .src_constraints = .{
7417 .{ .multiple_scalar_int = .{ .of = .xword, .is = .word } },
7418 .{ .multiple_scalar_int = .{ .of = .xword, .is = .word } },
7419 .any,
7420 },
7421 .patterns = &.{
7422 .{ .src = .{ .to_mem, .to_mem, .none } },
7423 },
7424 .extra_temps = .{
7425 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
7426 .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } },
7427 .unused,
7428 .unused,
7429 .unused,
7430 .unused,
7431 .unused,
7432 .unused,
7433 .unused,
7434 .unused,
7435 .unused,
7436 },
7437 .dst_temps = .{ .mem, .unused },
7438 .clobbers = .{ .eflags = true },
7439 .each = .{ .once = &.{
7440 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
7441 .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ },
7442 .{ ._, .p_w, .mull, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ },
7443 .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ },
7444 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
7445 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
7446 } },
7447 }, .{
7448 .src_constraints = .{
7449 .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } },
7450 .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } },
7451 .any,
7452 },
7453 .patterns = &.{
7454 .{ .src = .{ .to_mem, .to_mem, .none } },
7455 },
7456 .extra_temps = .{
7457 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
7458 .{ .type = .i16, .kind = .{ .rc = .general_purpose } },
7459 .unused,
7460 .unused,
7461 .unused,
7462 .unused,
7463 .unused,
7464 .unused,
7465 .unused,
7466 .unused,
7467 .unused,
7468 },
7469 .dst_temps = .{ .mem, .unused },
7470 .clobbers = .{ .eflags = true },
7471 .each = .{ .once = &.{
7472 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
7473 .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ },
7474 .{ ._, .i_, .mul, .tmp1w, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ },
7475 .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ },
7476 .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ },
7477 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
7478 } },
7479 }, .{
7480 .src_constraints = .{
7481 .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } },
7482 .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } },
7483 .any,
7484 },
7485 .patterns = &.{
7486 .{ .src = .{ .to_mem, .to_mem, .none } },
7487 },
7488 .extra_temps = .{
7489 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
7490 .{ .type = .u16, .kind = .{ .rc = .general_purpose } },
7491 .unused,
7492 .unused,
7493 .unused,
7494 .unused,
7495 .unused,
7496 .unused,
7497 .unused,
7498 .unused,
7499 .unused,
7500 },
7501 .dst_temps = .{ .mem, .unused },
7502 .clobbers = .{ .eflags = true },
7503 .each = .{ .once = &.{
7504 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
7505 .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ },
7506 .{ ._, .i_, .mul, .tmp1w, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ },
7507 .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ },
7508 .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ },
7509 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
7510 } },
7511 }, .{
7512 .required_features = .{ .avx, null, null, null },
7513 .src_constraints = .{
7514 .{ .scalar_int = .{ .of = .xword, .is = .dword } },
7515 .{ .scalar_int = .{ .of = .xword, .is = .dword } },
7516 .any,
7517 },
7518 .patterns = &.{
7519 .{ .src = .{ .to_sse, .mem, .none } },
7520 .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } },
7521 .{ .src = .{ .to_sse, .to_sse, .none } },
7522 },
7523 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
7524 .each = .{ .once = &.{
7525 .{ ._, .vp_d, .mull, .dst0x, .src0x, .src1x, ._ },
7526 } },
7527 }, .{
7528 .required_features = .{ .sse4_1, null, null, null },
7529 .src_constraints = .{
7530 .{ .scalar_int = .{ .of = .xword, .is = .dword } },
7531 .{ .scalar_int = .{ .of = .xword, .is = .dword } },
7532 .any,
7533 },
7534 .patterns = &.{
7535 .{ .src = .{ .to_mut_sse, .mem, .none } },
7536 .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } },
7537 .{ .src = .{ .to_mut_sse, .to_sse, .none } },
7538 },
7539 .dst_temps = .{ .{ .ref = .src0 }, .unused },
7540 .each = .{ .once = &.{
7541 .{ ._, .p_d, .mull, .dst0x, .src1x, ._, ._ },
7542 } },
7543 }, .{
7544 .required_features = .{ .avx2, null, null, null },
7545 .src_constraints = .{
7546 .{ .scalar_int = .{ .of = .yword, .is = .dword } },
7547 .{ .scalar_int = .{ .of = .yword, .is = .dword } },
7548 .any,
7549 },
7550 .patterns = &.{
7551 .{ .src = .{ .to_sse, .mem, .none } },
7552 .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } },
7553 .{ .src = .{ .to_sse, .to_sse, .none } },
7554 },
7555 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
7556 .each = .{ .once = &.{
7557 .{ ._, .vp_d, .mull, .dst0y, .src0y, .src1y, ._ },
7558 } },
7559 }, .{
7560 .required_features = .{ .avx2, null, null, null },
7561 .src_constraints = .{
7562 .{ .multiple_scalar_int = .{ .of = .yword, .is = .dword } },
7563 .{ .multiple_scalar_int = .{ .of = .yword, .is = .dword } },
7564 .any,
7565 },
7566 .patterns = &.{
7567 .{ .src = .{ .to_mem, .to_mem, .none } },
7568 },
7569 .extra_temps = .{
7570 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
7571 .{ .type = .vector_8_u32, .kind = .{ .rc = .sse } },
7572 .unused,
7573 .unused,
7574 .unused,
7575 .unused,
7576 .unused,
7577 .unused,
7578 .unused,
7579 .unused,
7580 .unused,
7581 },
7582 .dst_temps = .{ .mem, .unused },
7583 .clobbers = .{ .eflags = true },
7584 .each = .{ .once = &.{
7585 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
7586 .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ },
7587 .{ ._, .vp_d, .mull, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ },
7588 .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ },
7589 .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ },
7590 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
7591 } },
7592 }, .{
7593 .required_features = .{ .avx, null, null, null },
7594 .src_constraints = .{
7595 .{ .multiple_scalar_int = .{ .of = .xword, .is = .dword } },
7596 .{ .multiple_scalar_int = .{ .of = .xword, .is = .dword } },
7597 .any,
7598 },
7599 .patterns = &.{
7600 .{ .src = .{ .to_mem, .to_mem, .none } },
7601 },
7602 .extra_temps = .{
7603 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
7604 .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } },
7605 .unused,
7606 .unused,
7607 .unused,
7608 .unused,
7609 .unused,
7610 .unused,
7611 .unused,
7612 .unused,
7613 .unused,
7614 },
7615 .dst_temps = .{ .mem, .unused },
7616 .clobbers = .{ .eflags = true },
7617 .each = .{ .once = &.{
7618 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
7619 .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ },
7620 .{ ._, .vp_d, .mull, .tmp1x, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._ },
7621 .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ },
7622 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
7623 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
7624 } },
7625 }, .{
7626 .required_features = .{ .sse4_1, null, null, null },
7627 .src_constraints = .{
7628 .{ .multiple_scalar_int = .{ .of = .xword, .is = .dword } },
7629 .{ .multiple_scalar_int = .{ .of = .xword, .is = .dword } },
7630 .any,
7631 },
7632 .patterns = &.{
7633 .{ .src = .{ .to_mem, .to_mem, .none } },
7634 },
7635 .extra_temps = .{
7636 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
7637 .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } },
7638 .unused,
7639 .unused,
7640 .unused,
7641 .unused,
7642 .unused,
7643 .unused,
7644 .unused,
7645 .unused,
7646 .unused,
7647 },
7648 .dst_temps = .{ .mem, .unused },
7649 .clobbers = .{ .eflags = true },
7650 .each = .{ .once = &.{
7651 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
7652 .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ },
7653 .{ ._, .p_d, .mull, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ },
7654 .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ },
7655 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
7656 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
7657 } },
7658 }, .{
7659 .src_constraints = .{
7660 .{ .multiple_scalar_int = .{ .of = .dword, .is = .dword } },
7661 .{ .multiple_scalar_int = .{ .of = .dword, .is = .dword } },
7662 .any,
7663 },
7664 .patterns = &.{
7665 .{ .src = .{ .to_mem, .to_mem, .none } },
7666 },
7667 .extra_temps = .{
7668 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
7669 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
7670 .unused,
7671 .unused,
7672 .unused,
7673 .unused,
7674 .unused,
7675 .unused,
7676 .unused,
7677 .unused,
7678 .unused,
7679 },
7680 .dst_temps = .{ .mem, .unused },
7681 .clobbers = .{ .eflags = true },
7682 .each = .{ .once = &.{
7683 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
7684 .{ .@"0:", ._, .mov, .tmp1d, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ },
7685 .{ ._, .i_, .mul, .tmp1d, .memia(.src1d, .tmp0, .add_unaligned_size), ._, ._ },
7686 .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1d, ._, ._ },
7687 .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ },
7688 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
7689 } },
7690 }, .{
7691 .required_features = .{ .@"64bit", null, null, null },
7692 .src_constraints = .{
7693 .{ .multiple_scalar_int = .{ .of = .qword, .is = .qword } },
7694 .{ .multiple_scalar_int = .{ .of = .qword, .is = .qword } },
7695 .any,
7696 },
7697 .patterns = &.{
7698 .{ .src = .{ .to_mem, .to_mem, .none } },
7699 },
7700 .extra_temps = .{
7701 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
7702 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
7703 .unused,
7704 .unused,
7705 .unused,
7706 .unused,
7707 .unused,
7708 .unused,
7709 .unused,
7710 .unused,
7711 .unused,
7712 },
7713 .dst_temps = .{ .mem, .unused },
7714 .clobbers = .{ .eflags = true },
7715 .each = .{ .once = &.{
7716 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
7717 .{ .@"0:", ._, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ },
7718 .{ ._, .i_, .mul, .tmp1q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ },
5219 .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1q, ._, ._ },7719 .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1q, ._, ._ },
5220 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },7720 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
5221 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },7721 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
5222 } },7722 } },
5223 }, .{7723 }, .{
5224 .required_features = .{ .@"64bit", null, null, null },7724 .required_features = .{ .@"64bit", .bmi2, .adx, null },
5225 .src_constraints = .{7725 .src_constraints = .{
5226 .{ .scalar_remainder_int = .{ .of = .qword, .is = .qword } },7726 .{ .scalar_remainder_int = .{ .of = .qword, .is = .qword } },
5227 .{ .scalar_remainder_int = .{ .of = .qword, .is = .qword } },7727 .{ .scalar_remainder_int = .{ .of = .qword, .is = .qword } },
...@@ -5236,33 +7736,46 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -5236,33 +7736,46 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
5236 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },7736 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
5237 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },7737 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
5238 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },7738 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
7739 .{ .type = .u64, .kind = .{ .reg = .rdx } },
7740 .{ .type = .isize, .kind = .{ .reg = .rcx } },
7741 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
7742 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
5239 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },7743 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
5240 .unused,
5241 .unused,
5242 .unused,
5243 .unused,
5244 .unused,7744 .unused,
5245 },7745 },
5246 .dst_temps = .{ .mem, .unused },7746 .dst_temps = .{ .mem, .unused },
7747 .clobbers = .{ .eflags = true },
5247 .each = .{ .once = &.{7748 .each = .{ .once = &.{
5248 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },7749 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
5249 .{ .@"0:", ._, .lea, .tmp1p, .memia(.src0, .tmp0, .add_unaligned_size_add_elem_size), ._, ._ },7750 .{ .@"0:", ._, .lea, .tmp1p, .memia(.src0, .tmp0, .add_size), ._, ._ },
5250 .{ ._, ._, .lea, .tmp2p, .memia(.src1, .tmp0, .add_unaligned_size_add_elem_size), ._, ._ },7751 .{ ._, ._, .lea, .tmp2p, .memia(.src1, .tmp0, .add_size), ._, ._ },
5251 .{ ._, ._, .lea, .tmp3p, .memia(.dst0, .tmp0, .add_unaligned_size_add_elem_size), ._, ._ },7752 .{ ._, ._, .lea, .tmp3p, .memia(.dst0, .tmp0, .add_size), ._, ._ },
5252 .{ ._, ._, .mov, .tmp4p, .sa(.src0, .sub_elem_size_div_8), ._, ._ },7753 .{ ._, ._, .mov, .tmp4d, .sia(-8, .src0, .add_elem_size), ._, ._ },
5253 .{ ._, ._c, .cl, ._, ._, ._, ._ },7754 .{ .@"1:", ._, .xor, .tmp5d, .tmp5d, ._, ._ },
5254 .{ .@"1:", ._, .mov, .tmp5q, .leasi(.tmp1q, .@"8", .tmp4), ._, ._ },7755 .{ ._, ._, .@"or", .tmp5q, .leai(.tmp1q, .tmp4), ._, ._ },
5255 .{ ._, ._, .sbb, .tmp5q, .leasi(.tmp2q, .@"8", .tmp4), ._, ._ },7756 .{ ._, ._z, .j, .@"3f", ._, ._, ._ },
5256 .{ ._, ._, .mov, .leasi(.tmp3q, .@"8", .tmp4), .tmp5q, ._, ._ },7757 .{ ._, ._, .lea, .tmp6p, .leaad(.tmp4, .sub_src0_elem_size, 8), ._, ._ },
5257 .{ ._, ._c, .in, .tmp4p, ._, ._, ._ },7758 .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ },
5258 .{ ._, ._nz, .j, .@"1b", ._, ._, ._ },7759 .{ .@"2:", ._x, .mul, .tmp9q, .tmp8q, .leai(.tmp2q, .tmp6), ._ },
7760 .{ ._, ._x, .adc, .tmp8q, .tmp7q, ._, ._ },
7761 .{ ._, ._, .mov, .leaiad(.tmp3q, .tmp6, .add_dst0_elem_size, -8), .tmp8q, ._, ._ },
7762 .{ ._, ._rcxz, .j, .@"2f", ._, ._, ._ },
7763 .{ ._, ._x, .ado, .tmp9q, .leaia(.tmp3q, .tmp6, .add_dst0_elem_size), ._, ._ },
7764 .{ ._, ._, .mov, .tmp7q, .tmp9q, ._, ._ },
7765 .{ ._, ._, .lea, .tmp6p, .lead(.tmp6, 8), ._, ._ },
7766 .{ ._, ._mp, .j, .@"2b", ._, ._, ._ },
7767 .{ .@"3:", ._, .mov, .leai(.tmp3q, .tmp4), .tmp5q, ._, ._ },
7768 .{ .@"2:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ },
7769 .{ ._, ._, .sub, .tmp4d, .si(8), ._, ._ },
7770 .{ ._, ._ae, .j, .@"1b", ._, ._, ._ },
5259 .{ ._, ._, .add, .tmp0p, .sa(.src0, .add_elem_size), ._, ._ },7771 .{ ._, ._, .add, .tmp0p, .sa(.src0, .add_elem_size), ._, ._ },
5260 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },7772 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
5261 } },7773 } },
5262 }, .{7774 }, .{
7775 .required_features = .{ .@"64bit", .bmi2, .slow_incdec, null },
5263 .src_constraints = .{7776 .src_constraints = .{
5264 .{ .scalar_remainder_int = .{ .of = .dword, .is = .dword } },7777 .{ .scalar_remainder_int = .{ .of = .qword, .is = .qword } },
5265 .{ .scalar_remainder_int = .{ .of = .dword, .is = .dword } },7778 .{ .scalar_remainder_int = .{ .of = .qword, .is = .qword } },
5266 .any,7779 .any,
5267 },7780 },
5268 .patterns = &.{7781 .patterns = &.{
...@@ -5274,26 +7787,205 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -5274,26 +7787,205 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
5274 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },7787 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
5275 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },7788 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
5276 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },7789 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
5277 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },7790 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
5278 .unused,7791 .{ .type = .u64, .kind = .{ .reg = .rdx } },
5279 .unused,7792 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
5280 .unused,7793 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
5281 .unused,7794 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
5282 .unused,7795 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
5283 },7796 },
5284 .dst_temps = .{ .mem, .unused },7797 .dst_temps = .{ .mem, .unused },
7798 .clobbers = .{ .eflags = true },
5285 .each = .{ .once = &.{7799 .each = .{ .once = &.{
5286 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },7800 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
5287 .{ .@"0:", ._, .lea, .tmp1p, .memia(.src0, .tmp0, .add_unaligned_size_add_elem_size), ._, ._ },7801 .{ .@"0:", ._, .lea, .tmp1p, .memia(.src0, .tmp0, .add_size), ._, ._ },
5288 .{ ._, ._, .lea, .tmp2p, .memia(.src1, .tmp0, .add_unaligned_size_add_elem_size), ._, ._ },7802 .{ ._, ._, .lea, .tmp2p, .memiad(.src1, .tmp0, .add_size, 8), ._, ._ },
5289 .{ ._, ._, .lea, .tmp3p, .memia(.dst0, .tmp0, .add_unaligned_size_add_elem_size), ._, ._ },7803 .{ ._, ._, .lea, .tmp3p, .memia(.dst0, .tmp0, .add_size), ._, ._ },
5290 .{ ._, ._, .mov, .tmp4p, .sa(.src0, .sub_elem_size_div_4), ._, ._ },7804 .{ ._, ._, .mov, .tmp4d, .sia(-1, .src0, .add_elem_size_div_8), ._, ._ },
5291 .{ ._, ._c, .cl, ._, ._, ._, ._ },7805 .{ .@"1:", ._, .lea, .tmp5p, .leaa(.tmp4, .sub_src0_elem_size_div_8), ._, ._ },
5292 .{ .@"1:", ._, .mov, .tmp5d, .leasi(.tmp1d, .@"4", .tmp4), ._, ._ },7806 .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ },
5293 .{ ._, ._, .sbb, .tmp5d, .leasi(.tmp2d, .@"4", .tmp4), ._, ._ },7807 .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ },
5294 .{ ._, ._, .mov, .leasi(.tmp3d, .@"4", .tmp4), .tmp5d, ._, ._ },7808 .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ },
5295 .{ ._, ._c, .in, .tmp4p, ._, ._, ._ },7809 .{ ._, ._, .@"or", .tmp6q, .leasi(.tmp1q, .@"8", .tmp4), ._, ._ },
5296 .{ ._, ._nz, .j, .@"1b", ._, ._, ._ },7810 .{ ._, ._nz, .j, .@"3f", ._, ._, ._ },
7811 .{ ._, ._, .mov, .leasi(.tmp3q, .@"8", .tmp4), .tmp6q, ._, ._ },
7812 .{ ._, ._mp, .j, .@"4f", ._, ._, ._ },
7813 .{ .@"2:", ._, .adc, .tmp10q, .leasia(.tmp3q, .@"8", .tmp5, .add_src0_elem_size), ._, ._ },
7814 .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ },
7815 .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ },
7816 .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ },
7817 .{ .@"3:", ._x, .mul, .tmp10q, .tmp9q, .leasi(.tmp2q, .@"8", .tmp5), ._ },
7818 .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ },
7819 .{ ._, ._, .mov, .leasia(.tmp3q, .@"8", .tmp5, .add_src0_elem_size), .tmp9q, ._, ._ },
7820 .{ ._, ._c, .in, .tmp5p, ._, ._, ._ },
7821 .{ ._, ._nz, .j, .@"2b", ._, ._, ._ },
7822 .{ .@"4:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ },
7823 .{ ._, ._, .sub, .tmp4d, .si(1), ._, ._ },
7824 .{ ._, ._ae, .j, .@"1b", ._, ._, ._ },
7825 .{ ._, ._, .add, .tmp0p, .sa(.src0, .add_elem_size), ._, ._ },
7826 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
7827 } },
7828 }, .{
7829 .required_features = .{ .@"64bit", .bmi2, null, null },
7830 .src_constraints = .{
7831 .{ .scalar_remainder_int = .{ .of = .qword, .is = .qword } },
7832 .{ .scalar_remainder_int = .{ .of = .qword, .is = .qword } },
7833 .any,
7834 },
7835 .patterns = &.{
7836 .{ .src = .{ .to_mem, .to_mem, .none } },
7837 },
7838 .extra_temps = .{
7839 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
7840 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
7841 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
7842 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
7843 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
7844 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
7845 .{ .type = .u64, .kind = .{ .reg = .rdx } },
7846 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
7847 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
7848 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
7849 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
7850 },
7851 .dst_temps = .{ .mem, .unused },
7852 .clobbers = .{ .eflags = true },
7853 .each = .{ .once = &.{
7854 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
7855 .{ .@"0:", ._, .lea, .tmp1p, .memia(.src0, .tmp0, .add_size), ._, ._ },
7856 .{ ._, ._, .lea, .tmp2p, .memiad(.src1, .tmp0, .add_size, 8), ._, ._ },
7857 .{ ._, ._, .lea, .tmp3p, .memia(.dst0, .tmp0, .add_size), ._, ._ },
7858 .{ ._, ._, .mov, .tmp4d, .sia(-1, .src0, .add_elem_size_div_8), ._, ._ },
7859 .{ .@"1:", ._, .lea, .tmp5p, .leaa(.tmp4, .sub_src0_elem_size_div_8), ._, ._ },
7860 .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ },
7861 .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ },
7862 .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ },
7863 .{ ._, ._, .@"or", .tmp6q, .leasi(.tmp1q, .@"8", .tmp4), ._, ._ },
7864 .{ ._, ._nz, .j, .@"3f", ._, ._, ._ },
7865 .{ ._, ._, .mov, .leasi(.tmp3q, .@"8", .tmp4), .tmp6q, ._, ._ },
7866 .{ ._, ._mp, .j, .@"4f", ._, ._, ._ },
7867 .{ .@"2:", ._, .adc, .tmp10q, .leasia(.tmp3q, .@"8", .tmp5, .add_src0_elem_size), ._, ._ },
7868 .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ },
7869 .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ },
7870 .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ },
7871 .{ .@"3:", ._x, .mul, .tmp10q, .tmp9q, .leasi(.tmp2q, .@"8", .tmp5), ._ },
7872 .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ },
7873 .{ ._, ._, .mov, .leasia(.tmp3q, .@"8", .tmp5, .add_src0_elem_size), .tmp9q, ._, ._ },
7874 .{ ._, ._c, .in, .tmp5p, ._, ._, ._ },
7875 .{ ._, ._nz, .j, .@"2b", ._, ._, ._ },
7876 .{ .@"4:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ },
7877 .{ ._, ._c, .de, .tmp4d, ._, ._, ._ },
7878 .{ ._, ._ns, .j, .@"1b", ._, ._, ._ },
7879 .{ ._, ._, .add, .tmp0p, .sa(.src0, .add_elem_size), ._, ._ },
7880 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
7881 } },
7882 }, .{
7883 .required_features = .{ .@"64bit", .slow_incdec, null, null },
7884 .src_constraints = .{
7885 .{ .scalar_remainder_int = .{ .of = .qword, .is = .qword } },
7886 .{ .scalar_remainder_int = .{ .of = .qword, .is = .qword } },
7887 .any,
7888 },
7889 .patterns = &.{
7890 .{ .src = .{ .to_mem, .to_mem, .none } },
7891 },
7892 .extra_temps = .{
7893 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
7894 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
7895 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
7896 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
7897 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
7898 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
7899 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
7900 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
7901 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
7902 .{ .type = .u64, .kind = .{ .reg = .rax } },
7903 .{ .type = .u64, .kind = .{ .reg = .rdx } },
7904 },
7905 .dst_temps = .{ .mem, .unused },
7906 .clobbers = .{ .eflags = true },
7907 .each = .{ .once = &.{
7908 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
7909 .{ .@"0:", ._, .lea, .tmp1p, .memia(.src0, .tmp0, .add_size), ._, ._ },
7910 .{ ._, ._, .lea, .tmp2p, .memiad(.src1, .tmp0, .add_size, 8), ._, ._ },
7911 .{ ._, ._, .lea, .tmp3p, .memia(.dst0, .tmp0, .add_size), ._, ._ },
7912 .{ ._, ._, .mov, .tmp4d, .sia(-1, .src0, .add_elem_size_div_8), ._, ._ },
7913 .{ .@"1:", ._, .lea, .tmp5p, .leaa(.tmp4, .sub_src0_elem_size_div_8), ._, ._ },
7914 .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ },
7915 .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ },
7916 .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ },
7917 .{ ._, ._, .@"or", .tmp6q, .leasi(.tmp1q, .@"8", .tmp4), ._, ._ },
7918 .{ ._, ._nz, .j, .@"3f", ._, ._, ._ },
7919 .{ ._, ._, .mov, .leasi(.tmp3q, .@"8", .tmp4), .tmp6q, ._, ._ },
7920 .{ ._, ._mp, .j, .@"4f", ._, ._, ._ },
7921 .{ .@"2:", ._, .adc, .tmp10q, .leasia(.tmp3q, .@"8", .tmp5, .add_src0_elem_size), ._, ._ },
7922 .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ },
7923 .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ },
7924 .{ .@"3:", ._, .mov, .tmp9q, .tmp6q, ._, ._ },
7925 .{ ._, ._, .mul, .leasi(.tmp2q, .@"8", .tmp5), ._, ._, ._ },
7926 .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ },
7927 .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ },
7928 .{ ._, ._, .mov, .leasia(.tmp3q, .@"8", .tmp5, .add_src0_elem_size), .tmp9q, ._, ._ },
7929 .{ ._, ._c, .in, .tmp5p, ._, ._, ._ },
7930 .{ ._, ._nz, .j, .@"2b", ._, ._, ._ },
7931 .{ .@"4:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ },
7932 .{ ._, ._, .sub, .tmp4d, .si(1), ._, ._ },
7933 .{ ._, ._ae, .j, .@"1b", ._, ._, ._ },
7934 .{ ._, ._, .add, .tmp0p, .sa(.src0, .add_elem_size), ._, ._ },
7935 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
7936 } },
7937 }, .{
7938 .required_features = .{ .@"64bit", null, null, null },
7939 .src_constraints = .{
7940 .{ .scalar_remainder_int = .{ .of = .qword, .is = .qword } },
7941 .{ .scalar_remainder_int = .{ .of = .qword, .is = .qword } },
7942 .any,
7943 },
7944 .patterns = &.{
7945 .{ .src = .{ .to_mem, .to_mem, .none } },
7946 },
7947 .extra_temps = .{
7948 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
7949 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
7950 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
7951 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
7952 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
7953 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
7954 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
7955 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
7956 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
7957 .{ .type = .u64, .kind = .{ .reg = .rax } },
7958 .{ .type = .u64, .kind = .{ .reg = .rdx } },
7959 },
7960 .dst_temps = .{ .mem, .unused },
7961 .clobbers = .{ .eflags = true },
7962 .each = .{ .once = &.{
7963 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
7964 .{ .@"0:", ._, .lea, .tmp1p, .memia(.src0, .tmp0, .add_size), ._, ._ },
7965 .{ ._, ._, .lea, .tmp2p, .memiad(.src1, .tmp0, .add_size, 8), ._, ._ },
7966 .{ ._, ._, .lea, .tmp3p, .memia(.dst0, .tmp0, .add_size), ._, ._ },
7967 .{ ._, ._, .mov, .tmp4d, .sia(-1, .src0, .add_elem_size_div_8), ._, ._ },
7968 .{ .@"1:", ._, .lea, .tmp5p, .leaa(.tmp4, .sub_src0_elem_size_div_8), ._, ._ },
7969 .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ },
7970 .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ },
7971 .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ },
7972 .{ ._, ._, .@"or", .tmp6q, .leasi(.tmp1q, .@"8", .tmp4), ._, ._ },
7973 .{ ._, ._nz, .j, .@"3f", ._, ._, ._ },
7974 .{ ._, ._, .mov, .leasi(.tmp3q, .@"8", .tmp4), .tmp6q, ._, ._ },
7975 .{ ._, ._mp, .j, .@"4f", ._, ._, ._ },
7976 .{ .@"2:", ._, .adc, .tmp10q, .leasia(.tmp3q, .@"8", .tmp5, .add_src0_elem_size), ._, ._ },
7977 .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ },
7978 .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ },
7979 .{ .@"3:", ._, .mov, .tmp9q, .tmp6q, ._, ._ },
7980 .{ ._, ._, .mul, .leasi(.tmp2q, .@"8", .tmp5), ._, ._, ._ },
7981 .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ },
7982 .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ },
7983 .{ ._, ._, .mov, .leasia(.tmp3q, .@"8", .tmp5, .add_src0_elem_size), .tmp9q, ._, ._ },
7984 .{ ._, ._c, .in, .tmp5p, ._, ._, ._ },
7985 .{ ._, ._nz, .j, .@"2b", ._, ._, ._ },
7986 .{ .@"4:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ },
7987 .{ ._, ._c, .de, .tmp4d, ._, ._, ._ },
7988 .{ ._, ._ns, .j, .@"1b", ._, ._, ._ },
5297 .{ ._, ._, .add, .tmp0p, .sa(.src0, .add_elem_size), ._, ._ },7989 .{ ._, ._, .add, .tmp0p, .sa(.src0, .add_elem_size), ._, ._ },
5298 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },7990 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
5299 } },7991 } },
...@@ -5324,7 +8016,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -5324,7 +8016,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
5324 .each = .{ .once = &.{8016 .each = .{ .once = &.{
5325 .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ },8017 .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ },
5326 .{ ._, .v_ps, .cvtph2, .tmp0x, .src1q, ._, ._ },8018 .{ ._, .v_ps, .cvtph2, .tmp0x, .src1q, ._, ._ },
5327 .{ ._, .v_ss, .sub, .dst0x, .dst0x, .tmp0d, ._ },8019 .{ ._, .v_ss, .mul, .dst0x, .dst0x, .tmp0d, ._ },
5328 .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ },8020 .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ },
5329 } },8021 } },
5330 }, .{8022 }, .{
...@@ -5339,7 +8031,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -5339,7 +8031,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
5339 },8031 },
5340 .call_frame = .{ .alignment = .@"16" },8032 .call_frame = .{ .alignment = .@"16" },
5341 .extra_temps = .{8033 .extra_temps = .{
5342 .{ .type = .usize, .kind = .{ .symbol = &.{ .name = "__subhf3" } } },8034 .{ .type = .usize, .kind = .{ .symbol = &.{ .name = "__mulhf3" } } },
5343 .unused,8035 .unused,
5344 .unused,8036 .unused,
5345 .unused,8037 .unused,
...@@ -5386,7 +8078,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -5386,7 +8078,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
5386 .each = .{ .once = &.{8078 .each = .{ .once = &.{
5387 .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ },8079 .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ },
5388 .{ ._, .v_ps, .cvtph2, .tmp0x, .src1q, ._, ._ },8080 .{ ._, .v_ps, .cvtph2, .tmp0x, .src1q, ._, ._ },
5389 .{ ._, .v_ps, .sub, .dst0x, .dst0x, .tmp0x, ._ },8081 .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp0x, ._ },
5390 .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ },8082 .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ },
5391 } },8083 } },
5392 }, .{8084 }, .{
...@@ -5419,7 +8111,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -5419,7 +8111,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
5419 .each = .{ .once = &.{8111 .each = .{ .once = &.{
5420 .{ ._, .v_ps, .cvtph2, .dst0y, .src0x, ._, ._ },8112 .{ ._, .v_ps, .cvtph2, .dst0y, .src0x, ._, ._ },
5421 .{ ._, .v_ps, .cvtph2, .tmp0y, .src1x, ._, ._ },8113 .{ ._, .v_ps, .cvtph2, .tmp0y, .src1x, ._, ._ },
5422 .{ ._, .v_ps, .sub, .dst0y, .dst0y, .tmp0y, ._ },8114 .{ ._, .v_ps, .mul, .dst0y, .dst0y, .tmp0y, ._ },
5423 .{ ._, .v_, .cvtps2ph, .dst0x, .dst0y, .rm(.{}), ._ },8115 .{ ._, .v_, .cvtps2ph, .dst0x, .dst0y, .rm(.{}), ._ },
5424 } },8116 } },
5425 }, .{8117 }, .{
...@@ -5451,7 +8143,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -5451,7 +8143,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
5451 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },8143 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
5452 .{ .@"0:", .v_ps, .cvtph2, .tmp1y, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ },8144 .{ .@"0:", .v_ps, .cvtph2, .tmp1y, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ },
5453 .{ ._, .v_ps, .cvtph2, .tmp2y, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ },8145 .{ ._, .v_ps, .cvtph2, .tmp2y, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ },
5454 .{ ._, .v_ps, .sub, .tmp1y, .tmp1y, .tmp2y, ._ },8146 .{ ._, .v_ps, .mul, .tmp1y, .tmp1y, .tmp2y, ._ },
5455 .{ ._, .v_, .cvtps2ph, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1y, .rm(.{}), ._ },8147 .{ ._, .v_, .cvtps2ph, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1y, .rm(.{}), ._ },
5456 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },8148 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
5457 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },8149 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
...@@ -5471,7 +8163,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -5471,7 +8163,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
5471 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },8163 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
5472 .{ .type = .f16, .kind = .{ .reg = .xmm0 } },8164 .{ .type = .f16, .kind = .{ .reg = .xmm0 } },
5473 .{ .type = .f16, .kind = .{ .reg = .xmm1 } },8165 .{ .type = .f16, .kind = .{ .reg = .xmm1 } },
5474 .{ .type = .usize, .kind = .{ .symbol = &.{ .name = "__subhf3" } } },8166 .{ .type = .usize, .kind = .{ .symbol = &.{ .name = "__mulhf3" } } },
5475 .unused,8167 .unused,
5476 .unused,8168 .unused,
5477 .unused,8169 .unused,
...@@ -5507,7 +8199,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -5507,7 +8199,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
5507 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },8199 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
5508 .{ .type = .f16, .kind = .{ .reg = .xmm0 } },8200 .{ .type = .f16, .kind = .{ .reg = .xmm0 } },
5509 .{ .type = .f16, .kind = .{ .reg = .xmm1 } },8201 .{ .type = .f16, .kind = .{ .reg = .xmm1 } },
5510 .{ .type = .usize, .kind = .{ .symbol = &.{ .name = "__subhf3" } } },8202 .{ .type = .usize, .kind = .{ .symbol = &.{ .name = "__mulhf3" } } },
5511 .unused,8203 .unused,
5512 .unused,8204 .unused,
5513 .unused,8205 .unused,
...@@ -5544,7 +8236,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -5544,7 +8236,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
5544 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },8236 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
5545 .{ .type = .f16, .kind = .{ .reg = .xmm0 } },8237 .{ .type = .f16, .kind = .{ .reg = .xmm0 } },
5546 .{ .type = .f16, .kind = .{ .reg = .xmm1 } },8238 .{ .type = .f16, .kind = .{ .reg = .xmm1 } },
5547 .{ .type = .usize, .kind = .{ .symbol = &.{ .name = "__subhf3" } } },8239 .{ .type = .usize, .kind = .{ .symbol = &.{ .name = "__mulhf3" } } },
5548 .{ .type = .f16, .kind = .{ .reg = .ax } },8240 .{ .type = .f16, .kind = .{ .reg = .ax } },
5549 .unused,8241 .unused,
5550 .unused,8242 .unused,
...@@ -5584,7 +8276,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -5584,7 +8276,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
5584 .{ .type = .f32, .kind = .mem },8276 .{ .type = .f32, .kind = .mem },
5585 .{ .type = .f16, .kind = .{ .reg = .xmm0 } },8277 .{ .type = .f16, .kind = .{ .reg = .xmm0 } },
5586 .{ .type = .f16, .kind = .{ .reg = .xmm1 } },8278 .{ .type = .f16, .kind = .{ .reg = .xmm1 } },
5587 .{ .type = .usize, .kind = .{ .symbol = &.{ .name = "__subhf3" } } },8279 .{ .type = .usize, .kind = .{ .symbol = &.{ .name = "__mulhf3" } } },
5588 .unused,8280 .unused,
5589 .unused,8281 .unused,
5590 .unused,8282 .unused,
...@@ -5617,11 +8309,12 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -5617,11 +8309,12 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
5617 },8309 },
5618 .patterns = &.{8310 .patterns = &.{
5619 .{ .src = .{ .to_sse, .mem, .none } },8311 .{ .src = .{ .to_sse, .mem, .none } },
8312 .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } },
5620 .{ .src = .{ .to_sse, .to_sse, .none } },8313 .{ .src = .{ .to_sse, .to_sse, .none } },
5621 },8314 },
5622 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },8315 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
5623 .each = .{ .once = &.{8316 .each = .{ .once = &.{
5624 .{ ._, .v_ss, .sub, .dst0x, .src0x, .src1d, ._ },8317 .{ ._, .v_ss, .mul, .dst0x, .src0x, .src1d, ._ },
5625 } },8318 } },
5626 }, .{8319 }, .{
5627 .required_features = .{ .sse, null, null, null },8320 .required_features = .{ .sse, null, null, null },
...@@ -5632,11 +8325,12 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -5632,11 +8325,12 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
5632 },8325 },
5633 .patterns = &.{8326 .patterns = &.{
5634 .{ .src = .{ .to_mut_sse, .mem, .none } },8327 .{ .src = .{ .to_mut_sse, .mem, .none } },
8328 .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } },
5635 .{ .src = .{ .to_mut_sse, .to_sse, .none } },8329 .{ .src = .{ .to_mut_sse, .to_sse, .none } },
5636 },8330 },
5637 .dst_temps = .{ .{ .ref = .src0 }, .unused },8331 .dst_temps = .{ .{ .ref = .src0 }, .unused },
5638 .each = .{ .once = &.{8332 .each = .{ .once = &.{
5639 .{ ._, ._ss, .sub, .dst0x, .src1d, ._, ._ },8333 .{ ._, ._ss, .mul, .dst0x, .src1d, ._, ._ },
5640 } },8334 } },
5641 }, .{8335 }, .{
5642 .required_features = .{ .avx, null, null, null },8336 .required_features = .{ .avx, null, null, null },
...@@ -5647,11 +8341,12 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -5647,11 +8341,12 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
5647 },8341 },
5648 .patterns = &.{8342 .patterns = &.{
5649 .{ .src = .{ .to_sse, .mem, .none } },8343 .{ .src = .{ .to_sse, .mem, .none } },
8344 .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } },
5650 .{ .src = .{ .to_sse, .to_sse, .none } },8345 .{ .src = .{ .to_sse, .to_sse, .none } },
5651 },8346 },
5652 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },8347 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
5653 .each = .{ .once = &.{8348 .each = .{ .once = &.{
5654 .{ ._, .v_ps, .sub, .dst0x, .src0x, .src1x, ._ },8349 .{ ._, .v_ps, .mul, .dst0x, .src0x, .src1x, ._ },
5655 } },8350 } },
5656 }, .{8351 }, .{
5657 .required_features = .{ .sse, null, null, null },8352 .required_features = .{ .sse, null, null, null },
...@@ -5662,11 +8357,12 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -5662,11 +8357,12 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
5662 },8357 },
5663 .patterns = &.{8358 .patterns = &.{
5664 .{ .src = .{ .to_mut_sse, .mem, .none } },8359 .{ .src = .{ .to_mut_sse, .mem, .none } },
8360 .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } },
5665 .{ .src = .{ .to_mut_sse, .to_sse, .none } },8361 .{ .src = .{ .to_mut_sse, .to_sse, .none } },
5666 },8362 },
5667 .dst_temps = .{ .{ .ref = .src0 }, .unused },8363 .dst_temps = .{ .{ .ref = .src0 }, .unused },
5668 .each = .{ .once = &.{8364 .each = .{ .once = &.{
5669 .{ ._, ._ps, .sub, .dst0x, .src1x, ._, ._ },8365 .{ ._, ._ps, .mul, .dst0x, .src1x, ._, ._ },
5670 } },8366 } },
5671 }, .{8367 }, .{
5672 .required_features = .{ .avx, null, null, null },8368 .required_features = .{ .avx, null, null, null },
...@@ -5677,11 +8373,12 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -5677,11 +8373,12 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
5677 },8373 },
5678 .patterns = &.{8374 .patterns = &.{
5679 .{ .src = .{ .to_sse, .mem, .none } },8375 .{ .src = .{ .to_sse, .mem, .none } },
8376 .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } },
5680 .{ .src = .{ .to_sse, .to_sse, .none } },8377 .{ .src = .{ .to_sse, .to_sse, .none } },
5681 },8378 },
5682 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },8379 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
5683 .each = .{ .once = &.{8380 .each = .{ .once = &.{
5684 .{ ._, .v_ps, .sub, .dst0y, .src0y, .src1y, ._ },8381 .{ ._, .v_ps, .mul, .dst0y, .src0y, .src1y, ._ },
5685 } },8382 } },
5686 }, .{8383 }, .{
5687 .required_features = .{ .avx, null, null, null },8384 .required_features = .{ .avx, null, null, null },
...@@ -5711,7 +8408,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -5711,7 +8408,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
5711 .each = .{ .once = &.{8408 .each = .{ .once = &.{
5712 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },8409 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
5713 .{ .@"0:", .v_ps, .mova, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ },8410 .{ .@"0:", .v_ps, .mova, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ },
5714 .{ ._, .v_ps, .sub, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ },8411 .{ ._, .v_ps, .mul, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ },
5715 .{ ._, .v_ps, .mova, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ },8412 .{ ._, .v_ps, .mova, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ },
5716 .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ },8413 .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ },
5717 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },8414 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
...@@ -5744,7 +8441,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -5744,7 +8441,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
5744 .each = .{ .once = &.{8441 .each = .{ .once = &.{
5745 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },8442 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
5746 .{ .@"0:", ._ps, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ },8443 .{ .@"0:", ._ps, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ },
5747 .{ ._, ._ps, .sub, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ },8444 .{ ._, ._ps, .mul, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ },
5748 .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ },8445 .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ },
5749 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },8446 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
5750 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },8447 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
...@@ -5758,11 +8455,12 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -5758,11 +8455,12 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
5758 },8455 },
5759 .patterns = &.{8456 .patterns = &.{
5760 .{ .src = .{ .to_sse, .mem, .none } },8457 .{ .src = .{ .to_sse, .mem, .none } },
8458 .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } },
5761 .{ .src = .{ .to_sse, .to_sse, .none } },8459 .{ .src = .{ .to_sse, .to_sse, .none } },
5762 },8460 },
5763 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },8461 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
5764 .each = .{ .once = &.{8462 .each = .{ .once = &.{
5765 .{ ._, .v_sd, .sub, .dst0x, .src0x, .src1q, ._ },8463 .{ ._, .v_sd, .mul, .dst0x, .src0x, .src1q, ._ },
5766 } },8464 } },
5767 }, .{8465 }, .{
5768 .required_features = .{ .sse2, null, null, null },8466 .required_features = .{ .sse2, null, null, null },
...@@ -5773,11 +8471,12 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -5773,11 +8471,12 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
5773 },8471 },
5774 .patterns = &.{8472 .patterns = &.{
5775 .{ .src = .{ .to_mut_sse, .mem, .none } },8473 .{ .src = .{ .to_mut_sse, .mem, .none } },
8474 .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } },
5776 .{ .src = .{ .to_mut_sse, .to_sse, .none } },8475 .{ .src = .{ .to_mut_sse, .to_sse, .none } },
5777 },8476 },
5778 .dst_temps = .{ .{ .ref = .src0 }, .unused },8477 .dst_temps = .{ .{ .ref = .src0 }, .unused },
5779 .each = .{ .once = &.{8478 .each = .{ .once = &.{
5780 .{ ._, ._sd, .sub, .dst0x, .src1q, ._, ._ },8479 .{ ._, ._sd, .mul, .dst0x, .src1q, ._, ._ },
5781 } },8480 } },
5782 }, .{8481 }, .{
5783 .required_features = .{ .x87, null, null, null },8482 .required_features = .{ .x87, null, null, null },
...@@ -5787,7 +8486,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -5787,7 +8486,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
5787 .any,8486 .any,
5788 },8487 },
5789 .patterns = &.{8488 .patterns = &.{
5790 .{ .src = .{ .to_mem, .to_mem, .none } },8489 .{ .src = .{ .mem, .mem, .none } },
5791 },8490 },
5792 .extra_temps = .{8491 .extra_temps = .{
5793 .{ .type = .f64, .kind = .{ .reg = .st6 } },8492 .{ .type = .f64, .kind = .{ .reg = .st6 } },
...@@ -5805,7 +8504,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -5805,7 +8504,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
5805 .dst_temps = .{ .mem, .unused },8504 .dst_temps = .{ .mem, .unused },
5806 .each = .{ .once = &.{8505 .each = .{ .once = &.{
5807 .{ ._, .f_, .ld, .src0q, ._, ._, ._ },8506 .{ ._, .f_, .ld, .src0q, ._, ._, ._ },
5808 .{ ._, .f_, .sub, .src1q, ._, ._, ._ },8507 .{ ._, .f_, .mul, .src1q, ._, ._, ._ },
5809 .{ ._, .f_p, .st, .dst0q, ._, ._, ._ },8508 .{ ._, .f_p, .st, .dst0q, ._, ._, ._ },
5810 } },8509 } },
5811 }, .{8510 }, .{
...@@ -5817,11 +8516,12 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -5817,11 +8516,12 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
5817 },8516 },
5818 .patterns = &.{8517 .patterns = &.{
5819 .{ .src = .{ .to_sse, .mem, .none } },8518 .{ .src = .{ .to_sse, .mem, .none } },
8519 .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } },
5820 .{ .src = .{ .to_sse, .to_sse, .none } },8520 .{ .src = .{ .to_sse, .to_sse, .none } },
5821 },8521 },
5822 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },8522 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
5823 .each = .{ .once = &.{8523 .each = .{ .once = &.{
5824 .{ ._, .v_pd, .sub, .dst0x, .src0x, .src1x, ._ },8524 .{ ._, .v_pd, .mul, .dst0x, .src0x, .src1x, ._ },
5825 } },8525 } },
5826 }, .{8526 }, .{
5827 .required_features = .{ .sse2, null, null, null },8527 .required_features = .{ .sse2, null, null, null },
...@@ -5832,11 +8532,12 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -5832,11 +8532,12 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
5832 },8532 },
5833 .patterns = &.{8533 .patterns = &.{
5834 .{ .src = .{ .to_mut_sse, .mem, .none } },8534 .{ .src = .{ .to_mut_sse, .mem, .none } },
8535 .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } },
5835 .{ .src = .{ .to_mut_sse, .to_sse, .none } },8536 .{ .src = .{ .to_mut_sse, .to_sse, .none } },
5836 },8537 },
5837 .dst_temps = .{ .{ .ref = .src0 }, .unused },8538 .dst_temps = .{ .{ .ref = .src0 }, .unused },
5838 .each = .{ .once = &.{8539 .each = .{ .once = &.{
5839 .{ ._, ._pd, .sub, .dst0x, .src1x, ._, ._ },8540 .{ ._, ._pd, .mul, .dst0x, .src1x, ._, ._ },
5840 } },8541 } },
5841 }, .{8542 }, .{
5842 .required_features = .{ .avx, null, null, null },8543 .required_features = .{ .avx, null, null, null },
...@@ -5847,11 +8548,12 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -5847,11 +8548,12 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
5847 },8548 },
5848 .patterns = &.{8549 .patterns = &.{
5849 .{ .src = .{ .to_sse, .mem, .none } },8550 .{ .src = .{ .to_sse, .mem, .none } },
8551 .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } },
5850 .{ .src = .{ .to_sse, .to_sse, .none } },8552 .{ .src = .{ .to_sse, .to_sse, .none } },
5851 },8553 },
5852 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },8554 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
5853 .each = .{ .once = &.{8555 .each = .{ .once = &.{
5854 .{ ._, .v_pd, .sub, .dst0y, .src0y, .src1y, ._ },8556 .{ ._, .v_pd, .mul, .dst0y, .src0y, .src1y, ._ },
5855 } },8557 } },
5856 }, .{8558 }, .{
5857 .required_features = .{ .avx, null, null, null },8559 .required_features = .{ .avx, null, null, null },
...@@ -5881,7 +8583,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -5881,7 +8583,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
5881 .each = .{ .once = &.{8583 .each = .{ .once = &.{
5882 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },8584 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
5883 .{ .@"0:", .v_pd, .mova, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ },8585 .{ .@"0:", .v_pd, .mova, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ },
5884 .{ ._, .v_pd, .sub, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ },8586 .{ ._, .v_pd, .mul, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ },
5885 .{ ._, .v_pd, .mova, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ },8587 .{ ._, .v_pd, .mova, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ },
5886 .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ },8588 .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ },
5887 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },8589 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
...@@ -5914,7 +8616,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -5914,7 +8616,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
5914 .each = .{ .once = &.{8616 .each = .{ .once = &.{
5915 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },8617 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
5916 .{ .@"0:", ._pd, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ },8618 .{ .@"0:", ._pd, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ },
5917 .{ ._, ._pd, .sub, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ },8619 .{ ._, ._pd, .mul, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ },
5918 .{ ._, ._pd, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ },8620 .{ ._, ._pd, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ },
5919 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },8621 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
5920 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },8622 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
...@@ -5946,7 +8648,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -5946,7 +8648,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
5946 .each = .{ .once = &.{8648 .each = .{ .once = &.{
5947 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },8649 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
5948 .{ .@"0:", .f_, .ld, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._, ._ },8650 .{ .@"0:", .f_, .ld, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._, ._ },
5949 .{ ._, .f_, .sub, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._, ._ },8651 .{ ._, .f_, .mul, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._, ._ },
5950 .{ ._, .f_p, .st, .memia(.dst0q, .tmp0, .add_unaligned_size), ._, ._, ._ },8652 .{ ._, .f_p, .st, .memia(.dst0q, .tmp0, .add_unaligned_size), ._, ._, ._ },
5951 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },8653 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
5952 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },8654 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
...@@ -5978,7 +8680,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -5978,7 +8680,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
5978 .each = .{ .once = &.{8680 .each = .{ .once = &.{
5979 .{ ._, .f_, .ld, .src0t, ._, ._, ._ },8681 .{ ._, .f_, .ld, .src0t, ._, ._, ._ },
5980 .{ ._, .f_, .ld, .src1t, ._, ._, ._ },8682 .{ ._, .f_, .ld, .src1t, ._, ._, ._ },
5981 .{ ._, .f_p, .sub, ._, ._, ._, ._ },8683 .{ ._, .f_p, .mul, ._, ._, ._, ._ },
5982 .{ ._, .f_p, .st, .dst0t, ._, ._, ._ },8684 .{ ._, .f_p, .st, .dst0t, ._, ._, ._ },
5983 } },8685 } },
5984 }, .{8686 }, .{
...@@ -5990,34 +8692,6 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -5990,34 +8692,6 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
5990 },8692 },
5991 .patterns = &.{8693 .patterns = &.{
5992 .{ .src = .{ .to_x87, .mem, .none }, .commute = .{ 0, 1 } },8694 .{ .src = .{ .to_x87, .mem, .none }, .commute = .{ 0, 1 } },
5993 },
5994 .extra_temps = .{
5995 .{ .type = .f80, .kind = .{ .reg = .st7 } },
5996 .unused,
5997 .unused,
5998 .unused,
5999 .unused,
6000 .unused,
6001 .unused,
6002 .unused,
6003 .unused,
6004 .unused,
6005 .unused,
6006 },
6007 .dst_temps = .{ .{ .rc = .x87 }, .unused },
6008 .each = .{ .once = &.{
6009 .{ ._, .f_, .ld, .src0t, ._, ._, ._ },
6010 .{ ._, .f_, .subr, .tmp0t, .src1t, ._, ._ },
6011 .{ ._, .f_p, .st, .dst0t, ._, ._, ._ },
6012 } },
6013 }, .{
6014 .required_features = .{ .x87, null, null, null },
6015 .src_constraints = .{
6016 .{ .scalar_float = .{ .of = .xword, .is = .tbyte } },
6017 .{ .scalar_float = .{ .of = .xword, .is = .tbyte } },
6018 .any,
6019 },
6020 .patterns = &.{
6021 .{ .src = .{ .mem, .to_x87, .none } },8695 .{ .src = .{ .mem, .to_x87, .none } },
6022 .{ .src = .{ .to_x87, .to_x87, .none } },8696 .{ .src = .{ .to_x87, .to_x87, .none } },
6023 },8697 },
...@@ -6037,7 +8711,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -6037,7 +8711,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6037 .dst_temps = .{ .{ .rc = .x87 }, .unused },8711 .dst_temps = .{ .{ .rc = .x87 }, .unused },
6038 .each = .{ .once = &.{8712 .each = .{ .once = &.{
6039 .{ ._, .f_, .ld, .src0t, ._, ._, ._ },8713 .{ ._, .f_, .ld, .src0t, ._, ._, ._ },
6040 .{ ._, .f_, .sub, .tmp0t, .src1t, ._, ._ },8714 .{ ._, .f_, .mul, .tmp0t, .src1t, ._, ._ },
6041 .{ ._, .f_p, .st, .dst0t, ._, ._, ._ },8715 .{ ._, .f_p, .st, .dst0t, ._, ._, ._ },
6042 } },8716 } },
6043 }, .{8717 }, .{
...@@ -6068,7 +8742,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -6068,7 +8742,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6068 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },8742 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
6069 .{ .@"0:", .f_, .ld, .memia(.src0t, .tmp0, .add_unaligned_size), ._, ._, ._ },8743 .{ .@"0:", .f_, .ld, .memia(.src0t, .tmp0, .add_unaligned_size), ._, ._, ._ },
6070 .{ ._, .f_, .ld, .memia(.src1t, .tmp0, .add_unaligned_size), ._, ._, ._ },8744 .{ ._, .f_, .ld, .memia(.src1t, .tmp0, .add_unaligned_size), ._, ._, ._ },
6071 .{ ._, .f_p, .sub, ._, ._, ._, ._ },8745 .{ ._, .f_p, .mul, ._, ._, ._, ._ },
6072 .{ ._, .f_p, .st, .memia(.dst0t, .tmp0, .add_unaligned_size), ._, ._, ._ },8746 .{ ._, .f_p, .st, .memia(.dst0t, .tmp0, .add_unaligned_size), ._, ._, ._ },
6073 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },8747 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
6074 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },8748 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
...@@ -6085,7 +8759,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -6085,7 +8759,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6085 },8759 },
6086 .call_frame = .{ .alignment = .@"16" },8760 .call_frame = .{ .alignment = .@"16" },
6087 .extra_temps = .{8761 .extra_temps = .{
6088 .{ .type = .usize, .kind = .{ .symbol = &.{ .name = "__subtf3" } } },8762 .{ .type = .usize, .kind = .{ .symbol = &.{ .name = "__multf3" } } },
6089 .unused,8763 .unused,
6090 .unused,8764 .unused,
6091 .unused,8765 .unused,
...@@ -6117,7 +8791,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -6117,7 +8791,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6117 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },8791 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
6118 .{ .type = .f128, .kind = .{ .reg = .xmm0 } },8792 .{ .type = .f128, .kind = .{ .reg = .xmm0 } },
6119 .{ .type = .f128, .kind = .{ .reg = .xmm1 } },8793 .{ .type = .f128, .kind = .{ .reg = .xmm1 } },
6120 .{ .type = .usize, .kind = .{ .symbol = &.{ .name = "__subtf3" } } },8794 .{ .type = .usize, .kind = .{ .symbol = &.{ .name = "__multf3" } } },
6121 .unused,8795 .unused,
6122 .unused,8796 .unused,
6123 .unused,8797 .unused,
...@@ -6152,7 +8826,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -6152,7 +8826,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6152 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },8826 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
6153 .{ .type = .f128, .kind = .{ .reg = .xmm0 } },8827 .{ .type = .f128, .kind = .{ .reg = .xmm0 } },
6154 .{ .type = .f128, .kind = .{ .reg = .xmm1 } },8828 .{ .type = .f128, .kind = .{ .reg = .xmm1 } },
6155 .{ .type = .usize, .kind = .{ .symbol = &.{ .name = "__subtf3" } } },8829 .{ .type = .usize, .kind = .{ .symbol = &.{ .name = "__multf3" } } },
6156 .unused,8830 .unused,
6157 .unused,8831 .unused,
6158 .unused,8832 .unused,
...@@ -6187,7 +8861,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -6187,7 +8861,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6187 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },8861 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
6188 .{ .type = .f128, .kind = .{ .reg = .xmm0 } },8862 .{ .type = .f128, .kind = .{ .reg = .xmm0 } },
6189 .{ .type = .f128, .kind = .{ .reg = .xmm1 } },8863 .{ .type = .f128, .kind = .{ .reg = .xmm1 } },
6190 .{ .type = .usize, .kind = .{ .symbol = &.{ .name = "__subtf3" } } },8864 .{ .type = .usize, .kind = .{ .symbol = &.{ .name = "__multf3" } } },
6191 .unused,8865 .unused,
6192 .unused,8866 .unused,
6193 .unused,8867 .unused,
...@@ -6210,28 +8884,16 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -6210,28 +8884,16 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6210 } }) catch |err| switch (err) {8884 } }) catch |err| switch (err) {
6211 error.SelectFailed => return cg.fail("failed to select {s} {} {} {}", .{8885 error.SelectFailed => return cg.fail("failed to select {s} {} {} {}", .{
6212 @tagName(air_tag),8886 @tagName(air_tag),
6213 cg.typeOf(bin_op.lhs).fmt(pt),8887 ty.fmt(pt),
6214 ops[0].tracking(cg),8888 ops[0].tracking(cg),
6215 ops[1].tracking(cg),8889 ops[1].tracking(cg),
6216 }),8890 }),
6217 else => |e| return e,8891 else => |e| return e,
6218 };8892 };
6219 switch (air_tag) {
6220 else => unreachable,
6221 .sub, .sub_optimized => {},
6222 .sub_wrap => res[0].wrapInt(cg) catch |err| switch (err) {
6223 error.SelectFailed => return cg.fail("failed to select wrap {} {} {}", .{
6224 cg.typeOf(bin_op.lhs).fmt(pt),
6225 ops[0].tracking(cg),
6226 ops[1].tracking(cg),
6227 }),
6228 else => |e| return e,
6229 },
6230 }
6231 try res[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg);8893 try res[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg);
6232 },8894 },
6233 .sub_safe => unreachable,8895 .mul_safe => unreachable,
6234 .mul, .mul_optimized => |air_tag| if (use_old) try cg.airMulDivBinOp(inst, .mul) else {8896 .mul_wrap => |air_tag| if (use_old) try cg.airMulDivBinOp(inst, air_tag) else {
6235 const bin_op = air_datas[@intFromEnum(inst)].bin_op;8897 const bin_op = air_datas[@intFromEnum(inst)].bin_op;
6236 const ty = cg.typeOf(bin_op.lhs);8898 const ty = cg.typeOf(bin_op.lhs);
6237 var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs });8899 var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs });
...@@ -6628,6 +9290,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -6628,6 +9290,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6628 .{ .src = .{ .to_sse, .to_sse, .none } },9290 .{ .src = .{ .to_sse, .to_sse, .none } },
6629 },9291 },
6630 .extra_temps = .{9292 .extra_temps = .{
9293 .{ .type = .vector_16_u8, .kind = .{ .pshufb_trunc_mem = .{ .from = .word, .to = .byte } } },
9294 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
6631 .{ .type = .vector_8_i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } },9295 .{ .type = .vector_8_i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } },
6632 .unused,9296 .unused,
6633 .unused,9297 .unused,
...@@ -6637,15 +9301,14 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -6637,15 +9301,14 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6637 .unused,9301 .unused,
6638 .unused,9302 .unused,
6639 .unused,9303 .unused,
6640 .unused,
6641 .unused,
6642 },9304 },
6643 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },9305 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
6644 .each = .{ .once = &.{9306 .each = .{ .once = &.{
9307 .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ },
6645 .{ ._, .vp_w, .movsxb, .dst0x, .src0q, ._, ._ },9308 .{ ._, .vp_w, .movsxb, .dst0x, .src0q, ._, ._ },
6646 .{ ._, .vp_w, .movsxb, .tmp0x, .src1q, ._, ._ },9309 .{ ._, .vp_w, .movsxb, .tmp2x, .src1q, ._, ._ },
6647 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ },9310 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ },
6648 .{ ._, .vp_b, .ackssw, .dst0x, .dst0x, .dst0x, ._ },9311 .{ ._, .vp_b, .shuf, .dst0x, .dst0x, .lea(.tmp1x), ._ },
6649 } },9312 } },
6650 }, .{9313 }, .{
6651 .required_features = .{ .sse4_1, null, null, null },9314 .required_features = .{ .sse4_1, null, null, null },
...@@ -6661,6 +9324,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -6661,6 +9324,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6661 .{ .src = .{ .to_sse, .to_sse, .none } },9324 .{ .src = .{ .to_sse, .to_sse, .none } },
6662 },9325 },
6663 .extra_temps = .{9326 .extra_temps = .{
9327 .{ .type = .vector_16_u8, .kind = .{ .pshufb_trunc_mem = .{ .from = .word, .to = .byte } } },
9328 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
6664 .{ .type = .vector_8_i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } },9329 .{ .type = .vector_8_i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } },
6665 .unused,9330 .unused,
6666 .unused,9331 .unused,
...@@ -6670,29 +9335,28 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -6670,29 +9335,28 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6670 .unused,9335 .unused,
6671 .unused,9336 .unused,
6672 .unused,9337 .unused,
6673 .unused,
6674 .unused,
6675 },9338 },
6676 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },9339 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
6677 .each = .{ .once = &.{9340 .each = .{ .once = &.{
9341 .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ },
6678 .{ ._, .p_w, .movsxb, .dst0x, .src0q, ._, ._ },9342 .{ ._, .p_w, .movsxb, .dst0x, .src0q, ._, ._ },
6679 .{ ._, .p_w, .movsxb, .tmp0x, .src1q, ._, ._ },9343 .{ ._, .p_w, .movsxb, .tmp2x, .src1q, ._, ._ },
6680 .{ ._, .p_w, .mull, .dst0x, .tmp0x, ._, ._ },9344 .{ ._, .p_w, .mull, .dst0x, .tmp2x, ._, ._ },
6681 .{ ._, .p_b, .ackssw, .dst0x, .dst0x, ._, ._ },9345 .{ ._, .p_b, .shuf, .dst0x, .lea(.tmp1x), ._, ._ },
6682 } },9346 } },
6683 }, .{9347 }, .{
6684 .required_features = .{ .sse2, null, null, null },9348 .required_features = .{ .sse2, null, null, null },
6685 .src_constraints = .{9349 .src_constraints = .{
6686 .{ .scalar_signed_int = .{ .of = .qword, .is = .byte } },9350 .{ .scalar_int = .{ .of = .qword, .is = .byte } },
6687 .{ .scalar_signed_int = .{ .of = .qword, .is = .byte } },9351 .{ .scalar_int = .{ .of = .qword, .is = .byte } },
6688 .any,9352 .any,
6689 },9353 },
6690 .patterns = &.{9354 .patterns = &.{
6691 .{ .src = .{ .to_mut_sse, .to_mut_sse, .none } },9355 .{ .src = .{ .to_mut_sse, .to_mut_sse, .none } },
6692 },9356 },
6693 .extra_temps = .{9357 .extra_temps = .{
6694 .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } },9358 .{ .type = .vector_16_u8, .kind = .{ .pand_trunc_mem = .{ .from = .word, .to = .byte } } },
6695 .unused,9359 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
6696 .unused,9360 .unused,
6697 .unused,9361 .unused,
6698 .unused,9362 .unused,
...@@ -6705,14 +9369,12 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -6705,14 +9369,12 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6705 },9369 },
6706 .dst_temps = .{ .{ .ref = .src0 }, .unused },9370 .dst_temps = .{ .{ .ref = .src0 }, .unused },
6707 .each = .{ .once = &.{9371 .each = .{ .once = &.{
6708 .{ ._, .p_, .xor, .tmp0x, .tmp0x, ._, ._ },9372 .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ },
6709 .{ ._, .p_b, .cmpgt, .tmp0x, .src0x, ._, ._ },9373 .{ ._, .p_, .unpcklbw, .dst0x, .dst0x, ._, ._ },
6710 .{ ._, .p_, .unpcklbw, .dst0x, .tmp0x, ._, ._ },9374 .{ ._, .p_, .unpcklbw, .src1x, .src1x, ._, ._ },
6711 .{ ._, .p_, .xor, .tmp0x, .tmp0x, ._, ._ },
6712 .{ ._, .p_b, .cmpgt, .tmp0x, .src1x, ._, ._ },
6713 .{ ._, .p_, .unpcklbw, .src1x, .tmp0x, ._, ._ },
6714 .{ ._, .p_w, .mull, .dst0x, .src1x, ._, ._ },9375 .{ ._, .p_w, .mull, .dst0x, .src1x, ._, ._ },
6715 .{ ._, .p_b, .ackssw, .dst0x, .dst0x, ._, ._ },9376 .{ ._, .p_, .@"and", .dst0x, .lea(.tmp1x), ._, ._ },
9377 .{ ._, .p_b, .ackusw, .dst0x, .dst0x, ._, ._ },
6716 } },9378 } },
6717 }, .{9379 }, .{
6718 .required_features = .{ .avx2, null, null, null },9380 .required_features = .{ .avx2, null, null, null },
...@@ -6728,6 +9390,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -6728,6 +9390,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6728 .{ .src = .{ .to_sse, .to_sse, .none } },9390 .{ .src = .{ .to_sse, .to_sse, .none } },
6729 },9391 },
6730 .extra_temps = .{9392 .extra_temps = .{
9393 .{ .type = .vector_32_u8, .kind = .{ .pshufb_trunc_mem = .{ .from = .word, .to = .byte } } },
9394 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
6731 .{ .type = .vector_16_i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } },9395 .{ .type = .vector_16_i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } },
6732 .unused,9396 .unused,
6733 .unused,9397 .unused,
...@@ -6737,15 +9401,14 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -6737,15 +9401,14 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6737 .unused,9401 .unused,
6738 .unused,9402 .unused,
6739 .unused,9403 .unused,
6740 .unused,
6741 .unused,
6742 },9404 },
6743 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },9405 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
6744 .each = .{ .once = &.{9406 .each = .{ .once = &.{
9407 .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ },
6745 .{ ._, .vp_w, .movsxb, .dst0y, .src0x, ._, ._ },9408 .{ ._, .vp_w, .movsxb, .dst0y, .src0x, ._, ._ },
6746 .{ ._, .vp_w, .movsxb, .tmp0y, .src1x, ._, ._ },9409 .{ ._, .vp_w, .movsxb, .tmp2y, .src1x, ._, ._ },
6747 .{ ._, .vp_w, .mull, .dst0y, .dst0y, .tmp0y, ._ },9410 .{ ._, .vp_w, .mull, .dst0y, .dst0y, .tmp2y, ._ },
6748 .{ ._, .vp_b, .ackssw, .dst0y, .dst0y, .dst0y, ._ },9411 .{ ._, .vp_b, .shuf, .dst0y, .dst0y, .lea(.tmp1y), ._ },
6749 .{ ._, .v_q, .perm, .dst0y, .dst0y, .ui(0b10_00_10_00), ._ },9412 .{ ._, .v_q, .perm, .dst0y, .dst0y, .ui(0b10_00_10_00), ._ },
6750 } },9413 } },
6751 }, .{9414 }, .{
...@@ -6762,6 +9425,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -6762,6 +9425,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6762 .{ .src = .{ .to_sse, .to_sse, .none } },9425 .{ .src = .{ .to_sse, .to_sse, .none } },
6763 },9426 },
6764 .extra_temps = .{9427 .extra_temps = .{
9428 .{ .type = .vector_16_u8, .kind = .{ .pshufb_trunc_mem = .{ .from = .word, .to = .byte } } },
9429 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
6765 .{ .type = .vector_8_u16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } },9430 .{ .type = .vector_8_u16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } },
6766 .unused,9431 .unused,
6767 .unused,9432 .unused,
...@@ -6771,15 +9436,14 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -6771,15 +9436,14 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6771 .unused,9436 .unused,
6772 .unused,9437 .unused,
6773 .unused,9438 .unused,
6774 .unused,
6775 .unused,
6776 },9439 },
6777 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },9440 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
6778 .each = .{ .once = &.{9441 .each = .{ .once = &.{
9442 .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ },
6779 .{ ._, .vp_w, .movzxb, .dst0x, .src0q, ._, ._ },9443 .{ ._, .vp_w, .movzxb, .dst0x, .src0q, ._, ._ },
6780 .{ ._, .vp_w, .movzxb, .tmp0x, .src1q, ._, ._ },9444 .{ ._, .vp_w, .movzxb, .tmp2x, .src1q, ._, ._ },
6781 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ },9445 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ },
6782 .{ ._, .vp_b, .ackusw, .dst0x, .dst0x, .dst0x, ._ },9446 .{ ._, .vp_b, .shuf, .dst0x, .dst0x, .lea(.tmp1x), ._ },
6783 } },9447 } },
6784 }, .{9448 }, .{
6785 .required_features = .{ .sse4_1, null, null, null },9449 .required_features = .{ .sse4_1, null, null, null },
...@@ -6795,6 +9459,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -6795,6 +9459,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6795 .{ .src = .{ .to_sse, .to_sse, .none } },9459 .{ .src = .{ .to_sse, .to_sse, .none } },
6796 },9460 },
6797 .extra_temps = .{9461 .extra_temps = .{
9462 .{ .type = .vector_16_u8, .kind = .{ .pshufb_trunc_mem = .{ .from = .word, .to = .byte } } },
9463 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
6798 .{ .type = .vector_8_u16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } },9464 .{ .type = .vector_8_u16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } },
6799 .unused,9465 .unused,
6800 .unused,9466 .unused,
...@@ -6804,46 +9470,14 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -6804,46 +9470,14 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6804 .unused,9470 .unused,
6805 .unused,9471 .unused,
6806 .unused,9472 .unused,
6807 .unused,
6808 .unused,
6809 },9473 },
6810 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },9474 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
6811 .each = .{ .once = &.{9475 .each = .{ .once = &.{
9476 .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ },
6812 .{ ._, .p_w, .movzxb, .dst0x, .src0q, ._, ._ },9477 .{ ._, .p_w, .movzxb, .dst0x, .src0q, ._, ._ },
6813 .{ ._, .p_w, .movzxb, .tmp0x, .src1q, ._, ._ },9478 .{ ._, .p_w, .movzxb, .tmp2x, .src1q, ._, ._ },
6814 .{ ._, .p_w, .mull, .dst0x, .tmp0x, ._, ._ },9479 .{ ._, .p_w, .mull, .dst0x, .tmp2x, ._, ._ },
6815 .{ ._, .p_b, .ackusw, .dst0x, .dst0x, ._, ._ },9480 .{ ._, .p_b, .shuf, .dst0x, .lea(.tmp1x), ._, ._ },
6816 } },
6817 }, .{
6818 .required_features = .{ .sse2, null, null, null },
6819 .src_constraints = .{
6820 .{ .scalar_unsigned_int = .{ .of = .qword, .is = .byte } },
6821 .{ .scalar_unsigned_int = .{ .of = .qword, .is = .byte } },
6822 .any,
6823 },
6824 .patterns = &.{
6825 .{ .src = .{ .to_mut_sse, .to_mut_sse, .none } },
6826 },
6827 .extra_temps = .{
6828 .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } },
6829 .unused,
6830 .unused,
6831 .unused,
6832 .unused,
6833 .unused,
6834 .unused,
6835 .unused,
6836 .unused,
6837 .unused,
6838 .unused,
6839 },
6840 .dst_temps = .{ .{ .ref = .src0 }, .unused },
6841 .each = .{ .once = &.{
6842 .{ ._, .p_, .xor, .tmp0x, .tmp0x, ._, ._ },
6843 .{ ._, .p_, .unpcklbw, .dst0x, .tmp0x, ._, ._ },
6844 .{ ._, .p_, .unpcklbw, .src1x, .tmp0x, ._, ._ },
6845 .{ ._, .p_w, .mull, .dst0x, .src1x, ._, ._ },
6846 .{ ._, .p_b, .ackusw, .dst0x, .dst0x, ._, ._ },
6847 } },9481 } },
6848 }, .{9482 }, .{
6849 .required_features = .{ .avx2, null, null, null },9483 .required_features = .{ .avx2, null, null, null },
...@@ -6859,6 +9493,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -6859,6 +9493,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6859 .{ .src = .{ .to_sse, .to_sse, .none } },9493 .{ .src = .{ .to_sse, .to_sse, .none } },
6860 },9494 },
6861 .extra_temps = .{9495 .extra_temps = .{
9496 .{ .type = .vector_32_u8, .kind = .{ .pshufb_trunc_mem = .{ .from = .word, .to = .byte } } },
9497 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
6862 .{ .type = .vector_16_u16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } },9498 .{ .type = .vector_16_u16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } },
6863 .unused,9499 .unused,
6864 .unused,9500 .unused,
...@@ -6868,15 +9504,14 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -6868,15 +9504,14 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6868 .unused,9504 .unused,
6869 .unused,9505 .unused,
6870 .unused,9506 .unused,
6871 .unused,
6872 .unused,
6873 },9507 },
6874 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },9508 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
6875 .each = .{ .once = &.{9509 .each = .{ .once = &.{
9510 .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ },
6876 .{ ._, .vp_w, .movzxb, .dst0y, .src0x, ._, ._ },9511 .{ ._, .vp_w, .movzxb, .dst0y, .src0x, ._, ._ },
6877 .{ ._, .vp_w, .movzxb, .tmp0y, .src1x, ._, ._ },9512 .{ ._, .vp_w, .movzxb, .tmp2y, .src1x, ._, ._ },
6878 .{ ._, .vp_w, .mull, .dst0y, .dst0y, .tmp0y, ._ },9513 .{ ._, .vp_w, .mull, .dst0y, .dst0y, .tmp2y, ._ },
6879 .{ ._, .vp_b, .ackusw, .dst0y, .dst0y, .dst0y, ._ },9514 .{ ._, .vp_b, .shuf, .dst0y, .dst0y, .lea(.tmp1y), ._ },
6880 .{ ._, .v_q, .perm, .dst0y, .dst0y, .ui(0b10_00_10_00), ._ },9515 .{ ._, .v_q, .perm, .dst0y, .dst0y, .ui(0b10_00_10_00), ._ },
6881 } },9516 } },
6882 }, .{9517 }, .{
...@@ -6891,6 +9526,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -6891,6 +9526,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6891 },9526 },
6892 .extra_temps = .{9527 .extra_temps = .{
6893 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },9528 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
9529 .{ .type = .vector_8_u8, .kind = .{ .pshufb_trunc_mem = .{ .of = .xword, .from = .word, .to = .byte } } },
9530 .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } },
6894 .{ .type = .vector_16_i16, .kind = .{ .rc = .sse } },9531 .{ .type = .vector_16_i16, .kind = .{ .rc = .sse } },
6895 .{ .type = .vector_16_i16, .kind = .{ .rc = .sse } },9532 .{ .type = .vector_16_i16, .kind = .{ .rc = .sse } },
6896 .unused,9533 .unused,
...@@ -6899,19 +9536,19 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -6899,19 +9536,19 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6899 .unused,9536 .unused,
6900 .unused,9537 .unused,
6901 .unused,9538 .unused,
6902 .unused,
6903 .unused,
6904 },9539 },
6905 .dst_temps = .{ .mem, .unused },9540 .dst_temps = .{ .mem, .unused },
6906 .clobbers = .{ .eflags = true },9541 .clobbers = .{ .eflags = true },
6907 .each = .{ .once = &.{9542 .each = .{ .once = &.{
9543 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
9544 .{ ._, .vp_q, .broadcast, .tmp2y, .lea(.tmp0q), ._, ._ },
6908 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },9545 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
6909 .{ .@"0:", .vp_w, .movsxb, .tmp1y, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ },9546 .{ .@"0:", .vp_w, .movsxb, .tmp3y, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ },
6910 .{ ._, .vp_w, .movsxb, .tmp2y, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ },9547 .{ ._, .vp_w, .movsxb, .tmp4y, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ },
6911 .{ ._, .vp_w, .mull, .tmp1y, .tmp1y, .tmp2y, ._ },9548 .{ ._, .vp_w, .mull, .tmp3y, .tmp3y, .tmp4y, ._ },
6912 .{ ._, .vp_b, .ackssw, .tmp1y, .tmp1y, .tmp1y, ._ },9549 .{ ._, .vp_b, .shuf, .tmp3y, .tmp3y, .tmp2y, ._ },
6913 .{ ._, .v_q, .perm, .tmp1y, .tmp1y, .ui(0b10_00_10_00), ._ },9550 .{ ._, .v_q, .perm, .tmp3y, .tmp3y, .ui(0b10_00_10_00), ._ },
6914 .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ },9551 .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp3x, ._, ._ },
6915 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },9552 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
6916 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },9553 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
6917 } },9554 } },
...@@ -6927,6 +9564,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -6927,6 +9564,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6927 },9564 },
6928 .extra_temps = .{9565 .extra_temps = .{
6929 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },9566 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
9567 .{ .type = .vector_8_u8, .kind = .{ .pshufb_trunc_mem = .{ .of = .xword, .from = .word, .to = .byte } } },
9568 .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } },
6930 .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } },9569 .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } },
6931 .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } },9570 .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } },
6932 .unused,9571 .unused,
...@@ -6935,18 +9574,18 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -6935,18 +9574,18 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6935 .unused,9574 .unused,
6936 .unused,9575 .unused,
6937 .unused,9576 .unused,
6938 .unused,
6939 .unused,
6940 },9577 },
6941 .dst_temps = .{ .mem, .unused },9578 .dst_temps = .{ .mem, .unused },
6942 .clobbers = .{ .eflags = true },9579 .clobbers = .{ .eflags = true },
6943 .each = .{ .once = &.{9580 .each = .{ .once = &.{
9581 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
9582 .{ ._, .v_, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ },
6944 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },9583 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
6945 .{ .@"0:", .vp_w, .movsxb, .tmp1x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ },9584 .{ .@"0:", .vp_w, .movsxb, .tmp3x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ },
6946 .{ ._, .vp_w, .movsxb, .tmp2x, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ },9585 .{ ._, .vp_w, .movsxb, .tmp4x, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ },
6947 .{ ._, .vp_w, .mull, .tmp1x, .tmp1x, .tmp2x, ._ },9586 .{ ._, .vp_w, .mull, .tmp3x, .tmp3x, .tmp4x, ._ },
6948 .{ ._, .vp_b, .ackssw, .tmp1x, .tmp1x, .tmp1x, ._ },9587 .{ ._, .vp_b, .shuf, .tmp3x, .tmp3x, .tmp2x, ._ },
6949 .{ ._, .v_q, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ },9588 .{ ._, .v_q, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp3x, ._, ._ },
6950 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },9589 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
6951 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },9590 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
6952 } },9591 } },
...@@ -6962,6 +9601,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -6962,6 +9601,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6962 },9601 },
6963 .extra_temps = .{9602 .extra_temps = .{
6964 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },9603 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
9604 .{ .type = .vector_8_u8, .kind = .{ .pshufb_trunc_mem = .{ .of = .xword, .from = .word, .to = .byte } } },
9605 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
6965 .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } },9606 .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } },
6966 .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } },9607 .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } },
6967 .unused,9608 .unused,
...@@ -6970,26 +9611,26 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -6970,26 +9611,26 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6970 .unused,9611 .unused,
6971 .unused,9612 .unused,
6972 .unused,9613 .unused,
6973 .unused,
6974 .unused,
6975 },9614 },
6976 .dst_temps = .{ .mem, .unused },9615 .dst_temps = .{ .mem, .unused },
6977 .clobbers = .{ .eflags = true },9616 .clobbers = .{ .eflags = true },
6978 .each = .{ .once = &.{9617 .each = .{ .once = &.{
9618 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
9619 .{ ._, ._, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ },
6979 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },9620 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
6980 .{ .@"0:", .p_w, .movsxb, .tmp1x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ },9621 .{ .@"0:", .p_w, .movsxb, .tmp3x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ },
6981 .{ ._, .p_w, .movsxb, .tmp2x, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ },9622 .{ ._, .p_w, .movsxb, .tmp4x, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ },
6982 .{ ._, .p_w, .mull, .tmp1x, .tmp2x, ._, ._ },9623 .{ ._, .p_w, .mull, .tmp3x, .tmp4x, ._, ._ },
6983 .{ ._, .p_b, .ackssw, .tmp1x, .tmp1x, ._, ._ },9624 .{ ._, .p_b, .shuf, .tmp3x, .tmp2x, ._, ._ },
6984 .{ ._, ._q, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ },9625 .{ ._, ._q, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp3x, ._, ._ },
6985 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },9626 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
6986 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },9627 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
6987 } },9628 } },
6988 }, .{9629 }, .{
6989 .required_features = .{ .sse2, null, null, null },9630 .required_features = .{ .ssse3, null, null, null },
6990 .src_constraints = .{9631 .src_constraints = .{
6991 .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .byte } },9632 .{ .multiple_scalar_int = .{ .of = .qword, .is = .byte } },
6992 .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .byte } },9633 .{ .multiple_scalar_int = .{ .of = .qword, .is = .byte } },
6993 .any,9634 .any,
6994 },9635 },
6995 .patterns = &.{9636 .patterns = &.{
...@@ -6997,10 +9638,10 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -6997,10 +9638,10 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6997 },9638 },
6998 .extra_temps = .{9639 .extra_temps = .{
6999 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },9640 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
7000 .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } },9641 .{ .type = .vector_8_u8, .kind = .{ .pshufb_trunc_mem = .{ .of = .xword, .from = .word, .to = .byte } } },
7001 .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } },9642 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
7002 .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } },9643 .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } },
7003 .unused,9644 .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } },
7004 .unused,9645 .unused,
7005 .unused,9646 .unused,
7006 .unused,9647 .unused,
...@@ -7011,26 +9652,24 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -7011,26 +9652,24 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7011 .dst_temps = .{ .mem, .unused },9652 .dst_temps = .{ .mem, .unused },
7012 .clobbers = .{ .eflags = true },9653 .clobbers = .{ .eflags = true },
7013 .each = .{ .once = &.{9654 .each = .{ .once = &.{
9655 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
9656 .{ ._, ._, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ },
7014 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },9657 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
7015 .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ },9658 .{ .@"0:", ._q, .mov, .tmp3x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ },
7016 .{ ._, ._q, .mov, .tmp2x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ },9659 .{ ._, .p_, .unpcklbw, .tmp3x, .tmp3x, ._, ._ },
7017 .{ ._, .p_b, .cmpgt, .tmp1x, .tmp2x, ._, ._ },9660 .{ ._, ._q, .mov, .tmp4x, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ },
7018 .{ ._, .p_, .unpcklbw, .tmp2x, .tmp1x, ._, ._ },9661 .{ ._, .p_, .unpcklbw, .tmp4x, .tmp4x, ._, ._ },
7019 .{ ._, .p_, .xor, .tmp1x, .tmp1x, ._, ._ },9662 .{ ._, .p_w, .mull, .tmp3x, .tmp4x, ._, ._ },
7020 .{ ._, ._q, .mov, .tmp3x, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ },9663 .{ ._, .p_b, .shuf, .tmp3x, .tmp2x, ._, ._ },
7021 .{ ._, .p_b, .cmpgt, .tmp1x, .tmp3x, ._, ._ },9664 .{ ._, ._q, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp3x, ._, ._ },
7022 .{ ._, .p_, .unpcklbw, .tmp3x, .tmp1x, ._, ._ },
7023 .{ ._, .p_w, .mull, .tmp2x, .tmp3x, ._, ._ },
7024 .{ ._, .p_b, .ackssw, .tmp2x, .tmp2x, ._, ._ },
7025 .{ ._, ._q, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp2x, ._, ._ },
7026 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },9665 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
7027 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },9666 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
7028 } },9667 } },
7029 }, .{9668 }, .{
7030 .required_features = .{ .slow_incdec, null, null, null },9669 .required_features = .{ .sse3, null, null, null },
7031 .src_constraints = .{9670 .src_constraints = .{
7032 .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } },9671 .{ .multiple_scalar_int = .{ .of = .qword, .is = .byte } },
7033 .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } },9672 .{ .multiple_scalar_int = .{ .of = .qword, .is = .byte } },
7034 .any,9673 .any,
7035 },9674 },
7036 .patterns = &.{9675 .patterns = &.{
...@@ -7038,13 +9677,53 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -7038,13 +9677,53 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7038 },9677 },
7039 .extra_temps = .{9678 .extra_temps = .{
7040 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },9679 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
7041 .{ .type = .i8, .kind = .{ .reg = .al } },9680 .{ .type = .vector_8_u8, .kind = .{ .pand_trunc_mem = .{ .from = .word, .to = .byte } } },
9681 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
9682 .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } },
9683 .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } },
7042 .unused,9684 .unused,
7043 .unused,9685 .unused,
7044 .unused,9686 .unused,
7045 .unused,9687 .unused,
7046 .unused,9688 .unused,
7047 .unused,9689 .unused,
9690 },
9691 .dst_temps = .{ .mem, .unused },
9692 .clobbers = .{ .eflags = true },
9693 .each = .{ .once = &.{
9694 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
9695 .{ ._, ._, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ },
9696 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
9697 .{ .@"0:", ._q, .mov, .tmp3x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ },
9698 .{ ._, .p_, .unpcklbw, .tmp3x, .tmp3x, ._, ._ },
9699 .{ ._, ._q, .mov, .tmp4x, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ },
9700 .{ ._, .p_, .unpcklbw, .tmp4x, .tmp4x, ._, ._ },
9701 .{ ._, .p_w, .mull, .tmp3x, .tmp4x, ._, ._ },
9702 .{ ._, .p_, .@"and", .tmp3x, .tmp2x, ._, ._ },
9703 .{ ._, .p_b, .ackusw, .tmp3x, .tmp3x, ._, ._ },
9704 .{ ._, ._q, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp3x, ._, ._ },
9705 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
9706 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
9707 } },
9708 }, .{
9709 .required_features = .{ .sse2, null, null, null },
9710 .src_constraints = .{
9711 .{ .multiple_scalar_int = .{ .of = .qword, .is = .byte } },
9712 .{ .multiple_scalar_int = .{ .of = .qword, .is = .byte } },
9713 .any,
9714 },
9715 .patterns = &.{
9716 .{ .src = .{ .to_mem, .to_mem, .none } },
9717 },
9718 .extra_temps = .{
9719 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
9720 .{ .type = .vector_16_u8, .kind = .{ .pand_trunc_mem = .{ .from = .word, .to = .byte } } },
9721 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
9722 .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } },
9723 .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } },
9724 .unused,
9725 .unused,
9726 .unused,
7048 .unused,9727 .unused,
7049 .unused,9728 .unused,
7050 .unused,9729 .unused,
...@@ -7052,14 +9731,22 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -7052,14 +9731,22 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7052 .dst_temps = .{ .mem, .unused },9731 .dst_temps = .{ .mem, .unused },
7053 .clobbers = .{ .eflags = true },9732 .clobbers = .{ .eflags = true },
7054 .each = .{ .once = &.{9733 .each = .{ .once = &.{
9734 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
9735 .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ },
7055 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },9736 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
7056 .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ },9737 .{ .@"0:", ._q, .mov, .tmp3x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ },
7057 .{ ._, .i_, .mul, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._, ._ },9738 .{ ._, .p_, .unpcklbw, .tmp3x, .tmp3x, ._, ._ },
7058 .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ },9739 .{ ._, ._q, .mov, .tmp4x, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ },
7059 .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },9740 .{ ._, .p_, .unpcklbw, .tmp4x, .tmp4x, ._, ._ },
9741 .{ ._, .p_w, .mull, .tmp3x, .tmp4x, ._, ._ },
9742 .{ ._, .p_, .@"and", .tmp3x, .tmp2x, ._, ._ },
9743 .{ ._, .p_b, .ackusw, .tmp3x, .tmp3x, ._, ._ },
9744 .{ ._, ._q, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp3x, ._, ._ },
9745 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
7060 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },9746 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
7061 } },9747 } },
7062 }, .{9748 }, .{
9749 .required_features = .{ .slow_incdec, null, null, null },
7063 .src_constraints = .{9750 .src_constraints = .{
7064 .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } },9751 .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } },
7065 .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } },9752 .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } },
...@@ -7088,14 +9775,13 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -7088,14 +9775,13 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7088 .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ },9775 .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ },
7089 .{ ._, .i_, .mul, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._, ._ },9776 .{ ._, .i_, .mul, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._, ._ },
7090 .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ },9777 .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ },
7091 .{ ._, ._c, .in, .tmp0p, ._, ._, ._ },9778 .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
7092 .{ ._, ._nz, .j, .@"0b", ._, ._, ._ },9779 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
7093 } },9780 } },
7094 }, .{9781 }, .{
7095 .required_features = .{ .avx2, null, null, null },
7096 .src_constraints = .{9782 .src_constraints = .{
7097 .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .byte } },9783 .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } },
7098 .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .byte } },9784 .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } },
7099 .any,9785 .any,
7100 },9786 },
7101 .patterns = &.{9787 .patterns = &.{
...@@ -7103,8 +9789,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -7103,8 +9789,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7103 },9789 },
7104 .extra_temps = .{9790 .extra_temps = .{
7105 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },9791 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
7106 .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } },9792 .{ .type = .i8, .kind = .{ .reg = .al } },
7107 .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } },9793 .unused,
7108 .unused,9794 .unused,
7109 .unused,9795 .unused,
7110 .unused,9796 .unused,
...@@ -7118,20 +9804,17 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -7118,20 +9804,17 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7118 .clobbers = .{ .eflags = true },9804 .clobbers = .{ .eflags = true },
7119 .each = .{ .once = &.{9805 .each = .{ .once = &.{
7120 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },9806 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
7121 .{ .@"0:", .vp_w, .movzxb, .tmp1y, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ },9807 .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ },
7122 .{ ._, .vp_w, .movzxb, .tmp2y, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ },9808 .{ ._, .i_, .mul, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._, ._ },
7123 .{ ._, .vp_w, .mull, .tmp1y, .tmp1y, .tmp2y, ._ },9809 .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ },
7124 .{ ._, .vp_b, .ackusw, .tmp1y, .tmp1y, .tmp1y, ._ },9810 .{ ._, ._c, .in, .tmp0p, ._, ._, ._ },
7125 .{ ._, .v_q, .perm, .tmp1y, .tmp1y, .ui(0b10_00_10_00), ._ },9811 .{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
7126 .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ },
7127 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
7128 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
7129 } },9812 } },
7130 }, .{9813 }, .{
7131 .required_features = .{ .avx, null, null, null },9814 .required_features = .{ .avx2, null, null, null },
7132 .src_constraints = .{9815 .src_constraints = .{
7133 .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .byte } },9816 .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .byte } },
7134 .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .byte } },9817 .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .byte } },
7135 .any,9818 .any,
7136 },9819 },
7137 .patterns = &.{9820 .patterns = &.{
...@@ -7139,10 +9822,10 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -7139,10 +9822,10 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7139 },9822 },
7140 .extra_temps = .{9823 .extra_temps = .{
7141 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },9824 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
7142 .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } },9825 .{ .type = .vector_8_u8, .kind = .{ .pshufb_trunc_mem = .{ .of = .xword, .from = .word, .to = .byte } } },
7143 .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } },9826 .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } },
7144 .unused,9827 .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } },
7145 .unused,9828 .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } },
7146 .unused,9829 .unused,
7147 .unused,9830 .unused,
7148 .unused,9831 .unused,
...@@ -7153,17 +9836,20 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -7153,17 +9836,20 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7153 .dst_temps = .{ .mem, .unused },9836 .dst_temps = .{ .mem, .unused },
7154 .clobbers = .{ .eflags = true },9837 .clobbers = .{ .eflags = true },
7155 .each = .{ .once = &.{9838 .each = .{ .once = &.{
9839 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
9840 .{ ._, .vp_q, .broadcast, .tmp2y, .lea(.tmp0q), ._, ._ },
7156 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },9841 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
7157 .{ .@"0:", .vp_w, .movzxb, .tmp1x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ },9842 .{ .@"0:", .vp_w, .movzxb, .tmp3y, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ },
7158 .{ ._, .vp_w, .movzxb, .tmp2x, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ },9843 .{ ._, .vp_w, .movzxb, .tmp4y, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ },
7159 .{ ._, .vp_w, .mull, .tmp1x, .tmp1x, .tmp2x, ._ },9844 .{ ._, .vp_w, .mull, .tmp3y, .tmp3y, .tmp4y, ._ },
7160 .{ ._, .vp_b, .ackusw, .tmp1x, .tmp1x, .tmp1x, ._ },9845 .{ ._, .vp_b, .shuf, .tmp3y, .tmp3y, .tmp2y, ._ },
7161 .{ ._, .v_q, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ },9846 .{ ._, .v_q, .perm, .tmp3y, .tmp3y, .ui(0b10_00_10_00), ._ },
7162 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },9847 .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp3x, ._, ._ },
9848 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
7163 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },9849 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
7164 } },9850 } },
7165 }, .{9851 }, .{
7166 .required_features = .{ .sse4_1, null, null, null },9852 .required_features = .{ .avx, null, null, null },
7167 .src_constraints = .{9853 .src_constraints = .{
7168 .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .byte } },9854 .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .byte } },
7169 .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .byte } },9855 .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .byte } },
...@@ -7174,6 +9860,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -7174,6 +9860,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7174 },9860 },
7175 .extra_temps = .{9861 .extra_temps = .{
7176 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },9862 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
9863 .{ .type = .vector_8_u8, .kind = .{ .pshufb_trunc_mem = .{ .of = .xword, .from = .word, .to = .byte } } },
9864 .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } },
7177 .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } },9865 .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } },
7178 .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } },9866 .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } },
7179 .unused,9867 .unused,
...@@ -7182,23 +9870,23 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -7182,23 +9870,23 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7182 .unused,9870 .unused,
7183 .unused,9871 .unused,
7184 .unused,9872 .unused,
7185 .unused,
7186 .unused,
7187 },9873 },
7188 .dst_temps = .{ .mem, .unused },9874 .dst_temps = .{ .mem, .unused },
7189 .clobbers = .{ .eflags = true },9875 .clobbers = .{ .eflags = true },
7190 .each = .{ .once = &.{9876 .each = .{ .once = &.{
9877 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
9878 .{ ._, .v_, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ },
7191 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },9879 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
7192 .{ .@"0:", .p_w, .movzxb, .tmp1x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ },9880 .{ .@"0:", .vp_w, .movzxb, .tmp3x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ },
7193 .{ ._, .p_w, .movzxb, .tmp2x, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ },9881 .{ ._, .vp_w, .movzxb, .tmp4x, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ },
7194 .{ ._, .p_w, .mull, .tmp1x, .tmp2x, ._, ._ },9882 .{ ._, .vp_w, .mull, .tmp3x, .tmp3x, .tmp4x, ._ },
7195 .{ ._, .p_b, .ackusw, .tmp1x, .tmp1x, ._, ._ },9883 .{ ._, .vp_b, .shuf, .tmp3x, .tmp3x, .tmp2x, ._ },
7196 .{ ._, ._q, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ },9884 .{ ._, .v_q, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp3x, ._, ._ },
7197 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },9885 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
7198 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },9886 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
7199 } },9887 } },
7200 }, .{9888 }, .{
7201 .required_features = .{ .sse2, null, null, null },9889 .required_features = .{ .sse4_1, null, null, null },
7202 .src_constraints = .{9890 .src_constraints = .{
7203 .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .byte } },9891 .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .byte } },
7204 .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .byte } },9892 .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .byte } },
...@@ -7209,10 +9897,10 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -7209,10 +9897,10 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7209 },9897 },
7210 .extra_temps = .{9898 .extra_temps = .{
7211 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },9899 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
9900 .{ .type = .vector_8_u8, .kind = .{ .pshufb_trunc_mem = .{ .of = .xword, .from = .word, .to = .byte } } },
9901 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
7212 .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } },9902 .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } },
7213 .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } },9903 .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } },
7214 .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } },
7215 .unused,
7216 .unused,9904 .unused,
7217 .unused,9905 .unused,
7218 .unused,9906 .unused,
...@@ -7223,15 +9911,14 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -7223,15 +9911,14 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7223 .dst_temps = .{ .mem, .unused },9911 .dst_temps = .{ .mem, .unused },
7224 .clobbers = .{ .eflags = true },9912 .clobbers = .{ .eflags = true },
7225 .each = .{ .once = &.{9913 .each = .{ .once = &.{
9914 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
9915 .{ ._, ._, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ },
7226 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },9916 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
7227 .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ },9917 .{ .@"0:", .p_w, .movzxb, .tmp3x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ },
7228 .{ ._, ._q, .mov, .tmp2x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ },9918 .{ ._, .p_w, .movzxb, .tmp4x, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ },
7229 .{ ._, .p_, .unpcklbw, .tmp2x, .tmp1x, ._, ._ },9919 .{ ._, .p_w, .mull, .tmp3x, .tmp4x, ._, ._ },
7230 .{ ._, ._q, .mov, .tmp3x, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ },9920 .{ ._, .p_b, .shuf, .tmp3x, .tmp2x, ._, ._ },
7231 .{ ._, .p_, .unpcklbw, .tmp3x, .tmp1x, ._, ._ },9921 .{ ._, ._q, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp3x, ._, ._ },
7232 .{ ._, .p_w, .mull, .tmp2x, .tmp3x, ._, ._ },
7233 .{ ._, .p_b, .ackusw, .tmp2x, .tmp2x, ._, ._ },
7234 .{ ._, ._q, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp2x, ._, ._ },
7235 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },9922 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
7236 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },9923 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
7237 } },9924 } },
...@@ -8893,9 +11580,16 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -8893,9 +11580,16 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8893 }),11580 }),
8894 else => |e| return e,11581 else => |e| return e,
8895 };11582 };
11583 res[0].wrapInt(cg) catch |err| switch (err) {
11584 error.SelectFailed => return cg.fail("failed to select wrap {} {} {}", .{
11585 cg.typeOf(bin_op.lhs).fmt(pt),
11586 ops[0].tracking(cg),
11587 ops[1].tracking(cg),
11588 }),
11589 else => |e| return e,
11590 };
8896 try res[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg);11591 try res[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg);
8897 },11592 },
8898 .mul_safe => unreachable,
8899 .div_float, .div_float_optimized, .div_exact, .div_exact_optimized => |air_tag| if (use_old) try cg.airMulDivBinOp(inst, switch (air_tag) {11593 .div_float, .div_float_optimized, .div_exact, .div_exact_optimized => |air_tag| if (use_old) try cg.airMulDivBinOp(inst, switch (air_tag) {
8900 else => unreachable,11594 else => unreachable,
8901 .div_float, .div_float_optimized => .div_float,11595 .div_float, .div_float_optimized => .div_float,
...@@ -32065,7 +34759,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -32065,7 +34759,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
32065 .{ .src = .{ .to_sse, .none, .none } },34759 .{ .src = .{ .to_sse, .none, .none } },
32066 },34760 },
32067 .extra_temps = .{34761 .extra_temps = .{
32068 .{ .type = .vector_16_u8, .kind = .forward_bits },34762 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
32069 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },34763 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
32070 .unused,34764 .unused,
32071 .unused,34765 .unused,
...@@ -32089,7 +34783,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -32089,7 +34783,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
32089 .{ .src = .{ .to_mut_sse, .none, .none } },34783 .{ .src = .{ .to_mut_sse, .none, .none } },
32090 },34784 },
32091 .extra_temps = .{34785 .extra_temps = .{
32092 .{ .type = .vector_16_u8, .kind = .forward_bits },34786 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
32093 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },34787 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
32094 .unused,34788 .unused,
32095 .unused,34789 .unused,
...@@ -32113,7 +34807,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -32113,7 +34807,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
32113 .{ .src = .{ .to_sse, .none, .none } },34807 .{ .src = .{ .to_sse, .none, .none } },
32114 },34808 },
32115 .extra_temps = .{34809 .extra_temps = .{
32116 .{ .type = .vector_16_u8, .kind = .reverse_bits },34810 .{ .type = .vector_16_u8, .kind = .reverse_bits_mem },
32117 .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },34811 .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },
32118 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },34812 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
32119 .unused,34813 .unused,
...@@ -32141,7 +34835,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -32141,7 +34835,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
32141 .{ .src = .{ .to_sse, .none, .none } },34835 .{ .src = .{ .to_sse, .none, .none } },
32142 },34836 },
32143 .extra_temps = .{34837 .extra_temps = .{
32144 .{ .type = .vector_16_u8, .kind = .reverse_bits },34838 .{ .type = .vector_16_u8, .kind = .reverse_bits_mem },
32145 .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },34839 .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },
32146 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },34840 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
32147 .unused,34841 .unused,
...@@ -32170,7 +34864,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -32170,7 +34864,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
32170 .{ .src = .{ .to_mut_sse, .none, .none } },34864 .{ .src = .{ .to_mut_sse, .none, .none } },
32171 },34865 },
32172 .extra_temps = .{34866 .extra_temps = .{
32173 .{ .type = .vector_16_u8, .kind = .reverse_bits },34867 .{ .type = .vector_16_u8, .kind = .reverse_bits_mem },
32174 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },34868 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
32175 .unused,34869 .unused,
32176 .unused,34870 .unused,
...@@ -32199,7 +34893,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -32199,7 +34893,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
32199 .{ .src = .{ .to_mut_sse, .none, .none } },34893 .{ .src = .{ .to_mut_sse, .none, .none } },
32200 },34894 },
32201 .extra_temps = .{34895 .extra_temps = .{
32202 .{ .type = .vector_16_u8, .kind = .reverse_bits },34896 .{ .type = .vector_16_u8, .kind = .reverse_bits_mem },
32203 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },34897 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
32204 .unused,34898 .unused,
32205 .unused,34899 .unused,
...@@ -32266,7 +34960,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -32266,7 +34960,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
32266 .{ .src = .{ .to_sse, .none, .none } },34960 .{ .src = .{ .to_sse, .none, .none } },
32267 },34961 },
32268 .extra_temps = .{34962 .extra_temps = .{
32269 .{ .type = .vector_16_u8, .kind = .forward_bits },34963 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
32270 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },34964 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
32271 .unused,34965 .unused,
32272 .unused,34966 .unused,
...@@ -32292,7 +34986,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -32292,7 +34986,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
32292 .{ .src = .{ .to_mut_sse, .none, .none } },34986 .{ .src = .{ .to_mut_sse, .none, .none } },
32293 },34987 },
32294 .extra_temps = .{34988 .extra_temps = .{
32295 .{ .type = .vector_16_u8, .kind = .forward_bits },34989 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
32296 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },34990 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
32297 .unused,34991 .unused,
32298 .unused,34992 .unused,
...@@ -32318,7 +35012,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -32318,7 +35012,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
32318 .{ .src = .{ .to_sse, .none, .none } },35012 .{ .src = .{ .to_sse, .none, .none } },
32319 },35013 },
32320 .extra_temps = .{35014 .extra_temps = .{
32321 .{ .type = .vector_16_u8, .kind = .reverse_bits },35015 .{ .type = .vector_16_u8, .kind = .reverse_bits_mem },
32322 .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },35016 .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },
32323 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },35017 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
32324 .unused,35018 .unused,
...@@ -32348,7 +35042,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -32348,7 +35042,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
32348 .{ .src = .{ .to_sse, .none, .none } },35042 .{ .src = .{ .to_sse, .none, .none } },
32349 },35043 },
32350 .extra_temps = .{35044 .extra_temps = .{
32351 .{ .type = .vector_16_u8, .kind = .reverse_bits },35045 .{ .type = .vector_16_u8, .kind = .reverse_bits_mem },
32352 .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },35046 .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },
32353 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },35047 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
32354 .unused,35048 .unused,
...@@ -32379,7 +35073,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -32379,7 +35073,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
32379 .{ .src = .{ .to_mut_sse, .none, .none } },35073 .{ .src = .{ .to_mut_sse, .none, .none } },
32380 },35074 },
32381 .extra_temps = .{35075 .extra_temps = .{
32382 .{ .type = .vector_16_u8, .kind = .reverse_bits },35076 .{ .type = .vector_16_u8, .kind = .reverse_bits_mem },
32383 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },35077 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
32384 .unused,35078 .unused,
32385 .unused,35079 .unused,
...@@ -32410,7 +35104,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -32410,7 +35104,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
32410 .{ .src = .{ .to_mut_sse, .none, .none } },35104 .{ .src = .{ .to_mut_sse, .none, .none } },
32411 },35105 },
32412 .extra_temps = .{35106 .extra_temps = .{
32413 .{ .type = .vector_16_u8, .kind = .reverse_bits },35107 .{ .type = .vector_16_u8, .kind = .reverse_bits_mem },
32414 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },35108 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
32415 .unused,35109 .unused,
32416 .unused,35110 .unused,
...@@ -32480,7 +35174,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -32480,7 +35174,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
32480 .{ .src = .{ .to_sse, .none, .none } },35174 .{ .src = .{ .to_sse, .none, .none } },
32481 },35175 },
32482 .extra_temps = .{35176 .extra_temps = .{
32483 .{ .type = .vector_16_u8, .kind = .forward_bits },35177 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
32484 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },35178 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
32485 .unused,35179 .unused,
32486 .unused,35180 .unused,
...@@ -32506,7 +35200,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -32506,7 +35200,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
32506 .{ .src = .{ .to_mut_sse, .none, .none } },35200 .{ .src = .{ .to_mut_sse, .none, .none } },
32507 },35201 },
32508 .extra_temps = .{35202 .extra_temps = .{
32509 .{ .type = .vector_16_u8, .kind = .forward_bits },35203 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
32510 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },35204 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
32511 .unused,35205 .unused,
32512 .unused,35206 .unused,
...@@ -32532,7 +35226,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -32532,7 +35226,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
32532 .{ .src = .{ .to_sse, .none, .none } },35226 .{ .src = .{ .to_sse, .none, .none } },
32533 },35227 },
32534 .extra_temps = .{35228 .extra_temps = .{
32535 .{ .type = .vector_16_u8, .kind = .reverse_bits },35229 .{ .type = .vector_16_u8, .kind = .reverse_bits_mem },
32536 .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },35230 .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },
32537 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },35231 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
32538 .unused,35232 .unused,
...@@ -32562,7 +35256,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -32562,7 +35256,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
32562 .{ .src = .{ .to_sse, .none, .none } },35256 .{ .src = .{ .to_sse, .none, .none } },
32563 },35257 },
32564 .extra_temps = .{35258 .extra_temps = .{
32565 .{ .type = .vector_16_u8, .kind = .reverse_bits },35259 .{ .type = .vector_16_u8, .kind = .reverse_bits_mem },
32566 .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },35260 .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },
32567 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },35261 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
32568 .unused,35262 .unused,
...@@ -32593,7 +35287,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -32593,7 +35287,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
32593 .{ .src = .{ .to_mut_sse, .none, .none } },35287 .{ .src = .{ .to_mut_sse, .none, .none } },
32594 },35288 },
32595 .extra_temps = .{35289 .extra_temps = .{
32596 .{ .type = .vector_16_u8, .kind = .reverse_bits },35290 .{ .type = .vector_16_u8, .kind = .reverse_bits_mem },
32597 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },35291 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
32598 .unused,35292 .unused,
32599 .unused,35293 .unused,
...@@ -32624,7 +35318,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -32624,7 +35318,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
32624 .{ .src = .{ .to_mut_sse, .none, .none } },35318 .{ .src = .{ .to_mut_sse, .none, .none } },
32625 },35319 },
32626 .extra_temps = .{35320 .extra_temps = .{
32627 .{ .type = .vector_16_u8, .kind = .reverse_bits },35321 .{ .type = .vector_16_u8, .kind = .reverse_bits_mem },
32628 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },35322 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
32629 .unused,35323 .unused,
32630 .unused,35324 .unused,
...@@ -32694,7 +35388,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -32694,7 +35388,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
32694 .{ .src = .{ .to_sse, .none, .none } },35388 .{ .src = .{ .to_sse, .none, .none } },
32695 },35389 },
32696 .extra_temps = .{35390 .extra_temps = .{
32697 .{ .type = .vector_16_u8, .kind = .forward_bits },35391 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
32698 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .word } } },35392 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .word } } },
32699 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },35393 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
32700 .unused,35394 .unused,
...@@ -32720,7 +35414,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -32720,7 +35414,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
32720 .{ .src = .{ .to_mut_sse, .none, .none } },35414 .{ .src = .{ .to_mut_sse, .none, .none } },
32721 },35415 },
32722 .extra_temps = .{35416 .extra_temps = .{
32723 .{ .type = .vector_16_u8, .kind = .forward_bits },35417 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
32724 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .word } } },35418 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .word } } },
32725 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },35419 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
32726 .unused,35420 .unused,
...@@ -32746,7 +35440,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -32746,7 +35440,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
32746 .{ .src = .{ .to_mut_sse, .none, .none } },35440 .{ .src = .{ .to_mut_sse, .none, .none } },
32747 },35441 },
32748 .extra_temps = .{35442 .extra_temps = .{
32749 .{ .type = .vector_16_u8, .kind = .forward_bits },35443 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
32750 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },35444 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
32751 .unused,35445 .unused,
32752 .unused,35446 .unused,
...@@ -32773,7 +35467,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -32773,7 +35467,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
32773 },35467 },
32774 .extra_temps = .{35468 .extra_temps = .{
32775 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .word, .smear = 8 } } },35469 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .word, .smear = 8 } } },
32776 .{ .type = .vector_16_u8, .kind = .reverse_bits },35470 .{ .type = .vector_16_u8, .kind = .reverse_bits_mem },
32777 .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },35471 .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },
32778 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },35472 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
32779 .unused,35473 .unused,
...@@ -32802,7 +35496,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -32802,7 +35496,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
32802 },35496 },
32803 .extra_temps = .{35497 .extra_temps = .{
32804 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .word, .smear = 8 } } },35498 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .word, .smear = 8 } } },
32805 .{ .type = .vector_16_u8, .kind = .reverse_bits },35499 .{ .type = .vector_16_u8, .kind = .reverse_bits_mem },
32806 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },35500 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
32807 .unused,35501 .unused,
32808 .unused,35502 .unused,
...@@ -32871,7 +35565,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -32871,7 +35565,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
32871 .{ .src = .{ .to_sse, .none, .none } },35565 .{ .src = .{ .to_sse, .none, .none } },
32872 },35566 },
32873 .extra_temps = .{35567 .extra_temps = .{
32874 .{ .type = .vector_16_u8, .kind = .forward_bits },35568 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
32875 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .word } } },35569 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .word } } },
32876 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },35570 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
32877 .unused,35571 .unused,
...@@ -32898,7 +35592,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -32898,7 +35592,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
32898 .{ .src = .{ .to_mut_sse, .none, .none } },35592 .{ .src = .{ .to_mut_sse, .none, .none } },
32899 },35593 },
32900 .extra_temps = .{35594 .extra_temps = .{
32901 .{ .type = .vector_16_u8, .kind = .forward_bits },35595 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
32902 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .word } } },35596 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .word } } },
32903 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },35597 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
32904 .unused,35598 .unused,
...@@ -32925,7 +35619,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -32925,7 +35619,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
32925 .{ .src = .{ .to_mut_sse, .none, .none } },35619 .{ .src = .{ .to_mut_sse, .none, .none } },
32926 },35620 },
32927 .extra_temps = .{35621 .extra_temps = .{
32928 .{ .type = .vector_16_u8, .kind = .forward_bits },35622 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
32929 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },35623 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
32930 .unused,35624 .unused,
32931 .unused,35625 .unused,
...@@ -32953,7 +35647,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -32953,7 +35647,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
32953 },35647 },
32954 .extra_temps = .{35648 .extra_temps = .{
32955 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .word, .smear = 8 } } },35649 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .word, .smear = 8 } } },
32956 .{ .type = .vector_16_u8, .kind = .reverse_bits },35650 .{ .type = .vector_16_u8, .kind = .reverse_bits_mem },
32957 .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },35651 .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },
32958 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },35652 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
32959 .unused,35653 .unused,
...@@ -32984,7 +35678,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -32984,7 +35678,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
32984 },35678 },
32985 .extra_temps = .{35679 .extra_temps = .{
32986 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .word, .smear = 8 } } },35680 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .word, .smear = 8 } } },
32987 .{ .type = .vector_16_u8, .kind = .reverse_bits },35681 .{ .type = .vector_16_u8, .kind = .reverse_bits_mem },
32988 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },35682 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
32989 .unused,35683 .unused,
32990 .unused,35684 .unused,
...@@ -33056,7 +35750,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -33056,7 +35750,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
33056 .{ .src = .{ .to_sse, .none, .none } },35750 .{ .src = .{ .to_sse, .none, .none } },
33057 },35751 },
33058 .extra_temps = .{35752 .extra_temps = .{
33059 .{ .type = .vector_16_u8, .kind = .forward_bits },35753 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
33060 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .word } } },35754 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .word } } },
33061 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },35755 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
33062 .unused,35756 .unused,
...@@ -33083,7 +35777,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -33083,7 +35777,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
33083 .{ .src = .{ .to_mut_sse, .none, .none } },35777 .{ .src = .{ .to_mut_sse, .none, .none } },
33084 },35778 },
33085 .extra_temps = .{35779 .extra_temps = .{
33086 .{ .type = .vector_16_u8, .kind = .forward_bits },35780 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
33087 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .word } } },35781 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .word } } },
33088 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },35782 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
33089 .unused,35783 .unused,
...@@ -33110,7 +35804,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -33110,7 +35804,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
33110 .{ .src = .{ .to_mut_sse, .none, .none } },35804 .{ .src = .{ .to_mut_sse, .none, .none } },
33111 },35805 },
33112 .extra_temps = .{35806 .extra_temps = .{
33113 .{ .type = .vector_16_u8, .kind = .forward_bits },35807 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
33114 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },35808 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
33115 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },35809 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
33116 .unused,35810 .unused,
...@@ -33139,7 +35833,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -33139,7 +35833,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
33139 },35833 },
33140 .extra_temps = .{35834 .extra_temps = .{
33141 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .word, .smear = 8 } } },35835 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .word, .smear = 8 } } },
33142 .{ .type = .vector_16_u8, .kind = .reverse_bits },35836 .{ .type = .vector_16_u8, .kind = .reverse_bits_mem },
33143 .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },35837 .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },
33144 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },35838 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
33145 .unused,35839 .unused,
...@@ -33170,7 +35864,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -33170,7 +35864,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
33170 },35864 },
33171 .extra_temps = .{35865 .extra_temps = .{
33172 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .word, .smear = 8 } } },35866 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .word, .smear = 8 } } },
33173 .{ .type = .vector_16_u8, .kind = .reverse_bits },35867 .{ .type = .vector_16_u8, .kind = .reverse_bits_mem },
33174 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },35868 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
33175 .unused,35869 .unused,
33176 .unused,35870 .unused,
...@@ -33242,7 +35936,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -33242,7 +35936,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
33242 .{ .src = .{ .to_sse, .none, .none } },35936 .{ .src = .{ .to_sse, .none, .none } },
33243 },35937 },
33244 .extra_temps = .{35938 .extra_temps = .{
33245 .{ .type = .vector_16_u8, .kind = .forward_bits },35939 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
33246 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .dword } } },35940 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .dword } } },
33247 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },35941 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
33248 .unused,35942 .unused,
...@@ -33268,7 +35962,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -33268,7 +35962,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
33268 .{ .src = .{ .to_mut_sse, .none, .none } },35962 .{ .src = .{ .to_mut_sse, .none, .none } },
33269 },35963 },
33270 .extra_temps = .{35964 .extra_temps = .{
33271 .{ .type = .vector_16_u8, .kind = .forward_bits },35965 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
33272 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .dword } } },35966 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .dword } } },
33273 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },35967 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
33274 .unused,35968 .unused,
...@@ -33294,7 +35988,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -33294,7 +35988,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
33294 .{ .src = .{ .to_mut_sse, .none, .none } },35988 .{ .src = .{ .to_mut_sse, .none, .none } },
33295 },35989 },
33296 .extra_temps = .{35990 .extra_temps = .{
33297 .{ .type = .vector_16_u8, .kind = .forward_bits },35991 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
33298 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },35992 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
33299 .unused,35993 .unused,
33300 .unused,35994 .unused,
...@@ -33322,7 +36016,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -33322,7 +36016,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
33322 },36016 },
33323 .extra_temps = .{36017 .extra_temps = .{
33324 .{ .type = .vector_32_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .dword, .smear = 8 } } },36018 .{ .type = .vector_32_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .dword, .smear = 8 } } },
33325 .{ .type = .vector_32_u8, .kind = .reverse_bits },36019 .{ .type = .vector_32_u8, .kind = .reverse_bits_mem },
33326 .{ .type = .vector_32_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },36020 .{ .type = .vector_32_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },
33327 .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } },36021 .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } },
33328 .unused,36022 .unused,
...@@ -33392,7 +36086,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -33392,7 +36086,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
33392 .{ .src = .{ .to_sse, .none, .none } },36086 .{ .src = .{ .to_sse, .none, .none } },
33393 },36087 },
33394 .extra_temps = .{36088 .extra_temps = .{
33395 .{ .type = .vector_16_u8, .kind = .forward_bits },36089 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
33396 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .dword } } },36090 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .dword } } },
33397 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },36091 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
33398 .unused,36092 .unused,
...@@ -33419,7 +36113,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -33419,7 +36113,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
33419 .{ .src = .{ .to_mut_sse, .none, .none } },36113 .{ .src = .{ .to_mut_sse, .none, .none } },
33420 },36114 },
33421 .extra_temps = .{36115 .extra_temps = .{
33422 .{ .type = .vector_16_u8, .kind = .forward_bits },36116 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
33423 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .dword } } },36117 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .dword } } },
33424 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },36118 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
33425 .unused,36119 .unused,
...@@ -33446,7 +36140,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -33446,7 +36140,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
33446 .{ .src = .{ .to_mut_sse, .none, .none } },36140 .{ .src = .{ .to_mut_sse, .none, .none } },
33447 },36141 },
33448 .extra_temps = .{36142 .extra_temps = .{
33449 .{ .type = .vector_16_u8, .kind = .forward_bits },36143 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
33450 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },36144 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
33451 .unused,36145 .unused,
33452 .unused,36146 .unused,
...@@ -33475,7 +36169,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -33475,7 +36169,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
33475 },36169 },
33476 .extra_temps = .{36170 .extra_temps = .{
33477 .{ .type = .vector_32_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .dword, .smear = 8 } } },36171 .{ .type = .vector_32_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .dword, .smear = 8 } } },
33478 .{ .type = .vector_32_u8, .kind = .reverse_bits },36172 .{ .type = .vector_32_u8, .kind = .reverse_bits_mem },
33479 .{ .type = .vector_32_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },36173 .{ .type = .vector_32_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },
33480 .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } },36174 .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } },
33481 .unused,36175 .unused,
...@@ -33548,7 +36242,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -33548,7 +36242,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
33548 .{ .src = .{ .to_sse, .none, .none } },36242 .{ .src = .{ .to_sse, .none, .none } },
33549 },36243 },
33550 .extra_temps = .{36244 .extra_temps = .{
33551 .{ .type = .vector_16_u8, .kind = .forward_bits },36245 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
33552 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .dword } } },36246 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .dword } } },
33553 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },36247 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
33554 .unused,36248 .unused,
...@@ -33575,7 +36269,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -33575,7 +36269,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
33575 .{ .src = .{ .to_mut_sse, .none, .none } },36269 .{ .src = .{ .to_mut_sse, .none, .none } },
33576 },36270 },
33577 .extra_temps = .{36271 .extra_temps = .{
33578 .{ .type = .vector_16_u8, .kind = .forward_bits },36272 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
33579 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .dword } } },36273 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .dword } } },
33580 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },36274 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
33581 .unused,36275 .unused,
...@@ -33602,7 +36296,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -33602,7 +36296,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
33602 .{ .src = .{ .to_mut_sse, .none, .none } },36296 .{ .src = .{ .to_mut_sse, .none, .none } },
33603 },36297 },
33604 .extra_temps = .{36298 .extra_temps = .{
33605 .{ .type = .vector_16_u8, .kind = .forward_bits },36299 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
33606 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },36300 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
33607 .unused,36301 .unused,
33608 .unused,36302 .unused,
...@@ -33631,7 +36325,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -33631,7 +36325,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
33631 },36325 },
33632 .extra_temps = .{36326 .extra_temps = .{
33633 .{ .type = .vector_32_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .dword, .smear = 8 } } },36327 .{ .type = .vector_32_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .dword, .smear = 8 } } },
33634 .{ .type = .vector_32_u8, .kind = .reverse_bits },36328 .{ .type = .vector_32_u8, .kind = .reverse_bits_mem },
33635 .{ .type = .vector_32_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },36329 .{ .type = .vector_32_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },
33636 .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } },36330 .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } },
33637 .unused,36331 .unused,
...@@ -33704,7 +36398,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -33704,7 +36398,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
33704 .{ .src = .{ .to_sse, .none, .none } },36398 .{ .src = .{ .to_sse, .none, .none } },
33705 },36399 },
33706 .extra_temps = .{36400 .extra_temps = .{
33707 .{ .type = .vector_16_u8, .kind = .forward_bits },36401 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
33708 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .qword } } },36402 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .qword } } },
33709 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },36403 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
33710 .unused,36404 .unused,
...@@ -33730,7 +36424,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -33730,7 +36424,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
33730 .{ .src = .{ .to_mut_sse, .none, .none } },36424 .{ .src = .{ .to_mut_sse, .none, .none } },
33731 },36425 },
33732 .extra_temps = .{36426 .extra_temps = .{
33733 .{ .type = .vector_16_u8, .kind = .forward_bits },36427 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
33734 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .qword } } },36428 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .qword } } },
33735 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },36429 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
33736 .unused,36430 .unused,
...@@ -33756,7 +36450,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -33756,7 +36450,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
33756 .{ .src = .{ .to_mut_sse, .none, .none } },36450 .{ .src = .{ .to_mut_sse, .none, .none } },
33757 },36451 },
33758 .extra_temps = .{36452 .extra_temps = .{
33759 .{ .type = .vector_16_u8, .kind = .forward_bits },36453 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
33760 .unused,36454 .unused,
33761 .unused,36455 .unused,
33762 .unused,36456 .unused,
...@@ -33827,7 +36521,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -33827,7 +36521,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
33827 .{ .src = .{ .to_sse, .none, .none } },36521 .{ .src = .{ .to_sse, .none, .none } },
33828 },36522 },
33829 .extra_temps = .{36523 .extra_temps = .{
33830 .{ .type = .vector_16_u8, .kind = .forward_bits },36524 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
33831 .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },36525 .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },
33832 .unused,36526 .unused,
33833 .unused,36527 .unused,
...@@ -33855,7 +36549,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -33855,7 +36549,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
33855 .{ .src = .{ .to_mut_sse, .none, .none } },36549 .{ .src = .{ .to_mut_sse, .none, .none } },
33856 },36550 },
33857 .extra_temps = .{36551 .extra_temps = .{
33858 .{ .type = .vector_16_u8, .kind = .forward_bits },36552 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
33859 .unused,36553 .unused,
33860 .unused,36554 .unused,
33861 .unused,36555 .unused,
...@@ -33929,7 +36623,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -33929,7 +36623,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
33929 .{ .src = .{ .to_sse, .none, .none } },36623 .{ .src = .{ .to_sse, .none, .none } },
33930 },36624 },
33931 .extra_temps = .{36625 .extra_temps = .{
33932 .{ .type = .vector_16_u8, .kind = .forward_bits },36626 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
33933 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .qword } } },36627 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .qword } } },
33934 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },36628 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
33935 .unused,36629 .unused,
...@@ -33956,7 +36650,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -33956,7 +36650,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
33956 .{ .src = .{ .to_mut_sse, .none, .none } },36650 .{ .src = .{ .to_mut_sse, .none, .none } },
33957 },36651 },
33958 .extra_temps = .{36652 .extra_temps = .{
33959 .{ .type = .vector_16_u8, .kind = .forward_bits },36653 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
33960 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .qword } } },36654 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .qword } } },
33961 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },36655 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
33962 .unused,36656 .unused,
...@@ -33983,7 +36677,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -33983,7 +36677,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
33983 .{ .src = .{ .to_mut_sse, .none, .none } },36677 .{ .src = .{ .to_mut_sse, .none, .none } },
33984 },36678 },
33985 .extra_temps = .{36679 .extra_temps = .{
33986 .{ .type = .vector_16_u8, .kind = .forward_bits },36680 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
33987 .unused,36681 .unused,
33988 .unused,36682 .unused,
33989 .unused,36683 .unused,
...@@ -34057,7 +36751,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -34057,7 +36751,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
34057 .{ .src = .{ .to_sse, .none, .none } },36751 .{ .src = .{ .to_sse, .none, .none } },
34058 },36752 },
34059 .extra_temps = .{36753 .extra_temps = .{
34060 .{ .type = .vector_16_u8, .kind = .forward_bits },36754 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
34061 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .xword } } },36755 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .xword } } },
34062 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },36756 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
34063 .unused,36757 .unused,
...@@ -34083,7 +36777,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -34083,7 +36777,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
34083 .{ .src = .{ .to_mut_sse, .none, .none } },36777 .{ .src = .{ .to_mut_sse, .none, .none } },
34084 },36778 },
34085 .extra_temps = .{36779 .extra_temps = .{
34086 .{ .type = .vector_16_u8, .kind = .forward_bits },36780 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
34087 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .xword } } },36781 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .xword } } },
34088 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },36782 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
34089 .unused,36783 .unused,
...@@ -34109,7 +36803,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -34109,7 +36803,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
34109 .{ .src = .{ .to_sse, .none, .none } },36803 .{ .src = .{ .to_sse, .none, .none } },
34110 },36804 },
34111 .extra_temps = .{36805 .extra_temps = .{
34112 .{ .type = .vector_16_u8, .kind = .forward_bits },36806 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
34113 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .xword } } },36807 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .xword } } },
34114 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },36808 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
34115 .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },36809 .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },
...@@ -34142,7 +36836,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -34142,7 +36836,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
34142 .{ .src = .{ .to_mut_sse, .none, .none } },36836 .{ .src = .{ .to_mut_sse, .none, .none } },
34143 },36837 },
34144 .extra_temps = .{36838 .extra_temps = .{
34145 .{ .type = .vector_16_u8, .kind = .forward_bits },36839 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
34146 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .xword } } },36840 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .xword } } },
34147 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },36841 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
34148 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },36842 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
...@@ -34175,7 +36869,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -34175,7 +36869,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
34175 .{ .src = .{ .to_sse, .none, .none } },36869 .{ .src = .{ .to_sse, .none, .none } },
34176 },36870 },
34177 .extra_temps = .{36871 .extra_temps = .{
34178 .{ .type = .vector_16_u8, .kind = .forward_bits },36872 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
34179 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .xword } } },36873 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .xword } } },
34180 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },36874 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
34181 .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },36875 .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },
...@@ -34208,7 +36902,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -34208,7 +36902,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
34208 .{ .src = .{ .to_mut_sse, .none, .none } },36902 .{ .src = .{ .to_mut_sse, .none, .none } },
34209 },36903 },
34210 .extra_temps = .{36904 .extra_temps = .{
34211 .{ .type = .vector_16_u8, .kind = .forward_bits },36905 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
34212 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .xword } } },36906 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .xword } } },
34213 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },36907 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
34214 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },36908 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
...@@ -34241,7 +36935,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -34241,7 +36935,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
34241 .{ .src = .{ .to_sse, .none, .none } },36935 .{ .src = .{ .to_sse, .none, .none } },
34242 },36936 },
34243 .extra_temps = .{36937 .extra_temps = .{
34244 .{ .type = .vector_32_u8, .kind = .forward_bits },36938 .{ .type = .vector_32_u8, .kind = .forward_bits_mem },
34245 .{ .type = .vector_32_u8, .kind = .{ .pshufb_bswap_mem = .{ .repeat = 2, .size = .xword } } },36939 .{ .type = .vector_32_u8, .kind = .{ .pshufb_bswap_mem = .{ .repeat = 2, .size = .xword } } },
34246 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },36940 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
34247 .unused,36941 .unused,
...@@ -34268,7 +36962,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -34268,7 +36962,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
34268 .{ .src = .{ .to_sse, .none, .none } },36962 .{ .src = .{ .to_sse, .none, .none } },
34269 },36963 },
34270 .extra_temps = .{36964 .extra_temps = .{
34271 .{ .type = .vector_32_u8, .kind = .forward_bits },36965 .{ .type = .vector_32_u8, .kind = .forward_bits_mem },
34272 .{ .type = .vector_32_u8, .kind = .{ .pshufb_bswap_mem = .{ .repeat = 2, .size = .xword } } },36966 .{ .type = .vector_32_u8, .kind = .{ .pshufb_bswap_mem = .{ .repeat = 2, .size = .xword } } },
34273 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },36967 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
34274 .unused,36968 .unused,
...@@ -34297,7 +36991,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -34297,7 +36991,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
34297 .extra_temps = .{36991 .extra_temps = .{
34298 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },36992 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
34299 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },36993 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
34300 .{ .type = .vector_32_u8, .kind = .forward_bits },36994 .{ .type = .vector_32_u8, .kind = .forward_bits_mem },
34301 .{ .type = .vector_32_u8, .kind = .{ .pshufb_bswap_mem = .{ .repeat = 2, .size = .xword } } },36995 .{ .type = .vector_32_u8, .kind = .{ .pshufb_bswap_mem = .{ .repeat = 2, .size = .xword } } },
34302 .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } },36996 .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } },
34303 .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } },36997 .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } },
...@@ -34333,7 +37027,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -34333,7 +37027,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
34333 .extra_temps = .{37027 .extra_temps = .{
34334 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },37028 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
34335 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },37029 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
34336 .{ .type = .vector_32_u8, .kind = .forward_bits },37030 .{ .type = .vector_32_u8, .kind = .forward_bits_mem },
34337 .{ .type = .vector_32_u8, .kind = .{ .pshufb_bswap_mem = .{ .repeat = 2, .size = .xword } } },37031 .{ .type = .vector_32_u8, .kind = .{ .pshufb_bswap_mem = .{ .repeat = 2, .size = .xword } } },
34338 .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } },37032 .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } },
34339 .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } },37033 .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } },
...@@ -34369,7 +37063,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -34369,7 +37063,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
34369 .extra_temps = .{37063 .extra_temps = .{
34370 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },37064 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
34371 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },37065 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
34372 .{ .type = .vector_16_u8, .kind = .forward_bits },37066 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
34373 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .xword } } },37067 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .xword } } },
34374 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },37068 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
34375 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },37069 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
...@@ -106842,10 +109536,11 @@ const Select = struct {...@@ -106842,10 +109536,11 @@ const Select = struct {
106842 f64_0x1p52_0x1p84_mem,109536 f64_0x1p52_0x1p84_mem,
106843 u32_0x1p52_hi_0x1p84_hi_0_0_mem,109537 u32_0x1p52_hi_0x1p84_hi_0_0_mem,
106844 f32_0_0x1p64_mem,109538 f32_0_0x1p64_mem,
106845 pshufb_trunc_mem: struct { from: Memory.Size, to: Memory.Size },109539 pshufb_trunc_mem: struct { of: Memory.Size = .none, from: Memory.Size, to: Memory.Size },
109540 pand_trunc_mem: struct { from: Memory.Size, to: Memory.Size },
106846 pshufb_bswap_mem: struct { repeat: u4 = 1, size: Memory.Size, smear: u4 = 1 },109541 pshufb_bswap_mem: struct { repeat: u4 = 1, size: Memory.Size, smear: u4 = 1 },
106847 forward_bits,109542 forward_bits_mem,
106848 reverse_bits,109543 reverse_bits_mem,
106849 frame: FrameIndex,109544 frame: FrameIndex,
106850 lazy_symbol: struct { kind: link.File.LazySymbol.Kind, ref: Select.Operand.Ref = .none },109545 lazy_symbol: struct { kind: link.File.LazySymbol.Kind, ref: Select.Operand.Ref = .none },
106851 symbol: *const struct { lib: ?[]const u8 = null, name: []const u8 },109546 symbol: *const struct { lib: ?[]const u8 = null, name: []const u8 },
...@@ -107061,17 +109756,40 @@ const Select = struct {...@@ -107061,17 +109756,40 @@ const Select = struct {
107061 const zcu = pt.zcu;109756 const zcu = pt.zcu;
107062 assert(spec.type.isVector(zcu));109757 assert(spec.type.isVector(zcu));
107063 assert(spec.type.childType(zcu).toIntern() == .u8_type);109758 assert(spec.type.childType(zcu).toIntern() == .u8_type);
107064 var bytes: [16]u8 = @splat(1 << 7);109759 var elem_buf: [32]u8 = @splat(1 << 7);
109760 const elems = elem_buf[0..spec.type.vectorLen(zcu)];
109761 const of_bytes: u32 = @intCast(switch (trunc_spec.of) {
109762 .none => elems.len,
109763 else => |of| @divExact(of.bitSize(cg.target), 8),
109764 });
107065 const from_bytes: u32 = @intCast(@divExact(trunc_spec.from.bitSize(cg.target), 8));109765 const from_bytes: u32 = @intCast(@divExact(trunc_spec.from.bitSize(cg.target), 8));
107066 const to_bytes: u32 = @intCast(@divExact(trunc_spec.to.bitSize(cg.target), 8));109766 const to_bytes: u32 = @intCast(@divExact(trunc_spec.to.bitSize(cg.target), 8));
107067 var from_index: u32 = 0;109767 var from_index: u32 = 0;
107068 var to_index: u32 = 0;109768 var to_index: u32 = 0;
107069 while (from_index < bytes.len) {109769 while (from_index < of_bytes) : ({
107070 for (0..to_bytes) |byte_off| bytes[to_index + byte_off] = @intCast(from_index + byte_off);
107071 from_index += from_bytes;109770 from_index += from_bytes;
107072 to_index += to_bytes;109771 switch (from_index % 16) {
107073 }109772 0 => to_index = from_index,
107074 const elems = bytes[0..spec.type.vectorLen(zcu)];109773 else => to_index += to_bytes,
109774 }
109775 }) for (0..to_bytes) |byte_off| {
109776 elems[to_index + byte_off] = @as(u4, @truncate(from_index + byte_off));
109777 };
109778 return .{ try cg.tempMemFromValue(.fromInterned(try pt.intern(.{ .aggregate = .{
109779 .ty = spec.type.toIntern(),
109780 .storage = .{ .bytes = try zcu.intern_pool.getOrPutString(zcu.gpa, pt.tid, elems, .maybe_embedded_nulls) },
109781 } }))), true };
109782 },
109783 .pand_trunc_mem => |trunc_spec| {
109784 const zcu = pt.zcu;
109785 assert(spec.type.isVector(zcu));
109786 assert(spec.type.childType(zcu).toIntern() == .u8_type);
109787 var elem_buf: [32]u8 = @splat(0);
109788 const elems = elem_buf[0..spec.type.vectorLen(zcu)];
109789 const from_bytes: u32 = @intCast(@divExact(trunc_spec.from.bitSize(cg.target), 8));
109790 const to_bytes: u32 = @intCast(@divExact(trunc_spec.to.bitSize(cg.target), 8));
109791 var index: u32 = 0;
109792 while (index < elems.len) : (index += from_bytes) @memset(elems[index..][0..to_bytes], std.math.maxInt(u8));
107075 return .{ try cg.tempMemFromValue(.fromInterned(try pt.intern(.{ .aggregate = .{109793 return .{ try cg.tempMemFromValue(.fromInterned(try pt.intern(.{ .aggregate = .{
107076 .ty = spec.type.toIntern(),109794 .ty = spec.type.toIntern(),
107077 .storage = .{ .bytes = try zcu.intern_pool.getOrPutString(zcu.gpa, pt.tid, elems, .maybe_embedded_nulls) },109795 .storage = .{ .bytes = try zcu.intern_pool.getOrPutString(zcu.gpa, pt.tid, elems, .maybe_embedded_nulls) },
...@@ -107081,20 +109799,20 @@ const Select = struct {...@@ -107081,20 +109799,20 @@ const Select = struct {
107081 const zcu = pt.zcu;109799 const zcu = pt.zcu;
107082 assert(spec.type.isVector(zcu));109800 assert(spec.type.isVector(zcu));
107083 assert(spec.type.childType(zcu).toIntern() == .u8_type);109801 assert(spec.type.childType(zcu).toIntern() == .u8_type);
107084 var bytes: [32]u8 = @splat(1 << 7);109802 var elem_buf: [32]u8 = @splat(1 << 7);
109803 const elems = elem_buf[0..spec.type.vectorLen(zcu)];
107085 const len: usize = @intCast(@divExact(bswap_spec.size.bitSize(cg.target), 8));109804 const len: usize = @intCast(@divExact(bswap_spec.size.bitSize(cg.target), 8));
107086 var to_index: u32 = 0;109805 var to_index: u32 = 0;
107087 for (0..bswap_spec.repeat) |_| for (0..len) |from_index| {109806 for (0..bswap_spec.repeat) |_| for (0..len) |from_index| {
107088 @memset(bytes[to_index..][0..bswap_spec.smear], @intCast(len - 1 - from_index));109807 @memset(elems[to_index..][0..bswap_spec.smear], @intCast(len - 1 - from_index));
107089 to_index += bswap_spec.smear;109808 to_index += bswap_spec.smear;
107090 };109809 };
107091 const elems = bytes[0..spec.type.vectorLen(zcu)];
107092 return .{ try cg.tempMemFromValue(.fromInterned(try pt.intern(.{ .aggregate = .{109810 return .{ try cg.tempMemFromValue(.fromInterned(try pt.intern(.{ .aggregate = .{
107093 .ty = spec.type.toIntern(),109811 .ty = spec.type.toIntern(),
107094 .storage = .{ .bytes = try zcu.intern_pool.getOrPutString(zcu.gpa, pt.tid, elems, .maybe_embedded_nulls) },109812 .storage = .{ .bytes = try zcu.intern_pool.getOrPutString(zcu.gpa, pt.tid, elems, .maybe_embedded_nulls) },
107095 } }))), true };109813 } }))), true };
107096 },109814 },
107097 .forward_bits, .reverse_bits => {109815 .forward_bits_mem, .reverse_bits_mem => {
107098 const zcu = pt.zcu;109816 const zcu = pt.zcu;
107099 assert(spec.type.isVector(zcu));109817 assert(spec.type.isVector(zcu));
107100 assert(spec.type.childType(zcu).toIntern() == .u8_type);109818 assert(spec.type.childType(zcu).toIntern() == .u8_type);
...@@ -107102,8 +109820,8 @@ const Select = struct {...@@ -107102,8 +109820,8 @@ const Select = struct {
107102 const elems = bytes[0..spec.type.vectorLen(zcu)];109820 const elems = bytes[0..spec.type.vectorLen(zcu)];
107103 for (elems, 0..) |*elem, index| elem.* = switch (spec.kind) {109821 for (elems, 0..) |*elem, index| elem.* = switch (spec.kind) {
107104 else => unreachable,109822 else => unreachable,
107105 .forward_bits => @as(u8, 1 << 0) << @truncate(index),109823 .forward_bits_mem => @as(u8, 1 << 0) << @truncate(index),
107106 .reverse_bits => @as(u8, 1 << 7) >> @truncate(index),109824 .reverse_bits_mem => @as(u8, 1 << 7) >> @truncate(index),
107107 };109825 };
107108 return .{ try cg.tempMemFromValue(.fromInterned(try pt.intern(.{ .aggregate = .{109826 return .{ try cg.tempMemFromValue(.fromInterned(try pt.intern(.{ .aggregate = .{
107109 .ty = spec.type.toIntern(),109827 .ty = spec.type.toIntern(),
src/arch/x86_64/Lower.zig+1-1
...@@ -579,7 +579,7 @@ fn generic(lower: *Lower, inst: Mir.Inst) Error!void {...@@ -579,7 +579,7 @@ fn generic(lower: *Lower, inst: Mir.Inst) Error!void {
579 .rrri => inst.data.rrri.fixes,579 .rrri => inst.data.rrri.fixes,
580 .rri_s, .rri_u => inst.data.rri.fixes,580 .rri_s, .rri_u => inst.data.rri.fixes,
581 .ri_s, .ri_u, .ri_64, .ir => inst.data.ri.fixes,581 .ri_s, .ri_u, .ri_64, .ir => inst.data.ri.fixes,
582 .rm, .rmi_s, .mr => inst.data.rx.fixes,582 .rm, .rmi_s, .rmi_u, .mr => inst.data.rx.fixes,
583 .mrr, .rrm, .rmr => inst.data.rrx.fixes,583 .mrr, .rrm, .rmr => inst.data.rrx.fixes,
584 .rmi, .mri => inst.data.rix.fixes,584 .rmi, .mri => inst.data.rix.fixes,
585 .rrmr => inst.data.rrrx.fixes,585 .rrmr => inst.data.rrrx.fixes,
src/codegen/c/Type.zig+60
...@@ -1474,6 +1474,66 @@ pub const Pool = struct {...@@ -1474,6 +1474,66 @@ pub const Pool = struct {
1474 };1474 };
1475 return pool.fromFields(allocator, .@"struct", &fields, kind);1475 return pool.fromFields(allocator, .@"struct", &fields, kind);
1476 },1476 },
1477 .vector_1_u8_type => {
1478 const vector_ctype = try pool.getVector(allocator, .{
1479 .elem_ctype = .u8,
1480 .len = 1,
1481 });
1482 if (!kind.isParameter()) return vector_ctype;
1483 var fields = [_]Info.Field{
1484 .{
1485 .name = .{ .index = .array },
1486 .ctype = vector_ctype,
1487 .alignas = AlignAs.fromAbiAlignment(Type.u8.abiAlignment(zcu)),
1488 },
1489 };
1490 return pool.fromFields(allocator, .@"struct", &fields, kind);
1491 },
1492 .vector_2_u8_type => {
1493 const vector_ctype = try pool.getVector(allocator, .{
1494 .elem_ctype = .u8,
1495 .len = 2,
1496 });
1497 if (!kind.isParameter()) return vector_ctype;
1498 var fields = [_]Info.Field{
1499 .{
1500 .name = .{ .index = .array },
1501 .ctype = vector_ctype,
1502 .alignas = AlignAs.fromAbiAlignment(Type.u8.abiAlignment(zcu)),
1503 },
1504 };
1505 return pool.fromFields(allocator, .@"struct", &fields, kind);
1506 },
1507 .vector_4_u8_type => {
1508 const vector_ctype = try pool.getVector(allocator, .{
1509 .elem_ctype = .u8,
1510 .len = 4,
1511 });
1512 if (!kind.isParameter()) return vector_ctype;
1513 var fields = [_]Info.Field{
1514 .{
1515 .name = .{ .index = .array },
1516 .ctype = vector_ctype,
1517 .alignas = AlignAs.fromAbiAlignment(Type.u8.abiAlignment(zcu)),
1518 },
1519 };
1520 return pool.fromFields(allocator, .@"struct", &fields, kind);
1521 },
1522 .vector_8_u8_type => {
1523 const vector_ctype = try pool.getVector(allocator, .{
1524 .elem_ctype = .u8,
1525 .len = 8,
1526 });
1527 if (!kind.isParameter()) return vector_ctype;
1528 var fields = [_]Info.Field{
1529 .{
1530 .name = .{ .index = .array },
1531 .ctype = vector_ctype,
1532 .alignas = AlignAs.fromAbiAlignment(Type.u8.abiAlignment(zcu)),
1533 },
1534 };
1535 return pool.fromFields(allocator, .@"struct", &fields, kind);
1536 },
1477 .vector_16_u8_type => {1537 .vector_16_u8_type => {
1478 const vector_ctype = try pool.getVector(allocator, .{1538 const vector_ctype = try pool.getVector(allocator, .{
1479 .elem_ctype = .u8,1539 .elem_ctype = .u8,
test/behavior/x86_64/math.zig+9
...@@ -21684,6 +21684,15 @@ test mulUnsafe {...@@ -21684,6 +21684,15 @@ test mulUnsafe {
21684 try test_mul_unsafe.testIntVectors();21684 try test_mul_unsafe.testIntVectors();
21685}21685}
2168621686
21687inline fn mulWrap(comptime Type: type, lhs: Type, rhs: Type) Type {
21688 return lhs *% rhs;
21689}
21690test mulWrap {
21691 const test_mul_wrap = binary(mulWrap, .{});
21692 try test_mul_wrap.testInts();
21693 try test_mul_wrap.testIntVectors();
21694}
21695
21687inline fn multiply(comptime Type: type, lhs: Type, rhs: Type) @TypeOf(lhs * rhs) {21696inline fn multiply(comptime Type: type, lhs: Type, rhs: Type) @TypeOf(lhs * rhs) {
21688 return lhs * rhs;21697 return lhs * rhs;
21689}21698}
test/cases/compile_errors/@import_zon_bad_type.zig+3-3
...@@ -117,9 +117,9 @@ export fn testMutablePointer() void {...@@ -117,9 +117,9 @@ export fn testMutablePointer() void {
117// tmp.zig:37:38: note: imported here117// tmp.zig:37:38: note: imported here
118// neg_inf.zon:1:1: error: expected type '?u8'118// neg_inf.zon:1:1: error: expected type '?u8'
119// tmp.zig:57:28: note: imported here119// tmp.zig:57:28: note: imported here
120// neg_inf.zon:1:1: error: expected type 'tmp.testNonExhaustiveEnum__enum_487'120// neg_inf.zon:1:1: error: expected type 'tmp.testNonExhaustiveEnum__enum_491'
121// tmp.zig:62:39: note: imported here121// tmp.zig:62:39: note: imported here
122// neg_inf.zon:1:1: error: expected type 'tmp.testUntaggedUnion__union_489'122// neg_inf.zon:1:1: error: expected type 'tmp.testUntaggedUnion__union_493'
123// tmp.zig:67:44: note: imported here123// tmp.zig:67:44: note: imported here
124// neg_inf.zon:1:1: error: expected type 'tmp.testTaggedUnionVoid__union_492'124// neg_inf.zon:1:1: error: expected type 'tmp.testTaggedUnionVoid__union_496'
125// tmp.zig:72:50: note: imported here125// tmp.zig:72:50: note: imported here
test/cases/compile_errors/anytype_param_requires_comptime.zig+1-1
...@@ -15,6 +15,6 @@ pub export fn entry() void {...@@ -15,6 +15,6 @@ pub export fn entry() void {
15// error15// error
16//16//
17// :7:25: error: unable to resolve comptime value17// :7:25: error: unable to resolve comptime value
18// :7:25: note: initializer of comptime-only struct 'tmp.S.foo__anon_461.C' must be comptime-known18// :7:25: note: initializer of comptime-only struct 'tmp.S.foo__anon_465.C' must be comptime-known
19// :4:16: note: struct requires comptime because of this field19// :4:16: note: struct requires comptime because of this field
20// :4:16: note: types are not available at runtime20// :4:16: note: types are not available at runtime
test/cases/compile_errors/bogus_method_call_on_slice.zig+1-1
...@@ -16,5 +16,5 @@ pub export fn entry2() void {...@@ -16,5 +16,5 @@ pub export fn entry2() void {
16//16//
17// :3:6: error: no field or member function named 'copy' in '[]const u8'17// :3:6: error: no field or member function named 'copy' in '[]const u8'
18// :9:8: error: no field or member function named 'bar' in '@TypeOf(.{})'18// :9:8: error: no field or member function named 'bar' in '@TypeOf(.{})'
19// :12:18: error: no field or member function named 'bar' in 'tmp.entry2__struct_465'19// :12:18: error: no field or member function named 'bar' in 'tmp.entry2__struct_469'
20// :12:6: note: struct declared here20// :12:6: note: struct declared here
test/cases/compile_errors/coerce_anon_struct.zig+1-1
...@@ -6,6 +6,6 @@ export fn foo() void {...@@ -6,6 +6,6 @@ export fn foo() void {
66
7// error7// error
8//8//
9// :4:16: error: expected type 'tmp.T', found 'tmp.foo__struct_454'9// :4:16: error: expected type 'tmp.T', found 'tmp.foo__struct_458'
10// :3:16: note: struct declared here10// :3:16: note: struct declared here
11// :1:11: note: struct declared here11// :1:11: note: struct declared here
test/cases/compile_errors/redundant_try.zig+2-2
...@@ -44,9 +44,9 @@ comptime {...@@ -44,9 +44,9 @@ comptime {
44//44//
45// :5:23: error: expected error union type, found 'comptime_int'45// :5:23: error: expected error union type, found 'comptime_int'
46// :10:23: error: expected error union type, found '@TypeOf(.{})'46// :10:23: error: expected error union type, found '@TypeOf(.{})'
47// :15:23: error: expected error union type, found 'tmp.test2__struct_491'47// :15:23: error: expected error union type, found 'tmp.test2__struct_495'
48// :15:23: note: struct declared here48// :15:23: note: struct declared here
49// :20:27: error: expected error union type, found 'tmp.test3__struct_493'49// :20:27: error: expected error union type, found 'tmp.test3__struct_497'
50// :20:27: note: struct declared here50// :20:27: note: struct declared here
51// :25:23: error: expected error union type, found 'struct { comptime *const [5:0]u8 = "hello" }'51// :25:23: error: expected error union type, found 'struct { comptime *const [5:0]u8 = "hello" }'
52// :31:13: error: expected error union type, found 'u32'52// :31:13: error: expected error union type, found 'u32'