authorgravatar for mlugg@mlugg.co.ukMatthew Lugg <mlugg@mlugg.co.uk> 2025-09-30 13:28:37+01:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2025-09-30 13:28:37+01:00
log7adb15892eada307b43a6a7844d3e51720f8992d
tree6e8b78f77762cca0ec0721e0391586532f23c691
parentdf1909900c60a55cf93281ed7c476646c6d4c10c
parenta896a22932ec658d028c032744411cf93e18ee48
signaturebadge-check Signed by PGP key B5690EEEBB952194

Merge pull request #25353 from mlugg/x86_64-faster

x86_64: generate better constant memcpy code

4 files changed, 886 insertions(+), 298 deletions(-)

src/codegen/x86_64/CodeGen.zig+882-293
......@@ -2292,7 +2292,7 @@ fn genBodyBlock(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
22922292}
22932293
22942294fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2295 @setEvalBranchQuota(29_500);
2295 @setEvalBranchQuota(29_600);
22962296 const pt = cg.pt;
22972297 const zcu = pt.zcu;
22982298 const ip = &zcu.intern_pool;
......@@ -23288,10 +23288,10 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2328823288 .dst_temps = .{ .mem, .unused },
2328923289 .clobbers = .{ .eflags = true },
2329023290 .each = .{ .once = &.{
23291 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
23292 .{ .@"0:", ._, .lea, .tmp1p, .memia(.src0, .tmp0, .add_size), ._, ._ },
23293 .{ ._, ._, .lea, .tmp2p, .memia(.src1, .tmp0, .add_size), ._, ._ },
23294 .{ ._, ._, .lea, .tmp3p, .memia(.dst0, .tmp0, .add_size), ._, ._ },
23291 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
23292 .{ .@"0:", ._, .lea, .tmp1p, .memia(.src0, .tmp0, .add_unaligned_size), ._, ._ },
23293 .{ ._, ._, .lea, .tmp2p, .memia(.src1, .tmp0, .add_unaligned_size), ._, ._ },
23294 .{ ._, ._, .lea, .tmp3p, .memia(.dst0, .tmp0, .add_unaligned_size), ._, ._ },
2329523295 .{ ._, ._, .mov, .tmp4d, .sia(-8, .src0, .add_elem_size), ._, ._ },
2329623296 .{ .@"1:", ._, .xor, .tmp5d, .tmp5d, ._, ._ },
2329723297 .{ ._, ._, .@"or", .tmp5q, .leai(.tmp1q, .tmp4), ._, ._ },
......@@ -23339,10 +23339,10 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2333923339 .dst_temps = .{ .mem, .unused },
2334023340 .clobbers = .{ .eflags = true },
2334123341 .each = .{ .once = &.{
23342 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
23343 .{ .@"0:", ._, .lea, .tmp1p, .memia(.src0, .tmp0, .add_size), ._, ._ },
23344 .{ ._, ._, .lea, .tmp2p, .memiad(.src1, .tmp0, .add_size, 8), ._, ._ },
23345 .{ ._, ._, .lea, .tmp3p, .memia(.dst0, .tmp0, .add_size), ._, ._ },
23342 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
23343 .{ .@"0:", ._, .lea, .tmp1p, .memia(.src0, .tmp0, .add_unaligned_size), ._, ._ },
23344 .{ ._, ._, .lea, .tmp2p, .memiad(.src1, .tmp0, .add_unaligned_size, 8), ._, ._ },
23345 .{ ._, ._, .lea, .tmp3p, .memia(.dst0, .tmp0, .add_unaligned_size), ._, ._ },
2334623346 .{ ._, ._, .mov, .tmp4d, .sia(-1, .src0, .add_elem_size_div_8), ._, ._ },
2334723347 .{ .@"1:", ._, .lea, .tmp5p, .leaa(.tmp4, .sub_src0_elem_size_div_8), ._, ._ },
2334823348 .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ },
......@@ -23393,10 +23393,10 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2339323393 .dst_temps = .{ .mem, .unused },
2339423394 .clobbers = .{ .eflags = true },
2339523395 .each = .{ .once = &.{
23396 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
23397 .{ .@"0:", ._, .lea, .tmp1p, .memia(.src0, .tmp0, .add_size), ._, ._ },
23398 .{ ._, ._, .lea, .tmp2p, .memiad(.src1, .tmp0, .add_size, 8), ._, ._ },
23399 .{ ._, ._, .lea, .tmp3p, .memia(.dst0, .tmp0, .add_size), ._, ._ },
23396 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
23397 .{ .@"0:", ._, .lea, .tmp1p, .memia(.src0, .tmp0, .add_unaligned_size), ._, ._ },
23398 .{ ._, ._, .lea, .tmp2p, .memiad(.src1, .tmp0, .add_unaligned_size, 8), ._, ._ },
23399 .{ ._, ._, .lea, .tmp3p, .memia(.dst0, .tmp0, .add_unaligned_size), ._, ._ },
2340023400 .{ ._, ._, .mov, .tmp4d, .sia(-1, .src0, .add_elem_size_div_8), ._, ._ },
2340123401 .{ .@"1:", ._, .lea, .tmp5p, .leaa(.tmp4, .sub_src0_elem_size_div_8), ._, ._ },
2340223402 .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ },
......@@ -23447,10 +23447,10 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2344723447 .dst_temps = .{ .mem, .unused },
2344823448 .clobbers = .{ .eflags = true },
2344923449 .each = .{ .once = &.{
23450 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
23451 .{ .@"0:", ._, .lea, .tmp1p, .memia(.src0, .tmp0, .add_size), ._, ._ },
23452 .{ ._, ._, .lea, .tmp2p, .memiad(.src1, .tmp0, .add_size, 8), ._, ._ },
23453 .{ ._, ._, .lea, .tmp3p, .memia(.dst0, .tmp0, .add_size), ._, ._ },
23450 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
23451 .{ .@"0:", ._, .lea, .tmp1p, .memia(.src0, .tmp0, .add_unaligned_size), ._, ._ },
23452 .{ ._, ._, .lea, .tmp2p, .memiad(.src1, .tmp0, .add_unaligned_size, 8), ._, ._ },
23453 .{ ._, ._, .lea, .tmp3p, .memia(.dst0, .tmp0, .add_unaligned_size), ._, ._ },
2345423454 .{ ._, ._, .mov, .tmp4d, .sia(-1, .src0, .add_elem_size_div_8), ._, ._ },
2345523455 .{ .@"1:", ._, .lea, .tmp5p, .leaa(.tmp4, .sub_src0_elem_size_div_8), ._, ._ },
2345623456 .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ },
......@@ -23502,10 +23502,10 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2350223502 .dst_temps = .{ .mem, .unused },
2350323503 .clobbers = .{ .eflags = true },
2350423504 .each = .{ .once = &.{
23505 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
23506 .{ .@"0:", ._, .lea, .tmp1p, .memia(.src0, .tmp0, .add_size), ._, ._ },
23507 .{ ._, ._, .lea, .tmp2p, .memiad(.src1, .tmp0, .add_size, 8), ._, ._ },
23508 .{ ._, ._, .lea, .tmp3p, .memia(.dst0, .tmp0, .add_size), ._, ._ },
23505 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
23506 .{ .@"0:", ._, .lea, .tmp1p, .memia(.src0, .tmp0, .add_unaligned_size), ._, ._ },
23507 .{ ._, ._, .lea, .tmp2p, .memiad(.src1, .tmp0, .add_unaligned_size, 8), ._, ._ },
23508 .{ ._, ._, .lea, .tmp3p, .memia(.dst0, .tmp0, .add_unaligned_size), ._, ._ },
2350923509 .{ ._, ._, .mov, .tmp4d, .sia(-1, .src0, .add_elem_size_div_8), ._, ._ },
2351023510 .{ .@"1:", ._, .lea, .tmp5p, .leaa(.tmp4, .sub_src0_elem_size_div_8), ._, ._ },
2351123511 .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ },
......@@ -26107,10 +26107,10 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2610726107 .dst_temps = .{ .mem, .unused },
2610826108 .clobbers = .{ .eflags = true },
2610926109 .each = .{ .once = &.{
26110 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
26111 .{ .@"0:", ._, .lea, .tmp1p, .memia(.src0, .tmp0, .add_size), ._, ._ },
26112 .{ ._, ._, .lea, .tmp2p, .memia(.src1, .tmp0, .add_size), ._, ._ },
26113 .{ ._, ._, .lea, .tmp3p, .memia(.dst0, .tmp0, .add_size), ._, ._ },
26110 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
26111 .{ .@"0:", ._, .lea, .tmp1p, .memia(.src0, .tmp0, .add_unaligned_size), ._, ._ },
26112 .{ ._, ._, .lea, .tmp2p, .memia(.src1, .tmp0, .add_unaligned_size), ._, ._ },
26113 .{ ._, ._, .lea, .tmp3p, .memia(.dst0, .tmp0, .add_unaligned_size), ._, ._ },
2611426114 .{ ._, ._, .mov, .tmp4d, .sia(-8, .src0, .add_elem_size), ._, ._ },
2611526115 .{ .@"1:", ._, .xor, .tmp5d, .tmp5d, ._, ._ },
2611626116 .{ ._, ._, .@"or", .tmp5q, .leai(.tmp1q, .tmp4), ._, ._ },
......@@ -26158,10 +26158,10 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2615826158 .dst_temps = .{ .mem, .unused },
2615926159 .clobbers = .{ .eflags = true },
2616026160 .each = .{ .once = &.{
26161 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
26162 .{ .@"0:", ._, .lea, .tmp1p, .memia(.src0, .tmp0, .add_size), ._, ._ },
26163 .{ ._, ._, .lea, .tmp2p, .memiad(.src1, .tmp0, .add_size, 8), ._, ._ },
26164 .{ ._, ._, .lea, .tmp3p, .memia(.dst0, .tmp0, .add_size), ._, ._ },
26161 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
26162 .{ .@"0:", ._, .lea, .tmp1p, .memia(.src0, .tmp0, .add_unaligned_size), ._, ._ },
26163 .{ ._, ._, .lea, .tmp2p, .memiad(.src1, .tmp0, .add_unaligned_size, 8), ._, ._ },
26164 .{ ._, ._, .lea, .tmp3p, .memia(.dst0, .tmp0, .add_unaligned_size), ._, ._ },
2616526165 .{ ._, ._, .mov, .tmp4d, .sia(-1, .src0, .add_elem_size_div_8), ._, ._ },
2616626166 .{ .@"1:", ._, .lea, .tmp5p, .leaa(.tmp4, .sub_src0_elem_size_div_8), ._, ._ },
2616726167 .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ },
......@@ -26212,10 +26212,10 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2621226212 .dst_temps = .{ .mem, .unused },
2621326213 .clobbers = .{ .eflags = true },
2621426214 .each = .{ .once = &.{
26215 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
26216 .{ .@"0:", ._, .lea, .tmp1p, .memia(.src0, .tmp0, .add_size), ._, ._ },
26217 .{ ._, ._, .lea, .tmp2p, .memiad(.src1, .tmp0, .add_size, 8), ._, ._ },
26218 .{ ._, ._, .lea, .tmp3p, .memia(.dst0, .tmp0, .add_size), ._, ._ },
26215 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
26216 .{ .@"0:", ._, .lea, .tmp1p, .memia(.src0, .tmp0, .add_unaligned_size), ._, ._ },
26217 .{ ._, ._, .lea, .tmp2p, .memiad(.src1, .tmp0, .add_unaligned_size, 8), ._, ._ },
26218 .{ ._, ._, .lea, .tmp3p, .memia(.dst0, .tmp0, .add_unaligned_size), ._, ._ },
2621926219 .{ ._, ._, .mov, .tmp4d, .sia(-1, .src0, .add_elem_size_div_8), ._, ._ },
2622026220 .{ .@"1:", ._, .lea, .tmp5p, .leaa(.tmp4, .sub_src0_elem_size_div_8), ._, ._ },
2622126221 .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ },
......@@ -26266,10 +26266,10 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2626626266 .dst_temps = .{ .mem, .unused },
2626726267 .clobbers = .{ .eflags = true },
2626826268 .each = .{ .once = &.{
26269 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
26270 .{ .@"0:", ._, .lea, .tmp1p, .memia(.src0, .tmp0, .add_size), ._, ._ },
26271 .{ ._, ._, .lea, .tmp2p, .memiad(.src1, .tmp0, .add_size, 8), ._, ._ },
26272 .{ ._, ._, .lea, .tmp3p, .memia(.dst0, .tmp0, .add_size), ._, ._ },
26269 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
26270 .{ .@"0:", ._, .lea, .tmp1p, .memia(.src0, .tmp0, .add_unaligned_size), ._, ._ },
26271 .{ ._, ._, .lea, .tmp2p, .memiad(.src1, .tmp0, .add_unaligned_size, 8), ._, ._ },
26272 .{ ._, ._, .lea, .tmp3p, .memia(.dst0, .tmp0, .add_unaligned_size), ._, ._ },
2627326273 .{ ._, ._, .mov, .tmp4d, .sia(-1, .src0, .add_elem_size_div_8), ._, ._ },
2627426274 .{ .@"1:", ._, .lea, .tmp5p, .leaa(.tmp4, .sub_src0_elem_size_div_8), ._, ._ },
2627526275 .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ },
......@@ -26321,10 +26321,10 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2632126321 .dst_temps = .{ .mem, .unused },
2632226322 .clobbers = .{ .eflags = true },
2632326323 .each = .{ .once = &.{
26324 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
26325 .{ .@"0:", ._, .lea, .tmp1p, .memia(.src0, .tmp0, .add_size), ._, ._ },
26326 .{ ._, ._, .lea, .tmp2p, .memiad(.src1, .tmp0, .add_size, 8), ._, ._ },
26327 .{ ._, ._, .lea, .tmp3p, .memia(.dst0, .tmp0, .add_size), ._, ._ },
26324 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
26325 .{ .@"0:", ._, .lea, .tmp1p, .memia(.src0, .tmp0, .add_unaligned_size), ._, ._ },
26326 .{ ._, ._, .lea, .tmp2p, .memiad(.src1, .tmp0, .add_unaligned_size, 8), ._, ._ },
26327 .{ ._, ._, .lea, .tmp3p, .memia(.dst0, .tmp0, .add_unaligned_size), ._, ._ },
2632826328 .{ ._, ._, .mov, .tmp4d, .sia(-1, .src0, .add_elem_size_div_8), ._, ._ },
2632926329 .{ .@"1:", ._, .lea, .tmp5p, .leaa(.tmp4, .sub_src0_elem_size_div_8), ._, ._ },
2633026330 .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ },
......@@ -37912,7 +37912,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3791237912 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
3791337913 .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ },
3791437914 .{ ._, .i_, .div, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._, ._ },
37915 .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1h, ._, ._ },
37915 .{ ._, ._, .movsx, .tmp1d, .tmp1h, ._, ._ },
37916 .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1l, ._, ._ },
3791637917 .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
3791737918 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
3791837919 } },
......@@ -37944,7 +37945,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3794437945 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
3794537946 .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ },
3794637947 .{ ._, .i_, .div, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._, ._ },
37947 .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1h, ._, ._ },
37948 .{ ._, ._, .movsx, .tmp1d, .tmp1h, ._, ._ },
37949 .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1l, ._, ._ },
3794837950 .{ ._, ._c, .in, .tmp0p, ._, ._, ._ },
3794937951 .{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
3795037952 } },
......@@ -37977,7 +37979,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3797737979 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
3797837980 .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ },
3797937981 .{ ._, ._, .div, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._, ._ },
37980 .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1h, ._, ._ },
37982 .{ ._, ._, .movzx, .tmp1d, .tmp1h, ._, ._ },
37983 .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1l, ._, ._ },
3798137984 .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
3798237985 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
3798337986 } },
......@@ -38009,7 +38012,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3800938012 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
3801038013 .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ },
3801138014 .{ ._, ._, .div, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._, ._ },
38012 .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1h, ._, ._ },
38015 .{ ._, ._, .movzx, .tmp1d, .tmp1h, ._, ._ },
38016 .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1l, ._, ._ },
3801338017 .{ ._, ._c, .in, .tmp0p, ._, ._, ._ },
3801438018 .{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
3801538019 } },
......@@ -59429,9 +59433,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
5942959433 .required_features = .{ .mmx, null, null, null },
5943059434 .src_constraints = .{ .{ .size = .qword }, .{ .size = .qword }, .any },
5943159435 .patterns = &.{
59432 .{ .src = .{ .to_mut_mm, .mem, .none } },
59433 .{ .src = .{ .mem, .to_mut_mm, .none }, .commute = .{ 0, 1 } },
59434 .{ .src = .{ .to_mut_mm, .to_mm, .none } },
59436 .{ .src = .{ .to_mut_mmx, .mem, .none } },
59437 .{ .src = .{ .mem, .to_mut_mmx, .none }, .commute = .{ 0, 1 } },
59438 .{ .src = .{ .to_mut_mmx, .to_mmx, .none } },
5943559439 },
5943659440 .dst_temps = .{ .{ .ref = .src0 }, .unused },
5943759441 .each = .{ .once = &.{
......@@ -59441,9 +59445,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
5944159445 .required_features = .{ .avx, null, null, null },
5944259446 .src_constraints = .{ .{ .size = .xword }, .{ .size = .xword }, .any },
5944359447 .patterns = &.{
59444 .{ .src = .{ .to_xmm, .mem, .none } },
59445 .{ .src = .{ .mem, .to_xmm, .none }, .commute = .{ 0, 1 } },
59446 .{ .src = .{ .to_xmm, .to_xmm, .none } },
59448 .{ .src = .{ .to_sse, .mem, .none } },
59449 .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } },
59450 .{ .src = .{ .to_sse, .to_sse, .none } },
5944759451 },
5944859452 .dst_temps = .{ .{ .rc = .sse }, .unused },
5944959453 .each = .{ .once = &.{
......@@ -59453,9 +59457,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
5945359457 .required_features = .{ .sse2, null, null, null },
5945459458 .src_constraints = .{ .{ .size = .xword }, .{ .size = .xword }, .any },
5945559459 .patterns = &.{
59456 .{ .src = .{ .to_mut_xmm, .mem, .none } },
59457 .{ .src = .{ .mem, .to_mut_xmm, .none }, .commute = .{ 0, 1 } },
59458 .{ .src = .{ .to_mut_xmm, .to_xmm, .none } },
59460 .{ .src = .{ .to_mut_sse, .mem, .none } },
59461 .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } },
59462 .{ .src = .{ .to_mut_sse, .to_sse, .none } },
5945959463 },
5946059464 .dst_temps = .{ .{ .ref = .src0 }, .unused },
5946159465 .each = .{ .once = &.{
......@@ -59465,9 +59469,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
5946559469 .required_features = .{ .sse, null, null, null },
5946659470 .src_constraints = .{ .{ .size = .xword }, .{ .size = .xword }, .any },
5946759471 .patterns = &.{
59468 .{ .src = .{ .to_mut_xmm, .mem, .none } },
59469 .{ .src = .{ .mem, .to_mut_xmm, .none }, .commute = .{ 0, 1 } },
59470 .{ .src = .{ .to_mut_xmm, .to_xmm, .none } },
59472 .{ .src = .{ .to_mut_sse, .mem, .none } },
59473 .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } },
59474 .{ .src = .{ .to_mut_sse, .to_sse, .none } },
5947159475 },
5947259476 .dst_temps = .{ .{ .ref = .src0 }, .unused },
5947359477 .each = .{ .once = &.{
......@@ -59477,9 +59481,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
5947759481 .required_features = .{ .avx2, null, null, null },
5947859482 .src_constraints = .{ .{ .size = .yword }, .{ .size = .yword }, .any },
5947959483 .patterns = &.{
59480 .{ .src = .{ .to_ymm, .mem, .none } },
59481 .{ .src = .{ .mem, .to_ymm, .none }, .commute = .{ 0, 1 } },
59482 .{ .src = .{ .to_ymm, .to_ymm, .none } },
59484 .{ .src = .{ .to_sse, .mem, .none } },
59485 .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } },
59486 .{ .src = .{ .to_sse, .to_sse, .none } },
5948359487 },
5948459488 .dst_temps = .{ .{ .rc = .sse }, .unused },
5948559489 .each = .{ .once = &.{
......@@ -59489,9 +59493,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
5948959493 .required_features = .{ .avx, null, null, null },
5949059494 .src_constraints = .{ .{ .size = .yword }, .{ .size = .yword }, .any },
5949159495 .patterns = &.{
59492 .{ .src = .{ .to_ymm, .mem, .none } },
59493 .{ .src = .{ .mem, .to_ymm, .none }, .commute = .{ 0, 1 } },
59494 .{ .src = .{ .to_ymm, .to_ymm, .none } },
59496 .{ .src = .{ .to_sse, .mem, .none } },
59497 .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } },
59498 .{ .src = .{ .to_sse, .to_sse, .none } },
5949559499 },
5949659500 .dst_temps = .{ .{ .rc = .sse }, .unused },
5949759501 .each = .{ .once = &.{
......@@ -64077,6 +64081,16 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6407764081 var ops = try cg.tempsFromOperands(inst, .{ty_op.operand});
6407864082 var res: [1]Temp = undefined;
6407964083 cg.select(&res, &.{ty_op.ty.toType()}, &ops, comptime &.{ .{
64084 .src_constraints = .{ .{ .bool_vec = .byte }, .any, .any },
64085 .patterns = &.{
64086 .{ .src = .{ .mut_mem, .none, .none } },
64087 .{ .src = .{ .to_mut_gpr, .none, .none } },
64088 },
64089 .dst_temps = .{ .{ .ref = .src0 }, .unused },
64090 .each = .{ .once = &.{
64091 .{ ._, ._, .not, .dst0b, ._, ._, ._ },
64092 } },
64093 }, .{
6408064094 .src_constraints = .{ .{ .signed_or_exact_int = .byte }, .any, .any },
6408164095 .patterns = &.{
6408264096 .{ .src = .{ .mut_mem, .none, .none } },
......@@ -64097,6 +64111,16 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6409764111 .each = .{ .once = &.{
6409864112 .{ ._, ._, .xor, .dst0b, .sa(.src0, .add_umax), ._, ._ },
6409964113 } },
64114 }, .{
64115 .src_constraints = .{ .{ .bool_vec = .word }, .any, .any },
64116 .patterns = &.{
64117 .{ .src = .{ .mut_mem, .none, .none } },
64118 .{ .src = .{ .to_mut_gpr, .none, .none } },
64119 },
64120 .dst_temps = .{ .{ .ref = .src0 }, .unused },
64121 .each = .{ .once = &.{
64122 .{ ._, ._, .not, .dst0w, ._, ._, ._ },
64123 } },
6410064124 }, .{
6410164125 .src_constraints = .{ .{ .signed_or_exact_int = .word }, .any, .any },
6410264126 .patterns = &.{
......@@ -64118,6 +64142,16 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6411864142 .each = .{ .once = &.{
6411964143 .{ ._, ._, .xor, .dst0w, .sa(.src0, .add_umax), ._, ._ },
6412064144 } },
64145 }, .{
64146 .src_constraints = .{ .{ .bool_vec = .dword }, .any, .any },
64147 .patterns = &.{
64148 .{ .src = .{ .mut_mem, .none, .none } },
64149 .{ .src = .{ .to_mut_gpr, .none, .none } },
64150 },
64151 .dst_temps = .{ .{ .ref = .src0 }, .unused },
64152 .each = .{ .once = &.{
64153 .{ ._, ._, .not, .dst0d, ._, ._, ._ },
64154 } },
6412164155 }, .{
6412264156 .src_constraints = .{ .{ .signed_or_exact_int = .dword }, .any, .any },
6412364157 .patterns = &.{
......@@ -64139,6 +64173,17 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6413964173 .each = .{ .once = &.{
6414064174 .{ ._, ._, .xor, .dst0d, .sa(.src0, .add_umax), ._, ._ },
6414164175 } },
64176 }, .{
64177 .required_features = .{ .@"64bit", null, null, null },
64178 .src_constraints = .{ .{ .bool_vec = .qword }, .any, .any },
64179 .patterns = &.{
64180 .{ .src = .{ .mut_mem, .none, .none } },
64181 .{ .src = .{ .to_mut_gpr, .none, .none } },
64182 },
64183 .dst_temps = .{ .{ .ref = .src0 }, .unused },
64184 .each = .{ .once = &.{
64185 .{ ._, ._, .not, .dst0q, ._, ._, ._ },
64186 } },
6414264187 }, .{
6414364188 .required_features = .{ .@"64bit", null, null, null },
6414464189 .src_constraints = .{ .{ .signed_or_exact_int = .qword }, .any, .any },
......@@ -64162,12 +64207,24 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6416264207 .{ ._, ._, .mov, .dst0q, .ua(.src0, .add_umax), ._, ._ },
6416364208 .{ ._, ._, .xor, .dst0q, .src0q, ._, ._ },
6416464209 } },
64210 }, .{
64211 .required_features = .{ .mmx, null, null, null },
64212 .src_constraints = .{ .{ .bool_vec = .qword }, .any, .any },
64213 .patterns = &.{
64214 .{ .src = .{ .mem, .none, .none } },
64215 .{ .src = .{ .to_mmx, .none, .none } },
64216 },
64217 .dst_temps = .{ .{ .rc = .mmx }, .unused },
64218 .each = .{ .once = &.{
64219 .{ ._, .p_d, .cmpeq, .dst0q, .dst0q, ._, ._ },
64220 .{ ._, .p_, .xor, .dst0q, .src0q, ._, ._ },
64221 } },
6416564222 }, .{
6416664223 .required_features = .{ .mmx, null, null, null },
6416764224 .src_constraints = .{ .{ .signed_or_exact_int = .qword }, .any, .any },
6416864225 .patterns = &.{
6416964226 .{ .src = .{ .mem, .none, .none } },
64170 .{ .src = .{ .to_mm, .none, .none } },
64227 .{ .src = .{ .to_mmx, .none, .none } },
6417164228 },
6417264229 .dst_temps = .{ .{ .rc = .mmx }, .unused },
6417364230 .each = .{ .once = &.{
......@@ -64178,7 +64235,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6417864235 .required_features = .{ .mmx, null, null, null },
6417964236 .src_constraints = .{ .{ .unsigned_int = .qword }, .any, .any },
6418064237 .patterns = &.{
64181 .{ .src = .{ .to_mut_mm, .none, .none } },
64238 .{ .src = .{ .to_mut_mmx, .none, .none } },
6418264239 },
6418364240 .extra_temps = .{
6418464241 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
......@@ -64198,12 +64255,24 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6419864255 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
6419964256 .{ ._, .p_, .xor, .dst0q, .lea(.tmp0q), ._, ._ },
6420064257 } },
64258 }, .{
64259 .required_features = .{ .avx, null, null, null },
64260 .src_constraints = .{ .{ .bool_vec = .xword }, .any, .any },
64261 .patterns = &.{
64262 .{ .src = .{ .mem, .none, .none } },
64263 .{ .src = .{ .to_sse, .none, .none } },
64264 },
64265 .dst_temps = .{ .{ .rc = .sse }, .unused },
64266 .each = .{ .once = &.{
64267 .{ ._, .vp_q, .cmpeq, .dst0x, .dst0x, .dst0x, ._ },
64268 .{ ._, .vp_, .xor, .dst0x, .dst0x, .src0x, ._ },
64269 } },
6420164270 }, .{
6420264271 .required_features = .{ .avx, null, null, null },
6420364272 .src_constraints = .{ .{ .signed_or_exact_int = .xword }, .any, .any },
6420464273 .patterns = &.{
6420564274 .{ .src = .{ .mem, .none, .none } },
64206 .{ .src = .{ .to_xmm, .none, .none } },
64275 .{ .src = .{ .to_sse, .none, .none } },
6420764276 },
6420864277 .dst_temps = .{ .{ .rc = .sse }, .unused },
6420964278 .each = .{ .once = &.{
......@@ -64214,7 +64283,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6421464283 .required_features = .{ .avx, null, null, null },
6421564284 .src_constraints = .{ .{ .unsigned_int = .xword }, .any, .any },
6421664285 .patterns = &.{
64217 .{ .src = .{ .to_xmm, .none, .none } },
64286 .{ .src = .{ .to_sse, .none, .none } },
6421864287 },
6421964288 .extra_temps = .{
6422064289 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
......@@ -64234,12 +64303,24 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6423464303 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
6423564304 .{ ._, .vp_, .xor, .dst0x, .src0x, .lea(.tmp0x), ._ },
6423664305 } },
64306 }, .{
64307 .required_features = .{ .sse2, null, null, null },
64308 .src_constraints = .{ .{ .bool_vec = .xword }, .any, .any },
64309 .patterns = &.{
64310 .{ .src = .{ .mem, .none, .none } },
64311 .{ .src = .{ .to_sse, .none, .none } },
64312 },
64313 .dst_temps = .{ .{ .rc = .sse }, .unused },
64314 .each = .{ .once = &.{
64315 .{ ._, .p_d, .cmpeq, .dst0x, .dst0x, ._, ._ },
64316 .{ ._, .p_, .xor, .dst0x, .src0x, ._, ._ },
64317 } },
6423764318 }, .{
6423864319 .required_features = .{ .sse2, null, null, null },
6423964320 .src_constraints = .{ .{ .signed_or_exact_int = .xword }, .any, .any },
6424064321 .patterns = &.{
6424164322 .{ .src = .{ .mem, .none, .none } },
64242 .{ .src = .{ .to_xmm, .none, .none } },
64323 .{ .src = .{ .to_sse, .none, .none } },
6424364324 },
6424464325 .dst_temps = .{ .{ .rc = .sse }, .unused },
6424564326 .each = .{ .once = &.{
......@@ -64250,7 +64331,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6425064331 .required_features = .{ .sse2, null, null, null },
6425164332 .src_constraints = .{ .{ .unsigned_int = .xword }, .any, .any },
6425264333 .patterns = &.{
64253 .{ .src = .{ .to_mut_xmm, .none, .none } },
64334 .{ .src = .{ .to_mut_sse, .none, .none } },
6425464335 },
6425564336 .extra_temps = .{
6425664337 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
......@@ -64270,11 +64351,35 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6427064351 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
6427164352 .{ ._, .p_, .xor, .dst0x, .lea(.tmp0x), ._, ._ },
6427264353 } },
64354 }, .{
64355 .required_features = .{ .sse, null, null, null },
64356 .src_constraints = .{ .{ .bool_vec = .xword }, .any, .any },
64357 .patterns = &.{
64358 .{ .src = .{ .to_mut_sse, .none, .none } },
64359 },
64360 .extra_temps = .{
64361 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
64362 .{ .kind = .{ .umax_mem = .{ .ref = .src0 } } },
64363 .unused,
64364 .unused,
64365 .unused,
64366 .unused,
64367 .unused,
64368 .unused,
64369 .unused,
64370 .unused,
64371 .unused,
64372 },
64373 .dst_temps = .{ .{ .ref = .src0 }, .unused },
64374 .each = .{ .once = &.{
64375 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
64376 .{ ._, ._ps, .xor, .dst0x, .lea(.tmp0x), ._, ._ },
64377 } },
6427364378 }, .{
6427464379 .required_features = .{ .sse, null, null, null },
6427564380 .src_constraints = .{ .{ .int = .xword }, .any, .any },
6427664381 .patterns = &.{
64277 .{ .src = .{ .to_mut_xmm, .none, .none } },
64382 .{ .src = .{ .to_mut_sse, .none, .none } },
6427864383 },
6427964384 .extra_temps = .{
6428064385 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
......@@ -64294,12 +64399,24 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6429464399 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
6429564400 .{ ._, ._ps, .xor, .dst0x, .lea(.tmp0x), ._, ._ },
6429664401 } },
64402 }, .{
64403 .required_features = .{ .avx2, null, null, null },
64404 .src_constraints = .{ .{ .bool_vec = .yword }, .any, .any },
64405 .patterns = &.{
64406 .{ .src = .{ .mem, .none, .none } },
64407 .{ .src = .{ .to_sse, .none, .none } },
64408 },
64409 .dst_temps = .{ .{ .rc = .sse }, .unused },
64410 .each = .{ .once = &.{
64411 .{ ._, .vp_q, .cmpeq, .dst0y, .dst0y, .dst0y, ._ },
64412 .{ ._, .vp_, .xor, .dst0y, .dst0y, .src0y, ._ },
64413 } },
6429764414 }, .{
6429864415 .required_features = .{ .avx2, null, null, null },
6429964416 .src_constraints = .{ .{ .signed_or_exact_int = .yword }, .any, .any },
6430064417 .patterns = &.{
6430164418 .{ .src = .{ .mem, .none, .none } },
64302 .{ .src = .{ .to_ymm, .none, .none } },
64419 .{ .src = .{ .to_sse, .none, .none } },
6430364420 },
6430464421 .dst_temps = .{ .{ .rc = .sse }, .unused },
6430564422 .each = .{ .once = &.{
......@@ -64310,7 +64427,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6431064427 .required_features = .{ .avx2, null, null, null },
6431164428 .src_constraints = .{ .{ .unsigned_int = .yword }, .any, .any },
6431264429 .patterns = &.{
64313 .{ .src = .{ .to_ymm, .none, .none } },
64430 .{ .src = .{ .to_sse, .none, .none } },
6431464431 },
6431564432 .extra_temps = .{
6431664433 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
......@@ -64330,12 +64447,24 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6433064447 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
6433164448 .{ ._, .vp_, .xor, .dst0y, .src0y, .lea(.tmp0y), ._ },
6433264449 } },
64450 }, .{
64451 .required_features = .{ .avx, null, null, null },
64452 .src_constraints = .{ .{ .bool_vec = .yword }, .any, .any },
64453 .patterns = &.{
64454 .{ .src = .{ .mem, .none, .none } },
64455 .{ .src = .{ .to_sse, .none, .none } },
64456 },
64457 .dst_temps = .{ .{ .rc = .sse }, .unused },
64458 .each = .{ .once = &.{
64459 .{ ._, .v_pd, .cmp, .dst0y, .dst0y, .dst0y, .vp(.true) },
64460 .{ ._, .v_pd, .xor, .dst0y, .dst0y, .src0y, ._ },
64461 } },
6433364462 }, .{
6433464463 .required_features = .{ .avx, null, null, null },
6433564464 .src_constraints = .{ .{ .signed_or_exact_int = .yword }, .any, .any },
6433664465 .patterns = &.{
6433764466 .{ .src = .{ .mem, .none, .none } },
64338 .{ .src = .{ .to_ymm, .none, .none } },
64467 .{ .src = .{ .to_sse, .none, .none } },
6433964468 },
6434064469 .dst_temps = .{ .{ .rc = .sse }, .unused },
6434164470 .each = .{ .once = &.{
......@@ -64346,7 +64475,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6434664475 .required_features = .{ .avx, null, null, null },
6434764476 .src_constraints = .{ .{ .unsigned_int = .yword }, .any, .any },
6434864477 .patterns = &.{
64349 .{ .src = .{ .to_ymm, .none, .none } },
64478 .{ .src = .{ .to_sse, .none, .none } },
6435064479 },
6435164480 .extra_temps = .{
6435264481 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
......@@ -64366,6 +64495,37 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6436664495 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
6436764496 .{ ._, .v_pd, .xor, .dst0y, .src0y, .lea(.tmp0y), ._ },
6436864497 } },
64498 }, .{
64499 .required_features = .{ .avx2, null, null, null },
64500 .src_constraints = .{ .{ .remainder_bool_vec = .{ .of = .yword, .is = .xword } }, .any, .any },
64501 .patterns = &.{
64502 .{ .src = .{ .to_mem, .none, .none } },
64503 },
64504 .extra_temps = .{
64505 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
64506 .{ .kind = .{ .rc = .sse } },
64507 .{ .kind = .{ .rc = .sse } },
64508 .unused,
64509 .unused,
64510 .unused,
64511 .unused,
64512 .unused,
64513 .unused,
64514 .unused,
64515 .unused,
64516 },
64517 .dst_temps = .{ .mem, .unused },
64518 .clobbers = .{ .eflags = true },
64519 .each = .{ .once = &.{
64520 .{ ._, ._, .mov, .tmp0p, .sia(16, .src0, .sub_size), ._, ._ },
64521 .{ ._, .vp_q, .cmpeq, .tmp1y, .tmp1y, .tmp1y, ._ },
64522 .{ .@"0:", .vp_, .xor, .tmp2y, .tmp1y, .memiad(.src0y, .tmp0, .add_size, -16), ._ },
64523 .{ ._, .v_dqu, .mov, .memiad(.dst0y, .tmp0, .add_size, -16), .tmp2y, ._, ._ },
64524 .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ },
64525 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
64526 .{ .@"0:", .vp_, .xor, .tmp2x, .tmp1x, .memad(.src0x, .add_size, -16), ._ },
64527 .{ ._, .v_dqa, .mov, .memad(.dst0x, .add_size, -16), .tmp2x, ._, ._ },
64528 } },
6436964529 }, .{
6437064530 .required_features = .{ .avx2, null, null, null },
6437164531 .src_constraints = .{ .{ .signed_or_exact_remainder_int = .{ .of = .yword, .is = .xword } }, .any, .any },
......@@ -64397,6 +64557,35 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6439764557 .{ .@"0:", .vp_, .xor, .tmp2x, .tmp1x, .memad(.src0x, .add_size, -16), ._ },
6439864558 .{ ._, .v_dqa, .mov, .memad(.dst0x, .add_size, -16), .tmp2x, ._, ._ },
6439964559 } },
64560 }, .{
64561 .required_features = .{ .avx2, null, null, null },
64562 .src_constraints = .{ .{ .remainder_bool_vec = .{ .of = .yword, .is = .yword } }, .any, .any },
64563 .patterns = &.{
64564 .{ .src = .{ .to_mem, .none, .none } },
64565 },
64566 .extra_temps = .{
64567 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
64568 .{ .kind = .{ .rc = .sse } },
64569 .{ .kind = .{ .rc = .sse } },
64570 .unused,
64571 .unused,
64572 .unused,
64573 .unused,
64574 .unused,
64575 .unused,
64576 .unused,
64577 .unused,
64578 },
64579 .dst_temps = .{ .mem, .unused },
64580 .clobbers = .{ .eflags = true },
64581 .each = .{ .once = &.{
64582 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
64583 .{ ._, .vp_q, .cmpeq, .tmp1y, .tmp1y, .tmp1y, ._ },
64584 .{ .@"0:", .vp_, .xor, .tmp2y, .tmp1y, .memia(.src0y, .tmp0, .add_size), ._ },
64585 .{ ._, .v_dqu, .mov, .memia(.dst0y, .tmp0, .add_size), .tmp2y, ._, ._ },
64586 .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ },
64587 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
64588 } },
6440064589 }, .{
6440164590 .required_features = .{ .avx2, null, null, null },
6440264591 .src_constraints = .{ .{ .signed_or_exact_remainder_int = .{ .of = .yword, .is = .yword } }, .any, .any },
......@@ -64426,6 +64615,37 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6442664615 .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ },
6442764616 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
6442864617 } },
64618 }, .{
64619 .required_features = .{ .avx, null, null, null },
64620 .src_constraints = .{ .{ .remainder_bool_vec = .{ .of = .yword, .is = .xword } }, .any, .any },
64621 .patterns = &.{
64622 .{ .src = .{ .to_mem, .none, .none } },
64623 },
64624 .extra_temps = .{
64625 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
64626 .{ .kind = .{ .rc = .sse } },
64627 .{ .kind = .{ .rc = .sse } },
64628 .unused,
64629 .unused,
64630 .unused,
64631 .unused,
64632 .unused,
64633 .unused,
64634 .unused,
64635 .unused,
64636 },
64637 .dst_temps = .{ .mem, .unused },
64638 .clobbers = .{ .eflags = true },
64639 .each = .{ .once = &.{
64640 .{ ._, ._, .mov, .tmp0p, .sia(16, .src0, .sub_size), ._, ._ },
64641 .{ ._, .v_pd, .cmp, .tmp1y, .tmp1y, .tmp1y, .vp(.true) },
64642 .{ .@"0:", .v_pd, .xor, .tmp2y, .tmp1y, .memiad(.src0y, .tmp0, .add_size, -16), ._ },
64643 .{ ._, .v_pd, .movu, .memiad(.dst0y, .tmp0, .add_size, -16), .tmp2y, ._, ._ },
64644 .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ },
64645 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
64646 .{ .@"0:", .v_pd, .xor, .tmp2x, .tmp1x, .memad(.src0x, .add_size, -16), ._ },
64647 .{ ._, .v_pd, .mova, .memad(.dst0x, .add_size, -16), .tmp2x, ._, ._ },
64648 } },
6442964649 }, .{
6443064650 .required_features = .{ .avx, null, null, null },
6443164651 .src_constraints = .{ .{ .signed_or_exact_remainder_int = .{ .of = .yword, .is = .xword } }, .any, .any },
......@@ -64457,6 +64677,35 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6445764677 .{ .@"0:", .v_pd, .xor, .tmp2x, .tmp1x, .memad(.src0x, .add_size, -16), ._ },
6445864678 .{ ._, .v_pd, .mova, .memad(.dst0x, .add_size, -16), .tmp2x, ._, ._ },
6445964679 } },
64680 }, .{
64681 .required_features = .{ .avx, null, null, null },
64682 .src_constraints = .{ .{ .remainder_bool_vec = .{ .of = .yword, .is = .yword } }, .any, .any },
64683 .patterns = &.{
64684 .{ .src = .{ .to_mem, .none, .none } },
64685 },
64686 .extra_temps = .{
64687 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
64688 .{ .kind = .{ .rc = .sse } },
64689 .{ .kind = .{ .rc = .sse } },
64690 .unused,
64691 .unused,
64692 .unused,
64693 .unused,
64694 .unused,
64695 .unused,
64696 .unused,
64697 .unused,
64698 },
64699 .dst_temps = .{ .mem, .unused },
64700 .clobbers = .{ .eflags = true },
64701 .each = .{ .once = &.{
64702 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
64703 .{ ._, .v_pd, .cmp, .tmp1y, .tmp1y, .tmp1y, .vp(.true) },
64704 .{ .@"0:", .v_pd, .xor, .tmp2y, .tmp1y, .memia(.src0y, .tmp0, .add_size), ._ },
64705 .{ ._, .v_pd, .movu, .memia(.dst0y, .tmp0, .add_size), .tmp2y, ._, ._ },
64706 .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ },
64707 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
64708 } },
6446064709 }, .{
6446164710 .required_features = .{ .avx, null, null, null },
6446264711 .src_constraints = .{ .{ .signed_or_exact_remainder_int = .{ .of = .yword, .is = .yword } }, .any, .any },
......@@ -64486,6 +64735,35 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6448664735 .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ },
6448764736 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
6448864737 } },
64738 }, .{
64739 .required_features = .{ .avx, null, null, null },
64740 .src_constraints = .{ .{ .remainder_bool_vec = .{ .of = .xword, .is = .xword } }, .any, .any },
64741 .patterns = &.{
64742 .{ .src = .{ .to_mem, .none, .none } },
64743 },
64744 .extra_temps = .{
64745 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
64746 .{ .kind = .{ .rc = .sse } },
64747 .{ .kind = .{ .rc = .sse } },
64748 .unused,
64749 .unused,
64750 .unused,
64751 .unused,
64752 .unused,
64753 .unused,
64754 .unused,
64755 .unused,
64756 },
64757 .dst_temps = .{ .mem, .unused },
64758 .clobbers = .{ .eflags = true },
64759 .each = .{ .once = &.{
64760 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
64761 .{ ._, .vp_q, .cmpeq, .tmp1x, .tmp1x, .tmp1x, ._ },
64762 .{ .@"0:", .v_, .xor, .tmp2x, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._ },
64763 .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp2x, ._, ._ },
64764 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
64765 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
64766 } },
6448964767 }, .{
6449064768 .required_features = .{ .avx, null, null, null },
6449164769 .src_constraints = .{ .{ .signed_or_exact_remainder_int = .{ .of = .xword, .is = .xword } }, .any, .any },
......@@ -64515,6 +64793,36 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6451564793 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
6451664794 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
6451764795 } },
64796 }, .{
64797 .required_features = .{ .sse2, null, null, null },
64798 .src_constraints = .{ .{ .remainder_bool_vec = .{ .of = .xword, .is = .xword } }, .any, .any },
64799 .patterns = &.{
64800 .{ .src = .{ .to_mem, .none, .none } },
64801 },
64802 .extra_temps = .{
64803 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
64804 .{ .kind = .{ .rc = .sse } },
64805 .{ .kind = .{ .rc = .sse } },
64806 .unused,
64807 .unused,
64808 .unused,
64809 .unused,
64810 .unused,
64811 .unused,
64812 .unused,
64813 .unused,
64814 },
64815 .dst_temps = .{ .mem, .unused },
64816 .clobbers = .{ .eflags = true },
64817 .each = .{ .once = &.{
64818 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
64819 .{ ._, .p_d, .cmpeq, .tmp1x, .tmp1x, ._, ._ },
64820 .{ .@"0:", ._dqa, .mov, .tmp2x, .memia(.src0x, .tmp0, .add_size), ._, ._ },
64821 .{ ._, .p_, .xor, .tmp2x, .tmp1x, ._, ._ },
64822 .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp2x, ._, ._ },
64823 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
64824 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
64825 } },
6451864826 }, .{
6451964827 .required_features = .{ .sse2, null, null, null },
6452064828 .src_constraints = .{ .{ .signed_or_exact_remainder_int = .{ .of = .xword, .is = .xword } }, .any, .any },
......@@ -64545,6 +64853,34 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6454564853 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
6454664854 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
6454764855 } },
64856 }, .{
64857 .required_features = .{ .@"64bit", null, null, null },
64858 .src_constraints = .{ .{ .remainder_bool_vec = .{ .of = .xword, .is = .xword } }, .any, .any },
64859 .patterns = &.{
64860 .{ .src = .{ .mut_mem, .none, .none } },
64861 },
64862 .extra_temps = .{
64863 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
64864 .unused,
64865 .unused,
64866 .unused,
64867 .unused,
64868 .unused,
64869 .unused,
64870 .unused,
64871 .unused,
64872 .unused,
64873 .unused,
64874 },
64875 .dst_temps = .{ .{ .ref = .src0 }, .unused },
64876 .clobbers = .{ .eflags = true },
64877 .each = .{ .once = &.{
64878 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
64879 .{ .@"0:", ._, .not, .memia(.dst0q, .tmp0, .add_size), ._, ._, ._ },
64880 .{ ._, ._, .not, .memiad(.dst0q, .tmp0, .add_size, 8), ._, ._, ._ },
64881 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
64882 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
64883 } },
6454864884 }, .{
6454964885 .required_features = .{ .@"64bit", null, null, null },
6455064886 .src_constraints = .{ .{ .signed_or_exact_remainder_int = .{ .of = .xword, .is = .xword } }, .any, .any },
......@@ -64573,6 +64909,35 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6457364909 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
6457464910 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
6457564911 } },
64912 }, .{
64913 .required_features = .{ .@"64bit", null, null, null },
64914 .src_constraints = .{ .{ .remainder_bool_vec = .{ .of = .xword, .is = .xword } }, .any, .any },
64915 .patterns = &.{
64916 .{ .src = .{ .to_mem, .none, .none } },
64917 },
64918 .extra_temps = .{
64919 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
64920 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
64921 .unused,
64922 .unused,
64923 .unused,
64924 .unused,
64925 .unused,
64926 .unused,
64927 .unused,
64928 .unused,
64929 .unused,
64930 },
64931 .dst_temps = .{ .mem, .unused },
64932 .clobbers = .{ .eflags = true },
64933 .each = .{ .once = &.{
64934 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
64935 .{ .@"0:", ._, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_size), ._, ._ },
64936 .{ ._, ._, .not, .tmp1q, ._, ._, ._ },
64937 .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_size), .tmp1q, ._, ._ },
64938 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
64939 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
64940 } },
6457664941 }, .{
6457764942 .required_features = .{ .@"64bit", null, null, null },
6457864943 .src_constraints = .{ .{ .signed_or_exact_remainder_int = .{ .of = .xword, .is = .xword } }, .any, .any },
......@@ -65038,7 +65403,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6503865403 .src_constraints = .{ .{ .signed_int_or_full_vec = .qword }, .any, .any },
6503965404 .patterns = &.{
6504065405 .{ .src = .{ .mem, .none, .none } },
65041 .{ .src = .{ .to_mm, .none, .none } },
65406 .{ .src = .{ .to_mmx, .none, .none } },
6504265407 },
6504365408 .dst_temps = .{ .{ .rc = .mmx }, .unused },
6504465409 .each = .{ .once = &.{
......@@ -65049,7 +65414,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6504965414 .required_features = .{ .mmx, null, null, null },
6505065415 .src_constraints = .{ .{ .unsigned_int_vec = .qword }, .any, .any },
6505165416 .patterns = &.{
65052 .{ .src = .{ .to_mut_mm, .none, .none } },
65417 .{ .src = .{ .to_mut_mmx, .none, .none } },
6505365418 },
6505465419 .extra_temps = .{
6505565420 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
......@@ -65074,7 +65439,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6507465439 .src_constraints = .{ .{ .signed_int_or_full_vec = .xword }, .any, .any },
6507565440 .patterns = &.{
6507665441 .{ .src = .{ .mem, .none, .none } },
65077 .{ .src = .{ .to_xmm, .none, .none } },
65442 .{ .src = .{ .to_sse, .none, .none } },
6507865443 },
6507965444 .dst_temps = .{ .{ .rc = .sse }, .unused },
6508065445 .each = .{ .once = &.{
......@@ -65085,7 +65450,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6508565450 .required_features = .{ .avx, null, null, null },
6508665451 .src_constraints = .{ .{ .unsigned_int_vec = .xword }, .any, .any },
6508765452 .patterns = &.{
65088 .{ .src = .{ .to_xmm, .none, .none } },
65453 .{ .src = .{ .to_sse, .none, .none } },
6508965454 },
6509065455 .extra_temps = .{
6509165456 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
......@@ -65110,7 +65475,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6511065475 .src_constraints = .{ .{ .signed_int_or_full_vec = .xword }, .any, .any },
6511165476 .patterns = &.{
6511265477 .{ .src = .{ .mem, .none, .none } },
65113 .{ .src = .{ .to_xmm, .none, .none } },
65478 .{ .src = .{ .to_sse, .none, .none } },
6511465479 },
6511565480 .dst_temps = .{ .{ .rc = .sse }, .unused },
6511665481 .each = .{ .once = &.{
......@@ -65121,7 +65486,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6512165486 .required_features = .{ .sse2, null, null, null },
6512265487 .src_constraints = .{ .{ .unsigned_int_vec = .xword }, .any, .any },
6512365488 .patterns = &.{
65124 .{ .src = .{ .to_mut_xmm, .none, .none } },
65489 .{ .src = .{ .to_mut_sse, .none, .none } },
6512565490 },
6512665491 .extra_temps = .{
6512765492 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
......@@ -65145,7 +65510,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6514565510 .required_features = .{ .sse, null, null, null },
6514665511 .src_constraints = .{ .{ .unsigned_int_vec = .xword }, .any, .any },
6514765512 .patterns = &.{
65148 .{ .src = .{ .to_mut_xmm, .none, .none } },
65513 .{ .src = .{ .to_mut_sse, .none, .none } },
6514965514 },
6515065515 .extra_temps = .{
6515165516 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
......@@ -65170,7 +65535,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6517065535 .src_constraints = .{ .{ .signed_int_or_full_vec = .yword }, .any, .any },
6517165536 .patterns = &.{
6517265537 .{ .src = .{ .mem, .none, .none } },
65173 .{ .src = .{ .to_ymm, .none, .none } },
65538 .{ .src = .{ .to_sse, .none, .none } },
6517465539 },
6517565540 .dst_temps = .{ .{ .rc = .sse }, .unused },
6517665541 .each = .{ .once = &.{
......@@ -65181,7 +65546,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6518165546 .required_features = .{ .avx2, null, null, null },
6518265547 .src_constraints = .{ .{ .unsigned_int_vec = .yword }, .any, .any },
6518365548 .patterns = &.{
65184 .{ .src = .{ .to_ymm, .none, .none } },
65549 .{ .src = .{ .to_sse, .none, .none } },
6518565550 },
6518665551 .extra_temps = .{
6518765552 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
......@@ -65206,7 +65571,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6520665571 .src_constraints = .{ .{ .signed_int_or_full_vec = .yword }, .any, .any },
6520765572 .patterns = &.{
6520865573 .{ .src = .{ .mem, .none, .none } },
65209 .{ .src = .{ .to_ymm, .none, .none } },
65574 .{ .src = .{ .to_sse, .none, .none } },
6521065575 },
6521165576 .dst_temps = .{ .{ .rc = .sse }, .unused },
6521265577 .each = .{ .once = &.{
......@@ -65217,7 +65582,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6521765582 .required_features = .{ .avx, null, null, null },
6521865583 .src_constraints = .{ .{ .unsigned_int_vec = .yword }, .any, .any },
6521965584 .patterns = &.{
65220 .{ .src = .{ .to_ymm, .none, .none } },
65585 .{ .src = .{ .to_sse, .none, .none } },
6522165586 },
6522265587 .extra_temps = .{
6522365588 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
......@@ -65237,6 +65602,118 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6523765602 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
6523865603 .{ ._, .v_pd, .xor, .dst0y, .src0y, .lea(.tmp0y), ._ },
6523965604 } },
65605 }, .{
65606 .required_features = .{ .@"64bit", null, null, null },
65607 .src_constraints = .{ .{ .remainder_bool_vec = .{ .of = .xword, .is = .qword } }, .any, .any },
65608 .patterns = &.{
65609 .{ .src = .{ .to_mem, .none, .none } },
65610 },
65611 .extra_temps = .{
65612 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
65613 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
65614 .unused,
65615 .unused,
65616 .unused,
65617 .unused,
65618 .unused,
65619 .unused,
65620 .unused,
65621 .unused,
65622 .unused,
65623 },
65624 .dst_temps = .{ .mem, .unused },
65625 .each = .{ .once = &.{
65626 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_size), ._, ._ },
65627 .{ .@"0:", ._, .mov, .tmp1q, .memi(.src0q, .tmp0), ._, ._ },
65628 .{ ._, ._, .not, .tmp1q, ._, ._, ._ },
65629 .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp1q, ._, ._ },
65630 .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ },
65631 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
65632 } },
65633 }, .{
65634 .required_features = .{ .@"64bit", null, null, null },
65635 .src_constraints = .{ .{ .remainder_bool_vec = .{ .of = .xword, .is = .xword } }, .any, .any },
65636 .patterns = &.{
65637 .{ .src = .{ .to_mem, .none, .none } },
65638 },
65639 .extra_temps = .{
65640 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
65641 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
65642 .unused,
65643 .unused,
65644 .unused,
65645 .unused,
65646 .unused,
65647 .unused,
65648 .unused,
65649 .unused,
65650 .unused,
65651 },
65652 .dst_temps = .{ .mem, .unused },
65653 .each = .{ .once = &.{
65654 .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ },
65655 .{ .@"0:", ._, .mov, .tmp1q, .memi(.src0q, .tmp0), ._, ._ },
65656 .{ ._, ._, .not, .tmp1q, ._, ._, ._ },
65657 .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp1q, ._, ._ },
65658 .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ },
65659 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
65660 } },
65661 }, .{
65662 .required_features = .{ .@"64bit", null, null, null },
65663 .src_constraints = .{ .any_signed_int_or_full_vec, .any, .any },
65664 .patterns = &.{
65665 .{ .src = .{ .to_mem, .none, .none } },
65666 },
65667 .extra_temps = .{
65668 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
65669 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
65670 .unused,
65671 .unused,
65672 .unused,
65673 .unused,
65674 .unused,
65675 .unused,
65676 .unused,
65677 .unused,
65678 .unused,
65679 },
65680 .dst_temps = .{ .mem, .unused },
65681 .each = .{ .once = &.{
65682 .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size_up_8), ._, ._ },
65683 .{ .@"0:", ._, .mov, .tmp1q, .memi(.src0q, .tmp0), ._, ._ },
65684 .{ ._, ._, .not, .tmp1q, ._, ._, ._ },
65685 .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp1q, ._, ._ },
65686 .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ },
65687 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
65688 } },
65689 }, .{
65690 .required_features = .{ .@"64bit", null, null, null },
65691 .src_constraints = .{ .{ .signed_or_exact_remainder_int = .{ .of = .qword, .is = .qword } }, .any, .any },
65692 .patterns = &.{
65693 .{ .src = .{ .to_mem, .none, .none } },
65694 },
65695 .extra_temps = .{
65696 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
65697 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
65698 .unused,
65699 .unused,
65700 .unused,
65701 .unused,
65702 .unused,
65703 .unused,
65704 .unused,
65705 .unused,
65706 .unused,
65707 },
65708 .dst_temps = .{ .mem, .unused },
65709 .each = .{ .once = &.{
65710 .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ },
65711 .{ .@"0:", ._, .mov, .tmp1q, .memi(.src0q, .tmp0), ._, ._ },
65712 .{ ._, ._, .not, .tmp1q, ._, ._, ._ },
65713 .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp1q, ._, ._ },
65714 .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ },
65715 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
65716 } },
6524065717 }, .{
6524165718 .required_features = .{ .@"64bit", null, null, null },
6524265719 .patterns = &.{
......@@ -65257,14 +65734,94 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6525765734 },
6525865735 .dst_temps = .{ .mem, .unused },
6525965736 .each = .{ .once = &.{
65260 .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ },
65737 .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size_up_8), ._, ._ },
6526165738 .{ ._, ._, .lea, .tmp1p, .mem(.tmp3), ._, ._ },
6526265739 .{ .@"0:", ._, .mov, .tmp2q, .memi(.src0q, .tmp0), ._, ._ },
6526365740 .{ ._, ._, .xor, .tmp2q, .leai(.tmp1q, .tmp0), ._, ._ },
6526465741 .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp2q, ._, ._ },
65265 .{ ._, ._, .lea, .tmp0d, .lead(.tmp0, 8), ._, ._ },
65266 .{ ._, ._, .cmp, .tmp0d, .sa(.src0, .add_unaligned_size), ._, ._ },
65267 .{ ._, ._b, .j, .@"0b", ._, ._, ._ },
65742 .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ },
65743 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
65744 } },
65745 }, .{
65746 .src_constraints = .{ .{ .remainder_bool_vec = .{ .of = .dword, .is = .dword } }, .any, .any },
65747 .patterns = &.{
65748 .{ .src = .{ .to_mem, .none, .none } },
65749 },
65750 .extra_temps = .{
65751 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
65752 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
65753 .unused,
65754 .unused,
65755 .unused,
65756 .unused,
65757 .unused,
65758 .unused,
65759 .unused,
65760 .unused,
65761 .unused,
65762 },
65763 .dst_temps = .{ .mem, .unused },
65764 .each = .{ .once = &.{
65765 .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_size), ._, ._ },
65766 .{ .@"0:", ._, .mov, .tmp1d, .memi(.src0d, .tmp0), ._, ._ },
65767 .{ ._, ._, .not, .tmp1d, ._, ._, ._ },
65768 .{ ._, ._, .mov, .memi(.dst0d, .tmp0), .tmp1d, ._, ._ },
65769 .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ },
65770 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
65771 } },
65772 }, .{
65773 .src_constraints = .{ .any_signed_int_or_full_vec, .any, .any },
65774 .patterns = &.{
65775 .{ .src = .{ .to_mem, .none, .none } },
65776 },
65777 .extra_temps = .{
65778 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
65779 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
65780 .unused,
65781 .unused,
65782 .unused,
65783 .unused,
65784 .unused,
65785 .unused,
65786 .unused,
65787 .unused,
65788 .unused,
65789 },
65790 .dst_temps = .{ .mem, .unused },
65791 .each = .{ .once = &.{
65792 .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size_up_4), ._, ._ },
65793 .{ .@"0:", ._, .mov, .tmp1d, .memi(.src0d, .tmp0), ._, ._ },
65794 .{ ._, ._, .not, .tmp1d, ._, ._, ._ },
65795 .{ ._, ._, .mov, .memi(.dst0d, .tmp0), .tmp1d, ._, ._ },
65796 .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ },
65797 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
65798 } },
65799 }, .{
65800 .src_constraints = .{ .{ .signed_or_exact_remainder_int = .{ .of = .dword, .is = .dword } }, .any, .any },
65801 .patterns = &.{
65802 .{ .src = .{ .to_mem, .none, .none } },
65803 },
65804 .extra_temps = .{
65805 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
65806 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
65807 .unused,
65808 .unused,
65809 .unused,
65810 .unused,
65811 .unused,
65812 .unused,
65813 .unused,
65814 .unused,
65815 .unused,
65816 },
65817 .dst_temps = .{ .mem, .unused },
65818 .each = .{ .once = &.{
65819 .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_size), ._, ._ },
65820 .{ .@"0:", ._, .mov, .tmp1d, .memi(.src0d, .tmp0), ._, ._ },
65821 .{ ._, ._, .not, .tmp1d, ._, ._, ._ },
65822 .{ ._, ._, .mov, .memi(.dst0d, .tmp0), .tmp1d, ._, ._ },
65823 .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ },
65824 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
6526865825 } },
6526965826 }, .{
6527065827 .patterns = &.{
......@@ -65285,14 +65842,13 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6528565842 },
6528665843 .dst_temps = .{ .mem, .unused },
6528765844 .each = .{ .once = &.{
65288 .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ },
65845 .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size_up_8), ._, ._ },
6528965846 .{ ._, ._, .lea, .tmp1p, .mem(.tmp3), ._, ._ },
6529065847 .{ .@"0:", ._, .mov, .tmp2d, .memi(.src0d, .tmp0), ._, ._ },
6529165848 .{ ._, ._, .xor, .tmp2d, .leai(.tmp1d, .tmp0), ._, ._ },
6529265849 .{ ._, ._, .mov, .memi(.dst0d, .tmp0), .tmp2d, ._, ._ },
65293 .{ ._, ._, .lea, .tmp0d, .lead(.tmp0, 4), ._, ._ },
65294 .{ ._, ._, .cmp, .tmp0d, .sa(.src0, .add_unaligned_size), ._, ._ },
65295 .{ ._, ._b, .j, .@"0b", ._, ._, ._ },
65850 .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ },
65851 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
6529665852 } },
6529765853 } }) catch |err| switch (err) {
6529865854 error.SelectFailed => return cg.fail("failed to select {s} {f} {f}", .{
......@@ -75367,7 +75923,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7536775923 .src_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .byte } }, .any, .any },
7536875924 .patterns = &.{
7536975925 .{ .src = .{ .mem, .none, .none } },
75370 .{ .src = .{ .to_mm, .none, .none } },
75926 .{ .src = .{ .to_mmx, .none, .none } },
7537175927 },
7537275928 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .mmx } }, .unused },
7537375929 .each = .{ .once = &.{
......@@ -75378,7 +75934,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7537875934 .src_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .word } }, .any, .any },
7537975935 .patterns = &.{
7538075936 .{ .src = .{ .mem, .none, .none } },
75381 .{ .src = .{ .to_mm, .none, .none } },
75937 .{ .src = .{ .to_mmx, .none, .none } },
7538275938 },
7538375939 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .mmx } }, .unused },
7538475940 .each = .{ .once = &.{
......@@ -75389,7 +75945,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7538975945 .src_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .dword } }, .any, .any },
7539075946 .patterns = &.{
7539175947 .{ .src = .{ .mem, .none, .none } },
75392 .{ .src = .{ .to_mm, .none, .none } },
75948 .{ .src = .{ .to_mmx, .none, .none } },
7539375949 },
7539475950 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .mmx } }, .unused },
7539575951 .each = .{ .once = &.{
......@@ -112415,7 +112971,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
112415112971 .dst_temps = .{ .mem, .unused },
112416112972 .clobbers = .{ .eflags = true },
112417112973 .each = .{ .once = &.{
112418 .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ },
112974 .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size_up_8), ._, ._ },
112419112975 .{ .@"0:", .vp_d, .movsxb, .tmp1y, .memi(.src0q, .tmp0), ._, ._ },
112420112976 .{ ._, .v_ps, .cvtdq2, .tmp1y, .tmp1y, ._, ._ },
112421112977 .{ ._, .v_, .cvtps2ph, .memsi(.dst0x, .@"2", .tmp0), .tmp1y, .rm(.{}), ._ },
......@@ -112445,7 +113001,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
112445113001 .dst_temps = .{ .mem, .unused },
112446113002 .clobbers = .{ .eflags = true },
112447113003 .each = .{ .once = &.{
112448 .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_size), ._, ._ },
113004 .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size_up_4), ._, ._ },
112449113005 .{ .@"0:", .vp_d, .movsxb, .tmp1x, .memi(.src0d, .tmp0), ._, ._ },
112450113006 .{ ._, .v_ps, .cvtdq2, .tmp1x, .tmp1x, ._, ._ },
112451113007 .{ ._, .v_, .cvtps2ph, .memsi(.dst0q, .@"2", .tmp0), .tmp1x, .rm(.{}), ._ },
......@@ -112475,7 +113031,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
112475113031 .dst_temps = .{ .mem, .unused },
112476113032 .clobbers = .{ .eflags = true },
112477113033 .each = .{ .once = &.{
112478 .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ },
113034 .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size_up_8), ._, ._ },
112479113035 .{ .@"0:", .vp_d, .movzxb, .tmp1y, .memi(.src0q, .tmp0), ._, ._ },
112480113036 .{ ._, .v_ps, .cvtdq2, .tmp1y, .tmp1y, ._, ._ },
112481113037 .{ ._, .v_, .cvtps2ph, .memsi(.dst0x, .@"2", .tmp0), .tmp1y, .rm(.{}), ._ },
......@@ -112505,7 +113061,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
112505113061 .dst_temps = .{ .mem, .unused },
112506113062 .clobbers = .{ .eflags = true },
112507113063 .each = .{ .once = &.{
112508 .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_size), ._, ._ },
113064 .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size_up_4), ._, ._ },
112509113065 .{ .@"0:", .vp_d, .movzxb, .tmp1x, .memi(.src0d, .tmp0), ._, ._ },
112510113066 .{ ._, .v_ps, .cvtdq2, .tmp1x, .tmp1x, ._, ._ },
112511113067 .{ ._, .v_, .cvtps2ph, .memsi(.dst0q, .@"2", .tmp0), .tmp1x, .rm(.{}), ._ },
......@@ -113099,7 +113655,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
113099113655 .dst_temps = .{ .mem, .unused },
113100113656 .clobbers = .{ .eflags = true },
113101113657 .each = .{ .once = &.{
113102 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_size), ._, ._ },
113658 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size_up_16), ._, ._ },
113103113659 .{ .@"0:", .vp_d, .movsxw, .tmp1y, .memi(.src0x, .tmp0), ._, ._ },
113104113660 .{ ._, .v_ps, .cvtdq2, .tmp1y, .tmp1y, ._, ._ },
113105113661 .{ ._, .v_, .cvtps2ph, .memi(.dst0x, .tmp0), .tmp1y, .rm(.{}), ._ },
......@@ -113129,7 +113685,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
113129113685 .dst_temps = .{ .mem, .unused },
113130113686 .clobbers = .{ .eflags = true },
113131113687 .each = .{ .once = &.{
113132 .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ },
113688 .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size_up_8), ._, ._ },
113133113689 .{ .@"0:", .vp_d, .movsxw, .tmp1x, .memi(.src0q, .tmp0), ._, ._ },
113134113690 .{ ._, .v_ps, .cvtdq2, .tmp1x, .tmp1x, ._, ._ },
113135113691 .{ ._, .v_, .cvtps2ph, .memi(.dst0q, .tmp0), .tmp1x, .rm(.{}), ._ },
......@@ -113159,7 +113715,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
113159113715 .dst_temps = .{ .mem, .unused },
113160113716 .clobbers = .{ .eflags = true },
113161113717 .each = .{ .once = &.{
113162 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_size), ._, ._ },
113718 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size_up_16), ._, ._ },
113163113719 .{ .@"0:", .vp_d, .movzxw, .tmp1y, .memi(.src0x, .tmp0), ._, ._ },
113164113720 .{ ._, .v_ps, .cvtdq2, .tmp1y, .tmp1y, ._, ._ },
113165113721 .{ ._, .v_, .cvtps2ph, .memi(.dst0x, .tmp0), .tmp1y, .rm(.{}), ._ },
......@@ -113189,7 +113745,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
113189113745 .dst_temps = .{ .mem, .unused },
113190113746 .clobbers = .{ .eflags = true },
113191113747 .each = .{ .once = &.{
113192 .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ },
113748 .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size_up_8), ._, ._ },
113193113749 .{ .@"0:", .vp_d, .movzxw, .tmp1x, .memi(.src0q, .tmp0), ._, ._ },
113194113750 .{ ._, .v_ps, .cvtdq2, .tmp1x, .tmp1x, ._, ._ },
113195113751 .{ ._, .v_, .cvtps2ph, .memi(.dst0q, .tmp0), .tmp1x, .rm(.{}), ._ },
......@@ -113499,7 +114055,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
113499114055 .dst_temps = .{ .mem, .unused },
113500114056 .clobbers = .{ .eflags = true },
113501114057 .each = .{ .once = &.{
113502 .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_size), ._, ._ },
114058 .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_unaligned_size_up_16), ._, ._ },
113503114059 .{ .@"0:", .v_ps, .cvtdq2, .tmp1y, .memsi(.src0y, .@"2", .tmp0), ._, ._ },
113504114060 .{ ._, .v_, .cvtps2ph, .memi(.dst0x, .tmp0), .tmp1y, .rm(.{}), ._ },
113505114061 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
......@@ -113528,7 +114084,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
113528114084 .dst_temps = .{ .mem, .unused },
113529114085 .clobbers = .{ .eflags = true },
113530114086 .each = .{ .once = &.{
113531 .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_size), ._, ._ },
114087 .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_unaligned_size_up_8), ._, ._ },
113532114088 .{ .@"0:", .v_ps, .cvtdq2, .tmp1x, .memsi(.src0x, .@"2", .tmp0), ._, ._ },
113533114089 .{ ._, .v_, .cvtps2ph, .memi(.dst0q, .tmp0), .tmp1x, .rm(.{}), ._ },
113534114090 .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ },
......@@ -115542,7 +116098,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
115542116098 .dst_temps = .{ .mem, .unused },
115543116099 .clobbers = .{ .eflags = true },
115544116100 .each = .{ .once = &.{
115545 .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ },
116101 .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size_up_8), ._, ._ },
115546116102 .{ .@"0:", .vp_d, .movsxb, .tmp1y, .memi(.src0q, .tmp0), ._, ._ },
115547116103 .{ ._, .v_ps, .cvtdq2, .tmp1y, .tmp1y, ._, ._ },
115548116104 .{ ._, .v_ps, .mova, .memsi(.dst0y, .@"4", .tmp0), .tmp1y, ._, ._ },
......@@ -115572,7 +116128,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
115572116128 .dst_temps = .{ .mem, .unused },
115573116129 .clobbers = .{ .eflags = true },
115574116130 .each = .{ .once = &.{
115575 .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_size), ._, ._ },
116131 .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size_up_4), ._, ._ },
115576116132 .{ .@"0:", .vp_d, .movsxb, .tmp1x, .memi(.src0d, .tmp0), ._, ._ },
115577116133 .{ ._, .v_ps, .cvtdq2, .tmp1x, .tmp1x, ._, ._ },
115578116134 .{ ._, .v_ps, .mova, .memsi(.dst0x, .@"4", .tmp0), .tmp1x, ._, ._ },
......@@ -115602,7 +116158,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
115602116158 .dst_temps = .{ .mem, .unused },
115603116159 .clobbers = .{ .eflags = true },
115604116160 .each = .{ .once = &.{
115605 .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_size), ._, ._ },
116161 .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size_up_4), ._, ._ },
115606116162 .{ .@"0:", .p_d, .movsxb, .tmp1x, .memi(.src0d, .tmp0), ._, ._ },
115607116163 .{ ._, ._ps, .cvtdq2, .tmp1x, .tmp1x, ._, ._ },
115608116164 .{ ._, ._ps, .mova, .memsi(.dst0x, .@"4", .tmp0), .tmp1x, ._, ._ },
......@@ -115632,7 +116188,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
115632116188 .dst_temps = .{ .mem, .unused },
115633116189 .clobbers = .{ .eflags = true },
115634116190 .each = .{ .once = &.{
115635 .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ },
116191 .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size_up_8), ._, ._ },
115636116192 .{ .@"0:", .vp_d, .movzxb, .tmp1y, .memi(.src0q, .tmp0), ._, ._ },
115637116193 .{ ._, .v_ps, .cvtdq2, .tmp1y, .tmp1y, ._, ._ },
115638116194 .{ ._, .v_ps, .mova, .memsi(.dst0y, .@"4", .tmp0), .tmp1y, ._, ._ },
......@@ -115662,7 +116218,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
115662116218 .dst_temps = .{ .mem, .unused },
115663116219 .clobbers = .{ .eflags = true },
115664116220 .each = .{ .once = &.{
115665 .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_size), ._, ._ },
116221 .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size_up_4), ._, ._ },
115666116222 .{ .@"0:", .vp_d, .movzxb, .tmp1x, .memi(.src0d, .tmp0), ._, ._ },
115667116223 .{ ._, .v_ps, .cvtdq2, .tmp1x, .tmp1x, ._, ._ },
115668116224 .{ ._, .v_ps, .mova, .memsi(.dst0x, .@"4", .tmp0), .tmp1x, ._, ._ },
......@@ -115692,7 +116248,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
115692116248 .dst_temps = .{ .mem, .unused },
115693116249 .clobbers = .{ .eflags = true },
115694116250 .each = .{ .once = &.{
115695 .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_size), ._, ._ },
116251 .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size_up_4), ._, ._ },
115696116252 .{ .@"0:", .p_d, .movzxb, .tmp1x, .memi(.src0d, .tmp0), ._, ._ },
115697116253 .{ ._, ._ps, .cvtdq2, .tmp1x, .tmp1x, ._, ._ },
115698116254 .{ ._, ._ps, .mova, .memsi(.dst0x, .@"4", .tmp0), .tmp1x, ._, ._ },
......@@ -116048,7 +116604,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
116048116604 .dst_temps = .{ .mem, .unused },
116049116605 .clobbers = .{ .eflags = true },
116050116606 .each = .{ .once = &.{
116051 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_size), ._, ._ },
116607 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size_up_16), ._, ._ },
116052116608 .{ .@"0:", .vp_d, .movsxw, .tmp1y, .memi(.src0x, .tmp0), ._, ._ },
116053116609 .{ ._, .v_ps, .cvtdq2, .tmp1y, .tmp1y, ._, ._ },
116054116610 .{ ._, .v_ps, .mova, .memsi(.dst0y, .@"2", .tmp0), .tmp1y, ._, ._ },
......@@ -116078,7 +116634,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
116078116634 .dst_temps = .{ .mem, .unused },
116079116635 .clobbers = .{ .eflags = true },
116080116636 .each = .{ .once = &.{
116081 .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ },
116637 .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size_up_8), ._, ._ },
116082116638 .{ .@"0:", .vp_d, .movsxw, .tmp1x, .memi(.src0q, .tmp0), ._, ._ },
116083116639 .{ ._, .v_ps, .cvtdq2, .tmp1x, .tmp1x, ._, ._ },
116084116640 .{ ._, .v_ps, .mova, .memsi(.dst0x, .@"2", .tmp0), .tmp1x, ._, ._ },
......@@ -116108,7 +116664,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
116108116664 .dst_temps = .{ .mem, .unused },
116109116665 .clobbers = .{ .eflags = true },
116110116666 .each = .{ .once = &.{
116111 .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ },
116667 .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size_up_8), ._, ._ },
116112116668 .{ .@"0:", .p_d, .movsxw, .tmp1x, .memi(.src0q, .tmp0), ._, ._ },
116113116669 .{ ._, ._ps, .cvtdq2, .tmp1x, .tmp1x, ._, ._ },
116114116670 .{ ._, ._ps, .mova, .memsi(.dst0x, .@"2", .tmp0), .tmp1x, ._, ._ },
......@@ -116138,7 +116694,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
116138116694 .dst_temps = .{ .mem, .unused },
116139116695 .clobbers = .{ .eflags = true },
116140116696 .each = .{ .once = &.{
116141 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_size), ._, ._ },
116697 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size_up_16), ._, ._ },
116142116698 .{ .@"0:", .vp_d, .movzxw, .tmp1y, .memi(.src0x, .tmp0), ._, ._ },
116143116699 .{ ._, .v_ps, .cvtdq2, .tmp1y, .tmp1y, ._, ._ },
116144116700 .{ ._, .v_ps, .mova, .memsi(.dst0y, .@"2", .tmp0), .tmp1y, ._, ._ },
......@@ -116168,7 +116724,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
116168116724 .dst_temps = .{ .mem, .unused },
116169116725 .clobbers = .{ .eflags = true },
116170116726 .each = .{ .once = &.{
116171 .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ },
116727 .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size_up_8), ._, ._ },
116172116728 .{ .@"0:", .vp_d, .movzxw, .tmp1x, .memi(.src0q, .tmp0), ._, ._ },
116173116729 .{ ._, .v_ps, .cvtdq2, .tmp1x, .tmp1x, ._, ._ },
116174116730 .{ ._, .v_ps, .mova, .memsi(.dst0x, .@"2", .tmp0), .tmp1x, ._, ._ },
......@@ -116198,7 +116754,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
116198116754 .dst_temps = .{ .mem, .unused },
116199116755 .clobbers = .{ .eflags = true },
116200116756 .each = .{ .once = &.{
116201 .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ },
116757 .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size_up_8), ._, ._ },
116202116758 .{ .@"0:", .p_d, .movzxw, .tmp1x, .memi(.src0q, .tmp0), ._, ._ },
116203116759 .{ ._, ._ps, .cvtdq2, .tmp1x, .tmp1x, ._, ._ },
116204116760 .{ ._, ._ps, .mova, .memsi(.dst0x, .@"2", .tmp0), .tmp1x, ._, ._ },
......@@ -116388,7 +116944,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
116388116944 .dst_temps = .{ .mem, .unused },
116389116945 .clobbers = .{ .eflags = true },
116390116946 .each = .{ .once = &.{
116391 .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_size), ._, ._ },
116947 .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size_up_32), ._, ._ },
116392116948 .{ .@"0:", .v_ps, .cvtdq2, .tmp1y, .memi(.src0y, .tmp0), ._, ._ },
116393116949 .{ ._, .v_ps, .mova, .memi(.dst0y, .tmp0), .tmp1y, ._, ._ },
116394116950 .{ ._, ._, .sub, .tmp0d, .si(32), ._, ._ },
......@@ -116417,7 +116973,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
116417116973 .dst_temps = .{ .mem, .unused },
116418116974 .clobbers = .{ .eflags = true },
116419116975 .each = .{ .once = &.{
116420 .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_size), ._, ._ },
116976 .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_unaligned_size_up_16), ._, ._ },
116421116977 .{ .@"0:", .v_ps, .cvtdq2, .tmp1x, .memi(.src0x, .tmp0), ._, ._ },
116422116978 .{ ._, .v_ps, .mova, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ },
116423116979 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
......@@ -116446,7 +117002,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
116446117002 .dst_temps = .{ .mem, .unused },
116447117003 .clobbers = .{ .eflags = true },
116448117004 .each = .{ .once = &.{
116449 .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_size), ._, ._ },
117005 .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_unaligned_size_up_16), ._, ._ },
116450117006 .{ .@"0:", ._ps, .cvtdq2, .tmp1x, .memi(.src0x, .tmp0), ._, ._ },
116451117007 .{ ._, ._ps, .mova, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ },
116452117008 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
......@@ -118062,7 +118618,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
118062118618 .dst_temps = .{ .mem, .unused },
118063118619 .clobbers = .{ .eflags = true },
118064118620 .each = .{ .once = &.{
118065 .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_size), ._, ._ },
118621 .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size_up_4), ._, ._ },
118066118622 .{ .@"0:", .vp_d, .movsxb, .tmp1x, .memi(.src0d, .tmp0), ._, ._ },
118067118623 .{ ._, .v_pd, .cvtdq2, .tmp1y, .tmp1x, ._, ._ },
118068118624 .{ ._, .v_pd, .mova, .memsi(.dst0y, .@"8", .tmp0), .tmp1y, ._, ._ },
......@@ -118092,7 +118648,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
118092118648 .dst_temps = .{ .mem, .unused },
118093118649 .clobbers = .{ .eflags = true },
118094118650 .each = .{ .once = &.{
118095 .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_size), ._, ._ },
118651 .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size_up_2), ._, ._ },
118096118652 .{ .@"0:", .vp_, .xor, .tmp1x, .tmp1x, .tmp1x, ._ },
118097118653 .{ ._, .vp_w, .insr, .tmp1x, .tmp1x, .memi(.src0w, .tmp0), .ui(0) },
118098118654 .{ ._, .vp_d, .movsxb, .tmp1x, .tmp1d, ._, ._ },
......@@ -118124,7 +118680,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
118124118680 .dst_temps = .{ .mem, .unused },
118125118681 .clobbers = .{ .eflags = true },
118126118682 .each = .{ .once = &.{
118127 .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_size), ._, ._ },
118683 .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size_up_2), ._, ._ },
118128118684 .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ },
118129118685 .{ ._, .p_w, .insr, .tmp1x, .memi(.src0w, .tmp0), .ui(0), ._ },
118130118686 .{ ._, .p_d, .movsxb, .tmp1x, .tmp1d, ._, ._ },
......@@ -118156,7 +118712,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
118156118712 .dst_temps = .{ .mem, .unused },
118157118713 .clobbers = .{ .eflags = true },
118158118714 .each = .{ .once = &.{
118159 .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_size), ._, ._ },
118715 .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size_up_4), ._, ._ },
118160118716 .{ .@"0:", .vp_d, .movzxb, .tmp1x, .memi(.src0d, .tmp0), ._, ._ },
118161118717 .{ ._, .v_pd, .cvtdq2, .tmp1y, .tmp1x, ._, ._ },
118162118718 .{ ._, .v_pd, .mova, .memsi(.dst0y, .@"8", .tmp0), .tmp1y, ._, ._ },
......@@ -118186,7 +118742,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
118186118742 .dst_temps = .{ .mem, .unused },
118187118743 .clobbers = .{ .eflags = true },
118188118744 .each = .{ .once = &.{
118189 .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_size), ._, ._ },
118745 .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size_up_2), ._, ._ },
118190118746 .{ .@"0:", .vp_, .xor, .tmp1x, .tmp1x, .tmp1x, ._ },
118191118747 .{ ._, .vp_w, .insr, .tmp1x, .tmp1x, .memi(.src0w, .tmp0), .ui(0) },
118192118748 .{ ._, .vp_d, .movzxb, .tmp1x, .tmp1d, ._, ._ },
......@@ -118218,7 +118774,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
118218118774 .dst_temps = .{ .mem, .unused },
118219118775 .clobbers = .{ .eflags = true },
118220118776 .each = .{ .once = &.{
118221 .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_size), ._, ._ },
118777 .{ ._, ._, .mov, .tmp0d, .sia(-2, .src0, .add_unaligned_size_up_2), ._, ._ },
118222118778 .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ },
118223118779 .{ ._, .p_w, .insr, .tmp1x, .memi(.src0w, .tmp0), .ui(0), ._ },
118224118780 .{ ._, .p_d, .movzxb, .tmp1x, .tmp1d, ._, ._ },
......@@ -118700,7 +119256,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
118700119256 .dst_temps = .{ .mem, .unused },
118701119257 .clobbers = .{ .eflags = true },
118702119258 .each = .{ .once = &.{
118703 .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ },
119259 .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size_up_8), ._, ._ },
118704119260 .{ .@"0:", .vp_d, .movsxw, .tmp1x, .memi(.src0q, .tmp0), ._, ._ },
118705119261 .{ ._, .v_pd, .cvtdq2, .tmp1y, .tmp1x, ._, ._ },
118706119262 .{ ._, .v_pd, .mova, .memsi(.dst0y, .@"4", .tmp0), .tmp1y, ._, ._ },
......@@ -118730,7 +119286,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
118730119286 .dst_temps = .{ .mem, .unused },
118731119287 .clobbers = .{ .eflags = true },
118732119288 .each = .{ .once = &.{
118733 .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_size), ._, ._ },
119289 .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size_up_4), ._, ._ },
118734119290 .{ .@"0:", .v_d, .mov, .tmp1x, .memi(.src0d, .tmp0), ._, ._ },
118735119291 .{ ._, .vp_d, .movsxw, .tmp1x, .tmp1q, ._, ._ },
118736119292 .{ ._, .v_pd, .cvtdq2, .tmp1x, .tmp1q, ._, ._ },
......@@ -118761,7 +119317,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
118761119317 .dst_temps = .{ .mem, .unused },
118762119318 .clobbers = .{ .eflags = true },
118763119319 .each = .{ .once = &.{
118764 .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_size), ._, ._ },
119320 .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size_up_4), ._, ._ },
118765119321 .{ .@"0:", ._d, .mov, .tmp1x, .memi(.src0d, .tmp0), ._, ._ },
118766119322 .{ ._, .p_d, .movsxw, .tmp1x, .tmp1q, ._, ._ },
118767119323 .{ ._, ._pd, .cvtdq2, .tmp1x, .tmp1q, ._, ._ },
......@@ -118792,7 +119348,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
118792119348 .dst_temps = .{ .mem, .unused },
118793119349 .clobbers = .{ .eflags = true },
118794119350 .each = .{ .once = &.{
118795 .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ },
119351 .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size_up_8), ._, ._ },
118796119352 .{ .@"0:", .vp_d, .movzxw, .tmp1x, .memi(.src0q, .tmp0), ._, ._ },
118797119353 .{ ._, .v_pd, .cvtdq2, .tmp1y, .tmp1x, ._, ._ },
118798119354 .{ ._, .v_pd, .mova, .memsi(.dst0y, .@"4", .tmp0), .tmp1y, ._, ._ },
......@@ -118822,7 +119378,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
118822119378 .dst_temps = .{ .mem, .unused },
118823119379 .clobbers = .{ .eflags = true },
118824119380 .each = .{ .once = &.{
118825 .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_size), ._, ._ },
119381 .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size_up_4), ._, ._ },
118826119382 .{ .@"0:", .v_d, .mov, .tmp1x, .memi(.src0d, .tmp0), ._, ._ },
118827119383 .{ ._, .vp_d, .movzxw, .tmp1x, .tmp1q, ._, ._ },
118828119384 .{ ._, .v_pd, .cvtdq2, .tmp1x, .tmp1q, ._, ._ },
......@@ -118853,7 +119409,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
118853119409 .dst_temps = .{ .mem, .unused },
118854119410 .clobbers = .{ .eflags = true },
118855119411 .each = .{ .once = &.{
118856 .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_size), ._, ._ },
119412 .{ ._, ._, .mov, .tmp0d, .sia(-4, .src0, .add_unaligned_size_up_4), ._, ._ },
118857119413 .{ .@"0:", ._d, .mov, .tmp1x, .memi(.src0d, .tmp0), ._, ._ },
118858119414 .{ ._, .p_d, .movzxw, .tmp1x, .tmp1q, ._, ._ },
118859119415 .{ ._, ._pd, .cvtdq2, .tmp1x, .tmp1q, ._, ._ },
......@@ -119106,7 +119662,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
119106119662 .dst_temps = .{ .mem, .unused },
119107119663 .clobbers = .{ .eflags = true },
119108119664 .each = .{ .once = &.{
119109 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_size), ._, ._ },
119665 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size_up_16), ._, ._ },
119110119666 .{ .@"0:", .v_pd, .cvtdq2, .tmp1y, .memi(.src0x, .tmp0), ._, ._ },
119111119667 .{ ._, .v_pd, .mova, .memsi(.dst0y, .@"2", .tmp0), .tmp1y, ._, ._ },
119112119668 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
......@@ -119135,7 +119691,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
119135119691 .dst_temps = .{ .mem, .unused },
119136119692 .clobbers = .{ .eflags = true },
119137119693 .each = .{ .once = &.{
119138 .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ },
119694 .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size_up_8), ._, ._ },
119139119695 .{ .@"0:", .v_pd, .cvtdq2, .tmp1x, .memi(.src0q, .tmp0), ._, ._ },
119140119696 .{ ._, .v_pd, .mova, .memsi(.dst0x, .@"2", .tmp0), .tmp1x, ._, ._ },
119141119697 .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ },
......@@ -119164,7 +119720,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
119164119720 .dst_temps = .{ .mem, .unused },
119165119721 .clobbers = .{ .eflags = true },
119166119722 .each = .{ .once = &.{
119167 .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ },
119723 .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_unaligned_size_up_8), ._, ._ },
119168119724 .{ .@"0:", ._pd, .cvtdq2, .tmp1x, .memi(.src0q, .tmp0), ._, ._ },
119169119725 .{ ._, ._pd, .mova, .memsi(.dst0x, .@"2", .tmp0), .tmp1x, ._, ._ },
119170119726 .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ },
......@@ -168582,7 +169138,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
168582169138 .dst_temps = .{ .mem, .unused },
168583169139 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
168584169140 .each = .{ .once = &.{
168585 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .sub_unaligned_size), ._, ._ },
169141 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ },
168586169142 .{ .@"0:", ._ps, .mova, .tmp1x, .memi(.src0x, .tmp0), ._, ._ },
168587169143 .{ ._, ._ps, .mova, .tmp2x, .memi(.src1x, .tmp0), ._, ._ },
168588169144 .{ ._, ._ps, .mova, .tmp3x, .memi(.src2x, .tmp0), ._, ._ },
......@@ -182106,68 +182662,135 @@ fn genSetMem(
182106182662 }
182107182663}
182108182664
182109fn genInlineMemcpy(self: *CodeGen, dst_ptr: MCValue, src_ptr: MCValue, len: MCValue, opts: struct {
182665fn genInlineMemcpy(cg: *CodeGen, dst_ptr: MCValue, src_ptr: MCValue, len: MCValue, opts: struct {
182110182666 no_alias: bool,
182111182667}) InnerError!void {
182112 if (opts.no_alias and dst_ptr.isAddress() and src_ptr.isAddress()) switch (len) {
182113 else => {},
182114 .immediate => |len_imm| switch (len_imm) {
182115 else => {},
182116 1 => if (self.register_manager.tryAllocReg(null, abi.RegisterClass.gp)) |reg| {
182117 try self.asmRegisterMemory(.{ ._, .mov }, reg.to8(), try src_ptr.deref().mem(self, .{ .size = .byte }));
182118 try self.asmMemoryRegister(.{ ._, .mov }, try dst_ptr.deref().mem(self, .{ .size = .byte }), reg.to8());
182119 return;
182120 },
182121 2 => if (self.register_manager.tryAllocReg(null, abi.RegisterClass.gp)) |reg| {
182122 try self.asmRegisterMemory(.{ ._, .mov }, reg.to16(), try src_ptr.deref().mem(self, .{ .size = .word }));
182123 try self.asmMemoryRegister(.{ ._, .mov }, try dst_ptr.deref().mem(self, .{ .size = .word }), reg.to16());
182124 return;
182125 },
182126 4 => if (self.register_manager.tryAllocReg(null, abi.RegisterClass.gp)) |reg| {
182127 try self.asmRegisterMemory(.{ ._, .mov }, reg.to32(), try src_ptr.deref().mem(self, .{ .size = .dword }));
182128 try self.asmMemoryRegister(.{ ._, .mov }, try dst_ptr.deref().mem(self, .{ .size = .dword }), reg.to32());
182129 return;
182130 },
182131 8 => if (self.target.cpu.arch == .x86_64) {
182132 if (self.register_manager.tryAllocReg(null, abi.RegisterClass.gp)) |reg| {
182133 try self.asmRegisterMemory(.{ ._, .mov }, reg.to64(), try src_ptr.deref().mem(self, .{ .size = .qword }));
182134 try self.asmMemoryRegister(.{ ._, .mov }, try dst_ptr.deref().mem(self, .{ .size = .qword }), reg.to64());
182135 return;
182136 }
182137 },
182138 16 => if (self.hasFeature(.avx)) {
182139 if (self.register_manager.tryAllocReg(null, abi.RegisterClass.sse)) |reg| {
182140 try self.asmRegisterMemory(.{ .v_dqu, .mov }, reg.to128(), try src_ptr.deref().mem(self, .{ .size = .xword }));
182141 try self.asmMemoryRegister(.{ .v_dqu, .mov }, try dst_ptr.deref().mem(self, .{ .size = .xword }), reg.to128());
182142 return;
182143 }
182144 } else if (self.hasFeature(.sse2)) {
182145 if (self.register_manager.tryAllocReg(null, abi.RegisterClass.sse)) |reg| {
182146 try self.asmRegisterMemory(.{ ._dqu, .mov }, reg.to128(), try src_ptr.deref().mem(self, .{ .size = .xword }));
182147 try self.asmMemoryRegister(.{ ._dqu, .mov }, try dst_ptr.deref().mem(self, .{ .size = .xword }), reg.to128());
182148 return;
182149 }
182150 } else if (self.hasFeature(.sse)) {
182151 if (self.register_manager.tryAllocReg(null, abi.RegisterClass.sse)) |reg| {
182152 try self.asmRegisterMemory(.{ ._ps, .movu }, reg.to128(), try src_ptr.deref().mem(self, .{ .size = .xword }));
182153 try self.asmMemoryRegister(.{ ._ps, .movu }, try dst_ptr.deref().mem(self, .{ .size = .xword }), reg.to128());
182154 return;
182155 }
182156 },
182157 32 => if (self.hasFeature(.avx)) {
182158 if (self.register_manager.tryAllocReg(null, abi.RegisterClass.sse)) |reg| {
182159 try self.asmRegisterMemory(.{ .v_dqu, .mov }, reg.to256(), try src_ptr.deref().mem(self, .{ .size = .yword }));
182160 try self.asmMemoryRegister(.{ .v_dqu, .mov }, try dst_ptr.deref().mem(self, .{ .size = .yword }), reg.to256());
182161 return;
182162 }
182163 },
182164 },
182668 if (opts.no_alias and len == .immediate and cg.useConstMemcpyForSize(len.immediate)) {
182669 if (try cg.genConstMemcpy(dst_ptr, src_ptr, len.immediate)) {
182670 return;
182671 }
182672 }
182673
182674 try cg.spillRegisters(&.{ .rsi, .rdi, .rcx });
182675 try cg.genSetReg(.rsi, .usize, src_ptr, .{});
182676 try cg.genSetReg(.rdi, .usize, dst_ptr, .{});
182677 try cg.genSetReg(.rcx, .usize, len, .{});
182678 try cg.asmOpOnly(.{ .@"rep _sb", .mov });
182679}
182680
182681/// Returns `true` iff it would be efficient to use `genConstMemcpy` for a copy
182682/// of `len` bytes instead of using `rep movsb`.
182683fn useConstMemcpyForSize(cg: *CodeGen, len: u64) bool {
182684 // LLVM's threshold here is 8 load/store pairs. However, after that threshold, it typically
182685 // calls through to `memcpy` instead of emitting `rep movsb`; if we don't have the `fsrm` CPU
182686 // feature, `rep movsb` has a high startup cost, so is inefficient for small copies. Since we
182687 // don't currently call compiler_rt in the case of longer copies, we use a higher threshold if
182688 // we don't have the `fsrm` feature.
182689 return switch (len) {
182690 0...128 => true,
182691 129...256 => cg.hasFeature(.avx) or !cg.hasFeature(.fsrm), // if we have AVX, each pair is 32 bytes
182692 257...512 => !cg.hasFeature(.fsrm), // the threshold should probably be higher, but I'm being cautious
182693 else => false,
182165182694 };
182166 try self.spillRegisters(&.{ .rsi, .rdi, .rcx });
182167 try self.genSetReg(.rsi, .usize, src_ptr, .{});
182168 try self.genSetReg(.rdi, .usize, dst_ptr, .{});
182169 try self.genSetReg(.rcx, .usize, len, .{});
182170 try self.asmOpOnly(.{ .@"rep _sb", .mov });
182695}
182696/// If we can trivially copy `len` bytes from `orig_src_ptr` to `dst_ptr` using simple loads and
182697/// stored through already-free register[s] (i.e. without spilling anything), emits that code and
182698/// returns `true`. Otherwise, returns `false`, and the caller must lower the operation themselves,
182699/// for instance with `rep movsb`.
182700fn genConstMemcpy(
182701 cg: *CodeGen,
182702 dst_ptr: MCValue,
182703 orig_src_ptr: MCValue,
182704 len: u64,
182705) !bool {
182706 if (!dst_ptr.isAddress()) return false;
182707
182708 const src_reg: ?Register = r: {
182709 if (orig_src_ptr.isAddress()) break :r null;
182710 if (orig_src_ptr == .lea_uav or orig_src_ptr == .lea_nav) {
182711 // To "hack around linker relocation bugs", a `lea_uav` isn't considered an address, but
182712 // we want to avoid pessimising that case because it's common (e.g. returning constant
182713 // values over 8 bytes). So if there's a register free, load the source address into it.
182714 if (cg.register_manager.tryAllocReg(null, abi.RegisterClass.gp)) |src_reg| {
182715 // We'll actually do the load a little later, in case another register alloc fails.
182716 break :r src_reg;
182717 }
182718 }
182719 return false;
182720 };
182721
182722 // Prevent `src_reg` from aliasing `gp_reg` below.
182723 const src_lock: ?RegisterLock = if (src_reg) |r| cg.register_manager.lockReg(r) else null;
182724 defer if (src_lock) |l| cg.register_manager.unlockReg(l);
182725
182726 const want_sse_reg = len >= 16 and cg.hasFeature(.sse);
182727 const sse_reg: ?Register = if (want_sse_reg) r: {
182728 break :r cg.register_manager.tryAllocReg(null, abi.RegisterClass.sse);
182729 } else null;
182730
182731 const need_gp_reg = len % 16 != 0 or sse_reg == null;
182732 const gp_reg: Register = if (need_gp_reg) r: {
182733 break :r cg.register_manager.tryAllocReg(null, abi.RegisterClass.gp) orelse return false;
182734 } else undefined;
182735
182736 const src_ptr: MCValue = src_ptr: {
182737 const reg = src_reg orelse break :src_ptr orig_src_ptr;
182738 try cg.asmRegisterMemory(.{ ._, .lea }, reg.to64(), switch (orig_src_ptr) {
182739 .lea_uav => |uav| .{ .base = .{ .uav = uav } },
182740 .lea_nav => |nav| .{ .base = .{ .nav = nav } },
182741 else => unreachable,
182742 });
182743 break :src_ptr .{ .register = reg.to64() };
182744 };
182745
182746 var offset: u64 = 0;
182747
182748 if (sse_reg) |r| {
182749 if (cg.hasFeature(.avx)) {
182750 try cg.memcpyPart(dst_ptr, src_ptr, len, &offset, .{ .v_dqu, .mov }, r, .yword);
182751 try cg.memcpyPart(dst_ptr, src_ptr, len, &offset, .{ .v_dqu, .mov }, r, .xword);
182752 } else if (cg.hasFeature(.sse2)) {
182753 try cg.memcpyPart(dst_ptr, src_ptr, len, &offset, .{ ._dqu, .mov }, r, .xword);
182754 } else if (cg.hasFeature(.sse)) {
182755 try cg.memcpyPart(dst_ptr, src_ptr, len, &offset, .{ ._ps, .movu }, r, .xword);
182756 }
182757 }
182758
182759 if (need_gp_reg) {
182760 try cg.memcpyPart(dst_ptr, src_ptr, len, &offset, .{ ._, .mov }, gp_reg, .qword);
182761 try cg.memcpyPart(dst_ptr, src_ptr, len, &offset, .{ ._, .mov }, gp_reg, .dword);
182762 try cg.memcpyPart(dst_ptr, src_ptr, len, &offset, .{ ._, .mov }, gp_reg, .word);
182763 try cg.memcpyPart(dst_ptr, src_ptr, len, &offset, .{ ._, .mov }, gp_reg, .byte);
182764 }
182765
182766 assert(offset == len);
182767
182768 return true;
182769}
182770fn memcpyPart(
182771 cg: *CodeGen,
182772 dst_ptr: MCValue,
182773 src_ptr: MCValue,
182774 len: u64,
182775 offset: *u64,
182776 tag: Mir.Inst.FixedTag,
182777 temp_reg: Register,
182778 size: Memory.Size,
182779) !void {
182780 const bytes_len = @divExact(size.bitSize(cg.target), 8);
182781 while (len - offset.* >= bytes_len) {
182782 try cg.asmRegisterMemory(
182783 tag,
182784 temp_reg.toSize(size, cg.target),
182785 try src_ptr.deref().mem(cg, .{ .size = size, .disp = @intCast(offset.*) }),
182786 );
182787 try cg.asmMemoryRegister(
182788 tag,
182789 try dst_ptr.deref().mem(cg, .{ .size = size, .disp = @intCast(offset.*) }),
182790 temp_reg.toSize(size, cg.target),
182791 );
182792 offset.* += bytes_len;
182793 }
182171182794}
182172182795
182173182796fn genInlineMemset(
......@@ -186834,10 +187457,8 @@ const Temp = struct {
186834187457 },
186835187458 .memory, .indirect, .load_frame, .load_nav, .load_uav, .load_lazy_sym => {
186836187459 var val_ptr = try cg.tempInit(.usize, val_mcv.address());
186837 var len = try cg.tempInit(.usize, .{ .immediate = val_ty.abiSize(cg.pt.zcu) });
186838 try val_ptr.memcpy(ptr, &len, cg);
187460 try val_ptr.memcpy(ptr, val_ty.abiSize(cg.pt.zcu), cg);
186839187461 try val_ptr.die(cg);
186840 try len.die(cg);
186841187462 },
186842187463 }
186843187464 return val;
......@@ -186939,10 +187560,8 @@ const Temp = struct {
186939187560 .lea_frame, .lea_nav, .lea_uav, .lea_lazy_sym => continue :val_to_gpr,
186940187561 .memory, .indirect, .load_frame, .load_nav, .load_uav, .load_lazy_sym => {
186941187562 var val_ptr = try cg.tempInit(.usize, val_mcv.address());
186942 var len = try cg.tempInit(.usize, .{ .immediate = val_ty.abiSize(cg.pt.zcu) });
186943 try ptr.memcpy(&val_ptr, &len, cg);
187563 try ptr.memcpy(&val_ptr, val_ty.abiSize(cg.pt.zcu), cg);
186944187564 try val_ptr.die(cg);
186945 try len.die(cg);
186946187565 },
186947187566 }
186948187567 break;
......@@ -186981,11 +187600,9 @@ const Temp = struct {
186981187600 var val_ptr = try cg.tempInit(.usize, val_mcv.address());
186982187601 var src_ptr =
186983187602 try cg.tempInit(.usize, src.tracking(cg).short.address().offset(opts.disp));
186984 var len = try cg.tempInit(.usize, .{ .immediate = val_ty.abiSize(cg.pt.zcu) });
186985 try val_ptr.memcpy(&src_ptr, &len, cg);
187603 try val_ptr.memcpy(&src_ptr, val_ty.abiSize(cg.pt.zcu), cg);
186986187604 try val_ptr.die(cg);
186987187605 try src_ptr.die(cg);
186988 try len.die(cg);
186989187606 },
186990187607 }
186991187608 }
......@@ -187076,11 +187693,9 @@ const Temp = struct {
187076187693 var dst_ptr =
187077187694 try cg.tempInit(.usize, dst.tracking(cg).short.address().offset(opts.disp));
187078187695 var val_ptr = try cg.tempInit(.usize, val_mcv.address());
187079 var len = try cg.tempInit(.usize, .{ .immediate = val_ty.abiSize(cg.pt.zcu) });
187080 try dst_ptr.memcpy(&val_ptr, &len, cg);
187696 try dst_ptr.memcpy(&val_ptr, val_ty.abiSize(cg.pt.zcu), cg);
187081187697 try dst_ptr.die(cg);
187082187698 try val_ptr.die(cg);
187083 try len.die(cg);
187084187699 },
187085187700 }
187086187701 break;
......@@ -187182,10 +187797,8 @@ const Temp = struct {
187182187797 try ptr.toOffset(deferred_disp, cg);
187183187798 deferred_disp = 0;
187184187799 var src_ptr = try cg.tempInit(.usize, .{ .lea_frame = .{ .index = frame_index } });
187185 var len = try cg.tempInit(.usize, .{ .immediate = src_abi_size });
187186 try ptr.memcpy(&src_ptr, &len, cg);
187800 try ptr.memcpy(&src_ptr, src_abi_size, cg);
187187187801 try src_ptr.die(cg);
187188 try len.die(cg);
187189187802 }
187190187803 part_disp += part_size;
187191187804 deferred_disp += part_size;
......@@ -187224,11 +187837,9 @@ const Temp = struct {
187224187837 var dst_ptr = try cg.tempInit(.usize, dst.tracking(cg).short.address());
187225187838 try dst_ptr.toOffset(disp, cg);
187226187839 var src_ptr = try cg.tempInit(.usize, .{ .lea_frame = .{ .index = frame_index } });
187227 var len = try cg.tempInit(.usize, .{ .immediate = src_abi_size });
187228 try dst_ptr.memcpy(&src_ptr, &len, cg);
187840 try dst_ptr.memcpy(&src_ptr, src_abi_size, cg);
187229187841 try dst_ptr.die(cg);
187230187842 try src_ptr.die(cg);
187231 try len.die(cg);
187232187843 }
187233187844 }
187234187845
......@@ -187273,11 +187884,21 @@ const Temp = struct {
187273187884 assert(next_class_index == classes.len);
187274187885 }
187275187886
187276 fn memcpy(dst: *Temp, src: *Temp, len: *Temp, cg: *CodeGen) InnerError!void {
187277 while (true) for ([_]*Temp{ dst, src, len }, [_]Register{ .rdi, .rsi, .rcx }) |temp, reg| {
187278 if (try temp.toReg(reg, cg)) break;
187279 } else break;
187887 fn memcpy(dst: *Temp, src: *Temp, len: u64, cg: *CodeGen) InnerError!void {
187888 if (cg.useConstMemcpyForSize(len)) {
187889 while (try dst.toLea(cg) or try src.toLea(cg)) {} // `genConstMemcpy` wants these values to be address operands
187890 if (try cg.genConstMemcpy(dst.tracking(cg).short, src.tracking(cg).short, len)) {
187891 return;
187892 }
187893 }
187894 var temp = try cg.tempAllocReg(.usize, abi.RegisterClass.gp);
187895 while (try dst.toReg(.rdi, cg) or
187896 try src.toReg(.rsi, cg) or
187897 try temp.toReg(.rcx, cg))
187898 {}
187899 try cg.asmRegisterImmediate(.{ ._, .mov }, .rcx, .{ .unsigned = len });
187280187900 try cg.asmOpOnly(.{ .@"rep _sb", .mov });
187901 try temp.die(cg);
187281187902 }
187282187903
187283187904 fn memset(dst: *Temp, val: *Temp, len: *Temp, cg: *CodeGen) InnerError!void {
......@@ -190139,9 +190760,9 @@ const Temp = struct {
190139190760 .required_features = .{ .sse, .mmx, null, null },
190140190761 .src_constraints = .{ .{ .int = .qword }, .{ .int = .qword }, .any },
190141190762 .patterns = &.{
190142 .{ .src = .{ .to_mut_mm, .mem, .none } },
190143 .{ .src = .{ .mem, .to_mut_mm, .none }, .commute = .{ 0, 1 } },
190144 .{ .src = .{ .to_mut_mm, .to_mm, .none } },
190763 .{ .src = .{ .to_mut_mmx, .mem, .none } },
190764 .{ .src = .{ .mem, .to_mut_mmx, .none }, .commute = .{ 0, 1 } },
190765 .{ .src = .{ .to_mut_mmx, .to_mmx, .none } },
190145190766 },
190146190767 .extra_temps = .{
190147190768 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
......@@ -190169,9 +190790,9 @@ const Temp = struct {
190169190790 .required_features = .{ .avx, null, null, null },
190170190791 .src_constraints = .{ .{ .int = .xword }, .{ .int = .xword }, .any },
190171190792 .patterns = &.{
190172 .{ .src = .{ .to_xmm, .mem, .none } },
190173 .{ .src = .{ .mem, .to_xmm, .none }, .commute = .{ 0, 1 } },
190174 .{ .src = .{ .to_xmm, .to_xmm, .none } },
190793 .{ .src = .{ .to_sse, .mem, .none } },
190794 .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } },
190795 .{ .src = .{ .to_sse, .to_sse, .none } },
190175190796 },
190176190797 .extra_temps = .{
190177190798 .{ .kind = .{ .rc = .sse } },
......@@ -190196,9 +190817,9 @@ const Temp = struct {
190196190817 .required_features = .{ .sse4_1, null, null, null },
190197190818 .src_constraints = .{ .{ .int = .xword }, .{ .int = .xword }, .any },
190198190819 .patterns = &.{
190199 .{ .src = .{ .to_mut_xmm, .mem, .none } },
190200 .{ .src = .{ .mem, .to_mut_xmm, .none }, .commute = .{ 0, 1 } },
190201 .{ .src = .{ .to_mut_xmm, .to_xmm, .none } },
190820 .{ .src = .{ .to_mut_sse, .mem, .none } },
190821 .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } },
190822 .{ .src = .{ .to_mut_sse, .to_sse, .none } },
190202190823 },
190203190824 .dst_temps = .{ .{ .cc = .z }, .unused },
190204190825 .clobbers = .{ .eflags = true },
......@@ -190210,9 +190831,9 @@ const Temp = struct {
190210190831 .required_features = .{ .sse2, .fast_imm16, null, null },
190211190832 .src_constraints = .{ .{ .int = .xword }, .{ .int = .xword }, .any },
190212190833 .patterns = &.{
190213 .{ .src = .{ .to_mut_xmm, .mem, .none } },
190214 .{ .src = .{ .mem, .to_mut_xmm, .none }, .commute = .{ 0, 1 } },
190215 .{ .src = .{ .to_mut_xmm, .to_xmm, .none } },
190834 .{ .src = .{ .to_mut_sse, .mem, .none } },
190835 .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } },
190836 .{ .src = .{ .to_mut_sse, .to_sse, .none } },
190216190837 },
190217190838 .extra_temps = .{
190218190839 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
......@@ -190240,9 +190861,9 @@ const Temp = struct {
190240190861 .required_features = .{ .sse2, null, null, null },
190241190862 .src_constraints = .{ .{ .int = .xword }, .{ .int = .xword }, .any },
190242190863 .patterns = &.{
190243 .{ .src = .{ .to_mut_xmm, .mem, .none } },
190244 .{ .src = .{ .mem, .to_mut_xmm, .none }, .commute = .{ 0, 1 } },
190245 .{ .src = .{ .to_mut_xmm, .to_xmm, .none } },
190864 .{ .src = .{ .to_mut_sse, .mem, .none } },
190865 .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } },
190866 .{ .src = .{ .to_mut_sse, .to_sse, .none } },
190246190867 },
190247190868 .extra_temps = .{
190248190869 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
......@@ -190270,9 +190891,9 @@ const Temp = struct {
190270190891 .required_features = .{ .avx2, null, null, null },
190271190892 .src_constraints = .{ .{ .int = .yword }, .{ .int = .yword }, .any },
190272190893 .patterns = &.{
190273 .{ .src = .{ .to_ymm, .mem, .none } },
190274 .{ .src = .{ .mem, .to_ymm, .none }, .commute = .{ 0, 1 } },
190275 .{ .src = .{ .to_ymm, .to_ymm, .none } },
190894 .{ .src = .{ .to_sse, .mem, .none } },
190895 .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } },
190896 .{ .src = .{ .to_sse, .to_sse, .none } },
190276190897 },
190277190898 .extra_temps = .{
190278190899 .{ .kind = .{ .rc = .sse } },
......@@ -190297,9 +190918,9 @@ const Temp = struct {
190297190918 .required_features = .{ .avx, null, null, null },
190298190919 .src_constraints = .{ .{ .int = .yword }, .{ .int = .yword }, .any },
190299190920 .patterns = &.{
190300 .{ .src = .{ .to_ymm, .mem, .none } },
190301 .{ .src = .{ .mem, .to_ymm, .none }, .commute = .{ 0, 1 } },
190302 .{ .src = .{ .to_ymm, .to_ymm, .none } },
190921 .{ .src = .{ .to_sse, .mem, .none } },
190922 .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } },
190923 .{ .src = .{ .to_sse, .to_sse, .none } },
190303190924 },
190304190925 .extra_temps = .{
190305190926 .{ .kind = .{ .rc = .sse } },
......@@ -192279,6 +192900,7 @@ const Select = struct {
192279192900 any_scalar_int,
192280192901 any_scalar_signed_int,
192281192902 any_scalar_unsigned_int,
192903 any_signed_int_or_full_vec,
192282192904 any_float,
192283192905 po2_any,
192284192906 bool,
......@@ -192375,6 +192997,11 @@ const Select = struct {
192375192997 .any_scalar_int => cg.intInfo(ty.scalarType(zcu)) != null,
192376192998 .any_scalar_signed_int => if (cg.intInfo(ty.scalarType(zcu))) |int_info| int_info.signedness == .signed else false,
192377192999 .any_scalar_unsigned_int => if (cg.intInfo(ty.scalarType(zcu))) |int_info| int_info.signedness == .unsigned else false,
193000 .any_signed_int_or_full_vec => ty.isVector(zcu) and
193001 if (cg.intInfo(ty.childType(zcu))) |int_info| switch (int_info.signedness) {
193002 .signed => true,
193003 .unsigned => (int_info.bits >= 8 and std.math.isPowerOfTwo(int_info.bits)) or int_info.bits % 128 == 0,
193004 } else false,
192378193005 .any_float => ty.isRuntimeFloat(),
192379193006 .po2_any => std.math.isPowerOfTwo(ty.abiSize(zcu)),
192380193007 .bool => ty.toIntern() == .bool_type,
......@@ -192415,7 +193042,7 @@ const Select = struct {
192415193042 .signed_int_or_full_vec => |size| ty.isVector(zcu) and @divExact(size.bitSize(cg.target), 8) >= ty.abiSize(zcu) and
192416193043 if (cg.intInfo(ty.childType(zcu))) |int_info| switch (int_info.signedness) {
192417193044 .signed => true,
192418 .unsigned => int_info.bits >= 8 and std.math.isPowerOfTwo(int_info.bits),
193045 .unsigned => (int_info.bits >= 8 and std.math.isPowerOfTwo(int_info.bits)) or int_info.bits % 128 == 0,
192419193046 } else false,
192420193047 .unsigned_int_vec => |size| ty.isVector(zcu) and @divExact(size.bitSize(cg.target), 8) >= ty.abiSize(zcu) and
192421193048 if (cg.intInfo(ty.childType(zcu))) |int_info| int_info.signedness == .unsigned else false,
......@@ -192664,22 +193291,10 @@ const Select = struct {
192664193291 to_mmx,
192665193292 mut_mmx,
192666193293 to_mut_mmx,
192667 mm,
192668 to_mm,
192669 mut_mm,
192670 to_mut_mm,
192671193294 sse,
192672193295 to_sse,
192673193296 mut_sse,
192674193297 to_mut_sse,
192675 xmm,
192676 to_xmm,
192677 mut_xmm,
192678 to_mut_xmm,
192679 ymm,
192680 to_ymm,
192681 mut_ymm,
192682 to_mut_ymm,
192683193298 reg_mask: RegMaskSpec,
192684193299 all_reg_mask: RegMaskSpec,
192685193300
......@@ -192769,17 +193384,6 @@ const Select = struct {
192769193384 else => false,
192770193385 },
192771193386 .to_mmx, .to_mut_mmx => true,
192772 .mm => temp.typeOf(cg).abiSize(cg.pt.zcu) == 8 and switch (temp.tracking(cg).short) {
192773 .register => |reg| reg.isClass(.mmx),
192774 .register_offset => |reg_off| reg_off.reg.isClass(.mmx) and reg_off.off == 0,
192775 else => false,
192776 },
192777 .mut_mm => temp.isMut(cg) and temp.typeOf(cg).abiSize(cg.pt.zcu) == 8 and switch (temp.tracking(cg).short) {
192778 .register => |reg| reg.isClass(.mmx),
192779 .register_offset => |reg_off| reg_off.reg.isClass(.mmx) and reg_off.off == 0,
192780 else => false,
192781 },
192782 .to_mm, .to_mut_mm => temp.typeOf(cg).abiSize(cg.pt.zcu) == 8,
192783193387 .sse => switch (temp.tracking(cg).short) {
192784193388 .register => |reg| reg.isClass(.sse),
192785193389 .register_offset => |reg_off| reg_off.reg.isClass(.sse) and reg_off.off == 0,
......@@ -192791,28 +193395,6 @@ const Select = struct {
192791193395 else => false,
192792193396 },
192793193397 .to_sse, .to_mut_sse => true,
192794 .xmm => temp.typeOf(cg).abiSize(cg.pt.zcu) == 16 and switch (temp.tracking(cg).short) {
192795 .register => |reg| reg.isClass(.sse),
192796 .register_offset => |reg_off| reg_off.reg.isClass(.sse) and reg_off.off == 0,
192797 else => false,
192798 },
192799 .mut_xmm => temp.isMut(cg) and temp.typeOf(cg).abiSize(cg.pt.zcu) == 16 and switch (temp.tracking(cg).short) {
192800 .register => |reg| reg.isClass(.sse),
192801 .register_offset => |reg_off| reg_off.reg.isClass(.sse) and reg_off.off == 0,
192802 else => false,
192803 },
192804 .to_xmm, .to_mut_xmm => temp.typeOf(cg).abiSize(cg.pt.zcu) == 16,
192805 .ymm => temp.typeOf(cg).abiSize(cg.pt.zcu) == 32 and switch (temp.tracking(cg).short) {
192806 .register => |reg| reg.isClass(.sse),
192807 .register_offset => |reg_off| reg_off.reg.isClass(.sse) and reg_off.off == 0,
192808 else => false,
192809 },
192810 .mut_ymm => temp.isMut(cg) and temp.typeOf(cg).abiSize(cg.pt.zcu) == 32 and switch (temp.tracking(cg).short) {
192811 .register => |reg| reg.isClass(.sse),
192812 .register_offset => |reg_off| reg_off.reg.isClass(.sse) and reg_off.off == 0,
192813 else => false,
192814 },
192815 .to_ymm, .to_mut_ymm => temp.typeOf(cg).abiSize(cg.pt.zcu) == 32,
192816193398 .reg_mask => |mask_spec| switch (temp.tracking(cg).short) {
192817193399 .register_mask => |reg_mask| mask_spec.size.bitSize(cg.target) >=
192818193400 reg_mask.info.scalar.bitSize(cg.target) * temp.typeOf(cg).vectorLen(cg.pt.zcu) and
......@@ -192846,10 +193428,10 @@ const Select = struct {
192846193428 .to_mut_gphi => try temp.toRegClass(true, .gphi, cg),
192847193429 .x87, .to_x87 => try temp.toRegClass(false, .x87, cg),
192848193430 .mut_x87, .to_mut_x87 => try temp.toRegClass(true, .x87, cg),
192849 .mmx, .to_mmx, .mm, .to_mm => try temp.toRegClass(false, .mmx, cg),
192850 .mut_mmx, .to_mut_mmx, .mut_mm, .to_mut_mm => try temp.toRegClass(true, .mmx, cg),
192851 .sse, .to_sse, .xmm, .to_xmm, .ymm, .to_ymm => try temp.toRegClass(false, .sse, cg),
192852 .mut_sse, .to_mut_sse, .mut_xmm, .to_mut_xmm, .mut_ymm, .to_mut_ymm => try temp.toRegClass(true, .sse, cg),
193431 .mmx, .to_mmx => try temp.toRegClass(false, .mmx, cg),
193432 .mut_mmx, .to_mut_mmx => try temp.toRegClass(true, .mmx, cg),
193433 .sse, .to_sse => try temp.toRegClass(false, .sse, cg),
193434 .mut_sse, .to_mut_sse => try temp.toRegClass(true, .sse, cg),
192853193435 };
192854193436 }
192855193437 };
......@@ -193399,7 +193981,7 @@ const Select = struct {
193399193981 ref: Ref,
193400193982 scale: Memory.Scale = .@"1",
193401193983 } = .{ .ref = .none },
193402 unused: u2 = 0,
193984 unused: u1 = 0,
193403193985 },
193404193986 imm: i32 = 0,
193405193987
......@@ -193412,7 +193994,7 @@ const Select = struct {
193412193994 lea,
193413193995 mem,
193414193996 };
193415 const Adjust = packed struct(u11) {
193997 const Adjust = packed struct(u12) {
193416193998 sign: enum(u1) { neg, pos },
193417193999 lhs: enum(u6) {
193418194000 none,
......@@ -193449,7 +194031,7 @@ const Select = struct {
193449194031 umax_shr_src1,
193450194032 repeat,
193451194033 },
193452 op: enum(u2) { mul, div, div_8_down, rem_8_mul },
194034 op: enum(u3) { mul, div, div_8_down, rem_8_mul, up_2_mul, up_32_mul },
193453194035 rhs: Memory.Scale,
193454194036
193455194037 const none: Adjust = .{ .sign = .pos, .lhs = .none, .op = .mul, .rhs = .@"1" };
......@@ -193470,6 +194052,11 @@ const Select = struct {
193470194052 const add_delta_elem_size: Adjust = .{ .sign = .pos, .lhs = .delta_elem_size, .op = .mul, .rhs = .@"1" };
193471194053 const add_delta_elem_size_div_8: Adjust = .{ .sign = .pos, .lhs = .delta_elem_size, .op = .div, .rhs = .@"8" };
193472194054 const add_unaligned_size: Adjust = .{ .sign = .pos, .lhs = .unaligned_size, .op = .mul, .rhs = .@"1" };
194055 const add_unaligned_size_up_2: Adjust = .{ .sign = .pos, .lhs = .unaligned_size, .op = .up_2_mul, .rhs = .@"1" };
194056 const add_unaligned_size_up_4: Adjust = .{ .sign = .pos, .lhs = .unaligned_size, .op = .up_2_mul, .rhs = .@"2" };
194057 const add_unaligned_size_up_8: Adjust = .{ .sign = .pos, .lhs = .unaligned_size, .op = .up_2_mul, .rhs = .@"4" };
194058 const add_unaligned_size_up_16: Adjust = .{ .sign = .pos, .lhs = .unaligned_size, .op = .up_2_mul, .rhs = .@"8" };
194059 const add_unaligned_size_up_32: Adjust = .{ .sign = .pos, .lhs = .unaligned_size, .op = .up_32_mul, .rhs = .@"1" };
193473194060 const sub_unaligned_size: Adjust = .{ .sign = .neg, .lhs = .unaligned_size, .op = .mul, .rhs = .@"1" };
193474194061 const add_unaligned_size_add_elem_size: Adjust = .{ .sign = .pos, .lhs = .unaligned_size_add_elem_size, .op = .mul, .rhs = .@"1" };
193475194062 const add_unaligned_size_sub_elem_size: Adjust = .{ .sign = .pos, .lhs = .unaligned_size_sub_elem_size, .op = .mul, .rhs = .@"1" };
......@@ -194482,7 +195069,9 @@ const Select = struct {
194482195069 },
194483195070 .div => @shrExact(lhs, rhs),
194484195071 .div_8_down => lhs >> 3 & @as(SignedImm, -1) << rhs,
194485 .rem_8_mul => lhs & (@as(SignedImm, 1) << @intCast(@as(u3, 3) + rhs)) - 1,
195072 .rem_8_mul => lhs & ((@as(SignedImm, 8) << rhs) - 1),
195073 .up_2_mul => ((lhs - 1) & @as(SignedImm, -2) << rhs) + (@as(SignedImm, 2) << rhs),
195074 .up_32_mul => ((lhs - 1) & @as(SignedImm, -32) << rhs) + (@as(SignedImm, 32) << rhs),
194486195075 };
194487195076 const disp: SignedImm = @bitCast(@as(UnsignedImm, @as(u32, @bitCast(op.imm))));
194488195077 return switch (op.flags.adjust.sign) {
src/target.zig+1-5
......@@ -375,11 +375,7 @@ pub fn canBuildLibCompilerRt(target: *const std.Target) enum { no, yes, llvm_onl
375375 else => {},
376376 }
377377 return switch (zigBackend(target, false)) {
378 .stage2_aarch64 => .yes,
379 .stage2_x86_64 => switch (target.ofmt) {
380 .elf, .macho => .yes,
381 else => .llvm_only,
382 },
378 .stage2_aarch64, .stage2_x86_64 => .yes,
383379 else => .llvm_only,
384380 };
385381}
test/behavior/x86_64/build.zig+2
......@@ -5,6 +5,7 @@ pub fn build(b: *std.Build) void {
55 "test-filter",
66 "Skip tests that do not match any filter",
77 ) orelse &[0][]const u8{};
8 const strip = b.option(bool, "strip", "Omit debug information");
89
910 const compiler_rt_lib = b.addLibrary(.{
1011 .linkage = .static,
......@@ -171,6 +172,7 @@ pub fn build(b: *std.Build) void {
171172 const test_mod = b.createModule(.{
172173 .root_source_file = b.path(path),
173174 .target = target,
175 .strip = strip,
174176 });
175177 const test_exe = b.addTest(.{
176178 .name = std.fs.path.stem(path),
test/tests.zig+1
......@@ -1176,6 +1176,7 @@ const test_targets = blk: {
11761176 .abi = .musl,
11771177 },
11781178 .link_libc = true,
1179 .use_llvm = true,
11791180 .use_lld = false,
11801181 },
11811182