authorgravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2026-06-20 05:41:03-04:00
committergravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2026-06-20 21:08:23-04:00
log8ecc942ce890768f556e870bea1d66fe7bad8936
tree06be1303df607c8c377b1ef26ba250b19e1cbf14
parent2e7bd58abaafae8a029bc1a372085ace6738a365

x86_64: implement win64 abi for bool vectors

5 files changed, 1932 insertions(+), 283 deletions(-)

src/codegen/llvm/FuncGen.zig+4
......@@ -6790,6 +6790,7 @@ const ParamTypeIterator = struct {
67906790 }
67916791 },
67926792 .sse,
6793 .bool_vector_mask,
67936794 .integer_per_element,
67946795 .sse_per_element,
67956796 .sse_sse_x87_per_qword,
......@@ -6874,6 +6875,7 @@ const ParamTypeIterator = struct {
68746875 .none => break,
68756876 .memory => unreachable, // handled above
68766877 .win_i128 => unreachable, // windows only
6878 .bool_vector_mask,
68776879 .integer_per_element,
68786880 .sse_per_element,
68796881 .sse_sse_x87_per_qword,
......@@ -7091,6 +7093,7 @@ fn lowerWin64FnRetTy(o: *Object, fn_info: InternPool.Key.FuncType) Allocator.Err
70917093 }
70927094 },
70937095 .sse,
7096 .bool_vector_mask,
70947097 .integer_per_element,
70957098 .sse_per_element,
70967099 .sse_sse_x87_per_qword,
......@@ -7156,6 +7159,7 @@ fn lowerSystemVFnRetTy(o: *Object, fn_info: InternPool.Key.FuncType) Allocator.E
71567159 .none => break,
71577160 .memory => return .void,
71587161 .win_i128 => unreachable, // windows only
7162 .bool_vector_mask,
71597163 .integer_per_element,
71607164 .sse_per_element,
71617165 .sse_sse_x87_per_qword,
src/codegen/x86_64/CodeGen.zig+1911-277
......@@ -164,7 +164,7 @@ next_temp_index: Temp.Index = @enumFromInt(0),
164164temp_type: [Temp.Index.max]Type = undefined,
165165
166166const MaskInfo = packed struct {
167 kind: enum(u1) { sign, all },
167 kind: enum(u2) { lsb, msb, zero_extend, sign_extend },
168168 inverted: bool = false,
169169 scalar: Memory.Size,
170170};
......@@ -204,13 +204,15 @@ pub const MCValue = union(enum) {
204204 /// The value is a tuple { wrapped, overflow } where wrapped value is stored in the GP register.
205205 register_overflow: struct { reg: Register, eflags: Condition },
206206 /// The value is a bool vector stored in a vector register with a different scalar type.
207 register_mask: struct { reg: Register, info: MaskInfo },
207 register_mask: Mask,
208208 /// The value is in memory at a hard-coded address.
209209 /// If the type is a pointer, it means the pointer address is stored at this memory location.
210210 memory: u64,
211211 /// The value is in memory at a constant offset from the address in a register.
212212 indirect: bits.RegisterOffset,
213213 indirect_load_frame: bits.FrameAddr,
214 /// The value is a bool vector stored in memory with a different scalar type at the address in a register.
215 indirect_mask: Mask,
214216 /// The value stored at an offset from a frame index.
215217 /// Payload is a frame address.
216218 load_frame: bits.FrameAddr,
......@@ -243,6 +245,8 @@ pub const MCValue = union(enum) {
243245 reserved_frame: FrameIndex,
244246 air_ref: Air.Inst.Ref,
245247
248 const Mask = struct { reg: Register, info: MaskInfo };
249
246250 fn isModifiable(mcv: MCValue) bool {
247251 return switch (mcv) {
248252 .none,
......@@ -278,6 +282,7 @@ pub const MCValue = union(enum) {
278282 .register_quadruple,
279283 .memory,
280284 .indirect,
285 .indirect_mask,
281286 .load_nav,
282287 => true,
283288 .load_frame => |frame_addr| !frame_addr.index.isNamed(),
......@@ -346,9 +351,10 @@ pub const MCValue = union(enum) {
346351 .register_quadruple,
347352 => |*regs| regs,
348353 inline .register_offset,
349 .indirect,
350354 .register_overflow,
351355 .register_mask,
356 .indirect,
357 .indirect_mask,
352358 => |*pl| (&pl.reg)[0..1],
353359 else => &.{},
354360 };
......@@ -384,6 +390,7 @@ pub const MCValue = union(enum) {
384390 .register_offset,
385391 .register_overflow,
386392 .register_mask,
393 .indirect_mask,
387394 .lea_frame,
388395 .lea_nav,
389396 .lea_uav,
......@@ -427,6 +434,7 @@ pub const MCValue = union(enum) {
427434 .memory,
428435 .indirect,
429436 .indirect_load_frame,
437 .indirect_mask,
430438 .load_nav,
431439 .load_uav,
432440 .load_lazy_sym,
......@@ -476,6 +484,7 @@ pub const MCValue = union(enum) {
476484 .memory,
477485 .indirect,
478486 .indirect_load_frame,
487 .indirect_mask,
479488 .load_frame,
480489 .load_nav,
481490 .lea_nav,
......@@ -547,6 +556,10 @@ pub const MCValue = union(enum) {
547556 .disp = reg_off.off + mod_rm.disp,
548557 } },
549558 },
559 .indirect_mask => |reg_mask| .{
560 .base = .{ .reg = reg_mask.reg.toSize(.ptr, function.target) },
561 .mod = .{ .rm = mod_rm },
562 },
550563 .load_frame => |frame_addr| .{
551564 .base = .{ .frame = frame_addr.index },
552565 .mod = .{ .rm = .{
......@@ -568,7 +581,6 @@ pub const MCValue = union(enum) {
568581 switch (mcv) {
569582 .none, .unreach, .dead, .undef => try w.print("({s})", .{@tagName(mcv)}),
570583 .immediate => |pl| try w.print("0x{x}", .{pl}),
571 .memory => |pl| try w.print("[ds:0x{x}]", .{pl}),
572584 inline .eflags, .register => |pl| try w.print("{s}", .{@tagName(pl)}),
573585 .register_pair => |pl| try w.print("{s}:{s}", .{ @tagName(pl[1]), @tagName(pl[0]) }),
574586 .register_triple => |pl| try w.print("{s}:{s}:{s}", .{
......@@ -582,14 +594,21 @@ pub const MCValue = union(enum) {
582594 @tagName(pl.eflags),
583595 @tagName(pl.reg),
584596 }),
585 .register_mask => |pl| try w.print("mask({s},{f}):{c}{s}", .{
597 .register_mask => |pl| try w.print("mask({s},{f}):{s}{s}", .{
586598 @tagName(pl.info.kind),
587599 pl.info.scalar,
588 @as(u8, if (pl.info.inverted) '!' else ' '),
600 if (pl.info.inverted) "!" else "",
589601 @tagName(pl.reg),
590602 }),
603 .memory => |pl| try w.print("[ds:0x{x}]", .{pl}),
591604 .indirect => |pl| try w.print("[{s} + 0x{x}]", .{ @tagName(pl.reg), pl.off }),
592605 .indirect_load_frame => |pl| try w.print("[[{f} + 0x{x}]]", .{ pl.index, pl.off }),
606 .indirect_mask => |pl| try w.print("[mask({s},{f}):{s}{s}]", .{
607 @tagName(pl.info.kind),
608 pl.info.scalar,
609 if (pl.info.inverted) "!" else "",
610 @tagName(pl.reg),
611 }),
593612 .load_frame => |pl| try w.print("[{f} + 0x{x}]", .{ pl.index, pl.off }),
594613 .lea_frame => |pl| try w.print("{f} + 0x{x}", .{ pl.index, pl.off }),
595614 .load_nav => |pl| try w.print("[nav:{d}]", .{@intFromEnum(pl)}),
......@@ -666,6 +685,7 @@ const InstTracking = struct {
666685 .register_overflow,
667686 .register_mask,
668687 .indirect,
688 .indirect_mask,
669689 => .none,
670690 }, .short = result };
671691 }
......@@ -765,6 +785,7 @@ const InstTracking = struct {
765785 .register_overflow,
766786 .register_mask,
767787 .indirect,
788 .indirect_mask,
768789 .register_tee,
769790 .elementwise_gpr,
770791 .elementwise_sse,
......@@ -72517,7 +72538,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7251772538 .{ .src = .{ .to_sse, .none, .none } },
7251872539 },
7251972540 .extra_temps = .{
72520 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },
72541 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
7252172542 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
7252272543 .unused,
7252372544 .unused,
......@@ -72541,7 +72562,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7254172562 .{ .src = .{ .to_mut_sse, .none, .none } },
7254272563 },
7254372564 .extra_temps = .{
72544 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },
72565 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
7254572566 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
7254672567 .unused,
7254772568 .unused,
......@@ -72565,7 +72586,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7256572586 .{ .src = .{ .to_sse, .none, .none } },
7256672587 },
7256772588 .extra_temps = .{
72568 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .reverse } },
72589 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .reverse } } },
7256972590 .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },
7257072591 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
7257172592 .unused,
......@@ -72593,7 +72614,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7259372614 .{ .src = .{ .to_sse, .none, .none } },
7259472615 },
7259572616 .extra_temps = .{
72596 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .reverse } },
72617 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .reverse } } },
7259772618 .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },
7259872619 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
7259972620 .unused,
......@@ -72622,7 +72643,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7262272643 .{ .src = .{ .to_mut_sse, .none, .none } },
7262372644 },
7262472645 .extra_temps = .{
72625 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .reverse } },
72646 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .reverse } } },
7262672647 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
7262772648 .unused,
7262872649 .unused,
......@@ -72651,7 +72672,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7265172672 .{ .src = .{ .to_mut_sse, .none, .none } },
7265272673 },
7265372674 .extra_temps = .{
72654 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .reverse } },
72675 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .reverse } } },
7265572676 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
7265672677 .unused,
7265772678 .unused,
......@@ -72718,7 +72739,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7271872739 .{ .src = .{ .to_sse, .none, .none } },
7271972740 },
7272072741 .extra_temps = .{
72721 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },
72742 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
7272272743 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
7272372744 .unused,
7272472745 .unused,
......@@ -72744,7 +72765,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7274472765 .{ .src = .{ .to_mut_sse, .none, .none } },
7274572766 },
7274672767 .extra_temps = .{
72747 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },
72768 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
7274872769 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
7274972770 .unused,
7275072771 .unused,
......@@ -72770,7 +72791,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7277072791 .{ .src = .{ .to_sse, .none, .none } },
7277172792 },
7277272793 .extra_temps = .{
72773 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .reverse } },
72794 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .reverse } } },
7277472795 .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },
7277572796 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
7277672797 .unused,
......@@ -72800,7 +72821,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7280072821 .{ .src = .{ .to_sse, .none, .none } },
7280172822 },
7280272823 .extra_temps = .{
72803 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .reverse } },
72824 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .reverse } } },
7280472825 .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },
7280572826 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
7280672827 .unused,
......@@ -72831,7 +72852,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7283172852 .{ .src = .{ .to_mut_sse, .none, .none } },
7283272853 },
7283372854 .extra_temps = .{
72834 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .reverse } },
72855 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .reverse } } },
7283572856 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
7283672857 .unused,
7283772858 .unused,
......@@ -72862,7 +72883,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7286272883 .{ .src = .{ .to_mut_sse, .none, .none } },
7286372884 },
7286472885 .extra_temps = .{
72865 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .reverse } },
72886 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .reverse } } },
7286672887 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
7286772888 .unused,
7286872889 .unused,
......@@ -72932,7 +72953,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7293272953 .{ .src = .{ .to_sse, .none, .none } },
7293372954 },
7293472955 .extra_temps = .{
72935 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },
72956 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
7293672957 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
7293772958 .unused,
7293872959 .unused,
......@@ -72958,7 +72979,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7295872979 .{ .src = .{ .to_mut_sse, .none, .none } },
7295972980 },
7296072981 .extra_temps = .{
72961 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },
72982 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
7296272983 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
7296372984 .unused,
7296472985 .unused,
......@@ -72984,7 +73005,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7298473005 .{ .src = .{ .to_sse, .none, .none } },
7298573006 },
7298673007 .extra_temps = .{
72987 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .reverse } },
73008 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .reverse } } },
7298873009 .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },
7298973010 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
7299073011 .unused,
......@@ -73014,7 +73035,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7301473035 .{ .src = .{ .to_sse, .none, .none } },
7301573036 },
7301673037 .extra_temps = .{
73017 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .reverse } },
73038 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .reverse } } },
7301873039 .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },
7301973040 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
7302073041 .unused,
......@@ -73045,7 +73066,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7304573066 .{ .src = .{ .to_mut_sse, .none, .none } },
7304673067 },
7304773068 .extra_temps = .{
73048 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .reverse } },
73069 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .reverse } } },
7304973070 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
7305073071 .unused,
7305173072 .unused,
......@@ -73076,7 +73097,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7307673097 .{ .src = .{ .to_mut_sse, .none, .none } },
7307773098 },
7307873099 .extra_temps = .{
73079 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .reverse } },
73100 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .reverse } } },
7308073101 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
7308173102 .unused,
7308273103 .unused,
......@@ -73146,8 +73167,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7314673167 .{ .src = .{ .to_sse, .none, .none } },
7314773168 },
7314873169 .extra_temps = .{
73149 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },
73150 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .word } } },
73170 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
73171 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bytes_mem = .{ .direction = .reverse, .size = .word } } },
7315173172 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
7315273173 .unused,
7315373174 .unused,
......@@ -73172,8 +73193,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7317273193 .{ .src = .{ .to_mut_sse, .none, .none } },
7317373194 },
7317473195 .extra_temps = .{
73175 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },
73176 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .word } } },
73196 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
73197 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bytes_mem = .{ .direction = .reverse, .size = .word } } },
7317773198 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
7317873199 .unused,
7317973200 .unused,
......@@ -73198,7 +73219,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7319873219 .{ .src = .{ .to_mut_sse, .none, .none } },
7319973220 },
7320073221 .extra_temps = .{
73201 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },
73222 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
7320273223 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
7320373224 .unused,
7320473225 .unused,
......@@ -73224,8 +73245,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7322473245 .{ .src = .{ .to_sse, .none, .none } },
7322573246 },
7322673247 .extra_temps = .{
73227 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .word, .smear = 8 } } },
73228 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .reverse } },
73248 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bytes_mem = .{ .direction = .reverse, .size = .word, .smear = 8 } } },
73249 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .reverse } } },
7322973250 .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },
7323073251 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
7323173252 .unused,
......@@ -73253,8 +73274,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7325373274 .{ .src = .{ .to_mut_sse, .none, .none } },
7325473275 },
7325573276 .extra_temps = .{
73256 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .word, .smear = 8 } } },
73257 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .reverse } },
73277 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bytes_mem = .{ .direction = .reverse, .size = .word, .smear = 8 } } },
73278 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .reverse } } },
7325873279 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
7325973280 .unused,
7326073281 .unused,
......@@ -73323,8 +73344,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7332373344 .{ .src = .{ .to_sse, .none, .none } },
7332473345 },
7332573346 .extra_temps = .{
73326 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },
73327 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .word } } },
73347 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
73348 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bytes_mem = .{ .direction = .reverse, .size = .word } } },
7332873349 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
7332973350 .unused,
7333073351 .unused,
......@@ -73350,8 +73371,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7335073371 .{ .src = .{ .to_mut_sse, .none, .none } },
7335173372 },
7335273373 .extra_temps = .{
73353 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },
73354 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .word } } },
73374 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
73375 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bytes_mem = .{ .direction = .reverse, .size = .word } } },
7335573376 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
7335673377 .unused,
7335773378 .unused,
......@@ -73377,7 +73398,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7337773398 .{ .src = .{ .to_mut_sse, .none, .none } },
7337873399 },
7337973400 .extra_temps = .{
73380 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },
73401 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
7338173402 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
7338273403 .unused,
7338373404 .unused,
......@@ -73404,8 +73425,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7340473425 .{ .src = .{ .to_sse, .none, .none } },
7340573426 },
7340673427 .extra_temps = .{
73407 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .word, .smear = 8 } } },
73408 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .reverse } },
73428 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bytes_mem = .{ .direction = .reverse, .size = .word, .smear = 8 } } },
73429 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .reverse } } },
7340973430 .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },
7341073431 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
7341173432 .unused,
......@@ -73435,8 +73456,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7343573456 .{ .src = .{ .to_mut_sse, .none, .none } },
7343673457 },
7343773458 .extra_temps = .{
73438 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .word, .smear = 8 } } },
73439 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .reverse } },
73459 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bytes_mem = .{ .direction = .reverse, .size = .word, .smear = 8 } } },
73460 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .reverse } } },
7344073461 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
7344173462 .unused,
7344273463 .unused,
......@@ -73508,8 +73529,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7350873529 .{ .src = .{ .to_sse, .none, .none } },
7350973530 },
7351073531 .extra_temps = .{
73511 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },
73512 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .word } } },
73532 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
73533 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bytes_mem = .{ .direction = .reverse, .size = .word } } },
7351373534 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
7351473535 .unused,
7351573536 .unused,
......@@ -73535,8 +73556,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7353573556 .{ .src = .{ .to_mut_sse, .none, .none } },
7353673557 },
7353773558 .extra_temps = .{
73538 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },
73539 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .word } } },
73559 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
73560 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bytes_mem = .{ .direction = .reverse, .size = .word } } },
7354073561 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
7354173562 .unused,
7354273563 .unused,
......@@ -73562,7 +73583,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7356273583 .{ .src = .{ .to_mut_sse, .none, .none } },
7356373584 },
7356473585 .extra_temps = .{
73565 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },
73586 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
7356673587 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
7356773588 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
7356873589 .unused,
......@@ -73590,8 +73611,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7359073611 .{ .src = .{ .to_sse, .none, .none } },
7359173612 },
7359273613 .extra_temps = .{
73593 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .word, .smear = 8 } } },
73594 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .reverse } },
73614 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bytes_mem = .{ .direction = .reverse, .size = .word, .smear = 8 } } },
73615 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .reverse } } },
7359573616 .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },
7359673617 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
7359773618 .unused,
......@@ -73621,8 +73642,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7362173642 .{ .src = .{ .to_mut_sse, .none, .none } },
7362273643 },
7362373644 .extra_temps = .{
73624 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .word, .smear = 8 } } },
73625 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .reverse } },
73645 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bytes_mem = .{ .direction = .reverse, .size = .word, .smear = 8 } } },
73646 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .reverse } } },
7362673647 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
7362773648 .unused,
7362873649 .unused,
......@@ -73694,8 +73715,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7369473715 .{ .src = .{ .to_sse, .none, .none } },
7369573716 },
7369673717 .extra_temps = .{
73697 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },
73698 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .dword } } },
73718 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
73719 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bytes_mem = .{ .direction = .reverse, .size = .dword } } },
7369973720 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
7370073721 .unused,
7370173722 .unused,
......@@ -73720,8 +73741,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7372073741 .{ .src = .{ .to_mut_sse, .none, .none } },
7372173742 },
7372273743 .extra_temps = .{
73723 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },
73724 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .dword } } },
73744 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
73745 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bytes_mem = .{ .direction = .reverse, .size = .dword } } },
7372573746 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
7372673747 .unused,
7372773748 .unused,
......@@ -73746,7 +73767,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7374673767 .{ .src = .{ .to_mut_sse, .none, .none } },
7374773768 },
7374873769 .extra_temps = .{
73749 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },
73770 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
7375073771 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
7375173772 .unused,
7375273773 .unused,
......@@ -73773,8 +73794,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7377373794 .{ .src = .{ .to_sse, .none, .none } },
7377473795 },
7377573796 .extra_temps = .{
73776 .{ .type = .vector_32_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .dword, .smear = 8 } } },
73777 .{ .type = .vector_32_u8, .kind = .{ .bits_mem = .reverse } },
73797 .{ .type = .vector_32_u8, .kind = .{ .pshufb_bytes_mem = .{ .direction = .reverse, .size = .dword, .smear = 8 } } },
73798 .{ .type = .vector_32_u8, .kind = .{ .bits_mem = .{ .direction = .reverse } } },
7377873799 .{ .type = .vector_32_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },
7377973800 .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } },
7378073801 .unused,
......@@ -73844,8 +73865,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7384473865 .{ .src = .{ .to_sse, .none, .none } },
7384573866 },
7384673867 .extra_temps = .{
73847 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },
73848 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .dword } } },
73868 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
73869 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bytes_mem = .{ .direction = .reverse, .size = .dword } } },
7384973870 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
7385073871 .unused,
7385173872 .unused,
......@@ -73871,8 +73892,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7387173892 .{ .src = .{ .to_mut_sse, .none, .none } },
7387273893 },
7387373894 .extra_temps = .{
73874 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },
73875 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .dword } } },
73895 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
73896 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bytes_mem = .{ .direction = .reverse, .size = .dword } } },
7387673897 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
7387773898 .unused,
7387873899 .unused,
......@@ -73898,7 +73919,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7389873919 .{ .src = .{ .to_mut_sse, .none, .none } },
7389973920 },
7390073921 .extra_temps = .{
73901 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },
73922 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
7390273923 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
7390373924 .unused,
7390473925 .unused,
......@@ -73926,8 +73947,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7392673947 .{ .src = .{ .to_sse, .none, .none } },
7392773948 },
7392873949 .extra_temps = .{
73929 .{ .type = .vector_32_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .dword, .smear = 8 } } },
73930 .{ .type = .vector_32_u8, .kind = .{ .bits_mem = .reverse } },
73950 .{ .type = .vector_32_u8, .kind = .{ .pshufb_bytes_mem = .{ .direction = .reverse, .size = .dword, .smear = 8 } } },
73951 .{ .type = .vector_32_u8, .kind = .{ .bits_mem = .{ .direction = .reverse } } },
7393173952 .{ .type = .vector_32_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },
7393273953 .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } },
7393373954 .unused,
......@@ -74000,8 +74021,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7400074021 .{ .src = .{ .to_sse, .none, .none } },
7400174022 },
7400274023 .extra_temps = .{
74003 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },
74004 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .dword } } },
74024 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
74025 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bytes_mem = .{ .direction = .reverse, .size = .dword } } },
7400574026 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
7400674027 .unused,
7400774028 .unused,
......@@ -74027,8 +74048,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7402774048 .{ .src = .{ .to_mut_sse, .none, .none } },
7402874049 },
7402974050 .extra_temps = .{
74030 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },
74031 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .dword } } },
74051 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
74052 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bytes_mem = .{ .direction = .reverse, .size = .dword } } },
7403274053 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
7403374054 .unused,
7403474055 .unused,
......@@ -74054,7 +74075,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7405474075 .{ .src = .{ .to_mut_sse, .none, .none } },
7405574076 },
7405674077 .extra_temps = .{
74057 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },
74078 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
7405874079 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
7405974080 .unused,
7406074081 .unused,
......@@ -74082,8 +74103,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7408274103 .{ .src = .{ .to_sse, .none, .none } },
7408374104 },
7408474105 .extra_temps = .{
74085 .{ .type = .vector_32_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .dword, .smear = 8 } } },
74086 .{ .type = .vector_32_u8, .kind = .{ .bits_mem = .reverse } },
74106 .{ .type = .vector_32_u8, .kind = .{ .pshufb_bytes_mem = .{ .direction = .reverse, .size = .dword, .smear = 8 } } },
74107 .{ .type = .vector_32_u8, .kind = .{ .bits_mem = .{ .direction = .reverse } } },
7408774108 .{ .type = .vector_32_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },
7408874109 .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } },
7408974110 .unused,
......@@ -74156,8 +74177,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7415674177 .{ .src = .{ .to_sse, .none, .none } },
7415774178 },
7415874179 .extra_temps = .{
74159 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },
74160 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .qword } } },
74180 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
74181 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bytes_mem = .{ .direction = .reverse, .size = .qword } } },
7416174182 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
7416274183 .unused,
7416374184 .unused,
......@@ -74182,8 +74203,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7418274203 .{ .src = .{ .to_mut_sse, .none, .none } },
7418374204 },
7418474205 .extra_temps = .{
74185 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },
74186 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .qword } } },
74206 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
74207 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bytes_mem = .{ .direction = .reverse, .size = .qword } } },
7418774208 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
7418874209 .unused,
7418974210 .unused,
......@@ -74208,7 +74229,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7420874229 .{ .src = .{ .to_mut_sse, .none, .none } },
7420974230 },
7421074231 .extra_temps = .{
74211 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },
74232 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
7421274233 .unused,
7421374234 .unused,
7421474235 .unused,
......@@ -74279,7 +74300,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7427974300 .{ .src = .{ .to_sse, .none, .none } },
7428074301 },
7428174302 .extra_temps = .{
74282 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },
74303 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
7428374304 .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },
7428474305 .unused,
7428574306 .unused,
......@@ -74307,7 +74328,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7430774328 .{ .src = .{ .to_mut_sse, .none, .none } },
7430874329 },
7430974330 .extra_temps = .{
74310 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },
74331 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
7431174332 .unused,
7431274333 .unused,
7431374334 .unused,
......@@ -74381,8 +74402,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7438174402 .{ .src = .{ .to_sse, .none, .none } },
7438274403 },
7438374404 .extra_temps = .{
74384 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },
74385 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .qword } } },
74405 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
74406 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bytes_mem = .{ .direction = .reverse, .size = .qword } } },
7438674407 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
7438774408 .unused,
7438874409 .unused,
......@@ -74408,8 +74429,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7440874429 .{ .src = .{ .to_mut_sse, .none, .none } },
7440974430 },
7441074431 .extra_temps = .{
74411 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },
74412 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .qword } } },
74432 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
74433 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bytes_mem = .{ .direction = .reverse, .size = .qword } } },
7441374434 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
7441474435 .unused,
7441574436 .unused,
......@@ -74435,7 +74456,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7443574456 .{ .src = .{ .to_mut_sse, .none, .none } },
7443674457 },
7443774458 .extra_temps = .{
74438 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },
74459 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
7443974460 .unused,
7444074461 .unused,
7444174462 .unused,
......@@ -74509,8 +74530,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7450974530 .{ .src = .{ .to_sse, .none, .none } },
7451074531 },
7451174532 .extra_temps = .{
74512 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },
74513 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .xword } } },
74533 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
74534 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bytes_mem = .{ .direction = .reverse, .size = .xword } } },
7451474535 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
7451574536 .unused,
7451674537 .unused,
......@@ -74535,8 +74556,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7453574556 .{ .src = .{ .to_mut_sse, .none, .none } },
7453674557 },
7453774558 .extra_temps = .{
74538 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },
74539 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .xword } } },
74559 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
74560 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bytes_mem = .{ .direction = .reverse, .size = .xword } } },
7454074561 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
7454174562 .unused,
7454274563 .unused,
......@@ -74561,8 +74582,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7456174582 .{ .src = .{ .to_sse, .none, .none } },
7456274583 },
7456374584 .extra_temps = .{
74564 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },
74565 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .xword } } },
74585 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
74586 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bytes_mem = .{ .direction = .reverse, .size = .xword } } },
7456674587 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
7456774588 .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },
7456874589 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
......@@ -74594,8 +74615,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7459474615 .{ .src = .{ .to_mut_sse, .none, .none } },
7459574616 },
7459674617 .extra_temps = .{
74597 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },
74598 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .xword } } },
74618 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
74619 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bytes_mem = .{ .direction = .reverse, .size = .xword } } },
7459974620 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
7460074621 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
7460174622 .unused,
......@@ -74627,8 +74648,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7462774648 .{ .src = .{ .to_sse, .none, .none } },
7462874649 },
7462974650 .extra_temps = .{
74630 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },
74631 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .xword } } },
74651 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
74652 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bytes_mem = .{ .direction = .reverse, .size = .xword } } },
7463274653 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
7463374654 .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },
7463474655 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
......@@ -74660,8 +74681,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7466074681 .{ .src = .{ .to_mut_sse, .none, .none } },
7466174682 },
7466274683 .extra_temps = .{
74663 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },
74664 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .xword } } },
74684 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
74685 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bytes_mem = .{ .direction = .reverse, .size = .xword } } },
7466574686 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
7466674687 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
7466774688 .unused,
......@@ -74693,8 +74714,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7469374714 .{ .src = .{ .to_sse, .none, .none } },
7469474715 },
7469574716 .extra_temps = .{
74696 .{ .type = .vector_32_u8, .kind = .{ .bits_mem = .forward } },
74697 .{ .type = .vector_32_u8, .kind = .{ .pshufb_bswap_mem = .{ .repeat = 2, .size = .xword } } },
74717 .{ .type = .vector_32_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
74718 .{ .type = .vector_32_u8, .kind = .{ .pshufb_bytes_mem = .{ .direction = .reverse, .repeat = 2, .size = .xword } } },
7469874719 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
7469974720 .unused,
7470074721 .unused,
......@@ -74720,8 +74741,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7472074741 .{ .src = .{ .to_sse, .none, .none } },
7472174742 },
7472274743 .extra_temps = .{
74723 .{ .type = .vector_32_u8, .kind = .{ .bits_mem = .forward } },
74724 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .xword } } },
74744 .{ .type = .vector_32_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
74745 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bytes_mem = .{ .direction = .reverse, .size = .xword } } },
7472574746 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
7472674747 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
7472774748 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
......@@ -74752,8 +74773,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7475274773 .extra_temps = .{
7475374774 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
7475474775 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
74755 .{ .type = .vector_32_u8, .kind = .{ .bits_mem = .forward } },
74756 .{ .type = .vector_32_u8, .kind = .{ .pshufb_bswap_mem = .{ .repeat = 2, .size = .xword } } },
74776 .{ .type = .vector_32_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
74777 .{ .type = .vector_32_u8, .kind = .{ .pshufb_bytes_mem = .{ .direction = .reverse, .repeat = 2, .size = .xword } } },
7475774778 .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } },
7475874779 .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } },
7475974780 .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } },
......@@ -74788,8 +74809,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7478874809 .extra_temps = .{
7478974810 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
7479074811 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
74791 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },
74792 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .xword } } },
74812 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
74813 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bytes_mem = .{ .direction = .reverse, .size = .xword } } },
7479374814 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
7479474815 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
7479574816 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
......@@ -82067,7 +82088,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8206782088 .dst_temps = .{ .{ .mut_rc_mask = .{
8206882089 .ref = .src0,
8206982090 .rc = .sse,
82070 .info = .{ .kind = .all, .scalar = .dword },
82091 .info = .{ .kind = .sign_extend, .scalar = .dword },
8207182092 } }, .unused },
8207282093 .each = .{ .once = &.{
8207382094 .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ },
......@@ -82107,7 +82128,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8210782128 .dst_temps = .{ .{ .mut_rc_mask = .{
8210882129 .ref = .src0,
8210982130 .rc = .sse,
82110 .info = .{ .kind = .all, .scalar = .dword },
82131 .info = .{ .kind = .sign_extend, .scalar = .dword },
8211182132 } }, .unused },
8211282133 .each = .{ .once = &.{
8211382134 .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ },
......@@ -82147,7 +82168,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8214782168 .dst_temps = .{ .{ .mut_rc_mask = .{
8214882169 .ref = .src0,
8214982170 .rc = .sse,
82150 .info = .{ .kind = .all, .scalar = .dword },
82171 .info = .{ .kind = .sign_extend, .scalar = .dword },
8215182172 } }, .unused },
8215282173 .each = .{ .once = &.{
8215382174 .{ ._, .v_ps, .cvtph2, .dst0y, .src0x, ._, ._ },
......@@ -82171,7 +82192,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8217182192 .dst_temps = .{ .{ .mut_rc_mask = .{
8217282193 .ref = .src0,
8217382194 .rc = .sse,
82174 .info = .{ .kind = .all, .scalar = .dword },
82195 .info = .{ .kind = .sign_extend, .scalar = .dword },
8217582196 } }, .unused },
8217682197 .each = .{ .once = &.{
8217782198 .{ ._, .v_ss, .cmp, .dst0x, .src0x, .src1d, .vp(switch (cc) {
......@@ -82194,7 +82215,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8219482215 .dst_temps = .{ .{ .mut_rc_mask = .{
8219582216 .ref = .src0,
8219682217 .rc = .sse,
82197 .info = .{ .kind = .all, .scalar = .dword },
82218 .info = .{ .kind = .sign_extend, .scalar = .dword },
8219882219 } }, .unused },
8219982220 .each = .{ .once = &.{
8220082221 .{ ._, .v_ss, .cmp, .dst0x, .src0x, .src1d, .vp(switch (cc) {
......@@ -82216,7 +82237,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8221682237 },
8221782238 .dst_temps = .{ .{ .ref_mask = .{
8221882239 .ref = .src0,
82219 .info = .{ .kind = .all, .scalar = .dword },
82240 .info = .{ .kind = .sign_extend, .scalar = .dword },
8222082241 } }, .unused },
8222182242 .each = .{ .once = &.{
8222282243 .{ ._, ._ss, .cmp, .dst0x, .src1d, .sp(switch (cc) {
......@@ -82238,7 +82259,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8223882259 .dst_temps = .{ .{ .mut_rc_mask = .{
8223982260 .ref = .src0,
8224082261 .rc = .sse,
82241 .info = .{ .kind = .all, .scalar = .dword },
82262 .info = .{ .kind = .sign_extend, .scalar = .dword },
8224282263 } }, .unused },
8224382264 .each = .{ .once = &.{
8224482265 .{ ._, .v_ps, .cmp, .dst0x, .src0x, .src1x, .vp(switch (cc) {
......@@ -82261,7 +82282,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8226182282 .dst_temps = .{ .{ .mut_rc_mask = .{
8226282283 .ref = .src0,
8226382284 .rc = .sse,
82264 .info = .{ .kind = .all, .scalar = .dword },
82285 .info = .{ .kind = .sign_extend, .scalar = .dword },
8226582286 } }, .unused },
8226682287 .each = .{ .once = &.{
8226782288 .{ ._, .v_ps, .cmp, .dst0x, .src0x, .src1x, .vp(switch (cc) {
......@@ -82283,7 +82304,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8228382304 },
8228482305 .dst_temps = .{ .{ .ref_mask = .{
8228582306 .ref = .src0,
82286 .info = .{ .kind = .all, .scalar = .dword },
82307 .info = .{ .kind = .sign_extend, .scalar = .dword },
8228782308 } }, .unused },
8228882309 .each = .{ .once = &.{
8228982310 .{ ._, ._ps, .cmp, .dst0x, .src1x, .sp(switch (cc) {
......@@ -82305,7 +82326,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8230582326 .dst_temps = .{ .{ .mut_rc_mask = .{
8230682327 .ref = .src0,
8230782328 .rc = .sse,
82308 .info = .{ .kind = .all, .scalar = .dword },
82329 .info = .{ .kind = .sign_extend, .scalar = .dword },
8230982330 } }, .unused },
8231082331 .each = .{ .once = &.{
8231182332 .{ ._, .v_ps, .cmp, .dst0y, .src0y, .src1y, .vp(switch (cc) {
......@@ -82328,7 +82349,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8232882349 .dst_temps = .{ .{ .mut_rc_mask = .{
8232982350 .ref = .src0,
8233082351 .rc = .sse,
82331 .info = .{ .kind = .all, .scalar = .dword },
82352 .info = .{ .kind = .sign_extend, .scalar = .dword },
8233282353 } }, .unused },
8233382354 .each = .{ .once = &.{
8233482355 .{ ._, .v_ps, .cmp, .dst0y, .src0y, .src1y, .vp(switch (cc) {
......@@ -82350,7 +82371,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8235082371 .dst_temps = .{ .{ .mut_rc_mask = .{
8235182372 .ref = .src0,
8235282373 .rc = .sse,
82353 .info = .{ .kind = .all, .scalar = .qword },
82374 .info = .{ .kind = .sign_extend, .scalar = .qword },
8235482375 } }, .unused },
8235582376 .each = .{ .once = &.{
8235682377 .{ ._, .v_sd, .cmp, .dst0x, .src0x, .src1q, .vp(switch (cc) {
......@@ -82373,7 +82394,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8237382394 .dst_temps = .{ .{ .mut_rc_mask = .{
8237482395 .ref = .src0,
8237582396 .rc = .sse,
82376 .info = .{ .kind = .all, .scalar = .qword },
82397 .info = .{ .kind = .sign_extend, .scalar = .qword },
8237782398 } }, .unused },
8237882399 .each = .{ .once = &.{
8237982400 .{ ._, .v_sd, .cmp, .dst0x, .src0x, .src1q, .vp(switch (cc) {
......@@ -82395,7 +82416,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8239582416 },
8239682417 .dst_temps = .{ .{ .ref_mask = .{
8239782418 .ref = .src0,
82398 .info = .{ .kind = .all, .scalar = .qword },
82419 .info = .{ .kind = .sign_extend, .scalar = .qword },
8239982420 } }, .unused },
8240082421 .each = .{ .once = &.{
8240182422 .{ ._, ._sd, .cmp, .dst0x, .src1q, .sp(switch (cc) {
......@@ -82417,7 +82438,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8241782438 .dst_temps = .{ .{ .mut_rc_mask = .{
8241882439 .ref = .src0,
8241982440 .rc = .sse,
82420 .info = .{ .kind = .all, .scalar = .qword },
82441 .info = .{ .kind = .sign_extend, .scalar = .qword },
8242182442 } }, .unused },
8242282443 .each = .{ .once = &.{
8242382444 .{ ._, .v_pd, .cmp, .dst0x, .src0x, .src1x, .vp(switch (cc) {
......@@ -82440,7 +82461,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8244082461 .dst_temps = .{ .{ .mut_rc_mask = .{
8244182462 .ref = .src0,
8244282463 .rc = .sse,
82443 .info = .{ .kind = .all, .scalar = .qword },
82464 .info = .{ .kind = .sign_extend, .scalar = .qword },
8244482465 } }, .unused },
8244582466 .each = .{ .once = &.{
8244682467 .{ ._, .v_pd, .cmp, .dst0x, .src0x, .src1x, .vp(switch (cc) {
......@@ -82462,7 +82483,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8246282483 },
8246382484 .dst_temps = .{ .{ .ref_mask = .{
8246482485 .ref = .src0,
82465 .info = .{ .kind = .all, .scalar = .qword },
82486 .info = .{ .kind = .sign_extend, .scalar = .qword },
8246682487 } }, .unused },
8246782488 .each = .{ .once = &.{
8246882489 .{ ._, ._pd, .cmp, .dst0x, .src1x, .sp(switch (cc) {
......@@ -82484,7 +82505,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8248482505 .dst_temps = .{ .{ .mut_rc_mask = .{
8248582506 .ref = .src0,
8248682507 .rc = .sse,
82487 .info = .{ .kind = .all, .scalar = .qword },
82508 .info = .{ .kind = .sign_extend, .scalar = .qword },
8248882509 } }, .unused },
8248982510 .each = .{ .once = &.{
8249082511 .{ ._, .v_pd, .cmp, .dst0y, .src0y, .src1y, .vp(switch (cc) {
......@@ -82507,7 +82528,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8250782528 .dst_temps = .{ .{ .mut_rc_mask = .{
8250882529 .ref = .src0,
8250982530 .rc = .sse,
82510 .info = .{ .kind = .all, .scalar = .qword },
82531 .info = .{ .kind = .sign_extend, .scalar = .qword },
8251182532 } }, .unused },
8251282533 .each = .{ .once = &.{
8251382534 .{ ._, .v_pd, .cmp, .dst0y, .src0y, .src1y, .vp(switch (cc) {
......@@ -84711,7 +84732,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8471184732 .{ .src = .{ .to_sse, .to_sse, .none } },
8471284733 },
8471384734 .dst_temps = .{ .{ .mut_rc_mask = .{ .ref = .src0, .rc = .sse, .info = .{
84714 .kind = .all,
84735 .kind = .sign_extend,
8471584736 .inverted = switch (cc) {
8471684737 else => unreachable,
8471784738 .e => false,
......@@ -84735,7 +84756,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8473584756 .{ .src = .{ .to_sse, .to_sse, .none } },
8473684757 },
8473784758 .dst_temps = .{ .{ .mut_rc_mask = .{ .ref = .src0, .rc = .sse, .info = .{
84738 .kind = .all,
84759 .kind = .sign_extend,
8473984760 .inverted = switch (cc) {
8474084761 else => unreachable,
8474184762 .e => false,
......@@ -84759,7 +84780,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8475984780 .{ .src = .{ .to_sse, .to_sse, .none } },
8476084781 },
8476184782 .dst_temps = .{ .{ .mut_rc_mask = .{ .ref = .src0, .rc = .sse, .info = .{
84762 .kind = .all,
84783 .kind = .sign_extend,
8476384784 .inverted = switch (cc) {
8476484785 else => unreachable,
8476584786 .e => false,
......@@ -84783,7 +84804,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8478384804 .{ .src = .{ .to_sse, .to_sse, .none } },
8478484805 },
8478584806 .dst_temps = .{ .{ .mut_rc_mask = .{ .ref = .src0, .rc = .sse, .info = .{
84786 .kind = .all,
84807 .kind = .sign_extend,
8478784808 .inverted = switch (cc) {
8478884809 else => unreachable,
8478984810 .e => false,
......@@ -84807,7 +84828,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8480784828 .{ .src = .{ .to_mut_sse, .to_sse, .none } },
8480884829 },
8480984830 .dst_temps = .{ .{ .ref_mask = .{ .ref = .src0, .info = .{
84810 .kind = .all,
84831 .kind = .sign_extend,
8481184832 .inverted = switch (cc) {
8481284833 else => unreachable,
8481384834 .e => false,
......@@ -84831,7 +84852,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8483184852 .{ .src = .{ .to_mut_sse, .to_sse, .none } },
8483284853 },
8483384854 .dst_temps = .{ .{ .ref_mask = .{ .ref = .src0, .info = .{
84834 .kind = .all,
84855 .kind = .sign_extend,
8483584856 .inverted = switch (cc) {
8483684857 else => unreachable,
8483784858 .e => false,
......@@ -84855,7 +84876,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8485584876 .{ .src = .{ .to_mut_sse, .to_sse, .none } },
8485684877 },
8485784878 .dst_temps = .{ .{ .ref_mask = .{ .ref = .src0, .info = .{
84858 .kind = .all,
84879 .kind = .sign_extend,
8485984880 .inverted = switch (cc) {
8486084881 else => unreachable,
8486184882 .e => false,
......@@ -84879,7 +84900,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8487984900 .{ .src = .{ .to_mut_sse, .to_sse, .none } },
8488084901 },
8488184902 .dst_temps = .{ .{ .ref_mask = .{ .ref = .src0, .info = .{
84882 .kind = .all,
84903 .kind = .sign_extend,
8488384904 .inverted = switch (cc) {
8488484905 else => unreachable,
8488584906 .e => false,
......@@ -84903,7 +84924,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8490384924 .{ .src = .{ .to_mut_mmx, .to_mmx, .none } },
8490484925 },
8490584926 .dst_temps = .{ .{ .ref_mask = .{ .ref = .src0, .info = .{
84906 .kind = .all,
84927 .kind = .sign_extend,
8490784928 .inverted = switch (cc) {
8490884929 else => unreachable,
8490984930 .e => false,
......@@ -84927,7 +84948,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8492784948 .{ .src = .{ .to_mut_mmx, .to_mmx, .none } },
8492884949 },
8492984950 .dst_temps = .{ .{ .ref_mask = .{ .ref = .src0, .info = .{
84930 .kind = .all,
84951 .kind = .sign_extend,
8493184952 .inverted = switch (cc) {
8493284953 else => unreachable,
8493384954 .e => false,
......@@ -84951,7 +84972,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8495184972 .{ .src = .{ .to_mut_mmx, .to_mmx, .none } },
8495284973 },
8495384974 .dst_temps = .{ .{ .ref_mask = .{ .ref = .src0, .info = .{
84954 .kind = .all,
84975 .kind = .sign_extend,
8495584976 .inverted = switch (cc) {
8495684977 else => unreachable,
8495784978 .e => false,
......@@ -84975,7 +84996,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8497584996 .{ .src = .{ .to_sse, .to_sse, .none } },
8497684997 },
8497784998 .dst_temps = .{ .{ .mut_rc_mask = .{ .ref = .src0, .rc = .sse, .info = .{
84978 .kind = .all,
84999 .kind = .sign_extend,
8497985000 .inverted = switch (cc) {
8498085001 else => unreachable,
8498185002 .e => false,
......@@ -84999,7 +85020,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8499985020 .{ .src = .{ .to_sse, .to_sse, .none } },
8500085021 },
8500185022 .dst_temps = .{ .{ .mut_rc_mask = .{ .ref = .src0, .rc = .sse, .info = .{
85002 .kind = .all,
85023 .kind = .sign_extend,
8500385024 .inverted = switch (cc) {
8500485025 else => unreachable,
8500585026 .e => false,
......@@ -85023,7 +85044,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8502385044 .{ .src = .{ .to_sse, .to_sse, .none } },
8502485045 },
8502585046 .dst_temps = .{ .{ .mut_rc_mask = .{ .ref = .src0, .rc = .sse, .info = .{
85026 .kind = .all,
85047 .kind = .sign_extend,
8502785048 .inverted = switch (cc) {
8502885049 else => unreachable,
8502985050 .e => false,
......@@ -85047,7 +85068,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8504785068 .{ .src = .{ .to_sse, .to_sse, .none } },
8504885069 },
8504985070 .dst_temps = .{ .{ .mut_rc_mask = .{ .ref = .src0, .rc = .sse, .info = .{
85050 .kind = .all,
85071 .kind = .sign_extend,
8505185072 .inverted = switch (cc) {
8505285073 else => unreachable,
8505385074 .e => false,
......@@ -86699,7 +86720,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8669986720 .dst_temps = .{ .{ .mut_rc_mask = .{
8670086721 .ref = .src0,
8670186722 .rc = .sse,
86702 .info = .{ .kind = .all, .scalar = .dword },
86723 .info = .{ .kind = .sign_extend, .scalar = .dword },
8670386724 } }, .unused },
8670486725 .each = .{ .once = &.{
8670586726 .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ },
......@@ -86739,7 +86760,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8673986760 .dst_temps = .{ .{ .mut_rc_mask = .{
8674086761 .ref = .src0,
8674186762 .rc = .sse,
86742 .info = .{ .kind = .all, .scalar = .dword },
86763 .info = .{ .kind = .sign_extend, .scalar = .dword },
8674386764 } }, .unused },
8674486765 .each = .{ .once = &.{
8674586766 .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ },
......@@ -86779,7 +86800,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8677986800 .dst_temps = .{ .{ .mut_rc_mask = .{
8678086801 .ref = .src0,
8678186802 .rc = .sse,
86782 .info = .{ .kind = .all, .scalar = .dword },
86803 .info = .{ .kind = .sign_extend, .scalar = .dword },
8678386804 } }, .unused },
8678486805 .each = .{ .once = &.{
8678586806 .{ ._, .v_ps, .cvtph2, .dst0y, .src0x, ._, ._ },
......@@ -86805,7 +86826,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8680586826 .dst_temps = .{ .{ .mut_rc_mask = .{
8680686827 .ref = .src0,
8680786828 .rc = .sse,
86808 .info = .{ .kind = .all, .scalar = .dword },
86829 .info = .{ .kind = .sign_extend, .scalar = .dword },
8680986830 } }, .unused },
8681086831 .each = .{ .once = &.{
8681186832 .{ ._, .v_ss, .cmp, .dst0x, .src0x, .src1d, .vp(switch (cc) {
......@@ -86828,7 +86849,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8682886849 },
8682986850 .dst_temps = .{ .{ .ref_mask = .{
8683086851 .ref = .src0,
86831 .info = .{ .kind = .all, .scalar = .dword },
86852 .info = .{ .kind = .sign_extend, .scalar = .dword },
8683286853 } }, .unused },
8683386854 .each = .{ .once = &.{
8683486855 .{ ._, ._ss, .cmp, .dst0x, .src1d, .sp(switch (cc) {
......@@ -86852,7 +86873,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8685286873 .dst_temps = .{ .{ .mut_rc_mask = .{
8685386874 .ref = .src0,
8685486875 .rc = .sse,
86855 .info = .{ .kind = .all, .scalar = .dword },
86876 .info = .{ .kind = .sign_extend, .scalar = .dword },
8685686877 } }, .unused },
8685786878 .each = .{ .once = &.{
8685886879 .{ ._, .v_ps, .cmp, .dst0x, .src0x, .src1x, .vp(switch (cc) {
......@@ -86875,7 +86896,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8687586896 },
8687686897 .dst_temps = .{ .{ .ref_mask = .{
8687786898 .ref = .src0,
86878 .info = .{ .kind = .all, .scalar = .dword },
86899 .info = .{ .kind = .sign_extend, .scalar = .dword },
8687986900 } }, .unused },
8688086901 .each = .{ .once = &.{
8688186902 .{ ._, ._ps, .cmp, .dst0x, .src1x, .sp(switch (cc) {
......@@ -86899,7 +86920,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8689986920 .dst_temps = .{ .{ .mut_rc_mask = .{
8690086921 .ref = .src0,
8690186922 .rc = .sse,
86902 .info = .{ .kind = .all, .scalar = .dword },
86923 .info = .{ .kind = .sign_extend, .scalar = .dword },
8690386924 } }, .unused },
8690486925 .each = .{ .once = &.{
8690586926 .{ ._, .v_ps, .cmp, .dst0y, .src0y, .src1y, .vp(switch (cc) {
......@@ -86923,7 +86944,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8692386944 .dst_temps = .{ .{ .mut_rc_mask = .{
8692486945 .ref = .src0,
8692586946 .rc = .sse,
86926 .info = .{ .kind = .all, .scalar = .qword },
86947 .info = .{ .kind = .sign_extend, .scalar = .qword },
8692786948 } }, .unused },
8692886949 .each = .{ .once = &.{
8692986950 .{ ._, .v_sd, .cmp, .dst0x, .src0x, .src1q, .vp(switch (cc) {
......@@ -86946,7 +86967,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8694686967 },
8694786968 .dst_temps = .{ .{ .ref_mask = .{
8694886969 .ref = .src0,
86949 .info = .{ .kind = .all, .scalar = .qword },
86970 .info = .{ .kind = .sign_extend, .scalar = .qword },
8695086971 } }, .unused },
8695186972 .each = .{ .once = &.{
8695286973 .{ ._, ._sd, .cmp, .dst0x, .src1q, .sp(switch (cc) {
......@@ -86970,7 +86991,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8697086991 .dst_temps = .{ .{ .mut_rc_mask = .{
8697186992 .ref = .src0,
8697286993 .rc = .sse,
86973 .info = .{ .kind = .all, .scalar = .qword },
86994 .info = .{ .kind = .sign_extend, .scalar = .qword },
8697486995 } }, .unused },
8697586996 .each = .{ .once = &.{
8697686997 .{ ._, .v_pd, .cmp, .dst0x, .src0x, .src1x, .vp(switch (cc) {
......@@ -86993,7 +87014,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8699387014 },
8699487015 .dst_temps = .{ .{ .ref_mask = .{
8699587016 .ref = .src0,
86996 .info = .{ .kind = .all, .scalar = .qword },
87017 .info = .{ .kind = .sign_extend, .scalar = .qword },
8699787018 } }, .unused },
8699887019 .each = .{ .once = &.{
8699987020 .{ ._, ._pd, .cmp, .dst0x, .src1x, .sp(switch (cc) {
......@@ -87017,7 +87038,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8701787038 .dst_temps = .{ .{ .mut_rc_mask = .{
8701887039 .ref = .src0,
8701987040 .rc = .sse,
87020 .info = .{ .kind = .all, .scalar = .qword },
87041 .info = .{ .kind = .sign_extend, .scalar = .qword },
8702187042 } }, .unused },
8702287043 .each = .{ .once = &.{
8702387044 .{ ._, .v_pd, .cmp, .dst0y, .src0y, .src1y, .vp(switch (cc) {
......@@ -89392,6 +89413,25 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8939289413 .store, .store_safe => |air_tag| {
8939389414 const bin_op = air_datas[@intFromEnum(inst)].bin_op;
8939489415 var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs });
89416 switch (ops[1].tracking(cg).short) {
89417 else => {},
89418 .register_mask => |src_reg_mask| {
89419 const ty = ops[1].typeOf(cg);
89420 const new_op1 = try cg.tempAllocReg(ty, abi.RegisterClass.gp);
89421 try cg.genSetReg(
89422 new_op1.tracking(cg).short.register,
89423 ty,
89424 .{ .register_mask = src_reg_mask },
89425 .{ .safety = switch (air_tag) {
89426 else => unreachable,
89427 .store => false,
89428 .store_safe => true,
89429 } },
89430 );
89431 try ops[1].die(cg);
89432 ops[1] = new_op1;
89433 },
89434 }
8939589435 cg.select(&.{}, &.{}, &ops, comptime &.{ .{
8939689436 .src_constraints = .{ .{ .ptr_bool_vec_elem = .byte }, .bool, .any },
8939789437 .patterns = &.{
......@@ -127167,7 +127207,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
127167127207 .dst_constraints = .{ .bool, .any },
127168127208 .src_constraints = .{ .any_bool_vec, .any, .any },
127169127209 .patterns = &.{
127170 .{ .src = .{ .{ .all_reg_mask = .{ .size = .xword, .is = .inverted } }, .none, .none } },
127210 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .sign_extend, .is = .inverted } }, .none, .none } },
127171127211 },
127172127212 .dst_temps = .{ .{ .cc = .z }, .unused },
127173127213 .clobbers = .{ .eflags = true },
......@@ -127179,7 +127219,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
127179127219 .dst_constraints = .{ .bool, .any },
127180127220 .src_constraints = .{ .any_bool_vec, .any, .any },
127181127221 .patterns = &.{
127182 .{ .src = .{ .{ .all_reg_mask = .{ .size = .xword, .is = .inverted } }, .none, .none } },
127222 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .sign_extend, .is = .inverted } }, .none, .none } },
127183127223 },
127184127224 .dst_temps = .{ .{ .cc = .z }, .unused },
127185127225 .clobbers = .{ .eflags = true },
......@@ -127191,7 +127231,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
127191127231 .dst_constraints = .{ .bool, .any },
127192127232 .src_constraints = .{ .any_bool_vec, .any, .any },
127193127233 .patterns = &.{
127194 .{ .src = .{ .{ .all_reg_mask = .{ .size = .xword } }, .none, .none } },
127234 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .sign_extend } }, .none, .none } },
127195127235 },
127196127236 .extra_temps = .{
127197127237 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
......@@ -127217,7 +127257,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
127217127257 .dst_constraints = .{ .bool, .any },
127218127258 .src_constraints = .{ .any_bool_vec, .any, .any },
127219127259 .patterns = &.{
127220 .{ .src = .{ .{ .all_reg_mask = .{ .size = .xword } }, .none, .none } },
127260 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .sign_extend } }, .none, .none } },
127221127261 },
127222127262 .extra_temps = .{
127223127263 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
......@@ -127243,7 +127283,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
127243127283 .dst_constraints = .{ .bool, .any },
127244127284 .src_constraints = .{ .any_bool_vec, .any, .any },
127245127285 .patterns = &.{
127246 .{ .src = .{ .{ .reg_mask = .{ .size = .xword } }, .none, .none } },
127286 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .msb } }, .none, .none } },
127247127287 },
127248127288 .extra_temps = .{
127249127289 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
......@@ -127269,7 +127309,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
127269127309 .dst_constraints = .{ .bool, .any },
127270127310 .src_constraints = .{ .any_bool_vec, .any, .any },
127271127311 .patterns = &.{
127272 .{ .src = .{ .{ .reg_mask = .{ .size = .xword } }, .none, .none } },
127312 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .msb } }, .none, .none } },
127273127313 },
127274127314 .extra_temps = .{
127275127315 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
......@@ -127295,7 +127335,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
127295127335 .dst_constraints = .{ .bool, .any },
127296127336 .src_constraints = .{ .any_bool_vec, .any, .any },
127297127337 .patterns = &.{
127298 .{ .src = .{ .{ .all_reg_mask = .{ .size = .yword, .is = .inverted } }, .none, .none } },
127338 .{ .src = .{ .{ .reg_mask = .{ .size = .yword, .kind = .sign_extend, .is = .inverted } }, .none, .none } },
127299127339 },
127300127340 .dst_temps = .{ .{ .cc = .z }, .unused },
127301127341 .clobbers = .{ .eflags = true },
......@@ -127307,7 +127347,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
127307127347 .dst_constraints = .{ .bool, .any },
127308127348 .src_constraints = .{ .any_bool_vec, .any, .any },
127309127349 .patterns = &.{
127310 .{ .src = .{ .{ .all_reg_mask = .{ .size = .yword, .is = .inverted } }, .none, .none } },
127350 .{ .src = .{ .{ .reg_mask = .{ .size = .yword, .kind = .sign_extend, .is = .inverted } }, .none, .none } },
127311127351 },
127312127352 .dst_temps = .{ .{ .cc = .z }, .unused },
127313127353 .clobbers = .{ .eflags = true },
......@@ -127319,7 +127359,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
127319127359 .dst_constraints = .{ .bool, .any },
127320127360 .src_constraints = .{ .any_bool_vec, .any, .any },
127321127361 .patterns = &.{
127322 .{ .src = .{ .{ .all_reg_mask = .{ .size = .yword } }, .none, .none } },
127362 .{ .src = .{ .{ .reg_mask = .{ .size = .yword, .kind = .sign_extend } }, .none, .none } },
127323127363 },
127324127364 .extra_temps = .{
127325127365 .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } },
......@@ -127345,7 +127385,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
127345127385 .dst_constraints = .{ .bool, .any },
127346127386 .src_constraints = .{ .any_bool_vec, .any, .any },
127347127387 .patterns = &.{
127348 .{ .src = .{ .{ .all_reg_mask = .{ .size = .yword } }, .none, .none } },
127388 .{ .src = .{ .{ .reg_mask = .{ .size = .yword, .kind = .sign_extend } }, .none, .none } },
127349127389 },
127350127390 .extra_temps = .{
127351127391 .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } },
......@@ -127371,7 +127411,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
127371127411 .dst_constraints = .{ .bool, .any },
127372127412 .src_constraints = .{ .any_bool_vec, .any, .any },
127373127413 .patterns = &.{
127374 .{ .src = .{ .{ .reg_mask = .{ .size = .yword } }, .none, .none } },
127414 .{ .src = .{ .{ .reg_mask = .{ .size = .yword, .kind = .msb } }, .none, .none } },
127375127415 },
127376127416 .extra_temps = .{
127377127417 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
......@@ -127572,7 +127612,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
127572127612 .dst_constraints = .{ .bool, .any },
127573127613 .src_constraints = .{ .any_bool_vec, .any, .any },
127574127614 .patterns = &.{
127575 .{ .src = .{ .{ .all_reg_mask = .{ .size = .xword, .is = .uninverted } }, .none, .none } },
127615 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .sign_extend, .is = .uninverted } }, .none, .none } },
127576127616 },
127577127617 .dst_temps = .{ .{ .cc = .nz }, .unused },
127578127618 .clobbers = .{ .eflags = true },
......@@ -127584,7 +127624,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
127584127624 .dst_constraints = .{ .bool, .any },
127585127625 .src_constraints = .{ .any_bool_vec, .any, .any },
127586127626 .patterns = &.{
127587 .{ .src = .{ .{ .all_reg_mask = .{ .size = .xword, .is = .uninverted } }, .none, .none } },
127627 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .sign_extend, .is = .uninverted } }, .none, .none } },
127588127628 },
127589127629 .dst_temps = .{ .{ .cc = .nz }, .unused },
127590127630 .clobbers = .{ .eflags = true },
......@@ -127596,7 +127636,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
127596127636 .dst_constraints = .{ .bool, .any },
127597127637 .src_constraints = .{ .any_bool_vec, .any, .any },
127598127638 .patterns = &.{
127599 .{ .src = .{ .{ .all_reg_mask = .{ .size = .xword } }, .none, .none } },
127639 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .sign_extend } }, .none, .none } },
127600127640 },
127601127641 .extra_temps = .{
127602127642 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
......@@ -127622,7 +127662,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
127622127662 .dst_constraints = .{ .bool, .any },
127623127663 .src_constraints = .{ .any_bool_vec, .any, .any },
127624127664 .patterns = &.{
127625 .{ .src = .{ .{ .all_reg_mask = .{ .size = .xword } }, .none, .none } },
127665 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .sign_extend } }, .none, .none } },
127626127666 },
127627127667 .extra_temps = .{
127628127668 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
......@@ -127648,7 +127688,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
127648127688 .dst_constraints = .{ .bool, .any },
127649127689 .src_constraints = .{ .any_bool_vec, .any, .any },
127650127690 .patterns = &.{
127651 .{ .src = .{ .{ .reg_mask = .{ .size = .xword } }, .none, .none } },
127691 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .msb } }, .none, .none } },
127652127692 },
127653127693 .extra_temps = .{
127654127694 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
......@@ -127674,7 +127714,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
127674127714 .dst_constraints = .{ .bool, .any },
127675127715 .src_constraints = .{ .any_bool_vec, .any, .any },
127676127716 .patterns = &.{
127677 .{ .src = .{ .{ .reg_mask = .{ .size = .xword } }, .none, .none } },
127717 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .msb } }, .none, .none } },
127678127718 },
127679127719 .extra_temps = .{
127680127720 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
......@@ -127700,7 +127740,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
127700127740 .dst_constraints = .{ .bool, .any },
127701127741 .src_constraints = .{ .any_bool_vec, .any, .any },
127702127742 .patterns = &.{
127703 .{ .src = .{ .{ .all_reg_mask = .{ .size = .yword, .is = .uninverted } }, .none, .none } },
127743 .{ .src = .{ .{ .reg_mask = .{ .size = .yword, .kind = .sign_extend, .is = .uninverted } }, .none, .none } },
127704127744 },
127705127745 .dst_temps = .{ .{ .cc = .nz }, .unused },
127706127746 .clobbers = .{ .eflags = true },
......@@ -127712,7 +127752,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
127712127752 .dst_constraints = .{ .bool, .any },
127713127753 .src_constraints = .{ .any_bool_vec, .any, .any },
127714127754 .patterns = &.{
127715 .{ .src = .{ .{ .all_reg_mask = .{ .size = .yword, .is = .uninverted } }, .none, .none } },
127755 .{ .src = .{ .{ .reg_mask = .{ .size = .yword, .kind = .sign_extend, .is = .uninverted } }, .none, .none } },
127716127756 },
127717127757 .dst_temps = .{ .{ .cc = .nz }, .unused },
127718127758 .clobbers = .{ .eflags = true },
......@@ -127724,7 +127764,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
127724127764 .dst_constraints = .{ .bool, .any },
127725127765 .src_constraints = .{ .any_bool_vec, .any, .any },
127726127766 .patterns = &.{
127727 .{ .src = .{ .{ .all_reg_mask = .{ .size = .yword } }, .none, .none } },
127767 .{ .src = .{ .{ .reg_mask = .{ .size = .yword, .kind = .sign_extend } }, .none, .none } },
127728127768 },
127729127769 .extra_temps = .{
127730127770 .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } },
......@@ -127750,7 +127790,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
127750127790 .dst_constraints = .{ .bool, .any },
127751127791 .src_constraints = .{ .any_bool_vec, .any, .any },
127752127792 .patterns = &.{
127753 .{ .src = .{ .{ .all_reg_mask = .{ .size = .yword } }, .none, .none } },
127793 .{ .src = .{ .{ .reg_mask = .{ .size = .yword, .kind = .sign_extend } }, .none, .none } },
127754127794 },
127755127795 .extra_temps = .{
127756127796 .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } },
......@@ -127776,7 +127816,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
127776127816 .dst_constraints = .{ .bool, .any },
127777127817 .src_constraints = .{ .any_bool_vec, .any, .any },
127778127818 .patterns = &.{
127779 .{ .src = .{ .{ .reg_mask = .{ .size = .yword } }, .none, .none } },
127819 .{ .src = .{ .{ .reg_mask = .{ .size = .yword, .kind = .msb } }, .none, .none } },
127780127820 },
127781127821 .extra_temps = .{
127782127822 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
......@@ -174528,6 +174568,7 @@ fn load(self: *CodeGen, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) InnerE
174528174568 .register_overflow,
174529174569 .register_mask,
174530174570 .indirect_load_frame,
174571 .indirect_mask,
174531174572 .register_tee,
174532174573 .elementwise_gpr,
174533174574 .elementwise_sse,
......@@ -174586,6 +174627,7 @@ fn store(
174586174627 .register_overflow,
174587174628 .register_mask,
174588174629 .indirect_load_frame,
174630 .indirect_mask,
174589174631 .register_tee,
174590174632 .elementwise_gpr,
174591174633 .elementwise_sse,
......@@ -174636,6 +174678,7 @@ fn genUnOpMir(self: *CodeGen, mir_tag: Mir.Inst.FixedTag, dst_ty: Type, dst_mcv:
174636174678 .register_overflow,
174637174679 .register_mask,
174638174680 .indirect_load_frame,
174681 .indirect_mask,
174639174682 .lea_frame,
174640174683 .lea_nav,
174641174684 .lea_uav,
......@@ -175336,6 +175379,7 @@ fn genBinOpMir(
175336175379 .lea_lazy_sym,
175337175380 .lea_extern_func,
175338175381 .indirect_load_frame,
175382 .indirect_mask,
175339175383 .register_tee,
175340175384 .elementwise_gpr,
175341175385 .elementwise_sse,
......@@ -175378,6 +175422,7 @@ fn genBinOpMir(
175378175422 .register_overflow,
175379175423 .register_mask,
175380175424 .indirect_load_frame,
175425 .indirect_mask,
175381175426 .register_tee,
175382175427 .elementwise_gpr,
175383175428 .elementwise_sse,
......@@ -175555,6 +175600,7 @@ fn genBinOpMir(
175555175600 .register_overflow,
175556175601 .register_mask,
175557175602 .indirect_load_frame,
175603 .indirect_mask,
175558175604 .register_tee,
175559175605 .elementwise_gpr,
175560175606 .elementwise_sse,
......@@ -175664,6 +175710,7 @@ fn genBinOpMir(
175664175710 .register_overflow,
175665175711 .register_mask,
175666175712 .indirect_load_frame,
175713 .indirect_mask,
175667175714 .register_tee,
175668175715 .elementwise_gpr,
175669175716 .elementwise_sse,
......@@ -175793,7 +175840,13 @@ fn airArg(cg: *CodeGen, inst: Air.Inst.Index) !void {
175793175840 const result: MCValue = if (cg.mod.strip and cg.liveness.isUnused(inst)) .unreach else result: {
175794175841 const arg_ty = cg.typeOfIndex(inst);
175795175842 switch (src_mcv) {
175796 .register, .register_pair, .load_frame => {
175843 .register,
175844 .register_pair,
175845 .register_triple,
175846 .register_quadruple,
175847 .register_mask,
175848 .load_frame,
175849 => {
175797175850 for (src_mcv.getRegs()) |reg| cg.register_manager.getRegAssumeFree(reg, inst);
175798175851 break :result src_mcv;
175799175852 },
......@@ -175803,6 +175856,22 @@ fn airArg(cg: *CodeGen, inst: Air.Inst.Index) !void {
175803175856 try cg.genCopy(arg_ty, dst_mcv, src_mcv, .{});
175804175857 break :result dst_mcv;
175805175858 },
175859 .indirect_mask => |reg_mask| {
175860 cg.register_manager.getRegAssumeFree(reg_mask.reg, null);
175861 const dst_reg = try cg.register_manager.allocReg(inst, abi.RegisterClass.sse);
175862 const mask_size: u32 = @intCast(
175863 @divExact(reg_mask.info.scalar.bitSize(cg.target), 8) * arg_ty.vectorLen(zcu),
175864 );
175865 try cg.asmRegisterMemory(
175866 .{ if (cg.hasFeature(.avx)) .v_dqa else ._dqa, .mov },
175867 registerAlias(dst_reg, mask_size),
175868 .{
175869 .base = .{ .reg = reg_mask.reg },
175870 .mod = .{ .rm = .{ .size = .fromSize(mask_size) } },
175871 },
175872 );
175873 break :result .{ .register_mask = .{ .reg = dst_reg, .info = reg_mask.info } };
175874 },
175806175875 .indirect_load_frame => |frame_addr| {
175807175876 const dst_mcv = try cg.allocRegOrMem(inst, false);
175808175877 const ptr_reg = try cg.register_manager.allocReg(null, abi.RegisterClass.gp);
......@@ -176265,6 +176334,10 @@ fn genCall(cg: *CodeGen, info: union(enum) {
176265176334 for (regs) |reg| try cg.register_manager.getReg(reg, null);
176266176335 try reg_locks.appendSlice(allocator, &cg.register_manager.lockRegs(regs.len, regs));
176267176336 },
176337 .register_mask => |reg_mask| {
176338 try cg.register_manager.getReg(reg_mask.reg, null);
176339 try reg_locks.append(allocator, cg.register_manager.lockReg(reg_mask.reg));
176340 },
176268176341 .indirect => |reg_off| {
176269176342 frame_index.* = try cg.allocFrameIndex(.initType(arg_ty, zcu));
176270176343 try cg.genSetMem(.{ .frame = frame_index.* }, 0, arg_ty, src_arg, opts);
......@@ -176282,6 +176355,44 @@ fn genCall(cg: *CodeGen, info: union(enum) {
176282176355 opts,
176283176356 );
176284176357 },
176358 .indirect_mask => |reg_mask| {
176359 var src = switch (src_arg) {
176360 else => try cg.tempInit(arg_ty, src_arg),
176361 .air_ref => |src_ref| try cg.tempFromOperand(src_ref, false),
176362 };
176363 var dst = try cg.tempInit(arg_ty, .{ .register_mask = .{
176364 .reg = try cg.register_manager.allocReg(null, abi.RegisterClass.sse),
176365 .info = reg_mask.info,
176366 } });
176367 dst.copyToMask(&src, cg) catch |err| switch (err) {
176368 error.SelectFailed => return cg.fail("failed to select arg {f} {f} {f}", .{
176369 arg_ty.fmt(pt),
176370 dst.tracking(cg),
176371 src.tracking(cg),
176372 }),
176373 else => |e| return e,
176374 };
176375 try src.die(cg);
176376 const mask_size: u32 = @intCast(
176377 @divExact(reg_mask.info.scalar.bitSize(cg.target), 8) * arg_ty.vectorLen(zcu),
176378 );
176379 frame_index.* = try cg.allocFrameIndex(.init(.{
176380 .size = mask_size,
176381 .alignment = .fromByteUnits(std.math.ceilPowerOfTwoAssert(u32, mask_size)),
176382 }));
176383 try cg.asmMemoryRegister(
176384 .{ if (cg.hasFeature(.avx)) .v_dqa else ._dqa, .mov },
176385 .{
176386 .base = .{ .frame = frame_index.* },
176387 .mod = .{ .rm = .{ .size = .fromSize(mask_size) } },
176388 },
176389 registerAlias(dst.tracking(cg).short.register_mask.reg, mask_size),
176390 );
176391 try dst.die(cg);
176392
176393 try cg.register_manager.getReg(reg_mask.reg, null);
176394 try reg_locks.append(allocator, cg.register_manager.lockReg(reg_mask.reg));
176395 },
176285176396 .load_frame => {
176286176397 try cg.genCopy(arg_ty, dst_arg, src_arg, opts);
176287176398 try cg.freeValue(src_arg, .{});
......@@ -176396,7 +176507,7 @@ fn genCall(cg: *CodeGen, info: union(enum) {
176396176507
176397176508 var frame_offset: i32 = regs_frame_addr.info.frame_off;
176398176509 var arg_offset = elem_size * param_gpr_len;
176399 while (arg_size - arg_offset != 0) : ({
176510 while (arg_size - arg_offset > 0) : ({
176400176511 frame_offset += 8;
176401176512 arg_offset += elem_size;
176402176513 }) try cg.genSetMem(
......@@ -176458,8 +176569,28 @@ fn genCall(cg: *CodeGen, info: union(enum) {
176458176569 .register_triple,
176459176570 .register_quadruple,
176460176571 => try cg.genCopy(arg_ty, dst_arg, src_arg, opts),
176461 .indirect => |reg_off| try cg.genSetReg(reg_off.reg, .usize, .{
176462 .lea_frame = .{ .index = frame_index, .off = -reg_off.off },
176572 .register_mask => {
176573 var src = switch (src_arg) {
176574 else => try cg.tempInit(arg_ty, src_arg),
176575 .air_ref => |src_ref| try cg.tempFromOperand(src_ref, false),
176576 };
176577 var dst = try cg.tempInit(arg_ty, dst_arg);
176578 dst.copyToMask(&src, cg) catch |err| switch (err) {
176579 error.SelectFailed => return cg.fail("failed to select arg {f} {f} {f}", .{
176580 arg_ty.fmt(pt),
176581 dst.tracking(cg),
176582 src.tracking(cg),
176583 }),
176584 else => |e| return e,
176585 };
176586 try src.die(cg);
176587 try dst.die(cg);
176588 },
176589 .indirect => |dst_reg_off| try cg.genSetReg(dst_reg_off.reg, .usize, .{
176590 .lea_frame = .{ .index = frame_index, .off = -dst_reg_off.off },
176591 }, opts),
176592 .indirect_mask => |dst_reg_mask| try cg.genSetReg(dst_reg_mask.reg, .usize, .{
176593 .lea_frame = .{ .index = frame_index },
176463176594 }, opts),
176464176595 .register_tee => |dst_regs| {
176465176596 try cg.genSetReg(dst_regs[0], arg_ty, src_arg, opts);
......@@ -176570,31 +176701,43 @@ fn genCall(cg: *CodeGen, info: union(enum) {
176570176701 return call_info.return_value.short;
176571176702}
176572176703
176573fn airRet(self: *CodeGen, inst: Air.Inst.Index, safety: bool) !void {
176574 const pt = self.pt;
176575 const zcu = pt.zcu;
176576 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
176704fn airRet(cg: *CodeGen, inst: Air.Inst.Index, safety: bool) !void {
176705 const un_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
176577176706
176578 const ret_ty = self.fn_type.fnReturnType(zcu);
176579 switch (self.ret_mcv.short) {
176707 const ret_ty = cg.fn_type.fnReturnType(cg.pt.zcu);
176708 switch (cg.ret_mcv.short) {
176580176709 .none => {},
176581176710 .register => |reg| {
176582 const reg_lock = self.register_manager.lockRegAssumeUnused(reg);
176583 defer self.register_manager.unlockReg(reg_lock);
176584 try self.genCopy(ret_ty, self.ret_mcv.short, .{ .air_ref = un_op }, .{ .safety = safety });
176711 const reg_lock = cg.register_manager.lockRegAssumeUnused(reg);
176712 defer cg.register_manager.unlockReg(reg_lock);
176713 try cg.genCopy(ret_ty, cg.ret_mcv.short, .{ .air_ref = un_op }, .{ .safety = safety });
176585176714 },
176586176715 inline .register_pair, .register_triple, .register_quadruple => |regs| {
176587 const reg_locks = self.register_manager.lockRegsAssumeUnused(regs.len, regs);
176588 defer for (reg_locks) |reg_lock| self.register_manager.unlockReg(reg_lock);
176589 try self.genCopy(ret_ty, self.ret_mcv.short, .{ .air_ref = un_op }, .{ .safety = safety });
176716 const reg_locks = cg.register_manager.lockRegsAssumeUnused(regs.len, regs);
176717 defer for (reg_locks) |reg_lock| cg.register_manager.unlockReg(reg_lock);
176718 try cg.genCopy(ret_ty, cg.ret_mcv.short, .{ .air_ref = un_op }, .{ .safety = safety });
176719 },
176720 .register_mask => {
176721 var src = try cg.tempFromOperand(un_op, true);
176722 var dst = try cg.tempInit(ret_ty, cg.ret_mcv.short);
176723 dst.copyToMask(&src, cg) catch |err| switch (err) {
176724 error.SelectFailed => return cg.fail("failed to select ret {f} {f} {f}", .{
176725 ret_ty.fmt(cg.pt),
176726 dst.tracking(cg),
176727 src.tracking(cg),
176728 }),
176729 else => |e| return e,
176730 };
176731 try src.die(cg);
176732 try dst.die(cg);
176590176733 },
176591176734 .indirect => |reg_off| {
176592 try self.register_manager.getReg(reg_off.reg, null);
176593 const lock = self.register_manager.lockRegAssumeUnused(reg_off.reg);
176594 defer self.register_manager.unlockReg(lock);
176735 try cg.register_manager.getReg(reg_off.reg, null);
176736 const lock = cg.register_manager.lockRegAssumeUnused(reg_off.reg);
176737 defer cg.register_manager.unlockReg(lock);
176595176738
176596 try self.genSetReg(reg_off.reg, .usize, self.ret_mcv.long, .{});
176597 try self.genSetMem(
176739 try cg.genSetReg(reg_off.reg, .usize, cg.ret_mcv.long, .{});
176740 try cg.genSetMem(
176598176741 .{ .reg = reg_off.reg },
176599176742 reg_off.off,
176600176743 ret_ty,
......@@ -176604,59 +176747,78 @@ fn airRet(self: *CodeGen, inst: Air.Inst.Index, safety: bool) !void {
176604176747 },
176605176748 else => unreachable,
176606176749 }
176607 self.ret_mcv.liveOut(self, inst);
176750 cg.ret_mcv.liveOut(cg, inst);
176608176751
176609 if (self.err_ret_trace_reg != .none) {
176610 if (self.inst_tracking.getPtr(err_ret_trace_index)) |err_ret_trace| {
176752 if (cg.err_ret_trace_reg != .none) {
176753 if (cg.inst_tracking.getPtr(err_ret_trace_index)) |err_ret_trace| {
176611176754 if (switch (err_ret_trace.short) {
176612 .register => |reg| self.err_ret_trace_reg != reg,
176755 .register => |reg| cg.err_ret_trace_reg != reg,
176613176756 else => true,
176614 }) try self.genSetReg(self.err_ret_trace_reg, .usize, err_ret_trace.short, .{});
176615 err_ret_trace.liveOut(self, err_ret_trace_index);
176757 }) try cg.genSetReg(cg.err_ret_trace_reg, .usize, err_ret_trace.short, .{});
176758 err_ret_trace.liveOut(cg, err_ret_trace_index);
176616176759 }
176617176760 }
176618176761
176619 try self.finishAir(inst, .unreach, .{ un_op, .none, .none });
176762 try cg.finishAir(inst, .unreach, .{ un_op, .none, .none });
176620176763
176621176764 // TODO optimization opportunity: figure out when we can emit this as a 2 byte instruction
176622176765 // which is available if the jump is 127 bytes or less forward.
176623 const jmp_reloc = try self.asmJmpReloc(undefined);
176624 try self.epilogue_relocs.append(self.gpa, jmp_reloc);
176766 const jmp_reloc = try cg.asmJmpReloc(undefined);
176767 try cg.epilogue_relocs.append(cg.gpa, jmp_reloc);
176625176768}
176626176769
176627fn airRetLoad(self: *CodeGen, inst: Air.Inst.Index) !void {
176628 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
176629 const ptr = try self.resolveInst(un_op);
176630
176631 const ptr_ty = self.typeOf(un_op);
176632 switch (self.ret_mcv.short) {
176770fn airRetLoad(cg: *CodeGen, inst: Air.Inst.Index) !void {
176771 const un_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
176772 switch (cg.ret_mcv.short) {
176633176773 .none => {},
176634176774 .register,
176635176775 .register_pair,
176636176776 .register_triple,
176637176777 .register_quadruple,
176638 => try self.load(self.ret_mcv.short, ptr_ty, ptr),
176639 .indirect => |reg_off| try self.genSetReg(reg_off.reg, ptr_ty, ptr, .{}),
176778 => try cg.load(cg.ret_mcv.short, cg.typeOf(un_op), try cg.resolveInst(un_op)),
176779 .register_mask => {
176780 var ptr = try cg.tempFromOperand(un_op, true);
176781 const ret_ty = ptr.typeOf(cg).childType(cg.pt.zcu);
176782 var src = try ptr.load(ret_ty, .{}, cg);
176783 try ptr.die(cg);
176784 var dst = try cg.tempInit(ret_ty, cg.ret_mcv.short);
176785 dst.copyToMask(&src, cg) catch |err| switch (err) {
176786 error.SelectFailed => return cg.fail("failed to select ret_load {f} {f} {f}", .{
176787 ret_ty.fmt(cg.pt),
176788 dst.tracking(cg),
176789 src.tracking(cg),
176790 }),
176791 else => |e| return e,
176792 };
176793 try src.die(cg);
176794 try dst.die(cg);
176795 },
176796 .indirect => |dst_reg_off| try cg.genSetReg(
176797 dst_reg_off.reg,
176798 cg.typeOf(un_op),
176799 try cg.resolveInst(un_op),
176800 .{},
176801 ),
176640176802 else => unreachable,
176641176803 }
176642 self.ret_mcv.liveOut(self, inst);
176804 cg.ret_mcv.liveOut(cg, inst);
176643176805
176644 if (self.err_ret_trace_reg != .none) {
176645 if (self.inst_tracking.getPtr(err_ret_trace_index)) |err_ret_trace| {
176806 if (cg.err_ret_trace_reg != .none) {
176807 if (cg.inst_tracking.getPtr(err_ret_trace_index)) |err_ret_trace| {
176646176808 if (switch (err_ret_trace.short) {
176647 .register => |reg| self.err_ret_trace_reg != reg,
176809 .register => |reg| cg.err_ret_trace_reg != reg,
176648176810 else => true,
176649 }) try self.genSetReg(self.err_ret_trace_reg, .usize, err_ret_trace.short, .{});
176650 err_ret_trace.liveOut(self, err_ret_trace_index);
176811 }) try cg.genSetReg(cg.err_ret_trace_reg, .usize, err_ret_trace.short, .{});
176812 err_ret_trace.liveOut(cg, err_ret_trace_index);
176651176813 }
176652176814 }
176653176815
176654 try self.finishAir(inst, .unreach, .{ un_op, .none, .none });
176816 try cg.finishAir(inst, .unreach, .{ un_op, .none, .none });
176655176817
176656176818 // TODO optimization opportunity: figure out when we can emit this as a 2 byte instruction
176657176819 // which is available if the jump is 127 bytes or less forward.
176658 const jmp_reloc = try self.asmJmpReloc(undefined);
176659 try self.epilogue_relocs.append(self.gpa, jmp_reloc);
176820 const jmp_reloc = try cg.asmJmpReloc(undefined);
176821 try cg.epilogue_relocs.append(cg.gpa, jmp_reloc);
176660176822}
176661176823
176662176824fn airTry(self: *CodeGen, inst: Air.Inst.Index) !void {
......@@ -178640,6 +178802,7 @@ fn genCopy(self: *CodeGen, ty: Type, dst_mcv: MCValue, src_mcv: MCValue, opts: C
178640178802 .register_overflow,
178641178803 .register_mask,
178642178804 .indirect_load_frame,
178805 .indirect_mask,
178643178806 .lea_frame,
178644178807 .lea_nav,
178645178808 .lea_uav,
......@@ -178853,6 +179016,7 @@ fn genSetReg(
178853179016 .unreach,
178854179017 .dead,
178855179018 .indirect_load_frame,
179019 .indirect_mask,
178856179020 .register_tee,
178857179021 .elementwise_gpr,
178858179022 .elementwise_sse,
......@@ -179282,33 +179446,105 @@ fn genSetReg(
179282179446 const bits_lock = cg.register_manager.lockReg(bits_reg);
179283179447 defer if (bits_lock) |lock| cg.register_manager.unlockReg(lock);
179284179448
179285 const pack_reg = switch (src_reg_mask.info.scalar) {
179286 else => src_reg_mask.reg,
179287 .word => try cg.register_manager.allocReg(null, abi.RegisterClass.sse),
179449 var mask_size: u32 = @intCast(
179450 @divExact(src_reg_mask.info.scalar.bitSize(cg.target), 8) * ty.vectorLen(zcu),
179451 );
179452 const sign_reg = switch (src_reg_mask.info.kind) {
179453 .lsb => sign_reg: {
179454 const sign_reg = try cg.register_manager.allocReg(null, abi.RegisterClass.sse);
179455 const src_alias = registerAlias(src_reg_mask.reg, mask_size);
179456 const sign_alias = registerAlias(sign_reg, mask_size);
179457 if (cg.hasFeature(.avx2)) try cg.asmRegisterRegisterImmediate(
179458 .{ switch (src_reg_mask.info.scalar) {
179459 else => unreachable,
179460 .byte, .word => .vp_w,
179461 .dword => .vp_d,
179462 .qword => .vp_q,
179463 }, .sll },
179464 sign_alias,
179465 src_alias,
179466 .u(src_reg_mask.info.scalar.bitSize(cg.target) - 1),
179467 ) else {
179468 if (sign_alias != src_alias) try cg.asmRegisterRegister(
179469 .{ ._dqa, .mov },
179470 sign_alias,
179471 src_alias,
179472 );
179473 try cg.asmRegisterImmediate(
179474 .{ switch (src_reg_mask.info.scalar) {
179475 else => unreachable,
179476 .byte, .word => .p_w,
179477 .dword => .p_d,
179478 .qword => .p_q,
179479 }, .sll },
179480 sign_alias,
179481 .u(src_reg_mask.info.scalar.bitSize(cg.target) - 1),
179482 );
179483 }
179484 break :sign_reg sign_reg;
179485 },
179486 .zero_extend => sign_reg: {
179487 const sign_reg = try cg.register_manager.allocReg(null, abi.RegisterClass.sse);
179488 const src_alias = registerAlias(src_reg_mask.reg, mask_size);
179489 const sign_alias = registerAlias(sign_reg, mask_size);
179490 if (has_avx) {
179491 try cg.asmRegisterRegisterRegister(
179492 .{ .vp_, .xor },
179493 sign_alias,
179494 sign_alias,
179495 sign_alias,
179496 );
179497 try cg.asmRegisterRegisterRegister(.{ switch (src_reg_mask.info.scalar) {
179498 else => unreachable,
179499 .byte => .vp_b,
179500 .word => .vp_w,
179501 .dword => .vp_d,
179502 .qword => .vp_q,
179503 }, .sub }, sign_alias, sign_alias, src_alias);
179504 } else {
179505 try cg.asmRegisterRegister(.{ .p_, .xor }, sign_alias, sign_alias);
179506 try cg.asmRegisterRegister(.{ switch (src_reg_mask.info.scalar) {
179507 else => unreachable,
179508 .byte => .p_b,
179509 .word => .p_w,
179510 .dword => .p_d,
179511 .qword => .p_q,
179512 }, .sub }, sign_alias, src_alias);
179513 }
179514 break :sign_reg sign_reg;
179515 },
179516 .msb, .sign_extend => src_reg_mask.reg,
179288179517 };
179289 const pack_lock = cg.register_manager.lockReg(pack_reg);
179290 defer if (pack_lock) |lock| cg.register_manager.unlockReg(lock);
179518 const sign_lock = cg.register_manager.lockReg(sign_reg);
179519 defer if (sign_lock) |lock| cg.register_manager.unlockReg(lock);
179291179520
179292 var mask_size: u32 = @intCast(ty.vectorLen(zcu) * @divExact(src_reg_mask.info.scalar.bitSize(cg.target), 8));
179293 switch (src_reg_mask.info.scalar) {
179294 else => {},
179295 .word => {
179296 const src_alias = registerAlias(src_reg_mask.reg, mask_size);
179521 const pack_reg = switch (src_reg_mask.info.scalar) {
179522 else => sign_reg,
179523 .word => pack_reg: {
179524 const pack_reg = if (sign_reg == src_reg_mask.reg)
179525 try cg.register_manager.allocReg(null, abi.RegisterClass.sse)
179526 else
179527 sign_reg;
179528 const sign_alias = registerAlias(sign_reg, mask_size);
179297179529 const pack_alias = registerAlias(pack_reg, mask_size);
179298179530 if (has_avx) {
179299 try cg.asmRegisterRegisterRegister(.{ .vp_b, .ackssw }, pack_alias, src_alias, src_alias);
179531 try cg.asmRegisterRegisterRegister(.{ .vp_b, .ackssw }, pack_alias, sign_alias, sign_alias);
179300179532 } else {
179301 try cg.asmRegisterRegister(.{ ._dqa, .mov }, pack_alias, src_alias);
179533 if (pack_alias != sign_alias) try cg.asmRegisterRegister(.{ ._dqa, .mov }, pack_alias, sign_alias);
179302179534 try cg.asmRegisterRegister(.{ .p_b, .ackssw }, pack_alias, pack_alias);
179303179535 }
179304179536 mask_size = std.math.divCeil(u32, mask_size, 2) catch unreachable;
179537 break :pack_reg pack_reg;
179305179538 },
179306 }
179539 };
179540 const pack_lock = cg.register_manager.lockReg(pack_reg);
179541 defer if (pack_lock) |lock| cg.register_manager.unlockReg(lock);
179542
179307179543 try cg.asmRegisterRegister(.{ switch (src_reg_mask.info.scalar) {
179544 else => unreachable,
179308179545 .byte, .word => if (has_avx) .vp_b else .p_b,
179309179546 .dword => if (has_avx) .v_ps else ._ps,
179310179547 .qword => if (has_avx) .v_pd else ._pd,
179311 else => unreachable,
179312179548 }, .movmsk }, bits_reg.to32(), registerAlias(pack_reg, mask_size));
179313179549 if (src_reg_mask.info.inverted) try cg.asmRegister(.{ ._, .not }, registerAlias(bits_reg, abi_size));
179314179550 try cg.genSetReg(dst_reg, ty, .{ .register = bits_reg }, .{});
......@@ -179432,6 +179668,7 @@ fn genSetMem(
179432179668 .unreach,
179433179669 .dead,
179434179670 .indirect_load_frame,
179671 .indirect_mask,
179435179672 .register_tee,
179436179673 .elementwise_gpr,
179437179674 .elementwise_sse,
......@@ -180697,32 +180934,32 @@ fn airSelect(self: *CodeGen, inst: Air.Inst.Index) !void {
180697180934 const dst_lock = self.register_manager.lockReg(dst_reg);
180698180935 defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);
180699180936
180700 const mir_tag = @as(?Mir.Inst.FixedTag, if ((pred_reg_mask.info.kind == .all and
180937 const mir_tag = @as(?Mir.Inst.FixedTag, if ((pred_reg_mask.info.kind == .sign_extend and
180701180938 elem_ty.toIntern() != .f32_type and elem_ty.toIntern() != .f64_type) or pred_reg_mask.info.scalar == .byte)
180702180939 if (has_avx)
180703180940 .{ .vp_b, .blendv }
180704180941 else if (has_blend)
180705180942 .{ .p_b, .blendv }
180706 else if (pred_reg_mask.info.kind == .all)
180943 else if (pred_reg_mask.info.kind == .sign_extend)
180707180944 .{ .p_, undefined }
180708180945 else
180709180946 null
180710 else if ((pred_reg_mask.info.kind == .all and (elem_ty.toIntern() != .f64_type or !self.hasFeature(.sse2))) or
180947 else if ((pred_reg_mask.info.kind == .sign_extend and (elem_ty.toIntern() != .f64_type or !self.hasFeature(.sse2))) or
180711180948 pred_reg_mask.info.scalar == .dword)
180712180949 if (has_avx)
180713180950 .{ .v_ps, .blendv }
180714180951 else if (has_blend)
180715180952 .{ ._ps, .blendv }
180716 else if (pred_reg_mask.info.kind == .all)
180953 else if (pred_reg_mask.info.kind == .sign_extend)
180717180954 .{ ._ps, undefined }
180718180955 else
180719180956 null
180720 else if (pred_reg_mask.info.kind == .all or pred_reg_mask.info.scalar == .qword)
180957 else if (pred_reg_mask.info.kind == .sign_extend or pred_reg_mask.info.scalar == .qword)
180721180958 if (has_avx)
180722180959 .{ .v_pd, .blendv }
180723180960 else if (has_blend)
180724180961 .{ ._pd, .blendv }
180725 else if (pred_reg_mask.info.kind == .all)
180962 else if (pred_reg_mask.info.kind == .sign_extend)
180726180963 .{ ._pd, undefined }
180727180964 else
180728180965 null
......@@ -181635,6 +181872,17 @@ fn resolveCallingConventionValues(
181635181872 ret_gpr = ret_gpr[1..];
181636181873 param_gpr_index += 1;
181637181874 },
181875 .bool_vector_mask => {
181876 const len = ret_ty.vectorLen(zcu);
181877 const elem_size =
181878 @divExact(16, std.math.ceilPowerOfTwoAssert(u32, @min(len, 16)));
181879 ret_tracking[ret_tracking_len] = .init(.{ .register_mask = .{
181880 .reg = registerAlias(ret_sse[0], elem_size * len),
181881 .info = .{ .kind = .lsb, .scalar = .fromSize(elem_size) },
181882 } });
181883 ret_tracking_len += 1;
181884 ret_sse = ret_sse[1..];
181885 },
181638181886 .integer_per_element => {
181639181887 const len: u32 = @intCast(ret_ty.vectorLen(zcu));
181640181888 const alias_size: u32 = @intCast(@min(ret_ty.childType(zcu).abiSize(zcu), 8));
......@@ -181809,6 +182057,17 @@ fn resolveCallingConventionValues(
181809182057 else => unreachable,
181810182058 },
181811182059 .none => {},
182060 .bool_vector_mask => {
182061 arg_mcv[arg_mcv_len] = .{ .indirect_mask = .{
182062 .reg = param_gpr[param_gpr_index].to64(),
182063 .info = .{ .kind = .lsb, .scalar = .fromSize(@divExact(
182064 16,
182065 std.math.ceilPowerOfTwoAssert(u32, @min(ty.vectorLen(zcu), 16)),
182066 )) },
182067 } };
182068 arg_mcv_len += 1;
182069 param_gpr_index += 1;
182070 },
181812182071 .integer_per_element,
181813182072 .sse_per_element,
181814182073 .sse_per_xword,
......@@ -182441,6 +182700,7 @@ const Temp = struct {
182441182700 .memory,
182442182701 .indirect,
182443182702 .indirect_load_frame,
182703 .indirect_mask,
182444182704 .lea_frame,
182445182705 .load_nav,
182446182706 .lea_nav,
......@@ -182918,6 +183178,7 @@ const Temp = struct {
182918183178 .register_overflow,
182919183179 .register_mask,
182920183180 .indirect_load_frame,
183181 .indirect_mask,
182921183182 .register_tee,
182922183183 .elementwise_gpr,
182923183184 .elementwise_sse,
......@@ -183423,6 +183684,7 @@ const Temp = struct {
183423183684 .x87up,
183424183685 .none,
183425183686 .win_i128,
183687 .bool_vector_mask,
183426183688 .integer_per_element,
183427183689 .sse_per_element,
183428183690 .sse_sse_x87_per_qword,
......@@ -183440,6 +183702,7 @@ const Temp = struct {
183440183702 .x87up,
183441183703 .none,
183442183704 .win_i128,
183705 .bool_vector_mask,
183443183706 .integer_per_element,
183444183707 .sse_per_element,
183445183708 .sse_sse_x87_per_qword,
......@@ -183464,6 +183727,7 @@ const Temp = struct {
183464183727 .memory,
183465183728 .none,
183466183729 .win_i128,
183730 .bool_vector_mask,
183467183731 .integer_per_element,
183468183732 .sse_per_element,
183469183733 .sse_sse_x87_per_qword,
......@@ -183502,6 +183766,1331 @@ const Temp = struct {
183502183766 try cg.asmOpOnly(.{ .@"rep _sb", .sto });
183503183767 }
183504183768
183769 fn copyToMask(dst: *Temp, src: *Temp, cg: *CodeGen) Select.Error!void {
183770 var ops: [2]Temp = .{ dst.*, src.* };
183771 try cg.select(&.{}, &.{}, &ops, comptime &.{ .{
183772 .required_features = .{ .avx2, null, null, null },
183773 .src_constraints = .{ .{ .exact_bool_vec = 2 }, .{ .exact_bool_vec = 2 }, .any },
183774 .patterns = &.{
183775 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .from_sign_extend, .is = .uninverted } }, .mem, .none } },
183776 },
183777 .extra_temps = .{
183778 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward, .smear = 8 } } },
183779 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
183780 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
183781 .unused,
183782 .unused,
183783 .unused,
183784 .unused,
183785 .unused,
183786 .unused,
183787 .unused,
183788 .unused,
183789 },
183790 .each = .{ .once = &.{
183791 .{ ._, .vp_b, .broadcast, .src0x, .src1b, ._, ._ },
183792 .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ },
183793 .{ ._, .v_dqa, .mov, .tmp2x, .lea(.tmp1x), ._, ._ },
183794 .{ ._, .vp_, .@"and", .src0x, .src0x, .tmp2x, ._ },
183795 .{ ._, .vp_b, .cmpeq, .src0x, .src0x, .tmp2x, ._ },
183796 } },
183797 }, .{
183798 .required_features = .{ .avx2, null, null, null },
183799 .src_constraints = .{ .{ .exact_bool_vec = 2 }, .{ .exact_bool_vec = 2 }, .any },
183800 .patterns = &.{
183801 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .from_sign_extend, .is = .uninverted } }, .to_sse, .none } },
183802 },
183803 .extra_temps = .{
183804 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward, .smear = 8 } } },
183805 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
183806 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
183807 .unused,
183808 .unused,
183809 .unused,
183810 .unused,
183811 .unused,
183812 .unused,
183813 .unused,
183814 .unused,
183815 },
183816 .each = .{ .once = &.{
183817 .{ ._, .vp_b, .broadcast, .src0x, .src1x, ._, ._ },
183818 .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ },
183819 .{ ._, .v_dqa, .mov, .tmp2x, .lea(.tmp1x), ._, ._ },
183820 .{ ._, .vp_, .@"and", .src0x, .src0x, .tmp2x, ._ },
183821 .{ ._, .vp_b, .cmpeq, .src0x, .src0x, .tmp2x, ._ },
183822 } },
183823 }, .{
183824 .required_features = .{ .avx2, null, null, null },
183825 .src_constraints = .{ .{ .exact_bool_vec = 2 }, .{ .exact_bool_vec = 2 }, .any },
183826 .patterns = &.{
183827 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .from_zero_extend, .is = .uninverted } }, .mem, .none } },
183828 },
183829 .extra_temps = .{
183830 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward, .smear = 8 } } },
183831 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
183832 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
183833 .unused,
183834 .unused,
183835 .unused,
183836 .unused,
183837 .unused,
183838 .unused,
183839 .unused,
183840 .unused,
183841 },
183842 .each = .{ .once = &.{
183843 .{ ._, .vp_b, .broadcast, .src0x, .src1b, ._, ._ },
183844 .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ },
183845 .{ ._, .v_dqa, .mov, .tmp2x, .lea(.tmp1x), ._, ._ },
183846 .{ ._, .vp_, .@"and", .src0x, .src0x, .tmp2x, ._ },
183847 .{ ._, .vp_b, .cmpeq, .src0x, .src0x, .tmp2x, ._ },
183848 .{ ._, .vp_, .xor, .tmp2x, .tmp2x, .tmp2x, ._ },
183849 .{ ._, .vp_q, .sub, .src0x, .tmp2x, .src0x, ._ },
183850 } },
183851 }, .{
183852 .required_features = .{ .avx2, null, null, null },
183853 .src_constraints = .{ .{ .exact_bool_vec = 2 }, .{ .exact_bool_vec = 2 }, .any },
183854 .patterns = &.{
183855 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .from_zero_extend, .is = .uninverted } }, .to_sse, .none } },
183856 },
183857 .extra_temps = .{
183858 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward, .smear = 8 } } },
183859 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
183860 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
183861 .unused,
183862 .unused,
183863 .unused,
183864 .unused,
183865 .unused,
183866 .unused,
183867 .unused,
183868 .unused,
183869 },
183870 .each = .{ .once = &.{
183871 .{ ._, .vp_b, .broadcast, .src0x, .src1x, ._, ._ },
183872 .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ },
183873 .{ ._, .v_dqa, .mov, .tmp2x, .lea(.tmp1x), ._, ._ },
183874 .{ ._, .vp_, .@"and", .src0x, .src0x, .tmp2x, ._ },
183875 .{ ._, .vp_b, .cmpeq, .src0x, .src0x, .tmp2x, ._ },
183876 .{ ._, .vp_, .xor, .tmp2x, .tmp2x, .tmp2x, ._ },
183877 .{ ._, .vp_q, .sub, .src0x, .tmp2x, .src0x, ._ },
183878 } },
183879 }, .{
183880 .required_features = .{ .avx, null, null, null },
183881 .src_constraints = .{ .{ .exact_bool_vec = 2 }, .{ .exact_bool_vec = 2 }, .any },
183882 .patterns = &.{
183883 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .from_sign_extend, .is = .uninverted } }, .to_sse, .none } },
183884 },
183885 .extra_temps = .{
183886 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward, .smear = 8 } } },
183887 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
183888 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
183889 .unused,
183890 .unused,
183891 .unused,
183892 .unused,
183893 .unused,
183894 .unused,
183895 .unused,
183896 .unused,
183897 },
183898 .each = .{ .once = &.{
183899 .{ ._, .vp_, .xor, .tmp2x, .tmp2x, .tmp2x, ._ },
183900 .{ ._, .vp_b, .shuf, .src0x, .src1x, .tmp2x, ._ },
183901 .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ },
183902 .{ ._, .v_dqa, .mov, .tmp2x, .lea(.tmp1x), ._, ._ },
183903 .{ ._, .vp_, .@"and", .src0x, .src0x, .tmp2x, ._ },
183904 .{ ._, .vp_b, .cmpeq, .src0x, .src0x, .tmp2x, ._ },
183905 } },
183906 }, .{
183907 .required_features = .{ .avx, null, null, null },
183908 .src_constraints = .{ .{ .exact_bool_vec = 2 }, .{ .exact_bool_vec = 2 }, .any },
183909 .patterns = &.{
183910 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .from_zero_extend, .is = .uninverted } }, .to_sse, .none } },
183911 },
183912 .extra_temps = .{
183913 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward, .smear = 8 } } },
183914 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
183915 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
183916 .unused,
183917 .unused,
183918 .unused,
183919 .unused,
183920 .unused,
183921 .unused,
183922 .unused,
183923 .unused,
183924 },
183925 .each = .{ .once = &.{
183926 .{ ._, .vp_, .xor, .tmp2x, .tmp2x, .tmp2x, ._ },
183927 .{ ._, .vp_b, .shuf, .src0x, .src1x, .tmp2x, ._ },
183928 .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ },
183929 .{ ._, .v_dqa, .mov, .tmp2x, .lea(.tmp1x), ._, ._ },
183930 .{ ._, .vp_, .@"and", .src0x, .src0x, .tmp2x, ._ },
183931 .{ ._, .vp_b, .cmpeq, .src0x, .src0x, .tmp2x, ._ },
183932 .{ ._, .vp_, .xor, .tmp2x, .tmp2x, .tmp2x, ._ },
183933 .{ ._, .vp_q, .sub, .src0x, .tmp2x, .src0x, ._ },
183934 } },
183935 }, .{
183936 .required_features = .{ .ssse3, null, null, null },
183937 .src_constraints = .{ .{ .exact_bool_vec = 2 }, .{ .exact_bool_vec = 2 }, .any },
183938 .patterns = &.{
183939 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .from_sign_extend, .is = .uninverted } }, .to_sse, .none } },
183940 },
183941 .extra_temps = .{
183942 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward, .smear = 8 } } },
183943 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
183944 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
183945 .unused,
183946 .unused,
183947 .unused,
183948 .unused,
183949 .unused,
183950 .unused,
183951 .unused,
183952 .unused,
183953 },
183954 .each = .{ .once = &.{
183955 .{ ._, .p_, .xor, .tmp2x, .tmp2x, ._, ._ },
183956 .{ ._, ._dqa, .mov, .src0x, .src1x, ._, ._ },
183957 .{ ._, .p_b, .shuf, .src0x, .tmp2x, ._, ._ },
183958 .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ },
183959 .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp1x), ._, ._ },
183960 .{ ._, .p_, .@"and", .src0x, .tmp2x, ._, ._ },
183961 .{ ._, .p_b, .cmpeq, .src0x, .tmp2x, ._, ._ },
183962 } },
183963 }, .{
183964 .required_features = .{ .ssse3, null, null, null },
183965 .src_constraints = .{ .{ .exact_bool_vec = 2 }, .{ .exact_bool_vec = 2 }, .any },
183966 .patterns = &.{
183967 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .from_zero_extend, .is = .uninverted } }, .to_mut_sse, .none } },
183968 },
183969 .extra_temps = .{
183970 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward, .smear = 8 } } },
183971 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
183972 .unused,
183973 .unused,
183974 .unused,
183975 .unused,
183976 .unused,
183977 .unused,
183978 .unused,
183979 .unused,
183980 .unused,
183981 },
183982 .each = .{ .once = &.{
183983 .{ ._, .p_, .xor, .src0x, .src0x, ._, ._ },
183984 .{ ._, .p_b, .shuf, .src1x, .src0x, ._, ._ },
183985 .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ },
183986 .{ ._, ._dqa, .mov, .src0x, .lea(.tmp1x), ._, ._ },
183987 .{ ._, .p_, .@"and", .src1x, .src0x, ._, ._ },
183988 .{ ._, .p_b, .cmpeq, .src1x, .src0x, ._, ._ },
183989 .{ ._, .p_, .xor, .src0x, .src0x, ._, ._ },
183990 .{ ._, .p_q, .sub, .src0x, .src1x, ._, ._ },
183991 } },
183992 }, .{
183993 .required_features = .{ .sse2, null, null, null },
183994 .src_constraints = .{ .{ .exact_bool_vec = 2 }, .{ .exact_bool_vec = 2 }, .any },
183995 .patterns = &.{
183996 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .from_sign_extend, .is = .uninverted } }, .to_sse, .none } },
183997 },
183998 .extra_temps = .{
183999 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward, .smear = 8 } } },
184000 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
184001 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
184002 .unused,
184003 .unused,
184004 .unused,
184005 .unused,
184006 .unused,
184007 .unused,
184008 .unused,
184009 .unused,
184010 },
184011 .each = .{ .once = &.{
184012 .{ ._, ._dqa, .mov, .src0x, .src1x, ._, ._ },
184013 .{ ._, .p_, .unpcklbw, .src0x, .src0x, ._, ._ },
184014 .{ ._, .p_, .unpcklwd, .src0x, .src0x, ._, ._ },
184015 .{ ._, .p_, .unpckldq, .src0x, .src0x, ._, ._ },
184016 .{ ._, .p_, .unpcklqdq, .src0x, .src0x, ._, ._ },
184017 .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ },
184018 .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp1x), ._, ._ },
184019 .{ ._, .p_, .@"and", .src0x, .tmp2x, ._, ._ },
184020 .{ ._, .p_b, .cmpeq, .src0x, .tmp2x, ._, ._ },
184021 } },
184022 }, .{
184023 .required_features = .{ .sse2, null, null, null },
184024 .src_constraints = .{ .{ .exact_bool_vec = 2 }, .{ .exact_bool_vec = 2 }, .any },
184025 .patterns = &.{
184026 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .from_zero_extend, .is = .uninverted } }, .to_mut_sse, .none } },
184027 },
184028 .extra_temps = .{
184029 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward, .smear = 8 } } },
184030 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
184031 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
184032 .unused,
184033 .unused,
184034 .unused,
184035 .unused,
184036 .unused,
184037 .unused,
184038 .unused,
184039 .unused,
184040 },
184041 .each = .{ .once = &.{
184042 .{ ._, ._dqa, .mov, .tmp2x, .src1x, ._, ._ },
184043 .{ ._, .p_, .unpcklbw, .tmp2x, .tmp2x, ._, ._ },
184044 .{ ._, .p_, .unpcklwd, .tmp2x, .tmp2x, ._, ._ },
184045 .{ ._, .p_, .unpckldq, .tmp2x, .tmp2x, ._, ._ },
184046 .{ ._, .p_, .unpcklqdq, .tmp2x, .tmp2x, ._, ._ },
184047 .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ },
184048 .{ ._, ._dqa, .mov, .src0x, .lea(.tmp1x), ._, ._ },
184049 .{ ._, .p_, .@"and", .tmp2x, .src0x, ._, ._ },
184050 .{ ._, .p_b, .cmpeq, .tmp2x, .src0x, ._, ._ },
184051 .{ ._, .p_, .xor, .src0x, .src0x, ._, ._ },
184052 .{ ._, .p_q, .sub, .src0x, .tmp2x, ._, ._ },
184053 } },
184054 }, .{
184055 .required_features = .{ .avx2, null, null, null },
184056 .src_constraints = .{ .{ .exact_bool_vec = 4 }, .{ .exact_bool_vec = 4 }, .any },
184057 .patterns = &.{
184058 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .from_sign_extend, .is = .uninverted } }, .mem, .none } },
184059 },
184060 .extra_temps = .{
184061 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward, .smear = 4 } } },
184062 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
184063 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
184064 .unused,
184065 .unused,
184066 .unused,
184067 .unused,
184068 .unused,
184069 .unused,
184070 .unused,
184071 .unused,
184072 },
184073 .each = .{ .once = &.{
184074 .{ ._, .vp_b, .broadcast, .src0x, .src1b, ._, ._ },
184075 .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ },
184076 .{ ._, .v_dqa, .mov, .tmp2x, .lea(.tmp1x), ._, ._ },
184077 .{ ._, .vp_, .@"and", .src0x, .src0x, .tmp2x, ._ },
184078 .{ ._, .vp_b, .cmpeq, .src0x, .src0x, .tmp2x, ._ },
184079 } },
184080 }, .{
184081 .required_features = .{ .avx2, null, null, null },
184082 .src_constraints = .{ .{ .exact_bool_vec = 4 }, .{ .exact_bool_vec = 4 }, .any },
184083 .patterns = &.{
184084 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .from_sign_extend, .is = .uninverted } }, .to_sse, .none } },
184085 },
184086 .extra_temps = .{
184087 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward, .smear = 4 } } },
184088 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
184089 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
184090 .unused,
184091 .unused,
184092 .unused,
184093 .unused,
184094 .unused,
184095 .unused,
184096 .unused,
184097 .unused,
184098 },
184099 .each = .{ .once = &.{
184100 .{ ._, .vp_b, .broadcast, .src0x, .src1x, ._, ._ },
184101 .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ },
184102 .{ ._, .v_dqa, .mov, .tmp2x, .lea(.tmp1x), ._, ._ },
184103 .{ ._, .vp_, .@"and", .src0x, .src0x, .tmp2x, ._ },
184104 .{ ._, .vp_b, .cmpeq, .src0x, .src0x, .tmp2x, ._ },
184105 } },
184106 }, .{
184107 .required_features = .{ .avx2, null, null, null },
184108 .src_constraints = .{ .{ .exact_bool_vec = 4 }, .{ .exact_bool_vec = 4 }, .any },
184109 .patterns = &.{
184110 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .from_zero_extend, .is = .uninverted } }, .mem, .none } },
184111 },
184112 .extra_temps = .{
184113 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward, .smear = 4 } } },
184114 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
184115 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
184116 .unused,
184117 .unused,
184118 .unused,
184119 .unused,
184120 .unused,
184121 .unused,
184122 .unused,
184123 .unused,
184124 },
184125 .each = .{ .once = &.{
184126 .{ ._, .vp_b, .broadcast, .src0x, .src1b, ._, ._ },
184127 .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ },
184128 .{ ._, .v_dqa, .mov, .tmp2x, .lea(.tmp1x), ._, ._ },
184129 .{ ._, .vp_, .@"and", .src0x, .src0x, .tmp2x, ._ },
184130 .{ ._, .vp_b, .cmpeq, .src0x, .src0x, .tmp2x, ._ },
184131 .{ ._, .vp_, .xor, .tmp2x, .tmp2x, .tmp2x, ._ },
184132 .{ ._, .vp_b, .sub, .src0x, .tmp2x, .src0x, ._ },
184133 } },
184134 }, .{
184135 .required_features = .{ .avx2, null, null, null },
184136 .src_constraints = .{ .{ .exact_bool_vec = 4 }, .{ .exact_bool_vec = 4 }, .any },
184137 .patterns = &.{
184138 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .from_zero_extend, .is = .uninverted } }, .to_sse, .none } },
184139 },
184140 .extra_temps = .{
184141 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward, .smear = 4 } } },
184142 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
184143 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
184144 .unused,
184145 .unused,
184146 .unused,
184147 .unused,
184148 .unused,
184149 .unused,
184150 .unused,
184151 .unused,
184152 },
184153 .each = .{ .once = &.{
184154 .{ ._, .vp_b, .broadcast, .src0x, .src1x, ._, ._ },
184155 .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ },
184156 .{ ._, .v_dqa, .mov, .tmp2x, .lea(.tmp1x), ._, ._ },
184157 .{ ._, .vp_, .@"and", .src0x, .src0x, .tmp2x, ._ },
184158 .{ ._, .vp_b, .cmpeq, .src0x, .src0x, .tmp2x, ._ },
184159 .{ ._, .vp_, .xor, .tmp2x, .tmp2x, .tmp2x, ._ },
184160 .{ ._, .vp_b, .sub, .src0x, .tmp2x, .src0x, ._ },
184161 } },
184162 }, .{
184163 .required_features = .{ .avx, null, null, null },
184164 .src_constraints = .{ .{ .exact_bool_vec = 4 }, .{ .exact_bool_vec = 4 }, .any },
184165 .patterns = &.{
184166 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .from_sign_extend, .is = .uninverted } }, .to_sse, .none } },
184167 },
184168 .extra_temps = .{
184169 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward, .smear = 4 } } },
184170 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
184171 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
184172 .unused,
184173 .unused,
184174 .unused,
184175 .unused,
184176 .unused,
184177 .unused,
184178 .unused,
184179 .unused,
184180 },
184181 .each = .{ .once = &.{
184182 .{ ._, .vp_, .xor, .tmp2x, .tmp2x, .tmp2x, ._ },
184183 .{ ._, .vp_b, .shuf, .src0x, .src1x, .tmp2x, ._ },
184184 .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ },
184185 .{ ._, .v_dqa, .mov, .tmp2x, .lea(.tmp1x), ._, ._ },
184186 .{ ._, .vp_, .@"and", .src0x, .src0x, .tmp2x, ._ },
184187 .{ ._, .vp_b, .cmpeq, .src0x, .src0x, .tmp2x, ._ },
184188 } },
184189 }, .{
184190 .required_features = .{ .avx, null, null, null },
184191 .src_constraints = .{ .{ .exact_bool_vec = 4 }, .{ .exact_bool_vec = 4 }, .any },
184192 .patterns = &.{
184193 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .from_zero_extend, .is = .uninverted } }, .to_sse, .none } },
184194 },
184195 .extra_temps = .{
184196 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward, .smear = 4 } } },
184197 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
184198 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
184199 .unused,
184200 .unused,
184201 .unused,
184202 .unused,
184203 .unused,
184204 .unused,
184205 .unused,
184206 .unused,
184207 },
184208 .each = .{ .once = &.{
184209 .{ ._, .vp_, .xor, .tmp2x, .tmp2x, .tmp2x, ._ },
184210 .{ ._, .vp_b, .shuf, .src0x, .src1x, .tmp2x, ._ },
184211 .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ },
184212 .{ ._, .v_dqa, .mov, .tmp2x, .lea(.tmp1x), ._, ._ },
184213 .{ ._, .vp_, .@"and", .src0x, .src0x, .tmp2x, ._ },
184214 .{ ._, .vp_b, .cmpeq, .src0x, .src0x, .tmp2x, ._ },
184215 .{ ._, .vp_, .xor, .tmp2x, .tmp2x, .tmp2x, ._ },
184216 .{ ._, .vp_b, .sub, .src0x, .tmp2x, .src0x, ._ },
184217 } },
184218 }, .{
184219 .required_features = .{ .ssse3, null, null, null },
184220 .src_constraints = .{ .{ .exact_bool_vec = 4 }, .{ .exact_bool_vec = 4 }, .any },
184221 .patterns = &.{
184222 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .from_sign_extend, .is = .uninverted } }, .to_sse, .none } },
184223 },
184224 .extra_temps = .{
184225 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward, .smear = 4 } } },
184226 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
184227 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
184228 .unused,
184229 .unused,
184230 .unused,
184231 .unused,
184232 .unused,
184233 .unused,
184234 .unused,
184235 .unused,
184236 },
184237 .each = .{ .once = &.{
184238 .{ ._, .p_, .xor, .tmp2x, .tmp2x, ._, ._ },
184239 .{ ._, ._dqa, .mov, .src0x, .src1x, ._, ._ },
184240 .{ ._, .p_b, .shuf, .src0x, .tmp2x, ._, ._ },
184241 .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ },
184242 .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp1x), ._, ._ },
184243 .{ ._, .p_, .@"and", .src0x, .tmp2x, ._, ._ },
184244 .{ ._, .p_b, .cmpeq, .src0x, .tmp2x, ._, ._ },
184245 } },
184246 }, .{
184247 .required_features = .{ .ssse3, null, null, null },
184248 .src_constraints = .{ .{ .exact_bool_vec = 4 }, .{ .exact_bool_vec = 4 }, .any },
184249 .patterns = &.{
184250 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .from_zero_extend, .is = .uninverted } }, .to_mut_sse, .none } },
184251 },
184252 .extra_temps = .{
184253 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward, .smear = 4 } } },
184254 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
184255 .unused,
184256 .unused,
184257 .unused,
184258 .unused,
184259 .unused,
184260 .unused,
184261 .unused,
184262 .unused,
184263 .unused,
184264 },
184265 .each = .{ .once = &.{
184266 .{ ._, .p_, .xor, .src0x, .src0x, ._, ._ },
184267 .{ ._, .p_b, .shuf, .src1x, .src0x, ._, ._ },
184268 .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ },
184269 .{ ._, ._dqa, .mov, .src0x, .lea(.tmp1x), ._, ._ },
184270 .{ ._, .p_, .@"and", .src1x, .src0x, ._, ._ },
184271 .{ ._, .p_b, .cmpeq, .src1x, .src0x, ._, ._ },
184272 .{ ._, .p_, .xor, .src0x, .src0x, ._, ._ },
184273 .{ ._, .p_b, .sub, .src0x, .src1x, ._, ._ },
184274 } },
184275 }, .{
184276 .required_features = .{ .sse2, null, null, null },
184277 .src_constraints = .{ .{ .exact_bool_vec = 4 }, .{ .exact_bool_vec = 4 }, .any },
184278 .patterns = &.{
184279 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .from_sign_extend, .is = .uninverted } }, .to_sse, .none } },
184280 },
184281 .extra_temps = .{
184282 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward, .smear = 4 } } },
184283 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
184284 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
184285 .unused,
184286 .unused,
184287 .unused,
184288 .unused,
184289 .unused,
184290 .unused,
184291 .unused,
184292 .unused,
184293 },
184294 .each = .{ .once = &.{
184295 .{ ._, ._dqa, .mov, .src0x, .src1x, ._, ._ },
184296 .{ ._, .p_, .unpcklbw, .src0x, .src0x, ._, ._ },
184297 .{ ._, .p_, .unpcklwd, .src0x, .src0x, ._, ._ },
184298 .{ ._, .p_, .unpckldq, .src0x, .src0x, ._, ._ },
184299 .{ ._, .p_, .unpcklqdq, .src0x, .src0x, ._, ._ },
184300 .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ },
184301 .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp1x), ._, ._ },
184302 .{ ._, .p_, .@"and", .src0x, .tmp2x, ._, ._ },
184303 .{ ._, .p_b, .cmpeq, .src0x, .tmp2x, ._, ._ },
184304 } },
184305 }, .{
184306 .required_features = .{ .sse2, null, null, null },
184307 .src_constraints = .{ .{ .exact_bool_vec = 4 }, .{ .exact_bool_vec = 4 }, .any },
184308 .patterns = &.{
184309 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .from_zero_extend, .is = .uninverted } }, .to_sse, .none } },
184310 },
184311 .extra_temps = .{
184312 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward, .smear = 4 } } },
184313 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
184314 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
184315 .unused,
184316 .unused,
184317 .unused,
184318 .unused,
184319 .unused,
184320 .unused,
184321 .unused,
184322 .unused,
184323 },
184324 .each = .{ .once = &.{
184325 .{ ._, ._dqa, .mov, .tmp2x, .src1x, ._, ._ },
184326 .{ ._, .p_, .unpcklbw, .tmp2x, .tmp2x, ._, ._ },
184327 .{ ._, .p_, .unpcklwd, .tmp2x, .tmp2x, ._, ._ },
184328 .{ ._, .p_, .unpckldq, .tmp2x, .tmp2x, ._, ._ },
184329 .{ ._, .p_, .unpcklqdq, .tmp2x, .tmp2x, ._, ._ },
184330 .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ },
184331 .{ ._, ._dqa, .mov, .src0x, .lea(.tmp1x), ._, ._ },
184332 .{ ._, .p_, .@"and", .tmp2x, .src0x, ._, ._ },
184333 .{ ._, .p_b, .cmpeq, .tmp2x, .src0x, ._, ._ },
184334 .{ ._, .p_, .xor, .src0x, .src0x, ._, ._ },
184335 .{ ._, .p_b, .sub, .src0x, .tmp2x, ._, ._ },
184336 } },
184337 }, .{
184338 .required_features = .{ .avx2, null, null, null },
184339 .src_constraints = .{ .{ .exact_bool_vec = 8 }, .{ .exact_bool_vec = 8 }, .any },
184340 .patterns = &.{
184341 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .from_sign_extend, .is = .uninverted } }, .mem, .none } },
184342 },
184343 .extra_temps = .{
184344 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward, .smear = 2 } } },
184345 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
184346 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
184347 .unused,
184348 .unused,
184349 .unused,
184350 .unused,
184351 .unused,
184352 .unused,
184353 .unused,
184354 .unused,
184355 },
184356 .each = .{ .once = &.{
184357 .{ ._, .vp_b, .broadcast, .src0x, .src1b, ._, ._ },
184358 .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ },
184359 .{ ._, .v_dqa, .mov, .tmp2x, .lea(.tmp1x), ._, ._ },
184360 .{ ._, .vp_, .@"and", .src0x, .src0x, .tmp2x, ._ },
184361 .{ ._, .vp_b, .cmpeq, .src0x, .src0x, .tmp2x, ._ },
184362 } },
184363 }, .{
184364 .required_features = .{ .avx2, null, null, null },
184365 .src_constraints = .{ .{ .exact_bool_vec = 8 }, .{ .exact_bool_vec = 8 }, .any },
184366 .patterns = &.{
184367 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .from_sign_extend, .is = .uninverted } }, .to_sse, .none } },
184368 },
184369 .extra_temps = .{
184370 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward, .smear = 2 } } },
184371 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
184372 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
184373 .unused,
184374 .unused,
184375 .unused,
184376 .unused,
184377 .unused,
184378 .unused,
184379 .unused,
184380 .unused,
184381 },
184382 .each = .{ .once = &.{
184383 .{ ._, .vp_b, .broadcast, .src0x, .src1x, ._, ._ },
184384 .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ },
184385 .{ ._, .v_dqa, .mov, .tmp2x, .lea(.tmp1x), ._, ._ },
184386 .{ ._, .vp_, .@"and", .src0x, .src0x, .tmp2x, ._ },
184387 .{ ._, .vp_b, .cmpeq, .src0x, .src0x, .tmp2x, ._ },
184388 } },
184389 }, .{
184390 .required_features = .{ .avx2, null, null, null },
184391 .src_constraints = .{ .{ .exact_bool_vec = 8 }, .{ .exact_bool_vec = 8 }, .any },
184392 .patterns = &.{
184393 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .from_zero_extend, .is = .uninverted } }, .mem, .none } },
184394 },
184395 .extra_temps = .{
184396 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward, .smear = 2 } } },
184397 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
184398 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
184399 .unused,
184400 .unused,
184401 .unused,
184402 .unused,
184403 .unused,
184404 .unused,
184405 .unused,
184406 .unused,
184407 },
184408 .each = .{ .once = &.{
184409 .{ ._, .vp_b, .broadcast, .src0x, .src1b, ._, ._ },
184410 .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ },
184411 .{ ._, .v_dqa, .mov, .tmp2x, .lea(.tmp1x), ._, ._ },
184412 .{ ._, .vp_, .@"and", .src0x, .src0x, .tmp2x, ._ },
184413 .{ ._, .vp_b, .cmpeq, .src0x, .src0x, .tmp2x, ._ },
184414 .{ ._, .vp_, .xor, .tmp2x, .tmp2x, .tmp2x, ._ },
184415 .{ ._, .vp_b, .sub, .src0x, .tmp2x, .src0x, ._ },
184416 } },
184417 }, .{
184418 .required_features = .{ .avx2, null, null, null },
184419 .src_constraints = .{ .{ .exact_bool_vec = 8 }, .{ .exact_bool_vec = 8 }, .any },
184420 .patterns = &.{
184421 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .from_zero_extend, .is = .uninverted } }, .to_sse, .none } },
184422 },
184423 .extra_temps = .{
184424 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward, .smear = 2 } } },
184425 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
184426 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
184427 .unused,
184428 .unused,
184429 .unused,
184430 .unused,
184431 .unused,
184432 .unused,
184433 .unused,
184434 .unused,
184435 },
184436 .each = .{ .once = &.{
184437 .{ ._, .vp_b, .broadcast, .src0x, .src1x, ._, ._ },
184438 .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ },
184439 .{ ._, .v_dqa, .mov, .tmp2x, .lea(.tmp1x), ._, ._ },
184440 .{ ._, .vp_, .@"and", .src0x, .src0x, .tmp2x, ._ },
184441 .{ ._, .vp_b, .cmpeq, .src0x, .src0x, .tmp2x, ._ },
184442 .{ ._, .vp_, .xor, .tmp2x, .tmp2x, .tmp2x, ._ },
184443 .{ ._, .vp_b, .sub, .src0x, .tmp2x, .src0x, ._ },
184444 } },
184445 }, .{
184446 .required_features = .{ .avx, null, null, null },
184447 .src_constraints = .{ .{ .exact_bool_vec = 8 }, .{ .exact_bool_vec = 8 }, .any },
184448 .patterns = &.{
184449 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .from_sign_extend, .is = .uninverted } }, .to_sse, .none } },
184450 },
184451 .extra_temps = .{
184452 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward, .smear = 2 } } },
184453 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
184454 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
184455 .unused,
184456 .unused,
184457 .unused,
184458 .unused,
184459 .unused,
184460 .unused,
184461 .unused,
184462 .unused,
184463 },
184464 .each = .{ .once = &.{
184465 .{ ._, .vp_, .xor, .tmp2x, .tmp2x, .tmp2x, ._ },
184466 .{ ._, .vp_b, .shuf, .src0x, .src1x, .tmp2x, ._ },
184467 .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ },
184468 .{ ._, .v_dqa, .mov, .tmp2x, .lea(.tmp1x), ._, ._ },
184469 .{ ._, .vp_, .@"and", .src0x, .src0x, .tmp2x, ._ },
184470 .{ ._, .vp_b, .cmpeq, .src0x, .src0x, .tmp2x, ._ },
184471 } },
184472 }, .{
184473 .required_features = .{ .avx, null, null, null },
184474 .src_constraints = .{ .{ .exact_bool_vec = 8 }, .{ .exact_bool_vec = 8 }, .any },
184475 .patterns = &.{
184476 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .from_zero_extend, .is = .uninverted } }, .to_sse, .none } },
184477 },
184478 .extra_temps = .{
184479 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward, .smear = 2 } } },
184480 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
184481 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
184482 .unused,
184483 .unused,
184484 .unused,
184485 .unused,
184486 .unused,
184487 .unused,
184488 .unused,
184489 .unused,
184490 },
184491 .each = .{ .once = &.{
184492 .{ ._, .vp_, .xor, .tmp2x, .tmp2x, .tmp2x, ._ },
184493 .{ ._, .vp_b, .shuf, .src0x, .src1x, .tmp2x, ._ },
184494 .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ },
184495 .{ ._, .v_dqa, .mov, .tmp2x, .lea(.tmp1x), ._, ._ },
184496 .{ ._, .vp_, .@"and", .src0x, .src0x, .tmp2x, ._ },
184497 .{ ._, .vp_b, .cmpeq, .src0x, .src0x, .tmp2x, ._ },
184498 .{ ._, .vp_, .xor, .tmp2x, .tmp2x, .tmp2x, ._ },
184499 .{ ._, .vp_b, .sub, .src0x, .tmp2x, .src0x, ._ },
184500 } },
184501 }, .{
184502 .required_features = .{ .ssse3, null, null, null },
184503 .src_constraints = .{ .{ .exact_bool_vec = 8 }, .{ .exact_bool_vec = 8 }, .any },
184504 .patterns = &.{
184505 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .from_sign_extend, .is = .uninverted } }, .to_sse, .none } },
184506 },
184507 .extra_temps = .{
184508 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward, .smear = 2 } } },
184509 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
184510 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
184511 .unused,
184512 .unused,
184513 .unused,
184514 .unused,
184515 .unused,
184516 .unused,
184517 .unused,
184518 .unused,
184519 },
184520 .each = .{ .once = &.{
184521 .{ ._, .p_, .xor, .tmp2x, .tmp2x, ._, ._ },
184522 .{ ._, ._dqa, .mov, .src0x, .src1x, ._, ._ },
184523 .{ ._, .p_b, .shuf, .src0x, .tmp2x, ._, ._ },
184524 .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ },
184525 .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp1x), ._, ._ },
184526 .{ ._, .p_, .@"and", .src0x, .tmp2x, ._, ._ },
184527 .{ ._, .p_b, .cmpeq, .src0x, .tmp2x, ._, ._ },
184528 } },
184529 }, .{
184530 .required_features = .{ .ssse3, null, null, null },
184531 .src_constraints = .{ .{ .exact_bool_vec = 8 }, .{ .exact_bool_vec = 8 }, .any },
184532 .patterns = &.{
184533 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .from_zero_extend, .is = .uninverted } }, .to_mut_sse, .none } },
184534 },
184535 .extra_temps = .{
184536 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward, .smear = 2 } } },
184537 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
184538 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
184539 .unused,
184540 .unused,
184541 .unused,
184542 .unused,
184543 .unused,
184544 .unused,
184545 .unused,
184546 .unused,
184547 },
184548 .each = .{ .once = &.{
184549 .{ ._, .p_, .xor, .src0x, .src0x, ._, ._ },
184550 .{ ._, .p_b, .shuf, .src1x, .src0x, ._, ._ },
184551 .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ },
184552 .{ ._, ._dqa, .mov, .src0x, .lea(.tmp1x), ._, ._ },
184553 .{ ._, .p_, .@"and", .src1x, .src0x, ._, ._ },
184554 .{ ._, .p_b, .cmpeq, .src1x, .src0x, ._, ._ },
184555 .{ ._, .p_, .xor, .src0x, .src0x, ._, ._ },
184556 .{ ._, .p_b, .sub, .src0x, .src1x, ._, ._ },
184557 } },
184558 }, .{
184559 .required_features = .{ .sse2, null, null, null },
184560 .src_constraints = .{ .{ .exact_bool_vec = 8 }, .{ .exact_bool_vec = 8 }, .any },
184561 .patterns = &.{
184562 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .from_sign_extend, .is = .uninverted } }, .to_sse, .none } },
184563 },
184564 .extra_temps = .{
184565 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward, .smear = 2 } } },
184566 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
184567 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
184568 .unused,
184569 .unused,
184570 .unused,
184571 .unused,
184572 .unused,
184573 .unused,
184574 .unused,
184575 .unused,
184576 },
184577 .each = .{ .once = &.{
184578 .{ ._, ._dqa, .mov, .src0x, .src1x, ._, ._ },
184579 .{ ._, .p_, .unpcklbw, .src0x, .src0x, ._, ._ },
184580 .{ ._, .p_, .unpcklwd, .src0x, .src0x, ._, ._ },
184581 .{ ._, .p_, .unpckldq, .src0x, .src0x, ._, ._ },
184582 .{ ._, .p_, .unpcklqdq, .src0x, .src0x, ._, ._ },
184583 .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ },
184584 .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp1x), ._, ._ },
184585 .{ ._, .p_, .@"and", .src0x, .tmp2x, ._, ._ },
184586 .{ ._, .p_b, .cmpeq, .src0x, .tmp2x, ._, ._ },
184587 } },
184588 }, .{
184589 .required_features = .{ .sse2, null, null, null },
184590 .src_constraints = .{ .{ .exact_bool_vec = 8 }, .{ .exact_bool_vec = 8 }, .any },
184591 .patterns = &.{
184592 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .from_zero_extend, .is = .uninverted } }, .to_sse, .none } },
184593 },
184594 .extra_temps = .{
184595 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward, .smear = 2 } } },
184596 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
184597 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
184598 .unused,
184599 .unused,
184600 .unused,
184601 .unused,
184602 .unused,
184603 .unused,
184604 .unused,
184605 .unused,
184606 },
184607 .each = .{ .once = &.{
184608 .{ ._, ._dqa, .mov, .tmp2x, .src1x, ._, ._ },
184609 .{ ._, .p_, .unpcklbw, .tmp2x, .tmp2x, ._, ._ },
184610 .{ ._, .p_, .unpcklwd, .tmp2x, .tmp2x, ._, ._ },
184611 .{ ._, .p_, .unpckldq, .tmp2x, .tmp2x, ._, ._ },
184612 .{ ._, .p_, .unpcklqdq, .tmp2x, .tmp2x, ._, ._ },
184613 .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ },
184614 .{ ._, ._dqa, .mov, .src0x, .lea(.tmp1x), ._, ._ },
184615 .{ ._, .p_, .@"and", .tmp2x, .src0x, ._, ._ },
184616 .{ ._, .p_b, .cmpeq, .tmp2x, .src0x, ._, ._ },
184617 .{ ._, .p_, .xor, .src0x, .src0x, ._, ._ },
184618 .{ ._, .p_b, .sub, .src0x, .tmp2x, ._, ._ },
184619 } },
184620 }, .{
184621 .required_features = .{ .avx2, null, null, null },
184622 .src_constraints = .{ .{ .exact_bool_vec = 16 }, .{ .exact_bool_vec = 16 }, .any },
184623 .patterns = &.{
184624 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .from_sign_extend, .is = .uninverted } }, .to_sse, .none } },
184625 },
184626 .extra_temps = .{
184627 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bytes_mem = .{ .direction = .forward, .size = .word, .smear = 8 } } },
184628 .{ .type = .vector_8_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
184629 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
184630 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
184631 .unused,
184632 .unused,
184633 .unused,
184634 .unused,
184635 .unused,
184636 .unused,
184637 .unused,
184638 },
184639 .each = .{ .once = &.{
184640 .{ ._, ._, .lea, .tmp2p, .mem(.tmp0), ._, ._ },
184641 .{ ._, .vp_b, .shuf, .src0x, .src1x, .lea(.tmp2x), ._ },
184642 .{ ._, ._, .lea, .tmp2p, .mem(.tmp1), ._, ._ },
184643 .{ ._, .vp_q, .broadcast, .tmp3x, .lea(.tmp2q), ._, ._ },
184644 .{ ._, .vp_, .@"and", .src0x, .src0x, .tmp3x, ._ },
184645 .{ ._, .vp_b, .cmpeq, .src0x, .src0x, .tmp3x, ._ },
184646 } },
184647 }, .{
184648 .required_features = .{ .avx2, null, null, null },
184649 .src_constraints = .{ .{ .exact_bool_vec = 16 }, .{ .exact_bool_vec = 16 }, .any },
184650 .patterns = &.{
184651 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .from_zero_extend, .is = .uninverted } }, .to_sse, .none } },
184652 },
184653 .extra_temps = .{
184654 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bytes_mem = .{ .direction = .forward, .size = .word, .smear = 8 } } },
184655 .{ .type = .vector_8_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
184656 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
184657 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
184658 .unused,
184659 .unused,
184660 .unused,
184661 .unused,
184662 .unused,
184663 .unused,
184664 .unused,
184665 },
184666 .each = .{ .once = &.{
184667 .{ ._, ._, .lea, .tmp2p, .mem(.tmp0), ._, ._ },
184668 .{ ._, .vp_b, .shuf, .src0x, .src1x, .lea(.tmp2x), ._ },
184669 .{ ._, ._, .lea, .tmp2p, .mem(.tmp1), ._, ._ },
184670 .{ ._, .vp_q, .broadcast, .tmp3x, .lea(.tmp2q), ._, ._ },
184671 .{ ._, .vp_, .@"and", .src0x, .src0x, .tmp3x, ._ },
184672 .{ ._, .vp_b, .cmpeq, .src0x, .src0x, .tmp3x, ._ },
184673 .{ ._, .vp_, .xor, .tmp3x, .tmp3x, .tmp3x, ._ },
184674 .{ ._, .vp_b, .sub, .src0x, .tmp3x, .src0x, ._ },
184675 } },
184676 }, .{
184677 .required_features = .{ .avx, null, null, null },
184678 .src_constraints = .{ .{ .exact_bool_vec = 16 }, .{ .exact_bool_vec = 16 }, .any },
184679 .patterns = &.{
184680 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .from_sign_extend, .is = .uninverted } }, .to_sse, .none } },
184681 },
184682 .extra_temps = .{
184683 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bytes_mem = .{ .direction = .forward, .size = .word, .smear = 8 } } },
184684 .{ .type = .vector_8_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
184685 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
184686 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
184687 .unused,
184688 .unused,
184689 .unused,
184690 .unused,
184691 .unused,
184692 .unused,
184693 .unused,
184694 },
184695 .each = .{ .once = &.{
184696 .{ ._, ._, .lea, .tmp2p, .mem(.tmp0), ._, ._ },
184697 .{ ._, .vp_b, .shuf, .src0x, .src1x, .lea(.tmp2x), ._ },
184698 .{ ._, ._, .lea, .tmp2p, .mem(.tmp1), ._, ._ },
184699 .{ ._, .v_, .movddup, .tmp3x, .lea(.tmp2q), ._, ._ },
184700 .{ ._, .vp_, .@"and", .src0x, .src0x, .tmp3x, ._ },
184701 .{ ._, .vp_b, .cmpeq, .src0x, .src0x, .tmp3x, ._ },
184702 } },
184703 }, .{
184704 .required_features = .{ .avx, null, null, null },
184705 .src_constraints = .{ .{ .exact_bool_vec = 16 }, .{ .exact_bool_vec = 16 }, .any },
184706 .patterns = &.{
184707 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .from_zero_extend, .is = .uninverted } }, .to_sse, .none } },
184708 },
184709 .extra_temps = .{
184710 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bytes_mem = .{ .direction = .forward, .size = .word, .smear = 8 } } },
184711 .{ .type = .vector_8_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
184712 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
184713 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
184714 .unused,
184715 .unused,
184716 .unused,
184717 .unused,
184718 .unused,
184719 .unused,
184720 .unused,
184721 },
184722 .each = .{ .once = &.{
184723 .{ ._, ._, .lea, .tmp2p, .mem(.tmp0), ._, ._ },
184724 .{ ._, .vp_b, .shuf, .src0x, .src1x, .lea(.tmp2x), ._ },
184725 .{ ._, ._, .lea, .tmp2p, .mem(.tmp1), ._, ._ },
184726 .{ ._, .v_, .movddup, .tmp3x, .lea(.tmp2q), ._, ._ },
184727 .{ ._, .vp_, .@"and", .src0x, .src0x, .tmp3x, ._ },
184728 .{ ._, .vp_b, .cmpeq, .src0x, .src0x, .tmp3x, ._ },
184729 .{ ._, .vp_, .xor, .tmp3x, .tmp3x, .tmp3x, ._ },
184730 .{ ._, .vp_b, .sub, .src0x, .tmp3x, .src0x, ._ },
184731 } },
184732 }, .{
184733 .required_features = .{ .ssse3, null, null, null },
184734 .src_constraints = .{ .{ .exact_bool_vec = 16 }, .{ .exact_bool_vec = 16 }, .any },
184735 .patterns = &.{
184736 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .from_sign_extend, .is = .uninverted } }, .to_sse, .none } },
184737 },
184738 .extra_temps = .{
184739 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bytes_mem = .{ .direction = .forward, .size = .word, .smear = 8 } } },
184740 .{ .type = .vector_8_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
184741 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
184742 .{ .type = .usize, .kind = .{ .rc = .sse } },
184743 .unused,
184744 .unused,
184745 .unused,
184746 .unused,
184747 .unused,
184748 .unused,
184749 .unused,
184750 },
184751 .each = .{ .once = &.{
184752 .{ ._, ._, .lea, .tmp2p, .mem(.tmp0), ._, ._ },
184753 .{ ._, ._dqa, .mov, .src0x, .src1x, ._, ._ },
184754 .{ ._, .p_b, .shuf, .src0x, .lea(.tmp2x), ._, ._ },
184755 .{ ._, ._, .lea, .tmp2p, .mem(.tmp1), ._, ._ },
184756 .{ ._, ._, .movddup, .tmp3x, .lea(.tmp2q), ._, ._ },
184757 .{ ._, .p_, .@"and", .src0x, .tmp3x, ._, ._ },
184758 .{ ._, .p_b, .cmpeq, .src0x, .tmp3x, ._, ._ },
184759 } },
184760 }, .{
184761 .required_features = .{ .ssse3, null, null, null },
184762 .src_constraints = .{ .{ .exact_bool_vec = 16 }, .{ .exact_bool_vec = 16 }, .any },
184763 .patterns = &.{
184764 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .from_zero_extend, .is = .uninverted } }, .to_mut_sse, .none } },
184765 },
184766 .extra_temps = .{
184767 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bytes_mem = .{ .direction = .forward, .size = .word, .smear = 8 } } },
184768 .{ .type = .vector_8_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
184769 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
184770 .unused,
184771 .unused,
184772 .unused,
184773 .unused,
184774 .unused,
184775 .unused,
184776 .unused,
184777 .unused,
184778 },
184779 .each = .{ .once = &.{
184780 .{ ._, ._, .lea, .tmp2p, .mem(.tmp0), ._, ._ },
184781 .{ ._, .p_b, .shuf, .src1x, .lea(.tmp2x), ._, ._ },
184782 .{ ._, ._, .lea, .tmp2p, .mem(.tmp1), ._, ._ },
184783 .{ ._, ._, .movddup, .src0x, .lea(.tmp2q), ._, ._ },
184784 .{ ._, .p_, .@"and", .src1x, .src0x, ._, ._ },
184785 .{ ._, .p_b, .cmpeq, .src1x, .src0x, ._, ._ },
184786 .{ ._, .p_, .xor, .src0x, .src0x, ._, ._ },
184787 .{ ._, .p_b, .sub, .src0x, .src1x, ._, ._ },
184788 } },
184789 }, .{
184790 .required_features = .{ .sse3, null, null, null },
184791 .src_constraints = .{ .{ .exact_bool_vec = 16 }, .{ .exact_bool_vec = 16 }, .any },
184792 .patterns = &.{
184793 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .from_sign_extend, .is = .uninverted } }, .to_sse, .none } },
184794 },
184795 .extra_temps = .{
184796 .{ .type = .vector_8_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
184797 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
184798 .{ .type = .usize, .kind = .{ .rc = .sse } },
184799 .unused,
184800 .unused,
184801 .unused,
184802 .unused,
184803 .unused,
184804 .unused,
184805 .unused,
184806 .unused,
184807 },
184808 .each = .{ .once = &.{
184809 .{ ._, ._dqa, .mov, .src0x, .src1x, ._, ._ },
184810 .{ ._, .p_, .unpcklbw, .src0x, .src0x, ._, ._ },
184811 .{ ._, .p_, .unpcklwd, .src0x, .src0x, ._, ._ },
184812 .{ ._, .p_, .unpckldq, .src0x, .src0x, ._, ._ },
184813 .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ },
184814 .{ ._, ._, .movddup, .tmp2x, .lea(.tmp1q), ._, ._ },
184815 .{ ._, .p_, .@"and", .src0x, .tmp2x, ._, ._ },
184816 .{ ._, .p_b, .cmpeq, .src0x, .tmp2x, ._, ._ },
184817 } },
184818 }, .{
184819 .required_features = .{ .sse3, null, null, null },
184820 .src_constraints = .{ .{ .exact_bool_vec = 16 }, .{ .exact_bool_vec = 16 }, .any },
184821 .patterns = &.{
184822 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .from_zero_extend, .is = .uninverted } }, .to_sse, .none } },
184823 },
184824 .extra_temps = .{
184825 .{ .type = .vector_8_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
184826 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
184827 .{ .type = .usize, .kind = .{ .rc = .sse } },
184828 .unused,
184829 .unused,
184830 .unused,
184831 .unused,
184832 .unused,
184833 .unused,
184834 .unused,
184835 .unused,
184836 },
184837 .each = .{ .once = &.{
184838 .{ ._, ._dqa, .mov, .tmp2x, .src1x, ._, ._ },
184839 .{ ._, .p_, .unpcklbw, .tmp2x, .tmp2x, ._, ._ },
184840 .{ ._, .p_, .unpcklwd, .tmp2x, .tmp2x, ._, ._ },
184841 .{ ._, .p_, .unpckldq, .tmp2x, .tmp2x, ._, ._ },
184842 .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ },
184843 .{ ._, ._, .movddup, .src0x, .lea(.tmp1q), ._, ._ },
184844 .{ ._, .p_, .@"and", .tmp2x, .src0x, ._, ._ },
184845 .{ ._, .p_b, .cmpeq, .tmp2x, .src0x, ._, ._ },
184846 .{ ._, .p_, .xor, .src0x, .src0x, ._, ._ },
184847 .{ ._, .p_b, .sub, .src0x, .tmp2x, ._, ._ },
184848 } },
184849 }, .{
184850 .required_features = .{ .sse2, null, null, null },
184851 .src_constraints = .{ .{ .exact_bool_vec = 16 }, .{ .exact_bool_vec = 16 }, .any },
184852 .patterns = &.{
184853 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .from_sign_extend, .is = .uninverted } }, .to_sse, .none } },
184854 },
184855 .extra_temps = .{
184856 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
184857 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
184858 .{ .type = .usize, .kind = .{ .rc = .sse } },
184859 .unused,
184860 .unused,
184861 .unused,
184862 .unused,
184863 .unused,
184864 .unused,
184865 .unused,
184866 .unused,
184867 },
184868 .each = .{ .once = &.{
184869 .{ ._, ._dqa, .mov, .src0x, .src1x, ._, ._ },
184870 .{ ._, .p_, .unpcklbw, .src0x, .src0x, ._, ._ },
184871 .{ ._, .p_, .unpcklwd, .src0x, .src0x, ._, ._ },
184872 .{ ._, .p_, .unpckldq, .src0x, .src0x, ._, ._ },
184873 .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ },
184874 .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp1x), ._, ._ },
184875 .{ ._, .p_, .@"and", .src0x, .tmp2x, ._, ._ },
184876 .{ ._, .p_b, .cmpeq, .src0x, .tmp2x, ._, ._ },
184877 } },
184878 }, .{
184879 .required_features = .{ .sse2, null, null, null },
184880 .src_constraints = .{ .{ .exact_bool_vec = 16 }, .{ .exact_bool_vec = 16 }, .any },
184881 .patterns = &.{
184882 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .from_zero_extend, .is = .uninverted } }, .to_sse, .none } },
184883 },
184884 .extra_temps = .{
184885 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
184886 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
184887 .{ .type = .usize, .kind = .{ .rc = .sse } },
184888 .unused,
184889 .unused,
184890 .unused,
184891 .unused,
184892 .unused,
184893 .unused,
184894 .unused,
184895 .unused,
184896 },
184897 .each = .{ .once = &.{
184898 .{ ._, ._dqa, .mov, .tmp2x, .src1x, ._, ._ },
184899 .{ ._, .p_, .unpcklbw, .tmp2x, .tmp2x, ._, ._ },
184900 .{ ._, .p_, .unpcklwd, .tmp2x, .tmp2x, ._, ._ },
184901 .{ ._, .p_, .unpckldq, .tmp2x, .tmp2x, ._, ._ },
184902 .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ },
184903 .{ ._, ._dqa, .mov, .src0x, .lea(.tmp1x), ._, ._ },
184904 .{ ._, .p_, .@"and", .tmp2x, .src0x, ._, ._ },
184905 .{ ._, .p_b, .cmpeq, .tmp2x, .src0x, ._, ._ },
184906 .{ ._, .p_, .xor, .src0x, .src0x, ._, ._ },
184907 .{ ._, .p_b, .sub, .src0x, .tmp2x, ._, ._ },
184908 } },
184909 }, .{
184910 .required_features = .{ .avx2, null, null, null },
184911 .src_constraints = .{ .{ .exact_bool_vec = 32 }, .{ .exact_bool_vec = 32 }, .any },
184912 .patterns = &.{
184913 .{ .src = .{ .{ .reg_mask = .{ .size = .yword, .kind = .from_sign_extend, .is = .uninverted } }, .mem, .none } },
184914 },
184915 .extra_temps = .{
184916 .{ .type = .vector_32_u8, .kind = .{ .pshufb_bytes_mem = .{ .direction = .forward, .size = .dword, .smear = 8 } } },
184917 .{ .type = .vector_8_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
184918 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
184919 .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } },
184920 .unused,
184921 .unused,
184922 .unused,
184923 .unused,
184924 .unused,
184925 .unused,
184926 .unused,
184927 },
184928 .each = .{ .once = &.{
184929 .{ ._, .vp_d, .broadcast, .src0y, .src1d, ._, ._ },
184930 .{ ._, ._, .lea, .tmp2p, .mem(.tmp0), ._, ._ },
184931 .{ ._, .vp_b, .shuf, .src0y, .src0y, .lea(.tmp2y), ._ },
184932 .{ ._, ._, .lea, .tmp2p, .mem(.tmp1), ._, ._ },
184933 .{ ._, .vp_q, .broadcast, .tmp3y, .lea(.tmp2q), ._, ._ },
184934 .{ ._, .vp_, .@"and", .src0y, .src0y, .tmp3y, ._ },
184935 .{ ._, .vp_b, .cmpeq, .src0y, .src0y, .tmp3y, ._ },
184936 } },
184937 }, .{
184938 .required_features = .{ .avx2, null, null, null },
184939 .src_constraints = .{ .{ .exact_bool_vec = 32 }, .{ .exact_bool_vec = 32 }, .any },
184940 .patterns = &.{
184941 .{ .src = .{ .{ .reg_mask = .{ .size = .yword, .kind = .from_sign_extend, .is = .uninverted } }, .to_sse, .none } },
184942 },
184943 .extra_temps = .{
184944 .{ .type = .vector_32_u8, .kind = .{ .pshufb_bytes_mem = .{ .direction = .forward, .size = .dword, .smear = 8 } } },
184945 .{ .type = .vector_8_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
184946 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
184947 .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } },
184948 .unused,
184949 .unused,
184950 .unused,
184951 .unused,
184952 .unused,
184953 .unused,
184954 .unused,
184955 },
184956 .each = .{ .once = &.{
184957 .{ ._, .vp_d, .broadcast, .src0y, .src1x, ._, ._ },
184958 .{ ._, ._, .lea, .tmp2p, .mem(.tmp0), ._, ._ },
184959 .{ ._, .vp_b, .shuf, .src0y, .src0y, .lea(.tmp2y), ._ },
184960 .{ ._, ._, .lea, .tmp2p, .mem(.tmp1), ._, ._ },
184961 .{ ._, .vp_q, .broadcast, .tmp3y, .lea(.tmp2q), ._, ._ },
184962 .{ ._, .vp_, .@"and", .src0y, .src0y, .tmp3y, ._ },
184963 .{ ._, .vp_b, .cmpeq, .src0y, .src0y, .tmp3y, ._ },
184964 } },
184965 }, .{
184966 .required_features = .{ .avx2, null, null, null },
184967 .src_constraints = .{ .{ .exact_bool_vec = 32 }, .{ .exact_bool_vec = 32 }, .any },
184968 .patterns = &.{
184969 .{ .src = .{ .{ .reg_mask = .{ .size = .yword, .kind = .from_zero_extend, .is = .uninverted } }, .mem, .none } },
184970 },
184971 .extra_temps = .{
184972 .{ .type = .vector_32_u8, .kind = .{ .pshufb_bytes_mem = .{ .direction = .forward, .size = .dword, .smear = 8 } } },
184973 .{ .type = .vector_8_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
184974 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
184975 .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } },
184976 .unused,
184977 .unused,
184978 .unused,
184979 .unused,
184980 .unused,
184981 .unused,
184982 .unused,
184983 },
184984 .each = .{ .once = &.{
184985 .{ ._, .vp_d, .broadcast, .src0y, .src1d, ._, ._ },
184986 .{ ._, ._, .lea, .tmp2p, .mem(.tmp0), ._, ._ },
184987 .{ ._, .vp_b, .shuf, .src0y, .src0y, .lea(.tmp2y), ._ },
184988 .{ ._, ._, .lea, .tmp2p, .mem(.tmp1), ._, ._ },
184989 .{ ._, .vp_q, .broadcast, .tmp3y, .lea(.tmp2q), ._, ._ },
184990 .{ ._, .vp_, .@"and", .src0y, .src0y, .tmp3y, ._ },
184991 .{ ._, .vp_b, .cmpeq, .src0y, .src0y, .tmp3y, ._ },
184992 .{ ._, .vp_, .xor, .tmp3y, .tmp3y, .tmp3y, ._ },
184993 .{ ._, .vp_b, .sub, .src0y, .tmp3y, .src0y, ._ },
184994 } },
184995 }, .{
184996 .required_features = .{ .avx2, null, null, null },
184997 .src_constraints = .{ .{ .exact_bool_vec = 32 }, .{ .exact_bool_vec = 32 }, .any },
184998 .patterns = &.{
184999 .{ .src = .{ .{ .reg_mask = .{ .size = .yword, .kind = .from_zero_extend, .is = .uninverted } }, .to_sse, .none } },
185000 },
185001 .extra_temps = .{
185002 .{ .type = .vector_32_u8, .kind = .{ .pshufb_bytes_mem = .{ .direction = .forward, .size = .dword, .smear = 8 } } },
185003 .{ .type = .vector_8_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
185004 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
185005 .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } },
185006 .unused,
185007 .unused,
185008 .unused,
185009 .unused,
185010 .unused,
185011 .unused,
185012 .unused,
185013 },
185014 .each = .{ .once = &.{
185015 .{ ._, .vp_d, .broadcast, .src0y, .src1x, ._, ._ },
185016 .{ ._, ._, .lea, .tmp2p, .mem(.tmp0), ._, ._ },
185017 .{ ._, .vp_b, .shuf, .src0y, .src0y, .lea(.tmp2y), ._ },
185018 .{ ._, ._, .lea, .tmp2p, .mem(.tmp1), ._, ._ },
185019 .{ ._, .vp_q, .broadcast, .tmp3y, .lea(.tmp2q), ._, ._ },
185020 .{ ._, .vp_, .@"and", .src0y, .src0y, .tmp3y, ._ },
185021 .{ ._, .vp_b, .cmpeq, .src0y, .src0y, .tmp3y, ._ },
185022 .{ ._, .vp_, .xor, .tmp3y, .tmp3y, .tmp3y, ._ },
185023 .{ ._, .vp_b, .sub, .src0y, .tmp3y, .src0y, ._ },
185024 } },
185025 }, .{
185026 .required_features = .{ .avx, null, null, null },
185027 .src_constraints = .{ .{ .exact_bool_vec = 32 }, .{ .exact_bool_vec = 32 }, .any },
185028 .patterns = &.{
185029 .{ .src = .{ .{ .reg_mask = .{ .size = .yword, .kind = .from_sign_extend, .is = .uninverted } }, .to_sse, .none } },
185030 },
185031 .extra_temps = .{
185032 .{ .type = .vector_32_u8, .kind = .{ .pshufb_bytes_mem = .{ .direction = .forward, .size = .dword, .smear = 8 } } },
185033 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
185034 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
185035 .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } },
185036 .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } },
185037 .unused,
185038 .unused,
185039 .unused,
185040 .unused,
185041 .unused,
185042 .unused,
185043 },
185044 .each = .{ .once = &.{
185045 .{ ._, ._, .lea, .tmp2p, .mem(.tmp0), ._, ._ },
185046 .{ ._, .vp_b, .shuf, .tmp3x, .src1x, .lea(.tmp2y), ._ },
185047 .{ ._, .vp_b, .shuf, .src0x, .src1x, .lead(.tmp2y, 16), ._ },
185048 .{ ._, ._, .lea, .tmp2p, .mem(.tmp1), ._, ._ },
185049 .{ ._, .v_dqa, .mov, .tmp4x, .lea(.tmp2x), ._, ._ },
185050 .{ ._, .vp_, .@"and", .tmp3x, .tmp3x, .tmp4x, ._ },
185051 .{ ._, .vp_, .@"and", .src0x, .src0x, .tmp4x, ._ },
185052 .{ ._, .vp_b, .cmpeq, .tmp3x, .tmp3x, .tmp4x, ._ },
185053 .{ ._, .vp_b, .cmpeq, .src0x, .src0x, .tmp4x, ._ },
185054 .{ ._, .v_f128, .insert, .src0y, .tmp3y, .src0x, .ui(1) },
185055 } },
185056 }, .{
185057 .required_features = .{ .avx, null, null, null },
185058 .src_constraints = .{ .{ .exact_bool_vec = 32 }, .{ .exact_bool_vec = 32 }, .any },
185059 .patterns = &.{
185060 .{ .src = .{ .{ .reg_mask = .{ .size = .yword, .kind = .from_zero_extend, .is = .uninverted } }, .to_sse, .none } },
185061 },
185062 .extra_temps = .{
185063 .{ .type = .vector_32_u8, .kind = .{ .pshufb_bytes_mem = .{ .direction = .forward, .size = .dword, .smear = 8 } } },
185064 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
185065 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
185066 .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } },
185067 .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } },
185068 .unused,
185069 .unused,
185070 .unused,
185071 .unused,
185072 .unused,
185073 .unused,
185074 },
185075 .each = .{ .once = &.{
185076 .{ ._, ._, .lea, .tmp2p, .mem(.tmp0), ._, ._ },
185077 .{ ._, .vp_b, .shuf, .tmp3x, .src1x, .lea(.tmp2y), ._ },
185078 .{ ._, .vp_b, .shuf, .src0x, .src1x, .lead(.tmp2y, 16), ._ },
185079 .{ ._, ._, .lea, .tmp2p, .mem(.tmp1), ._, ._ },
185080 .{ ._, .v_dqa, .mov, .tmp4x, .lea(.tmp2x), ._, ._ },
185081 .{ ._, .vp_, .@"and", .tmp3x, .tmp3x, .tmp4x, ._ },
185082 .{ ._, .vp_, .@"and", .src0x, .src0x, .tmp4x, ._ },
185083 .{ ._, .vp_b, .cmpeq, .tmp3x, .tmp3x, .tmp4x, ._ },
185084 .{ ._, .vp_b, .cmpeq, .src0x, .src0x, .tmp4x, ._ },
185085 .{ ._, .vp_, .xor, .tmp4x, .tmp4x, .tmp4x, ._ },
185086 .{ ._, .vp_b, .sub, .tmp3x, .tmp4x, .tmp3x, ._ },
185087 .{ ._, .vp_b, .sub, .src0x, .tmp4x, .src0x, ._ },
185088 .{ ._, .v_f128, .insert, .src0y, .tmp3y, .src0x, .ui(1) },
185089 } },
185090 } });
185091 dst.*, src.* = ops;
185092 }
185093
183505185094 fn wrapInt(temp: *Temp, cg: *CodeGen) Select.Error!void {
183506185095 var ops: [1]Temp = .{temp.*};
183507185096 var res: [1]Temp = undefined;
......@@ -188030,6 +189619,7 @@ const Temp = struct {
188030189619 .unreach,
188031189620 .dead,
188032189621 .indirect_load_frame,
189622 .indirect_mask,
188033189623 .register_tee,
188034189624 .elementwise_gpr,
188035189625 .elementwise_sse,
......@@ -188885,25 +190475,60 @@ const Select = struct {
188885190475 to_sse,
188886190476 mut_sse,
188887190477 to_mut_sse,
188888 reg_mask: RegMaskSpec,
188889 all_reg_mask: RegMaskSpec,
188890
188891 const RegMaskSpec = struct {
190478 reg_mask: struct {
188892190479 size: Memory.Size,
190480 kind: enum {
190481 any,
190482 lsb,
190483 msb,
190484 zero_extend,
190485 sign_extend,
190486 from_lsb,
190487 from_msb,
190488 from_zero_extend,
190489 from_sign_extend,
190490 },
188893190491 is: enum {
188894190492 any,
188895190493 uninverted,
188896190494 inverted,
188897
188898 fn matches(is: @This(), inverted: bool) bool {
188899 return switch (is) {
188900 .any => true,
188901 .uninverted => !inverted,
188902 .inverted => inverted,
188903 };
188904 }
188905190495 } = .any,
188906 };
190496
190497 fn matches(spec: @This(), info: MaskInfo) bool {
190498 return switch (spec.kind) {
190499 .any => true,
190500 .lsb => switch (info.kind) {
190501 .lsb, .msb, .zero_extend, .sign_extend => true,
190502 },
190503 .msb => switch (info.kind) {
190504 .lsb, .zero_extend => false,
190505 .msb, .sign_extend => true,
190506 },
190507 .zero_extend => info.kind == .zero_extend,
190508 .sign_extend => info.kind == .sign_extend,
190509 .from_lsb => switch (info.kind) {
190510 .lsb => true,
190511 .msb, .zero_extend, .sign_extend => false,
190512 },
190513 .from_msb => switch (info.kind) {
190514 .lsb, .msb => true,
190515 .zero_extend, .sign_extend => false,
190516 },
190517 .from_sign_extend => switch (info.kind) {
190518 .lsb, .msb, .sign_extend => true,
190519 .zero_extend => false,
190520 },
190521 .from_zero_extend => switch (info.kind) {
190522 .lsb, .zero_extend => true,
190523 .msb, .sign_extend => false,
190524 },
190525 } and switch (spec.is) {
190526 .any => true,
190527 .uninverted => !info.inverted,
190528 .inverted => info.inverted,
190529 };
190530 }
190531 },
188907190532
188908190533 fn matches(src: Src, temp: Temp, cg: *CodeGen) bool {
188909190534 return switch (src) {
......@@ -188988,13 +190613,7 @@ const Select = struct {
188988190613 .reg_mask => |mask_spec| switch (temp.tracking(cg).short) {
188989190614 .register_mask => |reg_mask| mask_spec.size.bitSize(cg.target) >=
188990190615 reg_mask.info.scalar.bitSize(cg.target) * temp.typeOf(cg).vectorLen(cg.pt.zcu) and
188991 mask_spec.is.matches(reg_mask.info.inverted),
188992 else => false,
188993 },
188994 .all_reg_mask => |mask_spec| switch (temp.tracking(cg).short) {
188995 .register_mask => |reg_mask| mask_spec.size.bitSize(cg.target) ==
188996 reg_mask.info.scalar.bitSize(cg.target) * temp.typeOf(cg).vectorLen(cg.pt.zcu) and
188997 reg_mask.info.kind == .all and mask_spec.is.matches(reg_mask.info.inverted),
190616 mask_spec.matches(reg_mask.info),
188998190617 else => false,
188999190618 },
189000190619 };
......@@ -189002,7 +190621,7 @@ const Select = struct {
189002190621
189003190622 fn convert(src: Src, temp: *Temp, cg: *CodeGen) InnerError!bool {
189004190623 return switch (src) {
189005 .none, .any, .imm, .imm8, .imm16, .imm32, .simm32, .reg_mask, .all_reg_mask => false,
190624 .none, .any, .imm, .imm8, .imm16, .imm32, .simm32, .reg_mask => false,
189006190625 .mem, .to_mem => try temp.toBase(false, cg),
189007190626 .mut_mem, .to_mut_mem => try temp.toBase(true, cg),
189008190627 .to_reg => |reg| try temp.toReg(reg, cg),
......@@ -189073,8 +190692,8 @@ const Select = struct {
189073190692 pand_trunc_mem: struct { from: Memory.Size, to: Memory.Size },
189074190693 pand_mask_mem: struct { ref: Select.Operand.Ref, invert: bool = false },
189075190694 ptest_mask_mem: Select.Operand.Ref,
189076 pshufb_bswap_mem: struct { repeat: u4 = 1, size: Memory.Size, smear: u4 = 1 },
189077 bits_mem: enum { forward, reverse },
190695 pshufb_bytes_mem: struct { direction: Direction, repeat: u4 = 1, size: Memory.Size, smear: u4 = 1 },
190696 bits_mem: struct { direction: Direction, smear: u4 = 1 },
189078190697 splat_int_mem: struct { ref: Select.Operand.Ref, inside: enum { umin, smin, smax } = .umin, outside: enum { smin, smax } },
189079190698 splat_float_mem: struct { ref: Select.Operand.Ref, inside: enum { zero } = .zero, outside: f16 },
189080190699 frame: FrameIndex,
......@@ -189109,6 +190728,8 @@ const Select = struct {
189109190728 }
189110190729 };
189111190730
190731 const Direction = enum { forward, reverse };
190732
189112190733 fn lock(kind: Kind, cg: *CodeGen) ![2]?RegisterLock {
189113190734 var reg_locks: [2]?RegisterLock = @splat(null);
189114190735 const regs: [2]Register = switch (kind) {
......@@ -189430,11 +191051,15 @@ const Select = struct {
189430191051 var index: u7 = 0;
189431191052 for (0..@intCast(ref_ty.vectorLen(zcu))) |_| {
189432191053 switch (mask_info.kind) {
189433 .sign => {
191054 .lsb, .zero_extend => {
191055 elems[index] = 1;
191056 @memset(elems[index + 1 ..][0 .. elem_bytes - 1], std.math.minInt(u8));
191057 },
191058 .msb => {
189434191059 @memset(elems[index..][0 .. elem_bytes - 1], std.math.minInt(u8));
189435191060 elems[index + elem_bytes - 1] = @bitCast(@as(i8, std.math.minInt(i8)));
189436191061 },
189437 .all => @memset(elems[index..][0..elem_bytes], std.math.maxInt(u8)),
191062 .sign_extend => @memset(elems[index..][0..elem_bytes], std.math.maxInt(u8)),
189438191063 }
189439191064 index += elem_bytes;
189440191065 }
......@@ -189443,31 +191068,40 @@ const Select = struct {
189443191068 .storage = .{ .bytes = try zcu.intern_pool.getOrPutString(zcu.gpa, io, pt.tid, elems, .maybe_embedded_nulls) },
189444191069 } }))), true };
189445191070 },
189446 .pshufb_bswap_mem => |bswap_spec| {
191071 .pshufb_bytes_mem => |bytes_spec| {
189447191072 const zcu = pt.zcu;
189448191073 assert(spec.type.isVector(zcu) and spec.type.childType(zcu).toIntern() == .u8_type);
189449191074 var elem_buf: [32]u8 = @splat(1 << 7);
189450191075 const elems = elem_buf[0..spec.type.vectorLen(zcu)];
189451 const len: usize = @intCast(@divExact(bswap_spec.size.bitSize(cg.target), 8));
191076 const len: usize = @intCast(@divExact(bytes_spec.size.bitSize(cg.target), 8));
189452191077 var to_index: u6 = 0;
189453 for (0..bswap_spec.repeat) |_| for (0..len) |from_index| {
189454 @memset(elems[to_index..][0..bswap_spec.smear], @intCast(len - 1 - from_index));
189455 to_index += bswap_spec.smear;
191078 for (0..bytes_spec.repeat) |_| for (0..len) |from_index| {
191079 @memset(elems[to_index..][0..bytes_spec.smear], @intCast(switch (bytes_spec.direction) {
191080 .forward => from_index,
191081 .reverse => len - 1 - from_index,
191082 }));
191083 to_index += bytes_spec.smear;
189456191084 };
189457191085 return .{ try cg.tempMemFromValue(.fromInterned(try pt.intern(.{ .aggregate = .{
189458191086 .ty = spec.type.toIntern(),
189459191087 .storage = .{ .bytes = try zcu.intern_pool.getOrPutString(zcu.gpa, io, pt.tid, elems, .maybe_embedded_nulls) },
189460191088 } }))), true };
189461191089 },
189462 .bits_mem => |direction| {
191090 .bits_mem => |bits_spec| {
189463191091 const zcu = pt.zcu;
189464191092 assert(spec.type.isVector(zcu) and spec.type.childType(zcu).toIntern() == .u8_type);
189465 var bytes: [32]u8 = undefined;
189466 const elems = bytes[0..spec.type.vectorLen(zcu)];
189467 for (elems, 0..) |*elem, index| elem.* = switch (direction) {
189468 .forward => @as(u8, 1 << 0) << @truncate(index),
189469 .reverse => @as(u8, 1 << 7) >> @truncate(index),
189470 };
191093 var elem_buf: [32]u8 = @splat(1 << 7);
191094 const elems = elem_buf[0..spec.type.vectorLen(zcu)];
191095 var from_index: u6 = 0;
191096 var to_index: u6 = 0;
191097 while (elems.len - to_index > 0) {
191098 @memset(elems[to_index..][0..bits_spec.smear], @intCast(switch (bits_spec.direction) {
191099 .forward => @as(u8, 1 << 0) << @truncate(from_index),
191100 .reverse => @as(u8, 1 << 7) >> @truncate(from_index),
191101 }));
191102 from_index += 1;
191103 to_index += bits_spec.smear;
191104 }
189471191105 return .{ try cg.tempMemFromValue(.fromInterned(try pt.intern(.{ .aggregate = .{
189472191106 .ty = spec.type.toIntern(),
189473191107 .storage = .{ .bytes = try zcu.intern_pool.getOrPutString(zcu.gpa, io, pt.tid, elems, .maybe_embedded_nulls) },
src/codegen/x86_64/abi.zig+3-1
......@@ -24,6 +24,8 @@ pub const Class = enum {
2424 float,
2525 /// A `Class.sse` containing two `f32`s.
2626 float_combine,
27 /// Clang uses different element sizes depending on the vector length.
28 bool_vector_mask,
2729 /// Clang passes each vector element in a separate `Class.integer`.
2830 integer_per_element,
2931 /// Clang passes each vector element in a separate `Class.sse`.
......@@ -150,7 +152,7 @@ pub fn classifyWindows(init_ty: Type, zcu: *Zcu, target: *const std.Target, ctx:
150152 .{ 16, .sse_per_xword };
151153 if (elem_ty.toIntern() == .bool_type) {
152154 if (len > reg_size) return if (ctx == .arg) .integer_per_element else .memory;
153 return if (ctx == .arg) .memory else .sse;
155 return .bool_vector_mask;
154156 }
155157 const elem_size = elem_ty.abiSize(zcu);
156158 const unaligned_size = elem_size * len;
src/dev.zig+14
......@@ -52,6 +52,10 @@ pub const Env = enum {
5252 /// - `zig build-* -fincremental -fno-llvm -fno-lld -target x86_64-linux --listen=-`
5353 @"x86_64-linux",
5454
55 /// - sema
56 /// - `zig build-* -fincremental -fno-llvm -fno-lld -target x86_64-windows --listen=-`
57 @"x86_64-windows",
58
5559 pub inline fn supports(comptime dev_env: Env, comptime feature: Feature) bool {
5660 return switch (dev_env) {
5761 .full => true,
......@@ -216,6 +220,16 @@ pub const Env = enum {
216220 => true,
217221 else => Env.sema.supports(feature),
218222 },
223 .@"x86_64-windows" => switch (feature) {
224 .build_command,
225 .stdio_listen,
226 .incremental,
227 .legalize,
228 .x86_64_backend,
229 .coff2_linker,
230 => true,
231 else => Env.sema.supports(feature),
232 },
219233 };
220234 }
221235
test/c_abi/main.zig-5
......@@ -295,7 +295,6 @@ extern fn c_test_vector_2_bool() void;
295295
296296test "@Vector(2, bool)" {
297297 if (builtin.zig_backend == .stage2_llvm and (builtin.cpu.arch != .powerpc and builtin.cpu.arch != .wasm32)) return error.SkipZigTest;
298 if (builtin.zig_backend == .stage2_x86_64 and builtin.os.tag == .windows) return error.SkipZigTest;
299298
300299 const vec = c_ret_vector_2_bool();
301300 try expect(vec[0] == true);
......@@ -337,7 +336,6 @@ extern fn c_test_vector_4_bool() void;
337336
338337test "@Vector(4, bool)" {
339338 if (builtin.zig_backend == .stage2_llvm and (builtin.cpu.arch != .powerpc and builtin.cpu.arch != .wasm32)) return error.SkipZigTest;
340 if (builtin.zig_backend == .stage2_x86_64 and builtin.os.tag == .windows) return error.SkipZigTest;
341339
342340 const vec = c_ret_vector_4_bool();
343341 try expect(vec[0] == true);
......@@ -391,7 +389,6 @@ extern fn c_test_vector_8_bool() void;
391389
392390test "@Vector(8, bool)" {
393391 if (builtin.zig_backend == .stage2_llvm and (builtin.cpu.arch != .powerpc and builtin.cpu.arch != .wasm32)) return error.SkipZigTest;
394 if (builtin.zig_backend == .stage2_x86_64 and builtin.os.tag == .windows) return error.SkipZigTest;
395392
396393 const vec = c_ret_vector_8_bool();
397394 try expect(vec[0] == false);
......@@ -469,7 +466,6 @@ extern fn c_test_vector_16_bool() void;
469466
470467test "@Vector(16, bool)" {
471468 if (builtin.zig_backend == .stage2_llvm and (builtin.cpu.arch != .powerpc and builtin.cpu.arch != .wasm32)) return error.SkipZigTest;
472 if (builtin.zig_backend == .stage2_x86_64 and builtin.os.tag == .windows) return error.SkipZigTest;
473469
474470 const vec = c_ret_vector_16_bool();
475471 try expect(vec[0] == true);
......@@ -595,7 +591,6 @@ extern fn c_test_vector_32_bool() void;
595591
596592test "@Vector(32, bool)" {
597593 if (builtin.zig_backend == .stage2_llvm and (builtin.cpu.arch != .powerpc and builtin.cpu.arch != .wasm32)) return error.SkipZigTest;
598 if (builtin.zig_backend == .stage2_x86_64 and builtin.os.tag == .windows and builtin.cpu.has(.x86, .avx)) return error.SkipZigTest;
599594
600595 const vec = c_ret_vector_32_bool();
601596 try expect(vec[0] == true);