authorgravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2025-01-20 21:18:56-05:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-01-21 06:39:24-05:00
logd652dd065858c754f6d744beec3e7e5bc4ec058b
tree4787cb41816424b331b7e7eb8689776f49a3c220
parentf1ce1aff1139bab680963da4d43245407a9cba20

x86_64: rewrite `@abs` for scalar floats

4 files changed, 855 insertions(+), 260 deletions(-)

src/arch/x86_64/CodeGen.zig+538-195
......@@ -3257,7 +3257,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
32573257 },
32583258 .extra_temps = .{
32593259 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3260 .{ .kind = .{ .umax_mem = .src0 } },
3260 .{ .kind = .{ .umax_mem = .{ .ref = .src0 } } },
32613261 .unused,
32623262 .unused,
32633263 .unused,
......@@ -3288,7 +3288,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
32883288 },
32893289 .extra_temps = .{
32903290 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3291 .{ .kind = .{ .umax_mem = .src0 } },
3291 .{ .kind = .{ .umax_mem = .{ .ref = .src0 } } },
32923292 .unused,
32933293 .unused,
32943294 .unused,
......@@ -3319,7 +3319,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
33193319 },
33203320 .extra_temps = .{
33213321 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3322 .{ .kind = .{ .umax_mem = .src0 } },
3322 .{ .kind = .{ .umax_mem = .{ .ref = .src0 } } },
33233323 .unused,
33243324 .unused,
33253325 .unused,
......@@ -3338,7 +3338,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
33383338 },
33393339 .extra_temps = .{
33403340 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3341 .{ .kind = .{ .umax_mem = .src0 } },
3341 .{ .kind = .{ .umax_mem = .{ .ref = .src0 } } },
33423342 .unused,
33433343 .unused,
33443344 .unused,
......@@ -3369,7 +3369,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
33693369 },
33703370 .extra_temps = .{
33713371 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3372 .{ .kind = .{ .umax_mem = .src0 } },
3372 .{ .kind = .{ .umax_mem = .{ .ref = .src0 } } },
33733373 .unused,
33743374 .unused,
33753375 .unused,
......@@ -3400,7 +3400,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
34003400 },
34013401 .extra_temps = .{
34023402 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3403 .{ .kind = .{ .umax_mem = .src0 } },
3403 .{ .kind = .{ .umax_mem = .{ .ref = .src0 } } },
34043404 .unused,
34053405 .unused,
34063406 .unused,
......@@ -3988,7 +3988,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
39883988 },
39893989 .extra_temps = .{
39903990 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3991 .{ .kind = .{ .umax_mem = .src0 } },
3991 .{ .kind = .{ .umax_mem = .{ .ref = .src0 } } },
39923992 .unused,
39933993 .unused,
39943994 .unused,
......@@ -4019,7 +4019,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
40194019 },
40204020 .extra_temps = .{
40214021 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
4022 .{ .kind = .{ .umax_mem = .src0 } },
4022 .{ .kind = .{ .umax_mem = .{ .ref = .src0 } } },
40234023 .unused,
40244024 .unused,
40254025 .unused,
......@@ -4050,7 +4050,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
40504050 },
40514051 .extra_temps = .{
40524052 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
4053 .{ .kind = .{ .umax_mem = .src0 } },
4053 .{ .kind = .{ .umax_mem = .{ .ref = .src0 } } },
40544054 .unused,
40554055 .unused,
40564056 .unused,
......@@ -4069,7 +4069,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
40694069 },
40704070 .extra_temps = .{
40714071 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
4072 .{ .kind = .{ .umax_mem = .src0 } },
4072 .{ .kind = .{ .umax_mem = .{ .ref = .src0 } } },
40734073 .unused,
40744074 .unused,
40754075 .unused,
......@@ -4100,7 +4100,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
41004100 },
41014101 .extra_temps = .{
41024102 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
4103 .{ .kind = .{ .umax_mem = .src0 } },
4103 .{ .kind = .{ .umax_mem = .{ .ref = .src0 } } },
41044104 .unused,
41054105 .unused,
41064106 .unused,
......@@ -4131,7 +4131,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
41314131 },
41324132 .extra_temps = .{
41334133 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
4134 .{ .kind = .{ .umax_mem = .src0 } },
4134 .{ .kind = .{ .umax_mem = .{ .ref = .src0 } } },
41354135 .unused,
41364136 .unused,
41374137 .unused,
......@@ -4151,7 +4151,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
41514151 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
41524152 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
41534153 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
4154 .{ .kind = .{ .umax_mem = .src0 } },
4154 .{ .kind = .{ .umax_mem = .{ .ref = .src0 } } },
41554155 .unused,
41564156 .unused,
41574157 },
......@@ -4173,7 +4173,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
41734173 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
41744174 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
41754175 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
4176 .{ .kind = .{ .umax_mem = .src0 } },
4176 .{ .kind = .{ .umax_mem = .{ .ref = .src0 } } },
41774177 .unused,
41784178 .unused,
41794179 },
......@@ -8793,7 +8793,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
87938793
87948794 .abs => |air_tag| if (use_old) try cg.airAbs(inst) else fallback: {
87958795 const ty_op = air_datas[@intFromEnum(inst)].ty_op;
8796 if (ty_op.ty.toType().scalarType(zcu).isRuntimeFloat()) break :fallback try cg.airAbs(inst);
8796 if (ty_op.ty.toType().isVector(zcu) and ty_op.ty.toType().childType(zcu).isRuntimeFloat()) break :fallback try cg.airAbs(inst);
87978797 var ops = try cg.tempsFromOperands(inst, .{ty_op.operand});
87988798 var res: [1]Temp = undefined;
87998799 cg.select(&res, &.{ty_op.ty.toType()}, &ops, comptime &.{ .{
......@@ -8991,16 +8991,6 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
89918991 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
89928992 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
89938993 } },
8994 }, .{
8995 .required_features = .{ .mmx, .ssse3, null, null },
8996 .src_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .byte } }, .any },
8997 .patterns = &.{
8998 .{ .src = .{ .mut_mm, .none } },
8999 },
9000 .dst_temps = .{.{ .ref = .src0 }},
9001 .each = .{ .once = &.{
9002 .{ ._, .p_b, .abs, .dst0q, .src0q, ._, ._ },
9003 } },
90048994 }, .{
90058995 .required_features = .{ .mmx, .ssse3, null, null },
90068996 .src_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .byte } }, .any },
......@@ -9008,20 +8998,10 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
90088998 .{ .src = .{ .mem, .none } },
90098999 .{ .src = .{ .to_mm, .none } },
90109000 },
9011 .dst_temps = .{.{ .rc = .sse }},
9001 .dst_temps = .{.{ .mut_reg = .{ .ref = .src0, .rc = .mmx } }},
90129002 .each = .{ .once = &.{
90139003 .{ ._, .p_b, .abs, .dst0q, .src0q, ._, ._ },
90149004 } },
9015 }, .{
9016 .required_features = .{ .mmx, .ssse3, null, null },
9017 .src_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .word } }, .any },
9018 .patterns = &.{
9019 .{ .src = .{ .mut_mm, .none } },
9020 },
9021 .dst_temps = .{.{ .ref = .src0 }},
9022 .each = .{ .once = &.{
9023 .{ ._, .p_w, .abs, .dst0q, .src0q, ._, ._ },
9024 } },
90259005 }, .{
90269006 .required_features = .{ .mmx, .ssse3, null, null },
90279007 .src_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .word } }, .any },
......@@ -9029,20 +9009,10 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
90299009 .{ .src = .{ .mem, .none } },
90309010 .{ .src = .{ .to_mm, .none } },
90319011 },
9032 .dst_temps = .{.{ .rc = .sse }},
9012 .dst_temps = .{.{ .mut_reg = .{ .ref = .src0, .rc = .mmx } }},
90339013 .each = .{ .once = &.{
90349014 .{ ._, .p_w, .abs, .dst0q, .src0q, ._, ._ },
90359015 } },
9036 }, .{
9037 .required_features = .{ .mmx, .ssse3, null, null },
9038 .src_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .dword } }, .any },
9039 .patterns = &.{
9040 .{ .src = .{ .mut_mm, .none } },
9041 },
9042 .dst_temps = .{.{ .ref = .src0 }},
9043 .each = .{ .once = &.{
9044 .{ ._, .p_d, .abs, .dst0q, .src0q, ._, ._ },
9045 } },
90469016 }, .{
90479017 .required_features = .{ .mmx, .ssse3, null, null },
90489018 .src_constraints = .{ .{ .scalar_int = .{ .of = .qword, .is = .dword } }, .any },
......@@ -9050,196 +9020,106 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
90509020 .{ .src = .{ .mem, .none } },
90519021 .{ .src = .{ .to_mm, .none } },
90529022 },
9053 .dst_temps = .{.{ .rc = .sse }},
9023 .dst_temps = .{.{ .mut_reg = .{ .ref = .src0, .rc = .mmx } }},
90549024 .each = .{ .once = &.{
90559025 .{ ._, .p_d, .abs, .dst0q, .src0q, ._, ._ },
90569026 } },
9057 }, .{
9058 .required_features = .{ .ssse3, null, null, null },
9059 .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .byte } }, .any },
9060 .patterns = &.{
9061 .{ .src = .{ .mut_xmm, .none } },
9062 },
9063 .dst_temps = .{.{ .ref = .src0 }},
9064 .each = .{ .once = &.{
9065 .{ ._, .p_b, .abs, .dst0x, .src0x, ._, ._ },
9066 } },
90679027 }, .{
90689028 .required_features = .{ .ssse3, null, null, null },
90699029 .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .byte } }, .any },
90709030 .patterns = &.{
90719031 .{ .src = .{ .mem, .none } },
9072 .{ .src = .{ .to_xmm, .none } },
9032 .{ .src = .{ .to_sse, .none } },
90739033 },
9074 .dst_temps = .{.{ .rc = .sse }},
9034 .dst_temps = .{.{ .mut_reg = .{ .ref = .src0, .rc = .sse } }},
90759035 .each = .{ .once = &.{
90769036 .{ ._, .p_b, .abs, .dst0x, .src0x, ._, ._ },
90779037 } },
9078 }, .{
9079 .required_features = .{ .ssse3, null, null, null },
9080 .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .word } }, .any },
9081 .patterns = &.{
9082 .{ .src = .{ .mut_xmm, .none } },
9083 },
9084 .dst_temps = .{.{ .ref = .src0 }},
9085 .each = .{ .once = &.{
9086 .{ ._, .p_w, .abs, .dst0x, .src0x, ._, ._ },
9087 } },
90889038 }, .{
90899039 .required_features = .{ .ssse3, null, null, null },
90909040 .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .word } }, .any },
90919041 .patterns = &.{
90929042 .{ .src = .{ .mem, .none } },
9093 .{ .src = .{ .to_xmm, .none } },
9043 .{ .src = .{ .to_sse, .none } },
90949044 },
9095 .dst_temps = .{.{ .rc = .sse }},
9045 .dst_temps = .{.{ .mut_reg = .{ .ref = .src0, .rc = .sse } }},
90969046 .each = .{ .once = &.{
90979047 .{ ._, .p_w, .abs, .dst0x, .src0x, ._, ._ },
90989048 } },
9099 }, .{
9100 .required_features = .{ .ssse3, null, null, null },
9101 .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .dword } }, .any },
9102 .patterns = &.{
9103 .{ .src = .{ .mut_xmm, .none } },
9104 },
9105 .dst_temps = .{.{ .ref = .src0 }},
9106 .each = .{ .once = &.{
9107 .{ ._, .p_d, .abs, .dst0x, .src0x, ._, ._ },
9108 } },
91099049 }, .{
91109050 .required_features = .{ .ssse3, null, null, null },
91119051 .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .dword } }, .any },
91129052 .patterns = &.{
91139053 .{ .src = .{ .mem, .none } },
9114 .{ .src = .{ .to_xmm, .none } },
9054 .{ .src = .{ .to_sse, .none } },
91159055 },
9116 .dst_temps = .{.{ .rc = .sse }},
9056 .dst_temps = .{.{ .mut_reg = .{ .ref = .src0, .rc = .sse } }},
91179057 .each = .{ .once = &.{
91189058 .{ ._, .p_d, .abs, .dst0x, .src0x, ._, ._ },
91199059 } },
9120 }, .{
9121 .required_features = .{ .avx, null, null, null },
9122 .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .byte } }, .any },
9123 .patterns = &.{
9124 .{ .src = .{ .mut_xmm, .none } },
9125 },
9126 .dst_temps = .{.{ .ref = .src0 }},
9127 .each = .{ .once = &.{
9128 .{ ._, .vp_b, .abs, .dst0x, .src0x, ._, ._ },
9129 } },
91309060 }, .{
91319061 .required_features = .{ .avx, null, null, null },
91329062 .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .byte } }, .any },
91339063 .patterns = &.{
91349064 .{ .src = .{ .mem, .none } },
9135 .{ .src = .{ .to_xmm, .none } },
9065 .{ .src = .{ .to_sse, .none } },
91369066 },
9137 .dst_temps = .{.{ .rc = .sse }},
9067 .dst_temps = .{.{ .mut_reg = .{ .ref = .src0, .rc = .sse } }},
91389068 .each = .{ .once = &.{
91399069 .{ ._, .vp_b, .abs, .dst0x, .src0x, ._, ._ },
91409070 } },
9141 }, .{
9142 .required_features = .{ .avx, null, null, null },
9143 .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .word } }, .any },
9144 .patterns = &.{
9145 .{ .src = .{ .mut_xmm, .none } },
9146 },
9147 .dst_temps = .{.{ .ref = .src0 }},
9148 .each = .{ .once = &.{
9149 .{ ._, .vp_w, .abs, .dst0x, .src0x, ._, ._ },
9150 } },
91519071 }, .{
91529072 .required_features = .{ .avx, null, null, null },
91539073 .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .word } }, .any },
91549074 .patterns = &.{
91559075 .{ .src = .{ .mem, .none } },
9156 .{ .src = .{ .to_xmm, .none } },
9076 .{ .src = .{ .to_sse, .none } },
91579077 },
9158 .dst_temps = .{.{ .rc = .sse }},
9078 .dst_temps = .{.{ .mut_reg = .{ .ref = .src0, .rc = .sse } }},
91599079 .each = .{ .once = &.{
91609080 .{ ._, .vp_w, .abs, .dst0x, .src0x, ._, ._ },
91619081 } },
9162 }, .{
9163 .required_features = .{ .avx, null, null, null },
9164 .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .dword } }, .any },
9165 .patterns = &.{
9166 .{ .src = .{ .mut_xmm, .none } },
9167 },
9168 .dst_temps = .{.{ .ref = .src0 }},
9169 .each = .{ .once = &.{
9170 .{ ._, .vp_d, .abs, .dst0x, .src0x, ._, ._ },
9171 } },
91729082 }, .{
91739083 .required_features = .{ .avx, null, null, null },
91749084 .src_constraints = .{ .{ .scalar_int = .{ .of = .xword, .is = .dword } }, .any },
91759085 .patterns = &.{
91769086 .{ .src = .{ .mem, .none } },
9177 .{ .src = .{ .to_xmm, .none } },
9087 .{ .src = .{ .to_sse, .none } },
91789088 },
9179 .dst_temps = .{.{ .rc = .sse }},
9089 .dst_temps = .{.{ .mut_reg = .{ .ref = .src0, .rc = .sse } }},
91809090 .each = .{ .once = &.{
91819091 .{ ._, .vp_d, .abs, .dst0x, .src0x, ._, ._ },
91829092 } },
9183 }, .{
9184 .required_features = .{ .avx2, null, null, null },
9185 .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .byte } }, .any },
9186 .patterns = &.{
9187 .{ .src = .{ .mut_ymm, .none } },
9188 },
9189 .dst_temps = .{.{ .ref = .src0 }},
9190 .each = .{ .once = &.{
9191 .{ ._, .vp_b, .abs, .dst0y, .src0y, ._, ._ },
9192 } },
91939093 }, .{
91949094 .required_features = .{ .avx2, null, null, null },
91959095 .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .byte } }, .any },
91969096 .patterns = &.{
91979097 .{ .src = .{ .mem, .none } },
9198 .{ .src = .{ .to_ymm, .none } },
9098 .{ .src = .{ .to_sse, .none } },
91999099 },
9200 .dst_temps = .{.{ .rc = .sse }},
9100 .dst_temps = .{.{ .mut_reg = .{ .ref = .src0, .rc = .sse } }},
92019101 .each = .{ .once = &.{
92029102 .{ ._, .vp_b, .abs, .dst0y, .src0y, ._, ._ },
92039103 } },
9204 }, .{
9205 .required_features = .{ .avx2, null, null, null },
9206 .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .word } }, .any },
9207 .patterns = &.{
9208 .{ .src = .{ .mut_ymm, .none } },
9209 },
9210 .dst_temps = .{.{ .ref = .src0 }},
9211 .each = .{ .once = &.{
9212 .{ ._, .vp_w, .abs, .dst0y, .src0y, ._, ._ },
9213 } },
92149104 }, .{
92159105 .required_features = .{ .avx2, null, null, null },
92169106 .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .word } }, .any },
92179107 .patterns = &.{
92189108 .{ .src = .{ .mem, .none } },
9219 .{ .src = .{ .to_ymm, .none } },
9109 .{ .src = .{ .to_sse, .none } },
92209110 },
9221 .dst_temps = .{.{ .rc = .sse }},
9111 .dst_temps = .{.{ .mut_reg = .{ .ref = .src0, .rc = .sse } }},
92229112 .each = .{ .once = &.{
92239113 .{ ._, .vp_w, .abs, .dst0y, .src0y, ._, ._ },
92249114 } },
9225 }, .{
9226 .required_features = .{ .avx2, null, null, null },
9227 .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .dword } }, .any },
9228 .patterns = &.{
9229 .{ .src = .{ .mut_ymm, .none } },
9230 },
9231 .dst_temps = .{.{ .ref = .src0 }},
9232 .each = .{ .once = &.{
9233 .{ ._, .vp_d, .abs, .dst0y, .src0y, ._, ._ },
9234 } },
92359115 }, .{
92369116 .required_features = .{ .avx2, null, null, null },
92379117 .src_constraints = .{ .{ .scalar_int = .{ .of = .yword, .is = .dword } }, .any },
92389118 .patterns = &.{
92399119 .{ .src = .{ .mem, .none } },
9240 .{ .src = .{ .to_ymm, .none } },
9120 .{ .src = .{ .to_sse, .none } },
92419121 },
9242 .dst_temps = .{.{ .rc = .sse }},
9122 .dst_temps = .{.{ .mut_reg = .{ .ref = .src0, .rc = .sse } }},
92439123 .each = .{ .once = &.{
92449124 .{ ._, .vp_d, .abs, .dst0y, .src0y, ._, ._ },
92459125 } },
......@@ -9807,6 +9687,266 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
98079687 .{ ._, ._, .cmp, .tmp0d, .sa(.none, .add_src0_unaligned_size), ._, ._ },
98089688 .{ ._, ._b, .j, .@"0b", ._, ._, ._ },
98099689 } },
9690 }, .{
9691 .required_features = .{ .sse, null, null, null },
9692 .src_constraints = .{ .{ .scalar_exact_float = .{ .of = .xword, .is = .dword } }, .any },
9693 .patterns = &.{
9694 .{ .src = .{ .to_mut_sse, .none } },
9695 },
9696 .extra_temps = .{
9697 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
9698 .{ .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize = true } } },
9699 .unused,
9700 .unused,
9701 .unused,
9702 .unused,
9703 },
9704 .dst_temps = .{.{ .ref = .src0 }},
9705 .each = .{ .once = &.{
9706 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
9707 .{ ._, ._ps, .@"and", .dst0x, .lea(.xword, .tmp0), ._, ._ },
9708 } },
9709 }, .{
9710 .required_features = .{ .sse2, null, null, null },
9711 .src_constraints = .{ .{ .scalar_exact_float = .{ .of = .xword, .is = .qword } }, .any },
9712 .patterns = &.{
9713 .{ .src = .{ .to_mut_sse, .none } },
9714 },
9715 .extra_temps = .{
9716 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
9717 .{ .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize = true } } },
9718 .unused,
9719 .unused,
9720 .unused,
9721 .unused,
9722 },
9723 .dst_temps = .{.{ .ref = .src0 }},
9724 .each = .{ .once = &.{
9725 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
9726 .{ ._, ._pd, .@"and", .dst0x, .lea(.xword, .tmp0), ._, ._ },
9727 } },
9728 }, .{
9729 .required_features = .{ .sse, null, null, null },
9730 .src_constraints = .{ .{ .scalar_exact_float = .{ .of = .xword, .is = .qword } }, .any },
9731 .patterns = &.{
9732 .{ .src = .{ .to_mut_sse, .none } },
9733 },
9734 .extra_temps = .{
9735 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
9736 .{ .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize = true } } },
9737 .unused,
9738 .unused,
9739 .unused,
9740 .unused,
9741 },
9742 .dst_temps = .{.{ .ref = .src0 }},
9743 .each = .{ .once = &.{
9744 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
9745 .{ ._, ._ps, .@"and", .dst0x, .lea(.xword, .tmp0), ._, ._ },
9746 } },
9747 }, .{
9748 .required_features = .{ .avx, null, null, null },
9749 .src_constraints = .{ .{ .scalar_exact_float = .{ .of = .xword, .is = .dword } }, .any },
9750 .patterns = &.{
9751 .{ .src = .{ .to_sse, .none } },
9752 },
9753 .extra_temps = .{
9754 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
9755 .{ .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize = true } } },
9756 .unused,
9757 .unused,
9758 .unused,
9759 .unused,
9760 },
9761 .dst_temps = .{.{ .mut_reg = .{ .ref = .src0, .rc = .sse } }},
9762 .each = .{ .once = &.{
9763 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
9764 .{ ._, .v_ps, .@"and", .dst0x, .src0x, .lea(.xword, .tmp0), ._ },
9765 } },
9766 }, .{
9767 .required_features = .{ .avx, null, null, null },
9768 .src_constraints = .{ .{ .scalar_exact_float = .{ .of = .xword, .is = .qword } }, .any },
9769 .patterns = &.{
9770 .{ .src = .{ .to_sse, .none } },
9771 },
9772 .extra_temps = .{
9773 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
9774 .{ .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize = true } } },
9775 .unused,
9776 .unused,
9777 .unused,
9778 .unused,
9779 },
9780 .dst_temps = .{.{ .mut_reg = .{ .ref = .src0, .rc = .sse } }},
9781 .each = .{ .once = &.{
9782 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
9783 .{ ._, .v_pd, .@"and", .dst0x, .src0x, .lea(.xword, .tmp0), ._ },
9784 } },
9785 }, .{
9786 .required_features = .{ .avx, null, null, null },
9787 .src_constraints = .{ .{ .scalar_exact_float = .{ .of = .yword, .is = .dword } }, .any },
9788 .patterns = &.{
9789 .{ .src = .{ .to_sse, .none } },
9790 },
9791 .extra_temps = .{
9792 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
9793 .{ .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize = true } } },
9794 .unused,
9795 .unused,
9796 .unused,
9797 .unused,
9798 },
9799 .dst_temps = .{.{ .mut_reg = .{ .ref = .src0, .rc = .sse } }},
9800 .each = .{ .once = &.{
9801 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
9802 .{ ._, .v_ps, .@"and", .dst0y, .src0y, .lea(.yword, .tmp0), ._ },
9803 } },
9804 }, .{
9805 .required_features = .{ .avx, null, null, null },
9806 .src_constraints = .{ .{ .scalar_exact_float = .{ .of = .yword, .is = .qword } }, .any },
9807 .patterns = &.{
9808 .{ .src = .{ .to_sse, .none } },
9809 },
9810 .extra_temps = .{
9811 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
9812 .{ .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize = true } } },
9813 .unused,
9814 .unused,
9815 .unused,
9816 .unused,
9817 },
9818 .dst_temps = .{.{ .mut_reg = .{ .ref = .src0, .rc = .sse } }},
9819 .each = .{ .once = &.{
9820 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
9821 .{ ._, .v_pd, .@"and", .dst0y, .src0y, .lea(.yword, .tmp0), ._ },
9822 } },
9823 }, .{
9824 .required_features = .{ .x87, null, null, null },
9825 .src_constraints = .{ .{ .scalar_exact_float = .{ .of = .xword, .is = .tbyte } }, .any },
9826 .patterns = &.{
9827 .{ .src = .{ .to_x87, .none } },
9828 },
9829 .dst_temps = .{.{ .mut_reg = .{ .ref = .src0, .rc = .x87 } }},
9830 .clobbers = .{ .st = 1 },
9831 .each = .{ .once = &.{
9832 .{ ._, .f_, .ld, .src0t, ._, ._, ._ },
9833 .{ ._, .f_, .abs, ._, ._, ._, ._ },
9834 .{ ._, .f_p, .st, .dst0t, ._, ._, ._ },
9835 } },
9836 }, .{
9837 .required_features = .{ .avx2, null, null, null },
9838 .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .xword } }, .any },
9839 .patterns = &.{
9840 .{ .src = .{ .to_sse, .none } },
9841 },
9842 .extra_temps = .{
9843 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
9844 .{ .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize = true } } },
9845 .unused,
9846 .unused,
9847 .unused,
9848 .unused,
9849 },
9850 .dst_temps = .{.{ .mut_reg = .{ .ref = .src0, .rc = .sse } }},
9851 .each = .{ .once = &.{
9852 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
9853 .{ ._, .vp_, .@"and", .dst0x, .src0x, .lea(.xword, .tmp0), ._ },
9854 } },
9855 }, .{
9856 .required_features = .{ .avx, null, null, null },
9857 .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .xword } }, .any },
9858 .patterns = &.{
9859 .{ .src = .{ .to_sse, .none } },
9860 },
9861 .extra_temps = .{
9862 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
9863 .{ .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize = true } } },
9864 .unused,
9865 .unused,
9866 .unused,
9867 .unused,
9868 },
9869 .dst_temps = .{.{ .mut_reg = .{ .ref = .src0, .rc = .sse } }},
9870 .each = .{ .once = &.{
9871 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
9872 .{ ._, .vp_, .@"and", .dst0x, .src0x, .lea(.xword, .tmp0), ._ },
9873 } },
9874 }, .{
9875 .required_features = .{ .sse2, null, null, null },
9876 .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .xword } }, .any },
9877 .patterns = &.{
9878 .{ .src = .{ .to_mut_sse, .none } },
9879 },
9880 .extra_temps = .{
9881 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
9882 .{ .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize = true } } },
9883 .unused,
9884 .unused,
9885 .unused,
9886 .unused,
9887 },
9888 .dst_temps = .{.{ .ref = .src0 }},
9889 .each = .{ .once = &.{
9890 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
9891 .{ ._, .p_, .@"and", .dst0x, .lea(.xword, .tmp0), ._, ._ },
9892 } },
9893 }, .{
9894 .required_features = .{ .sse, null, null, null },
9895 .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .xword } }, .any },
9896 .patterns = &.{
9897 .{ .src = .{ .to_mut_sse, .none } },
9898 },
9899 .extra_temps = .{
9900 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
9901 .{ .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize = true } } },
9902 .unused,
9903 .unused,
9904 .unused,
9905 .unused,
9906 },
9907 .dst_temps = .{.{ .ref = .src0 }},
9908 .each = .{ .once = &.{
9909 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
9910 .{ ._, ._ps, .@"and", .dst0x, .lea(.xword, .tmp0), ._, ._ },
9911 } },
9912 }, .{
9913 .required_features = .{ .avx2, null, null, null },
9914 .src_constraints = .{ .{ .scalar_float = .{ .of = .yword, .is = .xword } }, .any },
9915 .patterns = &.{
9916 .{ .src = .{ .to_sse, .none } },
9917 },
9918 .extra_temps = .{
9919 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
9920 .{ .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize = true } } },
9921 .unused,
9922 .unused,
9923 .unused,
9924 .unused,
9925 },
9926 .dst_temps = .{.{ .mut_reg = .{ .ref = .src0, .rc = .sse } }},
9927 .each = .{ .once = &.{
9928 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
9929 .{ ._, .vp_, .@"and", .dst0y, .src0y, .lea(.yword, .tmp0), ._ },
9930 } },
9931 }, .{
9932 .required_features = .{ .avx, null, null, null },
9933 .src_constraints = .{ .{ .scalar_float = .{ .of = .yword, .is = .xword } }, .any },
9934 .patterns = &.{
9935 .{ .src = .{ .to_sse, .none } },
9936 },
9937 .extra_temps = .{
9938 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
9939 .{ .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize = true } } },
9940 .unused,
9941 .unused,
9942 .unused,
9943 .unused,
9944 },
9945 .dst_temps = .{.{ .mut_reg = .{ .ref = .src0, .rc = .sse } }},
9946 .each = .{ .once = &.{
9947 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
9948 .{ ._, .v_pd, .@"and", .dst0y, .src0y, .lea(.yword, .tmp0), ._ },
9949 } },
98109950 } }) catch |err| switch (err) {
98119951 error.SelectFailed => return cg.fail("failed to select {s} {} {}", .{
98129952 @tagName(air_tag),
......@@ -21372,10 +21512,11 @@ fn airCmp(self: *CodeGen, inst: Air.Inst.Index, op: std.math.CompareOperator) !v
2137221512 switch (ty.zigTypeTag(zcu)) {
2137321513 .float => {
2137421514 const float_bits = ty.floatBits(self.target.*);
21375 if (switch (float_bits) {
21376 16 => !self.hasFeature(.f16c),
21377 32, 64 => false,
21378 80, 128 => true,
21515 if (!switch (float_bits) {
21516 16 => self.hasFeature(.f16c),
21517 32 => self.hasFeature(.sse),
21518 64 => self.hasFeature(.sse2),
21519 80, 128 => false,
2137921520 else => unreachable,
2138021521 }) {
2138121522 var callee_buf: ["__???f2".len]u8 = undefined;
......@@ -23640,12 +23781,24 @@ const MoveStrategy = union(enum) {
2364023781 assert(dst_reg != .st7);
2364123782 try self.asmRegister(.{ .f_p, .st }, @enumFromInt(@intFromEnum(dst_reg) + 1));
2364223783 },
23643 .insert_extract => |ie| try self.asmRegisterMemoryImmediate(
23644 ie.insert,
23645 dst_reg,
23646 src_mem,
23647 .u(0),
23648 ),
23784 .insert_extract => |ie| if (ie.insert[0] != .p_w or self.hasFeature(.sse2))
23785 try self.asmRegisterMemoryImmediate(ie.insert, dst_reg, src_mem, .u(0))
23786 else {
23787 const tmp_frame_index = try self.allocFrameIndex(.init(.{
23788 .size = 16,
23789 .alignment = .@"16",
23790 }));
23791 const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
23792 try self.asmRegisterMemory(.{ ._, .movzx }, tmp_reg.to32(), src_mem);
23793 try self.asmMemoryRegister(.{ ._, .mov }, .{
23794 .base = .{ .frame = tmp_frame_index },
23795 .mod = .{ .rm = .{ .size = .word } },
23796 }, tmp_reg.to16());
23797 try self.asmRegisterMemory(.{ ._ps, .mova }, dst_reg.to128(), .{
23798 .base = .{ .frame = tmp_frame_index },
23799 .mod = .{ .rm = .{ .size = .xword } },
23800 });
23801 },
2364923802 .vex_insert_extract => |ie| try self.asmRegisterRegisterMemoryImmediate(
2365023803 ie.insert,
2365123804 dst_reg,
......@@ -23678,7 +23831,7 @@ const MoveStrategy = union(enum) {
2367823831 .mod = .{ .rm = .{ .size = .xword } },
2367923832 }, src_reg.to128());
2368023833 const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
23681 try self.asmRegisterMemory(.{ ._, .mov }, tmp_reg.to16(), .{
23834 try self.asmRegisterMemory(.{ ._, .movzx }, tmp_reg.to32(), .{
2368223835 .base = .{ .frame = tmp_frame_index },
2368323836 .mod = .{ .rm = .{ .size = .word } },
2368423837 });
......@@ -24122,7 +24275,18 @@ fn genSetReg(
2412224275 ),
2412324276 else => unreachable,
2412424277 },
24125 .segment, .x87, .mmx, .sse => try self.genSetReg(dst_reg, ty, try self.genTypedValue(try pt.undefValue(ty)), opts),
24278 .segment, .x87, .mmx, .sse => {
24279 const full_ty = try pt.vectorType(.{
24280 .len = self.vectorSize(.float),
24281 .child = .u8_type,
24282 });
24283 try self.genSetReg(dst_reg, full_ty, try self.genTypedValue(
24284 .fromInterned(try pt.intern(.{ .aggregate = .{
24285 .ty = full_ty.toIntern(),
24286 .storage = .{ .repeated_elem = (try pt.intValue(.u8, 0xaa)).toIntern() },
24287 } })),
24288 ), opts);
24289 },
2412624290 .ip, .cr, .dr => unreachable,
2412724291 },
2412824292 .eflags => |cc| try self.asmSetccRegister(cc, dst_reg.to8()),
......@@ -24171,13 +24335,19 @@ fn genSetReg(
2417124335 registerAlias(dst_reg, @max(abi_size, 4)),
2417224336 src_reg.to128(),
2417324337 ) else {
24338 const frame_size = std.math.ceilPowerOfTwoAssert(u32, @max(abi_size, 4));
2417424339 const frame_index = try self.allocFrameIndex(.init(.{
24175 .size = 4,
24176 .alignment = .@"4",
24340 .size = frame_size,
24341 .alignment = .fromNonzeroByteUnits(frame_size),
2417724342 }));
24178 try self.asmMemoryRegister(.{ ._ss, .mov }, .{
24343 try self.asmMemoryRegister(switch (frame_size) {
24344 4 => .{ ._ss, .mov },
24345 8 => .{ ._ps, .movl },
24346 16 => .{ ._ps, .mov },
24347 else => unreachable,
24348 }, .{
2417924349 .base = .{ .frame = frame_index },
24180 .mod = .{ .rm = .{ .size = .dword } },
24350 .mod = .{ .rm = .{ .size = .fromSize(frame_size) } },
2418124351 }, src_reg.to128());
2418224352 try self.asmRegisterMemory(.{ ._, .mov }, registerAlias(dst_reg, abi_size), .{
2418324353 .base = .{ .frame = frame_index },
......@@ -28572,7 +28742,7 @@ fn resolveCallingConventionValues(
2857228742 arg_mcv_i += 1;
2857328743 },
2857428744 .sse, .float, .float_combine => {
28575 const param_sse_regs = abi.getCAbiSseParamRegs(cc);
28745 const param_sse_regs = abi.getCAbiSseParamRegs(cc, self.target);
2857628746 const abi_size: u32 = @intCast(ty.abiSize(zcu));
2857728747 const reg_size = @min(abi_size, self.vectorSize(.float));
2857828748 var byte_offset: u32 = 0;
......@@ -28670,7 +28840,7 @@ fn resolveCallingConventionValues(
2867028840
2867128841 var param_gpr = abi.getCAbiIntParamRegs(cc);
2867228842 var param_x87 = abi.getCAbiX87ParamRegs(cc);
28673 var param_sse = abi.getCAbiSseParamRegs(cc);
28843 var param_sse = abi.getCAbiSseParamRegs(cc, self.target);
2867428844
2867528845 // Return values
2867628846 result.return_value = if (ret_ty.isNoReturn(zcu))
......@@ -29156,6 +29326,10 @@ fn intInfo(cg: *CodeGen, ty: Type) ?std.builtin.Type.Int {
2915629326 };
2915729327}
2915829328
29329fn floatBits(cg: *CodeGen, ty: Type) ?u16 {
29330 return if (ty.isRuntimeFloat()) ty.floatBits(cg.target.*) else null;
29331}
29332
2915929333const Temp = struct {
2916029334 index: Air.Inst.Index,
2916129335
......@@ -30234,6 +30408,7 @@ const Select = struct {
3023430408 backward: ?Mir.Inst.Index,
3023530409 forward: [1]?Mir.Inst.Index,
3023630410 },
30411 top: u3,
3023730412
3023830413 fn emitLabel(s: *Select, label_index: Label) void {
3023930414 if (label_index == ._) return;
......@@ -30269,6 +30444,92 @@ const Select = struct {
3026930444 },
3027030445 else => |e| return e,
3027130446 };
30447 switch (mir_tag[0]) {
30448 .f_ => switch (mir_tag[1]) {
30449 .abs, .st => {},
30450 .ld => s.top -%= 1,
30451 else => {
30452 const fixes = @tagName(mir_tag[0]);
30453 const fixes_blank = std.mem.indexOfScalar(u8, fixes, '_').?;
30454 std.debug.panic("{s}: {s}{s}{s}\n", .{
30455 @src().fn_name,
30456 fixes[0..fixes_blank],
30457 @tagName(mir_tag[1]),
30458 fixes[fixes_blank + 1 ..],
30459 });
30460 },
30461 },
30462 .f_p => switch (mir_tag[1]) {
30463 .st => s.top +%= 1,
30464 else => {
30465 const fixes = @tagName(mir_tag[0]);
30466 const fixes_blank = std.mem.indexOfScalar(u8, fixes, '_').?;
30467 std.debug.panic("{s}: {s}{s}{s}\n", .{
30468 @src().fn_name,
30469 fixes[0..fixes_blank],
30470 @tagName(mir_tag[1]),
30471 fixes[fixes_blank + 1 ..],
30472 });
30473 },
30474 },
30475 .f_1,
30476 => switch (mir_tag[1]) {
30477 .ld => s.top -%= 1,
30478 else => {
30479 const fixes = @tagName(mir_tag[0]);
30480 const fixes_blank = std.mem.indexOfScalar(u8, fixes, '_').?;
30481 std.debug.panic("{s}: {s}{s}{s}\n", .{
30482 @src().fn_name,
30483 fixes[0..fixes_blank],
30484 @tagName(mir_tag[1]),
30485 fixes[fixes_blank + 1 ..],
30486 });
30487 },
30488 },
30489 .f_l2e,
30490 .f_l2t,
30491 .f_lg2,
30492 .f_ln2,
30493 .f_pi,
30494 .f_z,
30495 => switch (mir_tag[1]) {
30496 .ld => s.top -%= 1,
30497 else => unreachable,
30498 },
30499 .f_b,
30500 .f_be,
30501 .f_cw,
30502 .f_e,
30503 .f_env,
30504 .f_nb,
30505 .f_nbe,
30506 .f_ne,
30507 .f_nu,
30508 .f_p1,
30509 .f_pp,
30510 .f_sw,
30511 .f_u,
30512 .fb_,
30513 .fb_p,
30514 .fi_,
30515 .fi_p,
30516 .fn_,
30517 .fn_cw,
30518 .fn_env,
30519 .fn_sw,
30520 => {},
30521 .f_cstp => switch (mir_tag[1]) {
30522 .de => s.top -%= 1,
30523 .in => s.top +%= 1,
30524 else => unreachable,
30525 },
30526 else => {},
30527 }
30528 }
30529
30530 fn lowerReg(s: *const Select, reg: Register) Register {
30531 if (reg.class() != .x87) return reg;
30532 return @enumFromInt(@intFromEnum(Register.st0) + (@as(u3, @intCast(reg.enc())) -% s.top));
3027230533 }
3027330534
3027430535 const Case = struct {
......@@ -30278,7 +30539,7 @@ const Select = struct {
3027830539 patterns: []const Select.Pattern,
3027930540 extra_temps: [@intFromEnum(Select.Operand.Ref.dst0) - @intFromEnum(Select.Operand.Ref.tmp0)]TempSpec = @splat(.unused),
3028030541 dst_temps: [@intFromEnum(Select.Operand.Ref.src0) - @intFromEnum(Select.Operand.Ref.dst0)]TempSpec.Kind = @splat(.unused),
30281 clobbers: struct { eflags: bool = false } = .{},
30542 clobbers: struct { eflags: bool = false, st: u3 = 0 } = .{},
3028230543 each: union(enum) {
3028330544 once: []const Instruction,
3028430545 },
......@@ -30305,6 +30566,8 @@ const Select = struct {
3030530566 scalar_signed_int: Memory.Size,
3030630567 scalar_unsigned_int: Memory.Size,
3030730568 scalar_remainder_int: struct { of: Memory.Size, is: Memory.Size },
30569 scalar_float: struct { of: Memory.Size, is: Memory.Size },
30570 scalar_exact_float: struct { of: Memory.Size, is: Memory.Size },
3030830571 multiple_scalar_int: struct { of: Memory.Size, is: Memory.Size },
3030930572 exact_int: u16,
3031030573 exact_signed_int: u16,
......@@ -30338,23 +30601,23 @@ const Select = struct {
3033830601 size.bitSize(cg.target) >= ty.vectorLen(zcu),
3033930602 .vec => |size| ty.isVector(zcu) and ty.scalarType(zcu).toIntern() != .bool_type and
3034030603 size.bitSize(cg.target) >= ty.abiSize(zcu),
30341 .signed_int_vec => |size| ty.isVector(zcu) and size.bitSize(cg.target) >= 8 * ty.abiSize(zcu) and
30604 .signed_int_vec => |size| ty.isVector(zcu) and @divExact(size.bitSize(cg.target), 8) >= ty.abiSize(zcu) and
3034230605 if (cg.intInfo(ty.childType(zcu))) |int_info| int_info.signedness == .signed else false,
30343 .signed_int_or_full_vec => |size| ty.isVector(zcu) and size.bitSize(cg.target) >= 8 * ty.abiSize(zcu) and
30606 .signed_int_or_full_vec => |size| ty.isVector(zcu) and @divExact(size.bitSize(cg.target), 8) >= ty.abiSize(zcu) and
3034430607 if (cg.intInfo(ty.childType(zcu))) |int_info| switch (int_info.signedness) {
3034530608 .signed => true,
3034630609 .unsigned => int_info.bits >= 8 and std.math.isPowerOfTwo(int_info.bits),
3034730610 } else false,
30348 .unsigned_int_vec => |size| ty.isVector(zcu) and size.bitSize(cg.target) >= 8 * ty.abiSize(zcu) and
30611 .unsigned_int_vec => |size| ty.isVector(zcu) and @divExact(size.bitSize(cg.target), 8) >= ty.abiSize(zcu) and
3034930612 if (cg.intInfo(ty.childType(zcu))) |int_info| int_info.signedness == .unsigned else false,
30350 .size => |size| size.bitSize(cg.target) >= 8 * ty.abiSize(zcu),
30351 .multiple_size => |size| size.bitSize(cg.target) % 8 * ty.abiSize(zcu) == 0,
30613 .size => |size| @divExact(size.bitSize(cg.target), 8) >= ty.abiSize(zcu),
30614 .multiple_size => |size| ty.abiSize(zcu) % @divExact(size.bitSize(cg.target), 8) == 0,
3035230615 .int => |size| if (cg.intInfo(ty)) |int_info| size.bitSize(cg.target) >= int_info.bits else false,
3035330616 .scalar_int_is => |size| if (cg.intInfo(ty.scalarType(zcu))) |int_info|
3035430617 size.bitSize(cg.target) >= int_info.bits
3035530618 else
3035630619 false,
30357 .scalar_int => |of_is| of_is.of.bitSize(cg.target) >= 8 * ty.abiSize(zcu) and
30620 .scalar_int => |of_is| @divExact(of_is.of.bitSize(cg.target), 8) >= ty.abiSize(zcu) and
3035830621 if (cg.intInfo(ty.scalarType(zcu))) |int_info| of_is.is.bitSize(cg.target) >= int_info.bits else false,
3035930622 .scalar_signed_int => |size| if (cg.intInfo(ty.scalarType(zcu))) |int_info| switch (int_info.signedness) {
3036030623 .signed => size.bitSize(cg.target) >= int_info.bits,
......@@ -30364,12 +30627,16 @@ const Select = struct {
3036430627 .signed => false,
3036530628 .unsigned => size.bitSize(cg.target) >= int_info.bits,
3036630629 } else false,
30367 .multiple_scalar_int => |of_is| of_is.of.bitSize(cg.target) % 8 * ty.abiSize(zcu) == 0 and
30630 .multiple_scalar_int => |of_is| ty.abiSize(zcu) % @divExact(of_is.of.bitSize(cg.target), 8) == 0 and
3036830631 if (cg.intInfo(ty.scalarType(zcu))) |int_info| of_is.is.bitSize(cg.target) >= int_info.bits else false,
3036930632 .scalar_remainder_int => |of_is| if (cg.intInfo(ty.scalarType(zcu))) |int_info|
3037030633 of_is.is.bitSize(cg.target) >= (int_info.bits - 1) % of_is.of.bitSize(cg.target) + 1
3037130634 else
3037230635 false,
30636 .scalar_float => |of_is| @divExact(of_is.of.bitSize(cg.target), 8) >= ty.abiSize(zcu) and
30637 if (cg.floatBits(ty.scalarType(zcu))) |float_bits| of_is.is.bitSize(cg.target) >= float_bits else false,
30638 .scalar_exact_float => |of_is| @divExact(of_is.of.bitSize(cg.target), 8) >= ty.abiSize(zcu) and
30639 if (cg.floatBits(ty.scalarType(zcu))) |float_bits| of_is.is.bitSize(cg.target) == float_bits else false,
3037330640 .exact_int => |bit_size| if (cg.intInfo(ty)) |int_info| bit_size == int_info.bits else false,
3037430641 .exact_signed_int => |bit_size| if (cg.intInfo(ty)) |int_info| switch (int_info.signedness) {
3037530642 .signed => bit_size == int_info.bits,
......@@ -30452,10 +30719,18 @@ const Select = struct {
3045230719 to_gpr,
3045330720 mut_gpr,
3045430721 to_mut_gpr,
30722 x87,
30723 to_x87,
30724 mut_x87,
30725 to_mut_x87,
3045530726 mm,
3045630727 to_mm,
3045730728 mut_mm,
3045830729 to_mut_mm,
30730 sse,
30731 to_sse,
30732 mut_sse,
30733 to_mut_sse,
3045930734 xmm,
3046030735 to_xmm,
3046130736 mut_xmm,
......@@ -30499,6 +30774,17 @@ const Select = struct {
3049930774 else => false,
3050030775 },
3050130776 .to_gpr, .to_mut_gpr => temp.typeOf(cg).abiSize(cg.pt.zcu) <= 8,
30777 .x87 => switch (temp.tracking(cg).short) {
30778 .register => |reg| reg.class() == .x87,
30779 .register_offset => |reg_off| reg_off.reg.class() == .x87 and reg_off.off == 0,
30780 else => false,
30781 },
30782 .mut_x87 => temp.isMut(cg) and switch (temp.tracking(cg).short) {
30783 .register => |reg| reg.class() == .x87,
30784 .register_offset => |reg_off| reg_off.reg.class() == .x87 and reg_off.off == 0,
30785 else => false,
30786 },
30787 .to_x87, .to_mut_x87 => true,
3050230788 .mm => temp.typeOf(cg).abiSize(cg.pt.zcu) == 8 and switch (temp.tracking(cg).short) {
3050330789 .register => |reg| reg.class() == .mmx,
3050430790 .register_offset => |reg_off| reg_off.reg.class() == .mmx and reg_off.off == 0,
......@@ -30510,6 +30796,17 @@ const Select = struct {
3051030796 else => false,
3051130797 },
3051230798 .to_mm, .to_mut_mm => temp.typeOf(cg).abiSize(cg.pt.zcu) == 8,
30799 .sse => switch (temp.tracking(cg).short) {
30800 .register => |reg| reg.class() == .sse,
30801 .register_offset => |reg_off| reg_off.reg.class() == .sse and reg_off.off == 0,
30802 else => false,
30803 },
30804 .mut_sse => temp.isMut(cg) and switch (temp.tracking(cg).short) {
30805 .register => |reg| reg.class() == .sse,
30806 .register_offset => |reg_off| reg_off.reg.class() == .sse and reg_off.off == 0,
30807 else => false,
30808 },
30809 .to_sse, .to_mut_sse => true,
3051330810 .xmm => temp.typeOf(cg).abiSize(cg.pt.zcu) == 16 and switch (temp.tracking(cg).short) {
3051430811 .register => |reg| reg.class() == .sse,
3051530812 .register_offset => |reg_off| reg_off.reg.class() == .sse and reg_off.off == 0,
......@@ -30542,10 +30839,12 @@ const Select = struct {
3054230839 .mem, .to_mem, .mut_mem, .to_mut_mem => try temp.toBase(cg),
3054330840 .gpr, .to_gpr => try temp.toRegClass(false, .general_purpose, cg),
3054430841 .mut_gpr, .to_mut_gpr => try temp.toRegClass(true, .general_purpose, cg),
30842 .x87, .to_x87 => try temp.toRegClass(false, .x87, cg),
30843 .mut_x87, .to_mut_x87 => try temp.toRegClass(true, .x87, cg),
3054530844 .mm, .to_mm => try temp.toRegClass(false, .mmx, cg),
3054630845 .mut_mm, .to_mut_mm => try temp.toRegClass(true, .mmx, cg),
30547 .xmm, .to_xmm, .ymm, .to_ymm => try temp.toRegClass(false, .sse, cg),
30548 .mut_xmm, .to_mut_xmm, .mut_ymm, .to_mut_ymm => try temp.toRegClass(true, .sse, cg),
30846 .sse, .to_sse, .xmm, .to_xmm, .ymm, .to_ymm => try temp.toRegClass(false, .sse, cg),
30847 .mut_sse, .to_mut_sse, .mut_xmm, .to_mut_xmm, .mut_ymm, .to_mut_ymm => try temp.toRegClass(true, .sse, cg),
3054930848 };
3055030849 }
3055130850 };
......@@ -30565,12 +30864,15 @@ const Select = struct {
3056530864 rc: Register.Class,
3056630865 rc_mask: struct { rc: Register.Class, info: MaskInfo },
3056730866 mem,
30568 smin_mem: Select.Operand.Ref,
30569 smax_mem: Select.Operand.Ref,
30570 umin_mem: Select.Operand.Ref,
30571 umax_mem: Select.Operand.Ref,
30867 smin_mem: ConstInfo,
30868 smax_mem: ConstInfo,
30869 umin_mem: ConstInfo,
30870 umax_mem: ConstInfo,
3057230871 ref: Select.Operand.Ref,
3057330872 ref_mask: struct { ref: Select.Operand.Ref, info: MaskInfo },
30873 mut_reg: struct { ref: Select.Operand.Ref, rc: Register.Class },
30874
30875 const ConstInfo = struct { ref: Select.Operand.Ref, vectorize: bool = false };
3057430876
3057530877 fn finish(kind: Kind, temp: Temp, s: *const Select) void {
3057630878 switch (kind) {
......@@ -30590,13 +30892,13 @@ const Select = struct {
3059030892 .rc => |rc| try cg.tempAllocReg(spec.type, regSetForRegClass(rc)),
3059130893 .rc_mask => |rc_mask| try cg.tempAllocReg(spec.type, regSetForRegClass(rc_mask.rc)),
3059230894 .mem => try cg.tempAllocMem(spec.type),
30593 .smin_mem, .smax_mem, .umin_mem, .umax_mem => |ty_ref| {
30895 .smin_mem, .smax_mem, .umin_mem, .umax_mem => |const_info| {
3059430896 const pt = cg.pt;
3059530897 const zcu = pt.zcu;
3059630898 const ip = &zcu.intern_pool;
30597 const ty = ty_ref.deref(s).typeOf(s.cg);
30598 const vector_len, const scalar_ty: Type = switch (ip.indexToKey(ty.toIntern())) {
30599 else => .{ null, ty },
30899 const ty = const_info.ref.deref(s).typeOf(s.cg);
30900 const vector_len: ?u32, const scalar_ty: Type = switch (ip.indexToKey(ty.toIntern())) {
30901 else => .{ if (const_info.vectorize) 1 else null, ty },
3060030902 .vector_type => |vector_type| .{ vector_type.len, .fromInterned(vector_type.child) },
3060130903 };
3060230904 const res_scalar_ty, const res_scalar_val: Value = res_scalar: switch (scalar_ty.toIntern()) {
......@@ -30609,7 +30911,10 @@ const Select = struct {
3060930911 }),
3061030912 },
3061130913 else => {
30612 const scalar_info = cg.intInfo(scalar_ty).?;
30914 const scalar_info: std.builtin.Type.Int = cg.intInfo(scalar_ty) orelse .{
30915 .signedness = .signed,
30916 .bits = cg.floatBits(scalar_ty).?,
30917 };
3061330918 const scalar_int_ty = try pt.intType(scalar_info.signedness, scalar_info.bits);
3061430919 if (scalar_info.bits <= 64) {
3061530920 const int_val: i64 = switch (spec.kind) {
......@@ -30651,6 +30956,15 @@ const Select = struct {
3065130956 },
3065230957 .ref => |ref| ref.deref(s),
3065330958 .ref_mask => |ref_mask| ref_mask.ref.deref(s),
30959 .mut_reg => |ref_rc| {
30960 const temp = ref_rc.ref.deref(s);
30961 if (temp.isMut(cg)) switch (temp.tracking(cg).short) {
30962 .register => |reg| if (reg.class() == ref_rc.rc) return temp,
30963 .register_offset => |reg_off| if (reg_off.off == 0 and reg_off.reg.class() == ref_rc.rc) return temp,
30964 else => {},
30965 };
30966 return try cg.tempAllocReg(spec.type, regSetForRegClass(ref_rc.rc));
30967 },
3065430968 };
3065530969 }
3065630970 };
......@@ -30759,6 +31073,7 @@ const Select = struct {
3075931073 const tmp0d: Sized = .{ .ref = .tmp0, .size = .dword };
3076031074 const tmp0p: Sized = .{ .ref = .tmp0, .size = .ptr };
3076131075 const tmp0q: Sized = .{ .ref = .tmp0, .size = .qword };
31076 const tmp0t: Sized = .{ .ref = .tmp0, .size = .tbyte };
3076231077 const tmp0x: Sized = .{ .ref = .tmp0, .size = .xword };
3076331078 const tmp0y: Sized = .{ .ref = .tmp0, .size = .yword };
3076431079
......@@ -30768,6 +31083,7 @@ const Select = struct {
3076831083 const tmp1d: Sized = .{ .ref = .tmp1, .size = .dword };
3076931084 const tmp1p: Sized = .{ .ref = .tmp1, .size = .ptr };
3077031085 const tmp1q: Sized = .{ .ref = .tmp1, .size = .qword };
31086 const tmp1t: Sized = .{ .ref = .tmp1, .size = .tbyte };
3077131087 const tmp1x: Sized = .{ .ref = .tmp1, .size = .xword };
3077231088 const tmp1y: Sized = .{ .ref = .tmp1, .size = .yword };
3077331089
......@@ -30777,6 +31093,7 @@ const Select = struct {
3077731093 const tmp2d: Sized = .{ .ref = .tmp2, .size = .dword };
3077831094 const tmp2p: Sized = .{ .ref = .tmp2, .size = .ptr };
3077931095 const tmp2q: Sized = .{ .ref = .tmp2, .size = .qword };
31096 const tmp2t: Sized = .{ .ref = .tmp2, .size = .tbyte };
3078031097 const tmp2x: Sized = .{ .ref = .tmp2, .size = .xword };
3078131098 const tmp2y: Sized = .{ .ref = .tmp2, .size = .yword };
3078231099
......@@ -30786,6 +31103,7 @@ const Select = struct {
3078631103 const tmp3d: Sized = .{ .ref = .tmp3, .size = .dword };
3078731104 const tmp3p: Sized = .{ .ref = .tmp3, .size = .ptr };
3078831105 const tmp3q: Sized = .{ .ref = .tmp3, .size = .qword };
31106 const tmp3t: Sized = .{ .ref = .tmp3, .size = .tbyte };
3078931107 const tmp3x: Sized = .{ .ref = .tmp3, .size = .xword };
3079031108 const tmp3y: Sized = .{ .ref = .tmp3, .size = .yword };
3079131109
......@@ -30795,6 +31113,7 @@ const Select = struct {
3079531113 const tmp4d: Sized = .{ .ref = .tmp4, .size = .dword };
3079631114 const tmp4p: Sized = .{ .ref = .tmp4, .size = .ptr };
3079731115 const tmp4q: Sized = .{ .ref = .tmp4, .size = .qword };
31116 const tmp4t: Sized = .{ .ref = .tmp4, .size = .tbyte };
3079831117 const tmp4x: Sized = .{ .ref = .tmp4, .size = .xword };
3079931118 const tmp4y: Sized = .{ .ref = .tmp4, .size = .yword };
3080031119
......@@ -30804,6 +31123,7 @@ const Select = struct {
3080431123 const tmp5d: Sized = .{ .ref = .tmp5, .size = .dword };
3080531124 const tmp5p: Sized = .{ .ref = .tmp5, .size = .ptr };
3080631125 const tmp5q: Sized = .{ .ref = .tmp5, .size = .qword };
31126 const tmp5t: Sized = .{ .ref = .tmp5, .size = .tbyte };
3080731127 const tmp5x: Sized = .{ .ref = .tmp5, .size = .xword };
3080831128 const tmp5y: Sized = .{ .ref = .tmp5, .size = .yword };
3080931129
......@@ -30813,6 +31133,7 @@ const Select = struct {
3081331133 const dst0d: Sized = .{ .ref = .dst0, .size = .dword };
3081431134 const dst0p: Sized = .{ .ref = .dst0, .size = .ptr };
3081531135 const dst0q: Sized = .{ .ref = .dst0, .size = .qword };
31136 const dst0t: Sized = .{ .ref = .dst0, .size = .tbyte };
3081631137 const dst0x: Sized = .{ .ref = .dst0, .size = .xword };
3081731138 const dst0y: Sized = .{ .ref = .dst0, .size = .yword };
3081831139
......@@ -30822,6 +31143,7 @@ const Select = struct {
3082231143 const src0d: Sized = .{ .ref = .src0, .size = .dword };
3082331144 const src0p: Sized = .{ .ref = .src0, .size = .ptr };
3082431145 const src0q: Sized = .{ .ref = .src0, .size = .qword };
31146 const src0t: Sized = .{ .ref = .src0, .size = .tbyte };
3082531147 const src0x: Sized = .{ .ref = .src0, .size = .xword };
3082631148 const src0y: Sized = .{ .ref = .src0, .size = .yword };
3082731149
......@@ -30831,6 +31153,7 @@ const Select = struct {
3083131153 const src1d: Sized = .{ .ref = .src1, .size = .dword };
3083231154 const src1p: Sized = .{ .ref = .src1, .size = .ptr };
3083331155 const src1q: Sized = .{ .ref = .src1, .size = .qword };
31156 const src1t: Sized = .{ .ref = .src1, .size = .tbyte };
3083431157 const src1x: Sized = .{ .ref = .src1, .size = .xword };
3083531158 const src1y: Sized = .{ .ref = .src1, .size = .yword };
3083631159 };
......@@ -30852,6 +31175,7 @@ const Select = struct {
3085231175 const tmp0d: Select.Operand = .{ .tag = .ref, .base = .tmp0d };
3085331176 const tmp0p: Select.Operand = .{ .tag = .ref, .base = .tmp0p };
3085431177 const tmp0q: Select.Operand = .{ .tag = .ref, .base = .tmp0q };
31178 const tmp0t: Select.Operand = .{ .tag = .ref, .base = .tmp0t };
3085531179 const tmp0x: Select.Operand = .{ .tag = .ref, .base = .tmp0x };
3085631180 const tmp0y: Select.Operand = .{ .tag = .ref, .base = .tmp0y };
3085731181
......@@ -30860,6 +31184,7 @@ const Select = struct {
3086031184 const tmp1d: Select.Operand = .{ .tag = .ref, .base = .tmp1d };
3086131185 const tmp1p: Select.Operand = .{ .tag = .ref, .base = .tmp1p };
3086231186 const tmp1q: Select.Operand = .{ .tag = .ref, .base = .tmp1q };
31187 const tmp1t: Select.Operand = .{ .tag = .ref, .base = .tmp1t };
3086331188 const tmp1x: Select.Operand = .{ .tag = .ref, .base = .tmp1x };
3086431189 const tmp1y: Select.Operand = .{ .tag = .ref, .base = .tmp1y };
3086531190
......@@ -30868,6 +31193,7 @@ const Select = struct {
3086831193 const tmp2d: Select.Operand = .{ .tag = .ref, .base = .tmp2d };
3086931194 const tmp2p: Select.Operand = .{ .tag = .ref, .base = .tmp2p };
3087031195 const tmp2q: Select.Operand = .{ .tag = .ref, .base = .tmp2q };
31196 const tmp2t: Select.Operand = .{ .tag = .ref, .base = .tmp2t };
3087131197 const tmp2x: Select.Operand = .{ .tag = .ref, .base = .tmp2x };
3087231198 const tmp2y: Select.Operand = .{ .tag = .ref, .base = .tmp2y };
3087331199
......@@ -30876,6 +31202,7 @@ const Select = struct {
3087631202 const tmp3d: Select.Operand = .{ .tag = .ref, .base = .tmp3d };
3087731203 const tmp3p: Select.Operand = .{ .tag = .ref, .base = .tmp3p };
3087831204 const tmp3q: Select.Operand = .{ .tag = .ref, .base = .tmp3q };
31205 const tmp3t: Select.Operand = .{ .tag = .ref, .base = .tmp3t };
3087931206 const tmp3x: Select.Operand = .{ .tag = .ref, .base = .tmp3x };
3088031207 const tmp3y: Select.Operand = .{ .tag = .ref, .base = .tmp3y };
3088131208
......@@ -30884,6 +31211,7 @@ const Select = struct {
3088431211 const tmp4d: Select.Operand = .{ .tag = .ref, .base = .tmp4d };
3088531212 const tmp4p: Select.Operand = .{ .tag = .ref, .base = .tmp4p };
3088631213 const tmp4q: Select.Operand = .{ .tag = .ref, .base = .tmp4q };
31214 const tmp4t: Select.Operand = .{ .tag = .ref, .base = .tmp4t };
3088731215 const tmp4x: Select.Operand = .{ .tag = .ref, .base = .tmp4x };
3088831216 const tmp4y: Select.Operand = .{ .tag = .ref, .base = .tmp4y };
3088931217
......@@ -30892,6 +31220,7 @@ const Select = struct {
3089231220 const tmp5d: Select.Operand = .{ .tag = .ref, .base = .tmp5d };
3089331221 const tmp5p: Select.Operand = .{ .tag = .ref, .base = .tmp5p };
3089431222 const tmp5q: Select.Operand = .{ .tag = .ref, .base = .tmp5q };
31223 const tmp5t: Select.Operand = .{ .tag = .ref, .base = .tmp5t };
3089531224 const tmp5x: Select.Operand = .{ .tag = .ref, .base = .tmp5x };
3089631225 const tmp5y: Select.Operand = .{ .tag = .ref, .base = .tmp5y };
3089731226
......@@ -30900,6 +31229,7 @@ const Select = struct {
3090031229 const dst0d: Select.Operand = .{ .tag = .ref, .base = .dst0d };
3090131230 const dst0p: Select.Operand = .{ .tag = .ref, .base = .dst0p };
3090231231 const dst0q: Select.Operand = .{ .tag = .ref, .base = .dst0q };
31232 const dst0t: Select.Operand = .{ .tag = .ref, .base = .dst0t };
3090331233 const dst0x: Select.Operand = .{ .tag = .ref, .base = .dst0x };
3090431234 const dst0y: Select.Operand = .{ .tag = .ref, .base = .dst0y };
3090531235
......@@ -30908,6 +31238,7 @@ const Select = struct {
3090831238 const src0d: Select.Operand = .{ .tag = .ref, .base = .src0d };
3090931239 const src0p: Select.Operand = .{ .tag = .ref, .base = .src0p };
3091031240 const src0q: Select.Operand = .{ .tag = .ref, .base = .src0q };
31241 const src0t: Select.Operand = .{ .tag = .ref, .base = .src0t };
3091131242 const src0x: Select.Operand = .{ .tag = .ref, .base = .src0x };
3091231243 const src0y: Select.Operand = .{ .tag = .ref, .base = .src0y };
3091331244
......@@ -30916,6 +31247,7 @@ const Select = struct {
3091631247 const src1d: Select.Operand = .{ .tag = .ref, .base = .src1d };
3091731248 const src1p: Select.Operand = .{ .tag = .ref, .base = .src1p };
3091831249 const src1q: Select.Operand = .{ .tag = .ref, .base = .src1q };
31250 const src1t: Select.Operand = .{ .tag = .ref, .base = .src1t };
3091931251 const src1x: Select.Operand = .{ .tag = .ref, .base = .src1x };
3092031252 const src1y: Select.Operand = .{ .tag = .ref, .base = .src1y };
3092131253
......@@ -31150,7 +31482,7 @@ const Select = struct {
3115031482 else => unreachable,
3115131483 } },
3115231484 else => |mcv| .{ .mem = try mcv.mem(s.cg, .{ .size = op.base.size }) },
31153 .register => |reg| .{ .reg = registerAlias(reg, @intCast(@divExact(op.base.size.bitSize(s.cg.target), 8))) },
31485 .register => |reg| .{ .reg = s.lowerReg(registerAlias(reg, @intCast(@divExact(op.base.size.bitSize(s.cg.target), 8)))) },
3115431486 },
3115531487 .simm => .{ .imm = .s(op.adjustedImm(i32, s)) },
3115631488 .uimm => .{ .imm = .u(@bitCast(op.adjustedImm(i64, s))) },
......@@ -31202,11 +31534,18 @@ fn select(
3120231534 .cg = cg,
3120331535 .temps = undefined,
3120431536 .labels = @splat(.{ .forward = @splat(null), .backward = null }),
31537 .top = 0,
3120531538 };
3120631539 const tmp_slots = s.temps[@intFromEnum(Select.Operand.Ref.tmp0)..@intFromEnum(Select.Operand.Ref.dst0)];
3120731540 const dst_slots = s.temps[@intFromEnum(Select.Operand.Ref.dst0)..@intFromEnum(Select.Operand.Ref.src0)];
3120831541 const src_slots = s.temps[@intFromEnum(Select.Operand.Ref.src0)..@intFromEnum(Select.Operand.Ref.none)];
3120931542
31543 for (0..case.clobbers.st -| 1) |i| {
31544 const tracked_index: RegisterManager.TrackedIndex = @intCast(RegisterManager.indexOfKnownRegIntoTracked(.st6).? - i);
31545 try cg.register_manager.getRegIndex(tracked_index, null);
31546 _ = cg.register_manager.lockRegIndexAssumeUnused(tracked_index);
31547 }
31548
3121031549 @memcpy(src_slots[0..src_temps.len], src_temps);
3121131550 std.mem.swap(Temp, &src_slots[pattern.commute[0]], &src_slots[pattern.commute[1]]);
3121231551 for (tmp_slots, case.extra_temps) |*slot, spec| slot.* = try spec.create(&s) orelse continue;
......@@ -31217,7 +31556,7 @@ fn select(
3121731556 @memcpy(src_slots[0..src_temps.len], src_temps);
3121831557 std.mem.swap(Temp, &src_slots[pattern.commute[0]], &src_slots[pattern.commute[1]]);
3121931558
31220 if (case.clobbers.eflags or case.each != .once) try cg.spillEflagsIfOccupied();
31559 if (case.clobbers.eflags) try cg.spillEflagsIfOccupied();
3122131560
3122231561 for (dst_temps, dst_tys, case.dst_temps[0..dst_temps.len]) |*dst_temp, dst_ty, dst_kind|
3122331562 dst_temp.* = (try Select.TempSpec.create(.{ .type = dst_ty, .kind = dst_kind }, &s)).?;
......@@ -31229,7 +31568,11 @@ fn select(
3122931568 s.emitLabel(.@"0:");
3123031569 },
3123131570 }
31571 assert(s.top == 0);
3123231572
31573 for (0..case.clobbers.st -| 1) |i| cg.register_manager.unlockReg(.{
31574 .tracked_index = @intCast(RegisterManager.indexOfKnownRegIntoTracked(.st6).? - i),
31575 });
3123331576 for (dst_temps, case.dst_temps[0..dst_temps.len]) |dst_temp, dst_kind| dst_kind.finish(dst_temp, &s);
3123431577 for (case.extra_temps, tmp_slots) |spec, temp| if (spec.kind != .unused) try temp.die(cg);
3123531578 return;
src/arch/x86_64/Mir.zig+4-6
......@@ -330,8 +330,8 @@ pub const Inst = struct {
330330 f_pi,
331331 /// Float ___ Pop Pop
332332 f_pp,
333 /// Float ___ stack-top pointer
334 f_stp,
333 /// Float ___ crement Stack-Top Pointer
334 f_cstp,
335335 /// Float ___ Status Word
336336 f_sw,
337337 /// Float ___ Unordered
......@@ -555,6 +555,7 @@ pub const Inst = struct {
555555 /// Decimal adjust AL after subtraction
556556 da,
557557 /// Decrement by 1
558 /// Decrement stack-top pointer
558559 /// Decrement shadow stack pointer
559560 de,
560561 /// Unsigned division
......@@ -587,6 +588,7 @@ pub const Inst = struct {
587588 /// Input from port
588589 /// Input from port to string
589590 /// Increment by 1
591 /// Increment stack-top pointer
590592 /// Increment shadow stack pointer
591593 in,
592594 /// Call to interrupt procedure
......@@ -792,14 +794,10 @@ pub const Inst = struct {
792794 comi,
793795 /// Cosine
794796 cos,
795 /// Decrement stack-top pointer
796 decstp,
797797 /// Reverse divide
798798 divr,
799799 /// Free floating-point register
800800 free,
801 /// Increment stack-top pointer
802 incstp,
803801 /// Initialize floating-point unit
804802 init,
805803 /// Load binary coded decimal integer
src/arch/x86_64/abi.zig+23-19
......@@ -427,7 +427,7 @@ pub const zigcc = struct {
427427
428428 const int_param_regs = gp_regs[0 .. volatile_gpr - 1];
429429 const x87_param_regs = x87_regs[0..volatile_x87];
430 const sse_param_regs = sse_avx_regs[0..volatile_sse];
430 const sse_param_regs = sse_avx_regs[0 .. volatile_sse / 2];
431431 const int_return_regs = gp_regs[0..volatile_gpr];
432432 const x87_return_regs = x87_regs[0..volatile_x87];
433433 const sse_return_regs = sse_avx_regs[0..volatile_gpr];
......@@ -443,11 +443,11 @@ pub const SysV = struct {
443443 pub const caller_preserved_regs = [_]Register{ .rax, .rcx, .rdx, .rsi, .rdi, .r8, .r9, .r10, .r11 } ++ x87_regs ++ sse_avx_regs;
444444
445445 pub const c_abi_int_param_regs = [_]Register{ .rdi, .rsi, .rdx, .rcx, .r8, .r9 };
446 pub const c_abi_x87_param_regs = x87_regs[0..0].*;
447 pub const c_abi_sse_param_regs = sse_avx_regs[0..8].*;
446 pub const c_abi_x87_param_regs = x87_regs[0..0];
447 pub const c_abi_sse_param_regs = sse_avx_regs[0..8];
448448 pub const c_abi_int_return_regs = [_]Register{ .rax, .rdx };
449 pub const c_abi_x87_return_regs = x87_regs[0..2].*;
450 pub const c_abi_sse_return_regs = sse_avx_regs[0..4].*;
449 pub const c_abi_x87_return_regs = x87_regs[0..2];
450 pub const c_abi_sse_return_regs = sse_avx_regs[0..4];
451451};
452452
453453pub const Win64 = struct {
......@@ -460,11 +460,11 @@ pub const Win64 = struct {
460460 pub const caller_preserved_regs = [_]Register{ .rax, .rcx, .rdx, .r8, .r9, .r10, .r11 } ++ x87_regs ++ sse_avx_regs;
461461
462462 pub const c_abi_int_param_regs = [_]Register{ .rcx, .rdx, .r8, .r9 };
463 pub const c_abi_x87_param_regs = x87_regs[0..0].*;
464 pub const c_abi_sse_param_regs = sse_avx_regs[0..4].*;
463 pub const c_abi_x87_param_regs = x87_regs[0..0];
464 pub const c_abi_sse_param_regs = sse_avx_regs[0..4];
465465 pub const c_abi_int_return_regs = [_]Register{.rax};
466 pub const c_abi_x87_return_regs = x87_regs[0..0].*;
467 pub const c_abi_sse_return_regs = sse_avx_regs[0..1].*;
466 pub const c_abi_x87_return_regs = x87_regs[0..0];
467 pub const c_abi_sse_return_regs = sse_avx_regs[0..1];
468468};
469469
470470pub fn getCalleePreservedRegs(cc: std.builtin.CallingConvention.Tag) []const Register {
......@@ -497,17 +497,21 @@ pub fn getCAbiIntParamRegs(cc: std.builtin.CallingConvention.Tag) []const Regist
497497pub fn getCAbiX87ParamRegs(cc: std.builtin.CallingConvention.Tag) []const Register {
498498 return switch (cc) {
499499 .auto => zigcc.x87_param_regs,
500 .x86_64_sysv => &SysV.c_abi_x87_param_regs,
501 .x86_64_win => &Win64.c_abi_x87_param_regs,
500 .x86_64_sysv => SysV.c_abi_x87_param_regs,
501 .x86_64_win => Win64.c_abi_x87_param_regs,
502502 else => unreachable,
503503 };
504504}
505505
506pub fn getCAbiSseParamRegs(cc: std.builtin.CallingConvention.Tag) []const Register {
506pub fn getCAbiSseParamRegs(cc: std.builtin.CallingConvention.Tag, target: *const std.Target) []const Register {
507507 return switch (cc) {
508 .auto => zigcc.sse_param_regs,
509 .x86_64_sysv => &SysV.c_abi_sse_param_regs,
510 .x86_64_win => &Win64.c_abi_sse_param_regs,
508 .auto => switch (target.cpu.arch) {
509 else => unreachable,
510 .x86 => zigcc.sse_param_regs[0 .. zigcc.sse_param_regs.len / 2],
511 .x86_64 => zigcc.sse_param_regs,
512 },
513 .x86_64_sysv => SysV.c_abi_sse_param_regs,
514 .x86_64_win => Win64.c_abi_sse_param_regs,
511515 else => unreachable,
512516 };
513517}
......@@ -524,8 +528,8 @@ pub fn getCAbiIntReturnRegs(cc: std.builtin.CallingConvention.Tag) []const Regis
524528pub fn getCAbiX87ReturnRegs(cc: std.builtin.CallingConvention.Tag) []const Register {
525529 return switch (cc) {
526530 .auto => zigcc.x87_return_regs,
527 .x86_64_sysv => &SysV.c_abi_x87_return_regs,
528 .x86_64_win => &Win64.c_abi_x87_return_regs,
531 .x86_64_sysv => SysV.c_abi_x87_return_regs,
532 .x86_64_win => Win64.c_abi_x87_return_regs,
529533 else => unreachable,
530534 };
531535}
......@@ -533,8 +537,8 @@ pub fn getCAbiX87ReturnRegs(cc: std.builtin.CallingConvention.Tag) []const Regis
533537pub fn getCAbiSseReturnRegs(cc: std.builtin.CallingConvention.Tag) []const Register {
534538 return switch (cc) {
535539 .auto => zigcc.sse_return_regs,
536 .x86_64_sysv => &SysV.c_abi_sse_return_regs,
537 .x86_64_win => &Win64.c_abi_sse_return_regs,
540 .x86_64_sysv => SysV.c_abi_sse_return_regs,
541 .x86_64_win => Win64.c_abi_sse_return_regs,
538542 else => unreachable,
539543 };
540544}
test/behavior/x86_64/math.zig+290-40
......@@ -1,22 +1,136 @@
1const builtin = @import("builtin");
2const inf = math.inf;
3const math = std.math;
4const max = math.floatMax;
5const min = math.floatMin;
6const nan = math.nan;
7const std = @import("std");
8const trueMin = math.floatTrueMin;
9
10const Gpr = switch (builtin.cpu.arch) {
11 else => unreachable,
12 .x86 => u32,
13 .x86_64 => u64,
14};
15const Sse = if (std.Target.x86.featureSetHas(builtin.cpu.features, .avx))
16 @Vector(32, u8)
17else
18 @Vector(16, u8);
19
20inline fn sign(rhs: anytype) bool {
21 return @call(.always_inline, math.signbit, .{rhs});
22}
23inline fn boolAnd(lhs: anytype, rhs: @TypeOf(lhs)) @TypeOf(lhs) {
24 switch (@typeInfo(@TypeOf(lhs))) {
25 .bool => return lhs and rhs,
26 .vector => |vector| switch (vector.child) {
27 bool => {
28 const Bits = @Vector(vector.len, u1);
29 const lhs_bits: Bits = @bitCast(lhs);
30 const rhs_bits: Bits = @bitCast(rhs);
31 return @bitCast(lhs_bits & rhs_bits);
32 },
33 else => {},
34 },
35 else => {},
36 }
37 @compileError("unsupported boolAnd type: " ++ @typeName(@TypeOf(lhs)));
38}
39inline fn boolOr(lhs: anytype, rhs: @TypeOf(lhs)) @TypeOf(lhs) {
40 switch (@typeInfo(@TypeOf(lhs))) {
41 .bool => return lhs or rhs,
42 .vector => |vector| switch (vector.child) {
43 bool => {
44 const Bits = @Vector(vector.len, u1);
45 const lhs_bits: Bits = @bitCast(lhs);
46 const rhs_bits: Bits = @bitCast(rhs);
47 return @bitCast(lhs_bits | rhs_bits);
48 },
49 else => {},
50 },
51 else => {},
52 }
53 @compileError("unsupported boolOr type: " ++ @typeName(@TypeOf(lhs)));
54}
55
56// noinline for a more helpful stack trace
57noinline fn checkExpected(expected: anytype, actual: @TypeOf(expected)) !void {
58 const info = @typeInfo(@TypeOf(expected));
59 const unexpected = switch (switch (info) {
60 else => info,
61 .vector => |vector| @typeInfo(vector.child),
62 }) {
63 else => expected != actual,
64 .float => boolOr(boolAnd(expected != actual, boolOr(expected == expected, actual == actual)), sign(expected) != sign(actual)),
65 };
66 if (switch (info) {
67 else => unexpected,
68 .vector => @reduce(.Or, unexpected),
69 }) return error.Unexpected;
70}
71test checkExpected {
72 if (checkExpected(nan(f32), nan(f32)) == error.Unexpected) return error.Unexpected;
73 if (checkExpected(nan(f32), -nan(f32)) != error.Unexpected) return error.Unexpected;
74 if (checkExpected(@as(f32, 0.0), @as(f32, 0.0)) == error.Unexpected) return error.Unexpected;
75 if (checkExpected(@as(f32, -0.0), @as(f32, -0.0)) == error.Unexpected) return error.Unexpected;
76 if (checkExpected(@as(f32, -0.0), @as(f32, 0.0)) != error.Unexpected) return error.Unexpected;
77 if (checkExpected(@as(f32, 0.0), @as(f32, -0.0)) != error.Unexpected) return error.Unexpected;
78}
79
180fn Unary(comptime op: anytype) type {
281 return struct {
3 fn testArgs(comptime Type: type, comptime imm_arg: Type) !void {
4 const expected = op(Type, imm_arg);
5 try struct {
6 fn checkExpected(actual: @TypeOf(expected)) !void {
7 if (switch (@typeInfo(@TypeOf(expected))) {
8 else => actual != expected,
9 .vector => @reduce(.Or, actual != expected),
10 }) return error.Unexpected;
11 }
12 noinline fn testArgKinds(mem_arg: Type) !void {
13 var reg_arg = mem_arg;
14 _ = .{&reg_arg};
15 try checkExpected(op(Type, reg_arg));
16 try checkExpected(op(Type, mem_arg));
17 try checkExpected(op(Type, imm_arg));
18 }
19 }.testArgKinds(imm_arg);
82 // noinline so that `mem_arg` is on the stack
83 noinline fn testArgKinds(
84 _: Gpr,
85 _: Gpr,
86 _: Gpr,
87 _: Gpr,
88 _: Gpr,
89 _: Gpr,
90 _: Gpr,
91 _: Gpr,
92 _: Sse,
93 _: Sse,
94 _: Sse,
95 _: Sse,
96 _: Sse,
97 _: Sse,
98 _: Sse,
99 _: Sse,
100 comptime Type: type,
101 comptime imm_arg: Type,
102 mem_arg: Type,
103 ) !void {
104 const expected = comptime op(Type, imm_arg);
105 var reg_arg = mem_arg;
106 _ = .{&reg_arg};
107 try checkExpected(expected, op(Type, reg_arg));
108 try checkExpected(expected, op(Type, mem_arg));
109 try checkExpected(expected, op(Type, imm_arg));
110 }
111 // noinline for a more helpful stack trace
112 noinline fn testArgs(comptime Type: type, comptime imm_arg: Type) !void {
113 try testArgKinds(
114 undefined,
115 undefined,
116 undefined,
117 undefined,
118 undefined,
119 undefined,
120 undefined,
121 undefined,
122 undefined,
123 undefined,
124 undefined,
125 undefined,
126 undefined,
127 undefined,
128 undefined,
129 undefined,
130 Type,
131 imm_arg,
132 imm_arg,
133 );
20134 }
21135 fn testIntTypes() !void {
22136 try testArgs(i1, -1);
......@@ -381,6 +495,102 @@ fn Unary(comptime op: anytype) type {
381495 try testArgs(u1025, 1 << 1023);
382496 try testArgs(u1025, 1 << 1024);
383497 }
498 fn testFloatTypes() !void {
499 try testArgs(f16, -nan(f16));
500 try testArgs(f16, -inf(f16));
501 try testArgs(f16, -max(f16));
502 try testArgs(f16, -10.0);
503 try testArgs(f16, -1.0);
504 try testArgs(f16, -0.1);
505 try testArgs(f16, -min(f16));
506 try testArgs(f16, -trueMin(f16));
507 try testArgs(f16, -0.0);
508 try testArgs(f16, 0.0);
509 try testArgs(f16, trueMin(f16));
510 try testArgs(f16, min(f16));
511 try testArgs(f16, 0.1);
512 try testArgs(f16, 1.0);
513 try testArgs(f16, 10.0);
514 try testArgs(f16, max(f16));
515 try testArgs(f16, inf(f16));
516 try testArgs(f16, nan(f16));
517
518 try testArgs(f32, -nan(f32));
519 try testArgs(f32, -inf(f32));
520 try testArgs(f32, -max(f32));
521 try testArgs(f32, -10.0);
522 try testArgs(f32, -1.0);
523 try testArgs(f32, -0.1);
524 try testArgs(f32, -min(f32));
525 try testArgs(f32, -trueMin(f32));
526 try testArgs(f32, -0.0);
527 try testArgs(f32, 0.0);
528 try testArgs(f32, trueMin(f32));
529 try testArgs(f32, min(f32));
530 try testArgs(f32, 0.1);
531 try testArgs(f32, 1.0);
532 try testArgs(f32, 10.0);
533 try testArgs(f32, max(f32));
534 try testArgs(f32, inf(f32));
535 try testArgs(f32, nan(f32));
536
537 try testArgs(f64, -nan(f64));
538 try testArgs(f64, -inf(f64));
539 try testArgs(f64, -max(f64));
540 try testArgs(f64, -10.0);
541 try testArgs(f64, -1.0);
542 try testArgs(f64, -0.1);
543 try testArgs(f64, -min(f64));
544 try testArgs(f64, -trueMin(f64));
545 try testArgs(f64, -0.0);
546 try testArgs(f64, 0.0);
547 try testArgs(f64, trueMin(f64));
548 try testArgs(f64, min(f64));
549 try testArgs(f64, 0.1);
550 try testArgs(f64, 1.0);
551 try testArgs(f64, 10.0);
552 try testArgs(f64, max(f64));
553 try testArgs(f64, inf(f64));
554 try testArgs(f64, nan(f64));
555
556 try testArgs(f80, -nan(f80));
557 try testArgs(f80, -inf(f80));
558 try testArgs(f80, -max(f80));
559 try testArgs(f80, -10.0);
560 try testArgs(f80, -1.0);
561 try testArgs(f80, -0.1);
562 try testArgs(f80, -min(f80));
563 try testArgs(f80, -trueMin(f80));
564 try testArgs(f80, -0.0);
565 try testArgs(f80, 0.0);
566 try testArgs(f80, trueMin(f80));
567 try testArgs(f80, min(f80));
568 try testArgs(f80, 0.1);
569 try testArgs(f80, 1.0);
570 try testArgs(f80, 10.0);
571 try testArgs(f80, max(f80));
572 try testArgs(f80, inf(f80));
573 try testArgs(f80, nan(f80));
574
575 try testArgs(f128, -nan(f128));
576 try testArgs(f128, -inf(f128));
577 try testArgs(f128, -max(f128));
578 try testArgs(f128, -10.0);
579 try testArgs(f128, -1.0);
580 try testArgs(f128, -0.1);
581 try testArgs(f128, -min(f128));
582 try testArgs(f128, -trueMin(f128));
583 try testArgs(f128, -0.0);
584 try testArgs(f128, 0.0);
585 try testArgs(f128, trueMin(f128));
586 try testArgs(f128, min(f128));
587 try testArgs(f128, 0.1);
588 try testArgs(f128, 1.0);
589 try testArgs(f128, 10.0);
590 try testArgs(f128, max(f128));
591 try testArgs(f128, inf(f128));
592 try testArgs(f128, nan(f128));
593 }
384594 fn testIntVectorTypes() !void {
385595 try testArgs(@Vector(3, i1), .{ -1 << 0, -1, 0 });
386596 try testArgs(@Vector(3, u1), .{ 0, 1, 1 << 0 });
......@@ -931,29 +1141,68 @@ fn Unary(comptime op: anytype) type {
9311141
9321142fn Binary(comptime op: anytype) type {
9331143 return struct {
934 fn testArgs(comptime Type: type, comptime imm_lhs: Type, comptime imm_rhs: Type) !void {
935 const expected = op(Type, imm_lhs, imm_rhs);
936 try struct {
937 fn checkExpected(actual: @TypeOf(expected)) !void {
938 if (switch (@typeInfo(@TypeOf(expected))) {
939 else => actual != expected,
940 .vector => @reduce(.Or, actual != expected),
941 }) return error.Unexpected;
942 }
943 noinline fn testArgKinds(mem_lhs: Type, mem_rhs: Type) !void {
944 var reg_lhs = mem_lhs;
945 var reg_rhs = mem_rhs;
946 _ = .{ &reg_lhs, &reg_rhs };
947 try checkExpected(op(Type, reg_lhs, reg_rhs));
948 try checkExpected(op(Type, reg_lhs, mem_rhs));
949 try checkExpected(op(Type, reg_lhs, imm_rhs));
950 try checkExpected(op(Type, mem_lhs, reg_rhs));
951 try checkExpected(op(Type, mem_lhs, mem_rhs));
952 try checkExpected(op(Type, mem_lhs, imm_rhs));
953 try checkExpected(op(Type, imm_lhs, reg_rhs));
954 try checkExpected(op(Type, imm_lhs, mem_rhs));
955 }
956 }.testArgKinds(imm_lhs, imm_rhs);
1144 // noinline so that `mem_lhs` and `mem_rhs` are on the stack
1145 noinline fn testArgKinds(
1146 _: Gpr,
1147 _: Gpr,
1148 _: Gpr,
1149 _: Gpr,
1150 _: Gpr,
1151 _: Gpr,
1152 _: Gpr,
1153 _: Gpr,
1154 _: Sse,
1155 _: Sse,
1156 _: Sse,
1157 _: Sse,
1158 _: Sse,
1159 _: Sse,
1160 _: Sse,
1161 _: Sse,
1162 comptime Type: type,
1163 comptime imm_lhs: Type,
1164 mem_lhs: Type,
1165 comptime imm_rhs: Type,
1166 mem_rhs: Type,
1167 ) !void {
1168 const expected = comptime op(Type, imm_lhs, imm_rhs);
1169 var reg_lhs = mem_lhs;
1170 var reg_rhs = mem_rhs;
1171 _ = .{ &reg_lhs, &reg_rhs };
1172 try checkExpected(expected, op(Type, reg_lhs, reg_rhs));
1173 try checkExpected(expected, op(Type, reg_lhs, mem_rhs));
1174 try checkExpected(expected, op(Type, reg_lhs, imm_rhs));
1175 try checkExpected(expected, op(Type, mem_lhs, reg_rhs));
1176 try checkExpected(expected, op(Type, mem_lhs, mem_rhs));
1177 try checkExpected(expected, op(Type, mem_lhs, imm_rhs));
1178 try checkExpected(expected, op(Type, imm_lhs, reg_rhs));
1179 try checkExpected(expected, op(Type, imm_lhs, mem_rhs));
1180 }
1181 // noinline for a more helpful stack trace
1182 noinline fn testArgs(comptime Type: type, comptime imm_lhs: Type, comptime imm_rhs: Type) !void {
1183 try testArgKinds(
1184 undefined,
1185 undefined,
1186 undefined,
1187 undefined,
1188 undefined,
1189 undefined,
1190 undefined,
1191 undefined,
1192 undefined,
1193 undefined,
1194 undefined,
1195 undefined,
1196 undefined,
1197 undefined,
1198 undefined,
1199 undefined,
1200 Type,
1201 imm_lhs,
1202 imm_lhs,
1203 imm_rhs,
1204 imm_rhs,
1205 );
9571206 }
9581207 fn testIntTypes() !void {
9591208 try testArgs(u8, 0xbb, 0x43);
......@@ -1308,6 +1557,7 @@ inline fn abs(comptime Type: type, rhs: Type) @TypeOf(@abs(rhs)) {
13081557test abs {
13091558 try Unary(abs).testIntTypes();
13101559 try Unary(abs).testIntVectorTypes();
1560 try Unary(abs).testFloatTypes();
13111561}
13121562
13131563inline fn clz(comptime Type: type, rhs: Type) @TypeOf(@clz(rhs)) {