authorgravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2025-01-27 13:37:08-05:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-01-29 22:00:08-08:00
loge4c049e4104ed16097d7481979e474479e45328b
tree1e527ce857f0b77fe96404d96b40a11b425a11a0
parentfecdc53a48970d55fc3ac1beb5be64b8589146cf

x86_64: rewrite comparisons

3 files changed, 3605 insertions(+), 2591 deletions(-)

src/arch/x86_64/CodeGen.zig+3118-2512
......@@ -635,6 +635,7 @@ const InstTracking = struct {
635635 }
636636 tracking_log.debug("spill {} from {} to {}", .{ inst, self.short, self.long });
637637 try cg.genCopy(cg.typeOfIndex(inst), self.long, self.short, .{});
638 for (self.short.getRegs()) |reg| if (reg.class() == .x87) try cg.asmRegister(.{ .f_, .free }, reg);
638639 }
639640
640641 fn reuseFrame(self: *InstTracking) void {
......@@ -649,7 +650,21 @@ const InstTracking = struct {
649650 }
650651
651652 fn trackSpill(self: *InstTracking, function: *CodeGen, inst: Air.Inst.Index) !void {
652 try function.freeValue(self.short);
653 switch (self.short) {
654 .register => |reg| function.register_manager.freeReg(reg),
655 inline .register_pair,
656 .register_triple,
657 .register_quadruple,
658 => |regs| for (regs) |reg| function.register_manager.freeReg(reg),
659 .register_offset, .indirect => |reg_off| function.register_manager.freeReg(reg_off.reg),
660 .register_overflow => |reg_ov| {
661 function.register_manager.freeReg(reg_ov.reg);
662 function.eflags_inst = null;
663 },
664 .register_mask => |reg_mask| function.register_manager.freeReg(reg_mask.reg),
665 .eflags => function.eflags_inst = null,
666 else => {}, // TODO process stack allocation death
667 }
653668 self.reuseFrame();
654669 tracking_log.debug("{} => {} (spilled)", .{ inst, self.* });
655670 }
......@@ -1063,10 +1078,16 @@ pub fn generateLazy(
10631078 .end_di_column = undefined, // no debug info yet
10641079 };
10651080 defer {
1081 function.inst_tracking.deinit(gpa);
10661082 function.mir_instructions.deinit(gpa);
10671083 function.mir_extra.deinit(gpa);
10681084 function.mir_table.deinit(gpa);
10691085 }
1086 try function.inst_tracking.ensureTotalCapacity(gpa, Temp.Index.max);
1087 for (0..Temp.Index.max) |temp_index| {
1088 const temp: Temp.Index = @enumFromInt(temp_index);
1089 function.inst_tracking.putAssumeCapacityNoClobber(temp.toIndex(), .init(.none));
1090 }
10701091
10711092 function.genLazy(lazy_sym) catch |err| switch (err) {
10721093 error.CodegenFail => return error.CodegenFail,
......@@ -2412,7 +2433,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
24122433
24132434 try cg.airArg(inst);
24142435
2415 cg.resetTemps();
2436 try cg.resetTemps();
24162437 cg.checkInvariantsAfterAirInst();
24172438 },
24182439 else => break,
......@@ -6348,7 +6369,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
63486369 .{ .@"1:", .f_p, .st, .dst0t, ._, ._, ._ },
63496370 } },
63506371 }, .{
6351 .required_features = .{ .@"64bit", .x87, null, null },
6372 .required_features = .{ .x87, null, null, null },
63526373 .src_constraints = .{
63536374 .{ .scalar_float = .{ .of = .xword, .is = .tbyte } },
63546375 .{ .scalar_float = .{ .of = .xword, .is = .tbyte } },
......@@ -6375,58 +6396,18 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
63756396 .{ ._, .f_, .ld, .src0t, ._, ._, ._ },
63766397 .{ ._, .f_, .xam, ._, ._, ._, ._ },
63776398 .{ ._, .fn_sw, .st, .tmp1w, ._, ._, ._ },
6378 .{ ._, ._, .@"test", .tmp1b, .si(0b0_1_000_100), ._, ._ },
6399 .{ ._, ._, .@"test", .tmp1b, .si(0b1_000_100), ._, ._ },
63796400 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
63806401 .{ ._, .f_, .xch, .src1t, ._, ._, ._ },
63816402 .{ ._, .f_, .ucom, .src1t, ._, ._, ._ },
63826403 .{ ._, .fn_sw, .st, .tmp1w, ._, ._, ._ },
63836404 .{ ._, .f_, .xch, .src1t, ._, ._, ._ },
6384 .{ ._, ._, .@"test", .tmp1b, .si(0b0_0_000_001), ._, ._ },
6405 .{ ._, ._, .@"test", .tmp1b, .si(0b0_000_001), ._, ._ },
63856406 .{ ._, ._nz, .j, .@"1f", ._, ._, ._ },
63866407 .{ .@"0:", .f_p, .st, .tmp0t, ._, ._, ._ },
63876408 .{ ._, .f_, .ld, .src1t, ._, ._, ._ },
63886409 .{ .@"1:", .f_p, .st, .dst0t, ._, ._, ._ },
63896410 } },
6390 }, .{
6391 .required_features = .{ .x87, null, null, null },
6392 .src_constraints = .{
6393 .{ .scalar_float = .{ .of = .xword, .is = .tbyte } },
6394 .{ .scalar_float = .{ .of = .xword, .is = .tbyte } },
6395 },
6396 .patterns = &.{
6397 .{ .src = .{ .to_x87, .mem }, .commute = .{ 0, 1 } },
6398 .{ .src = .{ .mem, .to_x87 } },
6399 .{ .src = .{ .to_x87, .to_x87 } },
6400 },
6401 .extra_temps = .{
6402 .{ .type = .f80, .kind = .{ .reg = .st7 } },
6403 .{ .type = .u8, .kind = .{ .reg = .ah } },
6404 .unused,
6405 .unused,
6406 .unused,
6407 .unused,
6408 .unused,
6409 .unused,
6410 .unused,
6411 },
6412 .dst_temps = .{.{ .mut_rc = .{ .ref = .src1, .rc = .x87 } }},
6413 .clobbers = .{ .eflags = true },
6414 .each = .{ .once = &.{
6415 .{ ._, .f_, .ld, .src0t, ._, ._, ._ },
6416 .{ ._, .f_, .ucom, .tmp0t, ._, ._, ._ },
6417 .{ ._, .fn_sw, .st, .tmp1w, ._, ._, ._ },
6418 .{ ._, ._, .sahf, ._, ._, ._, ._ },
6419 .{ ._, ._p, .j, .@"0f", ._, ._, ._ },
6420 .{ ._, .f_, .xch, .src1t, ._, ._, ._ },
6421 .{ ._, .f_, .ucom, .src1t, ._, ._, ._ },
6422 .{ ._, .fn_sw, .st, .tmp1w, ._, ._, ._ },
6423 .{ ._, .f_, .xch, .src1t, ._, ._, ._ },
6424 .{ ._, ._, .sahf, ._, ._, ._, ._ },
6425 .{ ._, ._b, .j, .@"1f", ._, ._, ._ },
6426 .{ .@"0:", .f_p, .st, .tmp0t, ._, ._, ._ },
6427 .{ ._, .f_, .ld, .src1t, ._, ._, ._ },
6428 .{ .@"1:", .f_p, .st, .dst0t, ._, ._, ._ },
6429 } },
64306411 }, .{
64316412 .required_features = .{ .x87, .cmov, null, null },
64326413 .src_constraints = .{
......@@ -6508,7 +6489,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
65086489 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
65096490 } },
65106491 }, .{
6511 .required_features = .{ .@"64bit", .x87, null, null },
6492 .required_features = .{ .x87, null, null, null },
65126493 .src_constraints = .{
65136494 .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } },
65146495 .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } },
......@@ -6535,13 +6516,13 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
65356516 .{ ._, .f_, .ld, .memia(.src0t, .tmp0, .add_size), ._, ._, ._ },
65366517 .{ ._, .f_, .xam, ._, ._, ._, ._ },
65376518 .{ ._, .fn_sw, .st, .tmp3w, ._, ._, ._ },
6538 .{ ._, ._, .@"test", .tmp3b, .si(0b0_1_000_100), ._, ._ },
6519 .{ ._, ._, .@"test", .tmp3b, .si(0b1_000_100), ._, ._ },
65396520 .{ ._, ._z, .j, .@"1f", ._, ._, ._ },
65406521 .{ ._, .f_, .xch, .tmp2t, ._, ._, ._ },
65416522 .{ ._, .f_, .ucom, .tmp2t, ._, ._, ._ },
65426523 .{ ._, .fn_sw, .st, .tmp3w, ._, ._, ._ },
65436524 .{ ._, .f_, .xch, .tmp2t, ._, ._, ._ },
6544 .{ ._, ._, .@"test", .tmp3b, .si(0b0_0_000_001), ._, ._ },
6525 .{ ._, ._, .@"test", .tmp3b, .si(0b0_000_001), ._, ._ },
65456526 .{ ._, ._nz, .j, .@"2f", ._, ._, ._ },
65466527 .{ .@"1:", .f_p, .st, .tmp1t, ._, ._, ._ },
65476528 .{ ._, .f_, .ld, .tmp2t, ._, ._, ._ },
......@@ -6550,49 +6531,6 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
65506531 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
65516532 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
65526533 } },
6553 }, .{
6554 .required_features = .{ .x87, null, null, null },
6555 .src_constraints = .{
6556 .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } },
6557 .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } },
6558 },
6559 .patterns = &.{
6560 .{ .src = .{ .to_mem, .to_mem } },
6561 },
6562 .extra_temps = .{
6563 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
6564 .{ .type = .f80, .kind = .{ .reg = .st6 } },
6565 .{ .type = .f80, .kind = .{ .reg = .st7 } },
6566 .{ .type = .u8, .kind = .{ .reg = .ah } },
6567 .unused,
6568 .unused,
6569 .unused,
6570 .unused,
6571 .unused,
6572 },
6573 .dst_temps = .{.mem},
6574 .clobbers = .{ .eflags = true },
6575 .each = .{ .once = &.{
6576 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
6577 .{ .@"0:", .f_, .ld, .memia(.src1t, .tmp0, .add_size), ._, ._, ._ },
6578 .{ ._, .f_, .ld, .memia(.src0t, .tmp0, .add_size), ._, ._, ._ },
6579 .{ ._, .f_, .ucom, .tmp1t, ._, ._, ._ },
6580 .{ ._, .fn_sw, .st, .tmp3w, ._, ._, ._ },
6581 .{ ._, ._, .sahf, ._, ._, ._, ._ },
6582 .{ ._, ._p, .j, .@"1f", ._, ._, ._ },
6583 .{ ._, .f_, .xch, .tmp2t, ._, ._, ._ },
6584 .{ ._, .f_, .ucom, .tmp2t, ._, ._, ._ },
6585 .{ ._, .fn_sw, .st, .tmp3w, ._, ._, ._ },
6586 .{ ._, .f_, .xch, .tmp2t, ._, ._, ._ },
6587 .{ ._, ._, .sahf, ._, ._, ._, ._ },
6588 .{ ._, ._b, .j, .@"2f", ._, ._, ._ },
6589 .{ .@"1:", .f_p, .st, .tmp1t, ._, ._, ._ },
6590 .{ ._, .f_, .ld, .tmp2t, ._, ._, ._ },
6591 .{ .@"2:", .f_p, .st, .memia(.dst0t, .tmp0, .add_size), ._, ._, ._ },
6592 .{ ._, .f_p, .st, .tmp2t, ._, ._, ._ },
6593 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
6594 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
6595 } },
65966534 }, .{
65976535 .required_features = .{ .sse, null, null, null },
65986536 .src_constraints = .{
......@@ -10271,7 +10209,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
1027110209 .{ .@"1:", .f_p, .st, .dst0t, ._, ._, ._ },
1027210210 } },
1027310211 }, .{
10274 .required_features = .{ .@"64bit", .x87, null, null },
10212 .required_features = .{ .x87, null, null, null },
1027510213 .src_constraints = .{
1027610214 .{ .scalar_float = .{ .of = .xword, .is = .tbyte } },
1027710215 .{ .scalar_float = .{ .of = .xword, .is = .tbyte } },
......@@ -10298,54 +10236,16 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
1029810236 .{ ._, .f_, .ld, .src0t, ._, ._, ._ },
1029910237 .{ ._, .f_, .xam, ._, ._, ._, ._ },
1030010238 .{ ._, .fn_sw, .st, .tmp1w, ._, ._, ._ },
10301 .{ ._, ._, .@"test", .tmp1b, .si(0b0_1_000_100), ._, ._ },
10239 .{ ._, ._, .@"test", .tmp1b, .si(0b1_000_100), ._, ._ },
1030210240 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
1030310241 .{ ._, .f_, .ucom, .src1t, ._, ._, ._ },
1030410242 .{ ._, .fn_sw, .st, .tmp1w, ._, ._, ._ },
10305 .{ ._, ._, .@"test", .tmp1b, .si(0b0_0_000_001), ._, ._ },
10243 .{ ._, ._, .@"test", .tmp1b, .si(0b0_000_001), ._, ._ },
1030610244 .{ ._, ._nz, .j, .@"1f", ._, ._, ._ },
1030710245 .{ .@"0:", .f_p, .st, .tmp0t, ._, ._, ._ },
1030810246 .{ ._, .f_, .ld, .src1t, ._, ._, ._ },
1030910247 .{ .@"1:", .f_p, .st, .dst0t, ._, ._, ._ },
1031010248 } },
10311 }, .{
10312 .required_features = .{ .x87, null, null, null },
10313 .src_constraints = .{
10314 .{ .scalar_float = .{ .of = .xword, .is = .tbyte } },
10315 .{ .scalar_float = .{ .of = .xword, .is = .tbyte } },
10316 },
10317 .patterns = &.{
10318 .{ .src = .{ .to_x87, .mem }, .commute = .{ 0, 1 } },
10319 .{ .src = .{ .mem, .to_x87 } },
10320 .{ .src = .{ .to_x87, .to_x87 } },
10321 },
10322 .extra_temps = .{
10323 .{ .type = .f80, .kind = .{ .reg = .st7 } },
10324 .{ .type = .u8, .kind = .{ .reg = .ah } },
10325 .unused,
10326 .unused,
10327 .unused,
10328 .unused,
10329 .unused,
10330 .unused,
10331 .unused,
10332 },
10333 .dst_temps = .{.{ .mut_rc = .{ .ref = .src1, .rc = .x87 } }},
10334 .clobbers = .{ .eflags = true },
10335 .each = .{ .once = &.{
10336 .{ ._, .f_, .ld, .src0t, ._, ._, ._ },
10337 .{ ._, .f_, .ucom, .tmp0t, ._, ._, ._ },
10338 .{ ._, .fn_sw, .st, .tmp1w, ._, ._, ._ },
10339 .{ ._, ._, .sahf, ._, ._, ._, ._ },
10340 .{ ._, ._p, .j, .@"0f", ._, ._, ._ },
10341 .{ ._, .f_, .ucom, .src1t, ._, ._, ._ },
10342 .{ ._, .fn_sw, .st, .tmp1w, ._, ._, ._ },
10343 .{ ._, ._, .sahf, ._, ._, ._, ._ },
10344 .{ ._, ._b, .j, .@"1f", ._, ._, ._ },
10345 .{ .@"0:", .f_p, .st, .tmp0t, ._, ._, ._ },
10346 .{ ._, .f_, .ld, .src1t, ._, ._, ._ },
10347 .{ .@"1:", .f_p, .st, .dst0t, ._, ._, ._ },
10348 } },
1034910249 }, .{
1035010250 .required_features = .{ .x87, .cmov, null, null },
1035110251 .src_constraints = .{
......@@ -10423,7 +10323,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
1042310323 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
1042410324 } },
1042510325 }, .{
10426 .required_features = .{ .@"64bit", .x87, null, null },
10326 .required_features = .{ .x87, null, null, null },
1042710327 .src_constraints = .{
1042810328 .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } },
1042910329 .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } },
......@@ -10450,11 +10350,11 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
1045010350 .{ ._, .f_, .ld, .memia(.src0t, .tmp0, .add_size), ._, ._, ._ },
1045110351 .{ ._, .f_, .xam, ._, ._, ._, ._ },
1045210352 .{ ._, .fn_sw, .st, .tmp3w, ._, ._, ._ },
10453 .{ ._, ._, .@"test", .tmp3b, .si(0b0_1_000_100), ._, ._ },
10353 .{ ._, ._, .@"test", .tmp3b, .si(0b1_000_100), ._, ._ },
1045410354 .{ ._, ._z, .j, .@"1f", ._, ._, ._ },
1045510355 .{ ._, .f_, .ucom, .tmp2t, ._, ._, ._ },
1045610356 .{ ._, .fn_sw, .st, .tmp3w, ._, ._, ._ },
10457 .{ ._, ._, .@"test", .tmp3b, .si(0b0_0_000_001), ._, ._ },
10357 .{ ._, ._, .@"test", .tmp3b, .si(0b0_000_001), ._, ._ },
1045810358 .{ ._, ._nz, .j, .@"2f", ._, ._, ._ },
1045910359 .{ .@"1:", .f_p, .st, .tmp1t, ._, ._, ._ },
1046010360 .{ ._, .f_, .ld, .tmp2t, ._, ._, ._ },
......@@ -10463,47 +10363,6 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
1046310363 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
1046410364 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
1046510365 } },
10466 }, .{
10467 .required_features = .{ .x87, null, null, null },
10468 .src_constraints = .{
10469 .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } },
10470 .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } },
10471 },
10472 .patterns = &.{
10473 .{ .src = .{ .to_mem, .to_mem } },
10474 },
10475 .extra_temps = .{
10476 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
10477 .{ .type = .f80, .kind = .{ .reg = .st6 } },
10478 .{ .type = .f80, .kind = .{ .reg = .st7 } },
10479 .{ .type = .u8, .kind = .{ .reg = .ah } },
10480 .unused,
10481 .unused,
10482 .unused,
10483 .unused,
10484 .unused,
10485 },
10486 .dst_temps = .{.mem},
10487 .clobbers = .{ .eflags = true },
10488 .each = .{ .once = &.{
10489 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
10490 .{ .@"0:", .f_, .ld, .memia(.src1t, .tmp0, .add_size), ._, ._, ._ },
10491 .{ ._, .f_, .ld, .memia(.src0t, .tmp0, .add_size), ._, ._, ._ },
10492 .{ ._, .f_, .ucom, .tmp1t, ._, ._, ._ },
10493 .{ ._, .fn_sw, .st, .tmp3w, ._, ._, ._ },
10494 .{ ._, ._, .sahf, ._, ._, ._, ._ },
10495 .{ ._, ._p, .j, .@"1f", ._, ._, ._ },
10496 .{ ._, .f_, .ucom, .tmp2t, ._, ._, ._ },
10497 .{ ._, .fn_sw, .st, .tmp3w, ._, ._, ._ },
10498 .{ ._, ._, .sahf, ._, ._, ._, ._ },
10499 .{ ._, ._b, .j, .@"2f", ._, ._, ._ },
10500 .{ .@"1:", .f_p, .st, .tmp1t, ._, ._, ._ },
10501 .{ ._, .f_, .ld, .tmp2t, ._, ._, ._ },
10502 .{ .@"2:", .f_p, .st, .memia(.dst0t, .tmp0, .add_size), ._, ._, ._ },
10503 .{ ._, .f_p, .st, .tmp2t, ._, ._, ._ },
10504 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
10505 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
10506 } },
1050710366 }, .{
1050810367 .required_features = .{ .sse, null, null, null },
1050910368 .src_constraints = .{
......@@ -12186,7 +12045,6 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
1218612045 .loop => if (use_old) try cg.airLoop(inst) else {
1218712046 const ty_pl = air_datas[@intFromEnum(inst)].ty_pl;
1218812047 const extra = cg.air.extraData(Air.Block, ty_pl.payload);
12189 cg.scope_generation += 1;
1219012048 try cg.loops.putNoClobber(cg.gpa, inst, .{
1219112049 .state = try cg.saveState(),
1219212050 .target = @intCast(cg.mir_instructions.len),
......@@ -15808,16 +15666,15 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
1580815666 .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
1580915667 .{ ._, ._nz, .j, .@"1f", ._, ._, ._ },
1581015668 .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
15811 .{ ._, ._r, .sh, .tmp3d, .si(3), ._, ._ },
15669 .{ ._, ._r, .sh, .tmp3d, .ui(3), ._, ._ },
1581215670 .{ ._, ._, .mov, .memid(.dst0b, .tmp3, -1), .tmp2b, ._, ._ },
1581315671 .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
1581415672 .{ .@"1:", ._, .add, .tmp0p, .si(32), ._, ._ },
1581515673 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
1581615674 .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
1581715675 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
15818 .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
15819 .{ ._, ._r, .sh, .tmp3d, .si(3), ._, ._ },
15820 .{ ._, ._, .mov, .memi(.dst0b, .tmp3), .tmp2b, ._, ._ },
15676 .{ ._, ._r, .sh, .tmp1d, .ui(3), ._, ._ },
15677 .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ },
1582115678 },
1582215679 .ne => &.{
1582315680 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
......@@ -15833,16 +15690,15 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
1583315690 .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
1583415691 .{ ._, ._nz, .j, .@"1f", ._, ._, ._ },
1583515692 .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
15836 .{ ._, ._r, .sh, .tmp3d, .si(3), ._, ._ },
15693 .{ ._, ._r, .sh, .tmp3d, .ui(3), ._, ._ },
1583715694 .{ ._, ._, .mov, .memid(.dst0b, .tmp3, -1), .tmp2b, ._, ._ },
1583815695 .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
1583915696 .{ .@"1:", ._, .add, .tmp0p, .si(32), ._, ._ },
1584015697 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
1584115698 .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
1584215699 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
15843 .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
15844 .{ ._, ._r, .sh, .tmp3d, .si(3), ._, ._ },
15845 .{ ._, ._, .mov, .memi(.dst0b, .tmp3), .tmp2b, ._, ._ },
15700 .{ ._, ._r, .sh, .tmp1d, .ui(3), ._, ._ },
15701 .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ },
1584615702 },
1584715703 } },
1584815704 }, .{
......@@ -15971,16 +15827,15 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
1597115827 .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
1597215828 .{ ._, ._nz, .j, .@"1f", ._, ._, ._ },
1597315829 .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
15974 .{ ._, ._r, .sh, .tmp3d, .si(3), ._, ._ },
15830 .{ ._, ._r, .sh, .tmp3d, .ui(3), ._, ._ },
1597515831 .{ ._, ._, .mov, .memid(.dst0b, .tmp3, -1), .tmp2b, ._, ._ },
1597615832 .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
1597715833 .{ .@"1:", ._, .add, .tmp0p, .si(16), ._, ._ },
1597815834 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
1597915835 .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
1598015836 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
15981 .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
15982 .{ ._, ._r, .sh, .tmp3d, .si(3), ._, ._ },
15983 .{ ._, ._, .mov, .memi(.dst0b, .tmp3), .tmp2b, ._, ._ },
15837 .{ ._, ._r, .sh, .tmp1d, .ui(3), ._, ._ },
15838 .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ },
1598415839 },
1598515840 .ne => &.{
1598615841 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
......@@ -15996,16 +15851,15 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
1599615851 .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
1599715852 .{ ._, ._nz, .j, .@"1f", ._, ._, ._ },
1599815853 .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
15999 .{ ._, ._r, .sh, .tmp3d, .si(3), ._, ._ },
15854 .{ ._, ._r, .sh, .tmp3d, .ui(3), ._, ._ },
1600015855 .{ ._, ._, .mov, .memid(.dst0b, .tmp3, -1), .tmp2b, ._, ._ },
1600115856 .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
1600215857 .{ .@"1:", ._, .add, .tmp0p, .si(16), ._, ._ },
1600315858 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
1600415859 .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
1600515860 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
16006 .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
16007 .{ ._, ._r, .sh, .tmp3d, .si(3), ._, ._ },
16008 .{ ._, ._, .mov, .memi(.dst0b, .tmp3), .tmp2b, ._, ._ },
15861 .{ ._, ._r, .sh, .tmp1d, .ui(3), ._, ._ },
15862 .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ },
1600915863 },
1601015864 } },
1601115865 }, .{
......@@ -16042,16 +15896,15 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
1604215896 .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
1604315897 .{ ._, ._nz, .j, .@"1f", ._, ._, ._ },
1604415898 .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
16045 .{ ._, ._r, .sh, .tmp3d, .si(3), ._, ._ },
15899 .{ ._, ._r, .sh, .tmp3d, .ui(3), ._, ._ },
1604615900 .{ ._, ._, .mov, .memid(.dst0b, .tmp3, -1), .tmp2b, ._, ._ },
1604715901 .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
1604815902 .{ .@"1:", ._, .add, .tmp0p, .si(16), ._, ._ },
1604915903 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
1605015904 .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
1605115905 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
16052 .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
16053 .{ ._, ._r, .sh, .tmp3d, .si(3), ._, ._ },
16054 .{ ._, ._, .mov, .memi(.dst0b, .tmp3), .tmp2b, ._, ._ },
15906 .{ ._, ._r, .sh, .tmp1d, .ui(3), ._, ._ },
15907 .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ },
1605515908 },
1605615909 .ne => &.{
1605715910 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
......@@ -16067,16 +15920,15 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
1606715920 .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
1606815921 .{ ._, ._nz, .j, .@"1f", ._, ._, ._ },
1606915922 .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
16070 .{ ._, ._r, .sh, .tmp3d, .si(3), ._, ._ },
15923 .{ ._, ._r, .sh, .tmp3d, .ui(3), ._, ._ },
1607115924 .{ ._, ._, .mov, .memid(.dst0b, .tmp3, -1), .tmp2b, ._, ._ },
1607215925 .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
1607315926 .{ .@"1:", ._, .add, .tmp0p, .si(16), ._, ._ },
1607415927 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
1607515928 .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
1607615929 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
16077 .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
16078 .{ ._, ._r, .sh, .tmp3d, .si(3), ._, ._ },
16079 .{ ._, ._, .mov, .memi(.dst0b, .tmp3), .tmp2b, ._, ._ },
15930 .{ ._, ._r, .sh, .tmp1d, .ui(3), ._, ._ },
15931 .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ },
1608015932 },
1608115933 } },
1608215934 }, .{
......@@ -16205,16 +16057,15 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
1620516057 .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
1620616058 .{ ._, ._nz, .j, .@"1f", ._, ._, ._ },
1620716059 .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
16208 .{ ._, ._r, .sh, .tmp3d, .si(3), ._, ._ },
16060 .{ ._, ._r, .sh, .tmp3d, .ui(3), ._, ._ },
1620916061 .{ ._, ._, .mov, .memid(.dst0b, .tmp3, -1), .tmp2b, ._, ._ },
1621016062 .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
1621116063 .{ .@"1:", ._, .add, .tmp0p, .si(16), ._, ._ },
1621216064 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
1621316065 .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
1621416066 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
16215 .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
16216 .{ ._, ._r, .sh, .tmp3d, .si(3), ._, ._ },
16217 .{ ._, ._, .mov, .memi(.dst0b, .tmp3), .tmp2b, ._, ._ },
16067 .{ ._, ._r, .sh, .tmp1d, .ui(3), ._, ._ },
16068 .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ },
1621816069 },
1621916070 .ne => &.{
1622016071 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
......@@ -16230,16 +16081,15 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
1623016081 .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
1623116082 .{ ._, ._nz, .j, .@"1f", ._, ._, ._ },
1623216083 .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
16233 .{ ._, ._r, .sh, .tmp3d, .si(3), ._, ._ },
16084 .{ ._, ._r, .sh, .tmp3d, .ui(3), ._, ._ },
1623416085 .{ ._, ._, .mov, .memid(.dst0b, .tmp3, -1), .tmp2b, ._, ._ },
1623516086 .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
1623616087 .{ .@"1:", ._, .add, .tmp0p, .si(16), ._, ._ },
1623716088 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
1623816089 .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
1623916090 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
16240 .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
16241 .{ ._, ._r, .sh, .tmp3d, .si(3), ._, ._ },
16242 .{ ._, ._, .mov, .memi(.dst0b, .tmp3), .tmp2b, ._, ._ },
16091 .{ ._, ._r, .sh, .tmp1d, .ui(3), ._, ._ },
16092 .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ },
1624316093 },
1624416094 } },
1624516095 }, .{
......@@ -16276,16 +16126,15 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
1627616126 .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
1627716127 .{ ._, ._nz, .j, .@"1f", ._, ._, ._ },
1627816128 .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
16279 .{ ._, ._r, .sh, .tmp3d, .si(3), ._, ._ },
16129 .{ ._, ._r, .sh, .tmp3d, .ui(3), ._, ._ },
1628016130 .{ ._, ._, .mov, .memid(.dst0b, .tmp3, -1), .tmp2b, ._, ._ },
1628116131 .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
1628216132 .{ .@"1:", ._, .add, .tmp0p, .si(16), ._, ._ },
1628316133 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
1628416134 .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
1628516135 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
16286 .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
16287 .{ ._, ._r, .sh, .tmp3d, .si(3), ._, ._ },
16288 .{ ._, ._, .mov, .memi(.dst0b, .tmp3), .tmp2b, ._, ._ },
16136 .{ ._, ._r, .sh, .tmp1d, .ui(3), ._, ._ },
16137 .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ },
1628916138 },
1629016139 .ne => &.{
1629116140 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
......@@ -16301,16 +16150,15 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
1630116150 .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
1630216151 .{ ._, ._nz, .j, .@"1f", ._, ._, ._ },
1630316152 .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
16304 .{ ._, ._r, .sh, .tmp3d, .si(3), ._, ._ },
16153 .{ ._, ._r, .sh, .tmp3d, .ui(3), ._, ._ },
1630516154 .{ ._, ._, .mov, .memid(.dst0b, .tmp3, -1), .tmp2b, ._, ._ },
1630616155 .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
1630716156 .{ .@"1:", ._, .add, .tmp0p, .si(16), ._, ._ },
1630816157 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
1630916158 .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
1631016159 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
16311 .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
16312 .{ ._, ._r, .sh, .tmp3d, .si(3), ._, ._ },
16313 .{ ._, ._, .mov, .memi(.dst0b, .tmp3), .tmp2b, ._, ._ },
16160 .{ ._, ._r, .sh, .tmp1d, .ui(3), ._, ._ },
16161 .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ },
1631416162 },
1631516163 } },
1631616164 }, .{
......@@ -16394,16 +16242,15 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
1639416242 .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
1639516243 .{ ._, ._nz, .j, .@"1f", ._, ._, ._ },
1639616244 .{ ._, ._, .mov, .tmp4d, .tmp1d, ._, ._ },
16397 .{ ._, ._r, .sh, .tmp4d, .si(3), ._, ._ },
16245 .{ ._, ._r, .sh, .tmp4d, .ui(3), ._, ._ },
1639816246 .{ ._, ._, .mov, .memid(.dst0b, .tmp4, -1), .tmp2b, ._, ._ },
1639916247 .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
1640016248 .{ .@"1:", ._, .add, .tmp0p, .si(8), ._, ._ },
1640116249 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
1640216250 .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
1640316251 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
16404 .{ ._, ._, .mov, .tmp4d, .tmp1d, ._, ._ },
16405 .{ ._, ._r, .sh, .tmp4d, .si(3), ._, ._ },
16406 .{ ._, ._, .mov, .memi(.dst0b, .tmp4), .tmp2b, ._, ._ },
16252 .{ ._, ._r, .sh, .tmp1d, .ui(3), ._, ._ },
16253 .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ },
1640716254 },
1640816255 .ne => &.{
1640916256 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
......@@ -16421,16 +16268,15 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
1642116268 .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
1642216269 .{ ._, ._nz, .j, .@"1f", ._, ._, ._ },
1642316270 .{ ._, ._, .mov, .tmp4d, .tmp1d, ._, ._ },
16424 .{ ._, ._r, .sh, .tmp4d, .si(3), ._, ._ },
16271 .{ ._, ._r, .sh, .tmp4d, .ui(3), ._, ._ },
1642516272 .{ ._, ._, .mov, .memid(.dst0b, .tmp4, -1), .tmp2b, ._, ._ },
1642616273 .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
1642716274 .{ .@"1:", ._, .add, .tmp0p, .si(8), ._, ._ },
1642816275 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
1642916276 .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
1643016277 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
16431 .{ ._, ._, .mov, .tmp4d, .tmp1d, ._, ._ },
16432 .{ ._, ._r, .sh, .tmp4d, .si(3), ._, ._ },
16433 .{ ._, ._, .mov, .memi(.dst0b, .tmp4), .tmp2b, ._, ._ },
16278 .{ ._, ._r, .sh, .tmp1d, .ui(3), ._, ._ },
16279 .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ },
1643416280 },
1643516281 } },
1643616282 }, .{
......@@ -16470,16 +16316,15 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
1647016316 .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
1647116317 .{ ._, ._nz, .j, .@"1f", ._, ._, ._ },
1647216318 .{ ._, ._, .mov, .tmp4d, .tmp1d, ._, ._ },
16473 .{ ._, ._r, .sh, .tmp4d, .si(3), ._, ._ },
16319 .{ ._, ._r, .sh, .tmp4d, .ui(3), ._, ._ },
1647416320 .{ ._, ._, .mov, .memid(.dst0b, .tmp4, -1), .tmp2b, ._, ._ },
1647516321 .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
1647616322 .{ .@"1:", ._, .add, .tmp0p, .si(8), ._, ._ },
1647716323 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
1647816324 .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
1647916325 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
16480 .{ ._, ._, .mov, .tmp4d, .tmp1d, ._, ._ },
16481 .{ ._, ._r, .sh, .tmp4d, .si(3), ._, ._ },
16482 .{ ._, ._, .mov, .memi(.dst0b, .tmp4), .tmp2b, ._, ._ },
16326 .{ ._, ._r, .sh, .tmp1d, .ui(3), ._, ._ },
16327 .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ },
1648316328 },
1648416329 .ne => &.{
1648516330 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
......@@ -16498,16 +16343,15 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
1649816343 .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
1649916344 .{ ._, ._nz, .j, .@"1f", ._, ._, ._ },
1650016345 .{ ._, ._, .mov, .tmp4d, .tmp1d, ._, ._ },
16501 .{ ._, ._r, .sh, .tmp4d, .si(3), ._, ._ },
16346 .{ ._, ._r, .sh, .tmp4d, .ui(3), ._, ._ },
1650216347 .{ ._, ._, .mov, .memid(.dst0b, .tmp4, -1), .tmp2b, ._, ._ },
1650316348 .{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ },
1650416349 .{ .@"1:", ._, .add, .tmp0p, .si(8), ._, ._ },
1650516350 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
1650616351 .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
1650716352 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
16508 .{ ._, ._, .mov, .tmp4d, .tmp1d, ._, ._ },
16509 .{ ._, ._r, .sh, .tmp4d, .si(3), ._, ._ },
16510 .{ ._, ._, .mov, .memi(.dst0b, .tmp4), .tmp2b, ._, ._ },
16353 .{ ._, ._r, .sh, .tmp1d, .ui(3), ._, ._ },
16354 .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ },
1651116355 },
1651216356 } },
1651316357 }, .{
......@@ -17059,16 +16903,15 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
1705916903 .{ ._, ._, .@"test", .tmp1d, .sia(-1, .none, .add_ptr_bit_size), ._, ._ },
1706016904 .{ ._, ._nz, .j, .@"1f", ._, ._, ._ },
1706116905 .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
17062 .{ ._, ._r, .sh, .tmp3d, .si(3), ._, ._ },
16906 .{ ._, ._r, .sh, .tmp3d, .ui(3), ._, ._ },
1706316907 .{ ._, ._, .mov, .memia(.dst0p, .tmp3, .sub_ptr_size), .tmp2p, ._, ._ },
1706416908 .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ },
1706516909 .{ .@"1:", ._, .add, .tmp0p, .si(1), ._, ._ },
1706616910 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
1706716911 .{ ._, ._, .@"test", .tmp1d, .sia(-1, .none, .add_ptr_bit_size), ._, ._ },
1706816912 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
17069 .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
17070 .{ ._, ._r, .sh, .tmp3d, .si(3), ._, ._ },
17071 .{ ._, ._, .mov, .memi(.dst0p, .tmp3), .tmp2p, ._, ._ },
16913 .{ ._, ._r, .sh, .tmp1d, .ui(6), ._, ._ },
16914 .{ ._, ._, .mov, .memsi(.dst0p, .@"8", .tmp1), .tmp2p, ._, ._ },
1707216915 } },
1707316916 }, .{
1707416917 .required_features = .{ .f16c, null, null, null },
......@@ -17809,9 +17652,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
1780917652 .{ ._, ._b, .j, .@"0b", ._, ._, ._ },
1781017653 .{ ._, ._, .@"test", .tmp0d, .si(0b111111), ._, ._ },
1781117654 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
17812 .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ },
17813 .{ ._, ._r, .sh, .tmp5d, .si(9), ._, ._ },
17814 .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp5), .tmp1q, ._, ._ },
17655 .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ },
17656 .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp1q, ._, ._ },
1781517657 } },
1781617658 }, .{
1781717659 .required_features = .{ .@"64bit", .avx, null, null },
......@@ -17864,9 +17706,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
1786417706 .{ ._, ._b, .j, .@"0b", ._, ._, ._ },
1786517707 .{ ._, ._, .@"test", .tmp0d, .si(0b111111), ._, ._ },
1786617708 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
17867 .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ },
17868 .{ ._, ._r, .sh, .tmp5d, .si(9), ._, ._ },
17869 .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp5), .tmp1q, ._, ._ },
17709 .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ },
17710 .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp1q, ._, ._ },
1787017711 } },
1787117712 }, .{
1787217713 .required_features = .{ .@"64bit", .sse2, .slow_incdec, null },
......@@ -17920,9 +17761,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
1792017761 .{ ._, ._b, .j, .@"0b", ._, ._, ._ },
1792117762 .{ ._, ._, .@"test", .tmp0d, .si(0b111111), ._, ._ },
1792217763 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
17923 .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ },
17924 .{ ._, ._r, .sh, .tmp5d, .si(9), ._, ._ },
17925 .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp5), .tmp1q, ._, ._ },
17764 .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ },
17765 .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp1q, ._, ._ },
1792617766 } },
1792717767 }, .{
1792817768 .required_features = .{ .@"64bit", .sse2, null, null },
......@@ -17976,9 +17816,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
1797617816 .{ ._, ._b, .j, .@"0b", ._, ._, ._ },
1797717817 .{ ._, ._, .@"test", .tmp0d, .si(0b111111), ._, ._ },
1797817818 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
17979 .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ },
17980 .{ ._, ._r, .sh, .tmp5d, .si(9), ._, ._ },
17981 .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp5), .tmp1q, ._, ._ },
17819 .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ },
17820 .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp1q, ._, ._ },
1798217821 } },
1798317822 }, .{
1798417823 .required_features = .{ .@"64bit", .sse, .slow_incdec, null },
......@@ -18034,9 +17873,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
1803417873 .{ ._, ._b, .j, .@"0b", ._, ._, ._ },
1803517874 .{ ._, ._, .@"test", .tmp0d, .si(0b111111), ._, ._ },
1803617875 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
18037 .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ },
18038 .{ ._, ._r, .sh, .tmp5d, .si(9), ._, ._ },
18039 .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp5), .tmp1q, ._, ._ },
17876 .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ },
17877 .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp1q, ._, ._ },
1804017878 } },
1804117879 }, .{
1804217880 .required_features = .{ .@"64bit", .sse, null, null },
......@@ -18092,9 +17930,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
1809217930 .{ ._, ._b, .j, .@"0b", ._, ._, ._ },
1809317931 .{ ._, ._, .@"test", .tmp0d, .si(0b111111), ._, ._ },
1809417932 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
18095 .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ },
18096 .{ ._, ._r, .sh, .tmp5d, .si(9), ._, ._ },
18097 .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp5), .tmp1q, ._, ._ },
17933 .{ ._, ._r, .sh, .tmp0d, .ui(6), ._, ._ },
17934 .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp1q, ._, ._ },
1809817935 } },
1809917936 }, .{
1810017937 .required_features = .{ .avx, .slow_incdec, null, null },
......@@ -18447,9 +18284,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
1844718284 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
1844818285 .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
1844918286 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
18450 .{ ._, ._, .mov, .tmp4d, .tmp1d, ._, ._ },
18451 .{ ._, ._r, .sh, .tmp4d, .ui(3), ._, ._ },
18452 .{ ._, ._, .mov, .memi(.dst0b, .tmp4), .tmp2b, ._, ._ },
18287 .{ ._, ._r, .sh, .tmp1d, .ui(3), ._, ._ },
18288 .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ },
1845318289 } },
1845418290 }, .{
1845518291 .required_features = .{ .sse2, null, null, null },
......@@ -18497,9 +18333,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
1849718333 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
1849818334 .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
1849918335 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
18500 .{ ._, ._, .mov, .tmp4d, .tmp1d, ._, ._ },
18501 .{ ._, ._r, .sh, .tmp4d, .ui(3), ._, ._ },
18502 .{ ._, ._, .mov, .memi(.dst0b, .tmp4), .tmp2b, ._, ._ },
18336 .{ ._, ._r, .sh, .tmp1d, .ui(3), ._, ._ },
18337 .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ },
1850318338 } },
1850418339 }, .{
1850518340 .required_features = .{ .x87, .cmov, .slow_incdec, null },
......@@ -18557,9 +18392,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
1855718392 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
1855818393 .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
1855918394 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
18560 .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
18561 .{ ._, ._r, .sh, .tmp3d, .ui(3), ._, ._ },
18562 .{ ._, ._, .mov, .memi(.dst0b, .tmp3), .tmp2b, ._, ._ },
18395 .{ ._, ._r, .sh, .tmp1d, .ui(3), ._, ._ },
18396 .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ },
1856318397 } },
1856418398 }, .{
1856518399 .required_features = .{ .x87, .cmov, null, null },
......@@ -18617,9 +18451,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
1861718451 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
1861818452 .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
1861918453 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
18620 .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
18621 .{ ._, ._r, .sh, .tmp3d, .ui(3), ._, ._ },
18622 .{ ._, ._, .mov, .memi(.dst0b, .tmp3), .tmp2b, ._, ._ },
18454 .{ ._, ._r, .sh, .tmp1d, .ui(3), ._, ._ },
18455 .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ },
1862318456 } },
1862418457 }, .{
1862518458 .required_features = .{ .x87, null, null, null },
......@@ -18652,8 +18485,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
1865218485 .{ ._, .f_, .ld, .memia(.src0q, .tmp0, .add_size), ._, ._, ._ },
1865318486 .{ ._, .f_pp, .ucom, ._, ._, ._, ._ },
1865418487 .{ ._, .fn_sw, .st, .tmp6w, ._, ._, ._ },
18655 .{ ._, ._, .xor, .tmp6b, .si(0b0_1_000_000), ._, ._ },
18656 .{ ._, ._, .@"test", .tmp6b, .si(0b0_1_000_100), ._, ._ },
18488 .{ ._, ._, .xor, .tmp6b, .si(0b1_000_000), ._, ._ },
18489 .{ ._, ._, .@"test", .tmp6b, .si(0b1_000_100), ._, ._ },
1865718490 .{ ._, .fromCond(cc), .set, .tmp3b, ._, ._, ._ },
1865818491 .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ },
1865918492 .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ },
......@@ -18668,72 +18501,70 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
1866818501 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
1866918502 .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
1867018503 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
18504 .{ ._, ._r, .sh, .tmp1d, .ui(3), ._, ._ },
18505 .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ },
18506 } },
18507 }, .{
18508 .required_features = .{ .x87, .cmov, .slow_incdec, null },
18509 .src_constraints = .{
18510 .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } },
18511 .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } },
18512 },
18513 .patterns = &.{
18514 .{ .src = .{ .to_mem, .to_mem } },
18515 },
18516 .extra_temps = .{
18517 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
18518 .{ .type = .u32, .kind = .{ .reg = .rcx } },
18519 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
18520 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
18521 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
18522 .{ .type = .f80, .kind = .{ .reg = .st6 } },
18523 .{ .type = .f80, .kind = .{ .reg = .st7 } },
18524 .unused,
18525 .unused,
18526 },
18527 .dst_temps = .{.mem},
18528 .clobbers = .{ .eflags = true },
18529 .each = .{ .once = &.{
18530 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
18531 .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
18532 .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ },
18533 .{ .@"0:", ._, .xor, .tmp3d, .tmp3d, ._, ._ },
18534 .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ },
18535 .{ ._, .f_, .ld, .memia(.src1t, .tmp0, .add_size), ._, ._, ._ },
18536 .{ ._, .f_, .ld, .memia(.src0t, .tmp0, .add_size), ._, ._, ._ },
18537 .{ ._, .f_p, .ucomi, .tmp5t, .tmp6t, ._, ._ },
18538 .{ ._, .f_p, .st, .tmp6t, ._, ._, ._ },
18539 .{ ._, .fromCond(cc), .set, .tmp3b, ._, ._, ._ },
18540 .{ ._, switch (cc) {
18541 else => unreachable,
18542 .e => ._np,
18543 .ne => ._p,
18544 }, .set, .tmp4b, ._, ._, ._ },
18545 .{ ._, ._, switch (cc) {
18546 else => unreachable,
18547 .e => .@"and",
18548 .ne => .@"or",
18549 }, .tmp3b, .tmp4b, ._, ._ },
18550 .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ },
18551 .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ },
18552 .{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 1), ._, ._ },
18553 .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
18554 .{ ._, ._nz, .j, .@"1f", ._, ._, ._ },
1867118555 .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
1867218556 .{ ._, ._r, .sh, .tmp3d, .ui(3), ._, ._ },
18673 .{ ._, ._, .mov, .memi(.dst0b, .tmp3), .tmp2b, ._, ._ },
18557 .{ ._, ._, .mov, .memid(.dst0b, .tmp3, -1), .tmp2b, ._, ._ },
18558 .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ },
18559 .{ .@"1:", ._, .add, .tmp0p, .si(16), ._, ._ },
18560 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
18561 .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
18562 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
18563 .{ ._, ._r, .sh, .tmp1d, .ui(3), ._, ._ },
18564 .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ },
1867418565 } },
1867518566 }, .{
18676 .required_features = .{ .x87, .cmov, .slow_incdec, null },
18677 .src_constraints = .{
18678 .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } },
18679 .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } },
18680 },
18681 .patterns = &.{
18682 .{ .src = .{ .to_mem, .to_mem } },
18683 },
18684 .extra_temps = .{
18685 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
18686 .{ .type = .u32, .kind = .{ .reg = .rcx } },
18687 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
18688 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
18689 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
18690 .{ .type = .f80, .kind = .{ .reg = .st6 } },
18691 .{ .type = .f80, .kind = .{ .reg = .st7 } },
18692 .unused,
18693 .unused,
18694 },
18695 .dst_temps = .{.mem},
18696 .clobbers = .{ .eflags = true },
18697 .each = .{ .once = &.{
18698 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
18699 .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ },
18700 .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ },
18701 .{ .@"0:", ._, .xor, .tmp3d, .tmp3d, ._, ._ },
18702 .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ },
18703 .{ ._, .f_, .ld, .memia(.src1t, .tmp0, .add_size), ._, ._, ._ },
18704 .{ ._, .f_, .ld, .memia(.src0t, .tmp0, .add_size), ._, ._, ._ },
18705 .{ ._, .f_p, .ucomi, .tmp5t, .tmp6t, ._, ._ },
18706 .{ ._, .f_p, .st, .tmp6t, ._, ._, ._ },
18707 .{ ._, .fromCond(cc), .set, .tmp3b, ._, ._, ._ },
18708 .{ ._, switch (cc) {
18709 else => unreachable,
18710 .e => ._np,
18711 .ne => ._p,
18712 }, .set, .tmp4b, ._, ._, ._ },
18713 .{ ._, ._, switch (cc) {
18714 else => unreachable,
18715 .e => .@"and",
18716 .ne => .@"or",
18717 }, .tmp3b, .tmp4b, ._, ._ },
18718 .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ },
18719 .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ },
18720 .{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 1), ._, ._ },
18721 .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
18722 .{ ._, ._nz, .j, .@"1f", ._, ._, ._ },
18723 .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
18724 .{ ._, ._r, .sh, .tmp3d, .ui(3), ._, ._ },
18725 .{ ._, ._, .mov, .memid(.dst0b, .tmp3, -1), .tmp2b, ._, ._ },
18726 .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ },
18727 .{ .@"1:", ._, .add, .tmp0p, .si(16), ._, ._ },
18728 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
18729 .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
18730 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
18731 .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
18732 .{ ._, ._r, .sh, .tmp3d, .ui(3), ._, ._ },
18733 .{ ._, ._, .mov, .memi(.dst0b, .tmp3), .tmp2b, ._, ._ },
18734 } },
18735 }, .{
18736 .required_features = .{ .x87, .cmov, null, null },
18567 .required_features = .{ .x87, .cmov, null, null },
1873718568 .src_constraints = .{
1873818569 .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } },
1873918570 .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } },
......@@ -18788,9 +18619,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
1878818619 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
1878918620 .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
1879018621 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
18791 .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
18792 .{ ._, ._r, .sh, .tmp3d, .ui(3), ._, ._ },
18793 .{ ._, ._, .mov, .memi(.dst0b, .tmp3), .tmp2b, ._, ._ },
18622 .{ ._, ._r, .sh, .tmp1d, .ui(3), ._, ._ },
18623 .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ },
1879418624 } },
1879518625 }, .{
1879618626 .required_features = .{ .x87, null, null, null },
......@@ -18823,8 +18653,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
1882318653 .{ ._, .f_, .ld, .memia(.src0t, .tmp0, .add_size), ._, ._, ._ },
1882418654 .{ ._, .f_pp, .ucom, ._, ._, ._, ._ },
1882518655 .{ ._, .fn_sw, .st, .tmp6w, ._, ._, ._ },
18826 .{ ._, ._, .xor, .tmp6b, .si(0b0_1_000_000), ._, ._ },
18827 .{ ._, ._, .@"test", .tmp6b, .si(0b0_1_000_100), ._, ._ },
18656 .{ ._, ._, .xor, .tmp6b, .si(0b1_000_000), ._, ._ },
18657 .{ ._, ._, .@"test", .tmp6b, .si(0b1_000_100), ._, ._ },
1882818658 .{ ._, .fromCond(cc), .set, .tmp3b, ._, ._, ._ },
1882918659 .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ },
1883018660 .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ },
......@@ -18839,9 +18669,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
1883918669 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
1884018670 .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ },
1884118671 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
18842 .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ },
18843 .{ ._, ._r, .sh, .tmp3d, .ui(3), ._, ._ },
18844 .{ ._, ._, .mov, .memi(.dst0b, .tmp3), .tmp2b, ._, ._ },
18672 .{ ._, ._r, .sh, .tmp1d, .ui(3), ._, ._ },
18673 .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp2b, ._, ._ },
1884518674 } },
1884618675 }, .{
1884718676 .required_features = .{ .avx, .slow_incdec, null, null },
......@@ -19164,9 +18993,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
1916418993 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
1916518994 .{ ._, ._, .@"test", .tmp1d, .si(0b111111), ._, ._ },
1916618995 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
19167 .{ ._, ._, .mov, .tmp6d, .tmp1d, ._, ._ },
19168 .{ ._, ._r, .sh, .tmp6d, .si(9), ._, ._ },
19169 .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp6), .tmp2q, ._, ._ },
18996 .{ ._, ._r, .sh, .tmp1d, .ui(6), ._, ._ },
18997 .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp1), .tmp2q, ._, ._ },
1917018998 } },
1917118999 }, .{
1917219000 .required_features = .{ .@"64bit", .avx, null, null },
......@@ -19219,9 +19047,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
1921919047 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
1922019048 .{ ._, ._, .@"test", .tmp1d, .si(0b111111), ._, ._ },
1922119049 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
19222 .{ ._, ._, .mov, .tmp6d, .tmp1d, ._, ._ },
19223 .{ ._, ._r, .sh, .tmp6d, .si(9), ._, ._ },
19224 .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp6), .tmp2q, ._, ._ },
19050 .{ ._, ._r, .sh, .tmp1d, .ui(6), ._, ._ },
19051 .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp1), .tmp2q, ._, ._ },
1922519052 } },
1922619053 }, .{
1922719054 .required_features = .{ .@"64bit", .sse2, .slow_incdec, null },
......@@ -19274,9 +19101,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
1927419101 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
1927519102 .{ ._, ._, .@"test", .tmp1d, .si(0b111111), ._, ._ },
1927619103 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
19277 .{ ._, ._, .mov, .tmp6d, .tmp1d, ._, ._ },
19278 .{ ._, ._r, .sh, .tmp6d, .si(9), ._, ._ },
19279 .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp6), .tmp2q, ._, ._ },
19104 .{ ._, ._r, .sh, .tmp1d, .ui(6), ._, ._ },
19105 .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp1), .tmp2q, ._, ._ },
1928019106 } },
1928119107 }, .{
1928219108 .required_features = .{ .@"64bit", .sse2, null, null },
......@@ -19329,9 +19155,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
1932919155 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
1933019156 .{ ._, ._, .@"test", .tmp1d, .si(0b111111), ._, ._ },
1933119157 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
19332 .{ ._, ._, .mov, .tmp6d, .tmp1d, ._, ._ },
19333 .{ ._, ._r, .sh, .tmp6d, .si(9), ._, ._ },
19334 .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp6), .tmp2q, ._, ._ },
19158 .{ ._, ._r, .sh, .tmp1d, .ui(6), ._, ._ },
19159 .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp1), .tmp2q, ._, ._ },
1933519160 } },
1933619161 }, .{
1933719162 .required_features = .{ .@"64bit", .sse, .slow_incdec, null },
......@@ -19384,9 +19209,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
1938419209 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
1938519210 .{ ._, ._, .@"test", .tmp1d, .si(0b111111), ._, ._ },
1938619211 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
19387 .{ ._, ._, .mov, .tmp6d, .tmp1d, ._, ._ },
19388 .{ ._, ._r, .sh, .tmp6d, .si(9), ._, ._ },
19389 .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp6), .tmp2q, ._, ._ },
19212 .{ ._, ._r, .sh, .tmp1d, .ui(6), ._, ._ },
19213 .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp1), .tmp2q, ._, ._ },
1939019214 } },
1939119215 }, .{
1939219216 .required_features = .{ .@"64bit", .sse, null, null },
......@@ -19439,9 +19263,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
1943919263 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
1944019264 .{ ._, ._, .@"test", .tmp1d, .si(0b111111), ._, ._ },
1944119265 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
19442 .{ ._, ._, .mov, .tmp6d, .tmp1d, ._, ._ },
19443 .{ ._, ._r, .sh, .tmp6d, .si(9), ._, ._ },
19444 .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp6), .tmp2q, ._, ._ },
19266 .{ ._, ._r, .sh, .tmp1d, .ui(6), ._, ._ },
19267 .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp1), .tmp2q, ._, ._ },
1944519268 } },
1944619269 } },
1944719270 }) catch |err| switch (err) {
......@@ -19651,11 +19474,11 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
1965119474 .each = .{ .once = &.{
1965219475 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
1965319476 .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ },
19654 .{ ._, ._r, .sa, .tmp1q, .si(63), ._, ._ },
19477 .{ ._, ._r, .sa, .tmp1q, .ui(63), ._, ._ },
1965519478 .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ },
1965619479 .{ .@"0:", ._, .mov, .tmp3q, .memia(.src0q, .tmp0, .add_size), ._, ._ },
1965719480 .{ ._, ._, .xor, .tmp3q, .tmp1q, ._, ._ },
19658 .{ ._, ._r, .sh, .tmp2b, .si(1), ._, ._ },
19481 .{ ._, ._r, .sh, .tmp2b, .ui(1), ._, ._ },
1965919482 .{ ._, ._, .sbb, .tmp3q, .tmp1q, ._, ._ },
1966019483 .{ ._, ._c, .set, .tmp2b, ._, ._, ._ },
1966119484 .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_size), .tmp3q, ._, ._ },
......@@ -20413,11 +20236,11 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2041320236 .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ },
2041420237 .{ .@"0:", ._, .mov, .tmp1d, .sa(.none, .add_src0_elem_size), ._, ._ },
2041520238 .{ ._, ._, .mov, .tmp2q, .memiad(.src0q, .tmp0, .add_src0_elem_size, -8), ._, ._ },
20416 .{ ._, ._r, .sa, .tmp2q, .si(63), ._, ._ },
20239 .{ ._, ._r, .sa, .tmp2q, .ui(63), ._, ._ },
2041720240 .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ },
2041820241 .{ .@"1:", ._, .mov, .tmp4q, .memi(.src0q, .tmp0), ._, ._ },
2041920242 .{ ._, ._, .xor, .tmp4q, .tmp2q, ._, ._ },
20420 .{ ._, ._r, .sh, .tmp3b, .si(1), ._, ._ },
20243 .{ ._, ._r, .sh, .tmp3b, .ui(1), ._, ._ },
2042120244 .{ ._, ._, .sbb, .tmp4q, .tmp2q, ._, ._ },
2042220245 .{ ._, ._c, .set, .tmp3b, ._, ._, ._ },
2042320246 .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp4q, ._, ._ },
......@@ -20932,185 +20755,405 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2093220755 .cmp_gte_optimized,
2093320756 .cmp_gt,
2093420757 .cmp_gt_optimized,
20935 => |air_tag| if (use_old) try cg.airCmp(inst, switch (air_tag) {
20936 else => unreachable,
20937 .cmp_lt, .cmp_lt_optimized => .lt,
20938 .cmp_lte, .cmp_lte_optimized => .lte,
20939 .cmp_gte, .cmp_gte_optimized => .gte,
20940 .cmp_gt, .cmp_gt_optimized => .gt,
20941 }) else fallback: {
20758 => |air_tag| if (use_old) try cg.airCmp(inst, air_tag.toCmpOp().?) else {
2094220759 const bin_op = air_datas[@intFromEnum(inst)].bin_op;
20943 const scalar_ty = cg.typeOf(bin_op.lhs).scalarType(zcu);
20944 if (scalar_ty.isRuntimeFloat()) break :fallback try cg.airCmp(inst, switch (air_tag) {
20945 else => unreachable,
20946 .cmp_lt, .cmp_lt_optimized => .lt,
20947 .cmp_lte, .cmp_lte_optimized => .lte,
20948 .cmp_gte, .cmp_gte_optimized => .gte,
20949 .cmp_gt, .cmp_gt_optimized => .gt,
20950 });
20951 const signedness = if (scalar_ty.isAbiInt(zcu))
20952 scalar_ty.intInfo(zcu).signedness
20953 else
20954 .unsigned;
20760 const cmp_op = air_tag.toCmpOp().?;
2095520761 var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs });
2095620762 var res: [1]Temp = undefined;
20957 cg.select(&res, &.{.bool}, &ops, switch (@as(Condition, switch (signedness) {
20958 .signed => switch (air_tag) {
20763 (if (cg.floatBits(cg.typeOf(bin_op.lhs))) |_| err: {
20764 switch (cmp_op) {
2095920765 else => unreachable,
20960 .cmp_lt, .cmp_lt_optimized => .l,
20961 .cmp_lte, .cmp_lte_optimized => .le,
20962 .cmp_gte, .cmp_gte_optimized => .ge,
20963 .cmp_gt, .cmp_gt_optimized => .g,
20964 },
20965 .unsigned => switch (air_tag) {
20766 .lt, .lte => {},
20767 .gt, .gte => std.mem.swap(Temp, &ops[0], &ops[1]),
20768 }
20769 break :err cg.select(&res, &.{.bool}, &ops, switch (switch (cmp_op) {
2096620770 else => unreachable,
20967 .cmp_lt, .cmp_lt_optimized => .b,
20968 .cmp_lte, .cmp_lte_optimized => .be,
20969 .cmp_gte, .cmp_gte_optimized => .ae,
20970 .cmp_gt, .cmp_gt_optimized => .a,
20971 },
20972 })) {
20973 else => unreachable,
20974 inline .l, .le, .ge, .g, .b, .be, .ae, .a => |cc| comptime &.{ .{
20975 .src_constraints = .{ .{ .int = .byte }, .{ .int = .byte } },
20976 .patterns = &.{
20977 .{ .src = .{ .imm8, .mem }, .commute = .{ 0, 1 } },
20978 .{ .src = .{ .imm8, .to_gpr }, .commute = .{ 0, 1 } },
20979 .{ .src = .{ .mem, .to_gpr }, .commute = .{ 0, 1 } },
20980 },
20981 .dst_temps = .{.{ .cc = cc.commute() }},
20982 .clobbers = .{ .eflags = true },
20983 .each = .{ .once = &.{
20984 .{ ._, ._, .cmp, .src0b, .src1b, ._, ._ },
20985 } },
20986 }, .{
20987 .src_constraints = .{ .{ .int = .byte }, .{ .int = .byte } },
20988 .patterns = &.{
20989 .{ .src = .{ .mem, .imm8 } },
20990 .{ .src = .{ .to_gpr, .imm8 } },
20991 .{ .src = .{ .to_gpr, .mem } },
20992 .{ .src = .{ .to_gpr, .to_gpr } },
20993 },
20994 .dst_temps = .{.{ .cc = cc }},
20995 .clobbers = .{ .eflags = true },
20996 .each = .{ .once = &.{
20997 .{ ._, ._, .cmp, .src0b, .src1b, ._, ._ },
20998 } },
20999 }, .{
21000 .src_constraints = .{ .{ .int = .word }, .{ .int = .word } },
21001 .patterns = &.{
21002 .{ .src = .{ .imm16, .mem }, .commute = .{ 0, 1 } },
21003 .{ .src = .{ .imm16, .to_gpr }, .commute = .{ 0, 1 } },
21004 .{ .src = .{ .mem, .to_gpr }, .commute = .{ 0, 1 } },
21005 },
21006 .dst_temps = .{.{ .cc = cc.commute() }},
21007 .clobbers = .{ .eflags = true },
21008 .each = .{ .once = &.{
21009 .{ ._, ._, .cmp, .src0w, .src1w, ._, ._ },
21010 } },
21011 }, .{
21012 .src_constraints = .{ .{ .int = .word }, .{ .int = .word } },
21013 .patterns = &.{
21014 .{ .src = .{ .mem, .imm16 } },
21015 .{ .src = .{ .to_gpr, .imm16 } },
21016 .{ .src = .{ .to_gpr, .mem } },
21017 .{ .src = .{ .to_gpr, .to_gpr } },
21018 },
21019 .dst_temps = .{.{ .cc = cc }},
21020 .clobbers = .{ .eflags = true },
21021 .each = .{ .once = &.{
21022 .{ ._, ._, .cmp, .src0w, .src1w, ._, ._ },
21023 } },
21024 }, .{
21025 .src_constraints = .{ .{ .int = .dword }, .{ .int = .dword } },
21026 .patterns = &.{
21027 .{ .src = .{ .imm32, .mem }, .commute = .{ 0, 1 } },
21028 .{ .src = .{ .imm32, .to_gpr }, .commute = .{ 0, 1 } },
21029 .{ .src = .{ .mem, .to_gpr }, .commute = .{ 0, 1 } },
21030 },
21031 .dst_temps = .{.{ .cc = cc.commute() }},
21032 .clobbers = .{ .eflags = true },
21033 .each = .{ .once = &.{
21034 .{ ._, ._, .cmp, .src0d, .src1d, ._, ._ },
21035 } },
21036 }, .{
21037 .src_constraints = .{ .{ .int = .dword }, .{ .int = .dword } },
21038 .patterns = &.{
21039 .{ .src = .{ .mem, .imm32 } },
21040 .{ .src = .{ .to_gpr, .imm32 } },
21041 .{ .src = .{ .to_gpr, .mem } },
21042 .{ .src = .{ .to_gpr, .to_gpr } },
21043 },
21044 .dst_temps = .{.{ .cc = cc }},
21045 .clobbers = .{ .eflags = true },
21046 .each = .{ .once = &.{
21047 .{ ._, ._, .cmp, .src0d, .src1d, ._, ._ },
21048 } },
21049 }, .{
21050 .required_features = .{ .@"64bit", null, null, null },
21051 .src_constraints = .{ .{ .int = .qword }, .{ .int = .qword } },
21052 .patterns = &.{
21053 .{ .src = .{ .simm32, .mem }, .commute = .{ 0, 1 } },
21054 .{ .src = .{ .simm32, .to_gpr }, .commute = .{ 0, 1 } },
21055 .{ .src = .{ .mem, .to_gpr }, .commute = .{ 0, 1 } },
21056 },
21057 .dst_temps = .{.{ .cc = cc.commute() }},
21058 .clobbers = .{ .eflags = true },
21059 .each = .{ .once = &.{
21060 .{ ._, ._, .cmp, .src0q, .src1q, ._, ._ },
21061 } },
21062 }, .{
21063 .required_features = .{ .@"64bit", null, null, null },
21064 .src_constraints = .{ .{ .int = .qword }, .{ .int = .qword } },
21065 .patterns = &.{
21066 .{ .src = .{ .mem, .simm32 } },
21067 .{ .src = .{ .to_gpr, .simm32 } },
21068 .{ .src = .{ .to_gpr, .mem } },
21069 .{ .src = .{ .to_gpr, .to_gpr } },
21070 },
21071 .dst_temps = .{.{ .cc = cc }},
21072 .clobbers = .{ .eflags = true },
21073 .each = .{ .once = &.{
21074 .{ ._, ._, .cmp, .src0q, .src1q, ._, ._ },
21075 } },
21076 }, .{
21077 .src_constraints = .{ .any_int, .any_int },
21078 .patterns = &.{
21079 .{ .src = .{ .to_mem, .to_mem }, .commute = switch (cc) {
21080 else => unreachable,
21081 .l, .ge, .b, .ae => .{ 0, 0 },
21082 .le, .g, .be, .a => .{ 0, 1 },
20771 .lt, .gt => true,
20772 .lte, .gte => false,
20773 }) {
20774 inline false, true => |strict| comptime &.{ .{
20775 .required_features = .{ .f16c, null, null, null },
20776 .src_constraints = .{ .{ .float = .word }, .{ .float = .word } },
20777 .patterns = &.{
20778 .{ .src = .{ .to_sse, .to_sse }, .commute = .{ 0, 1 } },
20779 },
20780 .extra_temps = .{
20781 .{ .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },
20782 .{ .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } },
20783 .unused,
20784 .unused,
20785 .unused,
20786 .unused,
20787 .unused,
20788 .unused,
20789 .unused,
20790 },
20791 .dst_temps = .{.{ .cc = switch (strict) {
20792 true => .a,
20793 false => .ae,
20794 } }},
20795 .clobbers = .{ .eflags = true },
20796 .each = .{ .once = &.{
20797 .{ ._, .v_ps, .cvtph2, .tmp0x, .src0q, ._, ._ },
20798 .{ ._, .v_ps, .cvtph2, .tmp1x, .src1q, ._, ._ },
20799 .{ ._, .v_ss, .ucomi, .tmp0x, .tmp1d, ._, ._ },
20800 } },
20801 }, .{
20802 .required_features = .{ .sse, null, null, null },
20803 .src_constraints = .{ .{ .float = .word }, .{ .float = .word } },
20804 .patterns = &.{
20805 .{ .src = .{ .{ .to_reg = .xmm0 }, .{ .to_reg = .xmm1 } } },
20806 },
20807 .call_frame = .{ .alignment = .@"16" },
20808 .extra_temps = .{
20809 .{ .type = .usize, .kind = .{ .symbol = &.{ .name = "__cmphf2" } } },
20810 .{ .type = .i32, .kind = .{ .reg = .eax } },
20811 .unused,
20812 .unused,
20813 .unused,
20814 .unused,
20815 .unused,
20816 .unused,
20817 .unused,
20818 },
20819 .dst_temps = .{.{ .cc = switch (strict) {
20820 true => .l,
20821 false => .le,
20822 } }},
20823 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
20824 .each = .{ .once = &.{
20825 .{ ._, ._, .call, .tmp0d, ._, ._, ._ },
20826 .{ ._, ._, .@"test", .tmp1d, .tmp1d, ._, ._ },
20827 } },
20828 }, .{
20829 .required_features = .{ .avx, null, null, null },
20830 .src_constraints = .{ .{ .float = .dword }, .{ .float = .dword } },
20831 .patterns = &.{
20832 .{ .src = .{ .mem, .to_sse }, .commute = .{ 0, 1 } },
20833 .{ .src = .{ .to_sse, .to_sse }, .commute = .{ 0, 1 } },
20834 },
20835 .dst_temps = .{.{ .cc = switch (strict) {
20836 true => .a,
20837 false => .ae,
20838 } }},
20839 .clobbers = .{ .eflags = true },
20840 .each = .{ .once = &.{
20841 .{ ._, .v_ss, .ucomi, .src0x, .src1d, ._, ._ },
20842 } },
20843 }, .{
20844 .required_features = .{ .sse, null, null, null },
20845 .src_constraints = .{ .{ .float = .dword }, .{ .float = .dword } },
20846 .patterns = &.{
20847 .{ .src = .{ .mem, .to_sse }, .commute = .{ 0, 1 } },
20848 .{ .src = .{ .to_sse, .to_sse }, .commute = .{ 0, 1 } },
20849 },
20850 .dst_temps = .{.{ .cc = switch (strict) {
20851 true => .a,
20852 false => .ae,
20853 } }},
20854 .clobbers = .{ .eflags = true },
20855 .each = .{ .once = &.{
20856 .{ ._, ._ss, .ucomi, .src0x, .src1d, ._, ._ },
20857 } },
20858 }, .{
20859 .required_features = .{ .avx, null, null, null },
20860 .src_constraints = .{ .{ .float = .qword }, .{ .float = .qword } },
20861 .patterns = &.{
20862 .{ .src = .{ .mem, .to_sse }, .commute = .{ 0, 1 } },
20863 .{ .src = .{ .to_sse, .to_sse }, .commute = .{ 0, 1 } },
20864 },
20865 .dst_temps = .{.{ .cc = switch (strict) {
20866 true => .a,
20867 false => .ae,
20868 } }},
20869 .clobbers = .{ .eflags = true },
20870 .each = .{ .once = &.{
20871 .{ ._, .v_sd, .ucomi, .src0x, .src1q, ._, ._ },
20872 } },
20873 }, .{
20874 .required_features = .{ .sse2, null, null, null },
20875 .src_constraints = .{ .{ .float = .qword }, .{ .float = .qword } },
20876 .patterns = &.{
20877 .{ .src = .{ .mem, .to_sse }, .commute = .{ 0, 1 } },
20878 .{ .src = .{ .to_sse, .to_sse }, .commute = .{ 0, 1 } },
20879 },
20880 .dst_temps = .{.{ .cc = switch (strict) {
20881 true => .a,
20882 false => .ae,
20883 } }},
20884 .clobbers = .{ .eflags = true },
20885 .each = .{ .once = &.{
20886 .{ ._, ._sd, .ucomi, .src0x, .src1q, ._, ._ },
20887 } },
20888 }, .{
20889 .required_features = .{ .x87, .cmov, null, null },
20890 .src_constraints = .{ .{ .float = .qword }, .{ .float = .qword } },
20891 .patterns = &.{
20892 .{ .src = .{ .to_mem, .to_mem }, .commute = .{ 0, 1 } },
20893 },
20894 .extra_temps = .{
20895 .{ .type = .f80, .kind = .{ .reg = .st6 } },
20896 .{ .type = .f80, .kind = .{ .reg = .st7 } },
20897 .unused,
20898 .unused,
20899 .unused,
20900 .unused,
20901 .unused,
20902 .unused,
20903 .unused,
20904 },
20905 .dst_temps = .{.{ .cc = switch (strict) {
20906 true => .a,
20907 false => .ae,
20908 } }},
20909 .clobbers = .{ .eflags = true },
20910 .each = .{ .once = &.{
20911 .{ ._, .f_, .ld, .src1q, ._, ._, ._ },
20912 .{ ._, .f_, .ld, .src0q, ._, ._, ._ },
20913 .{ ._, .f_p, .ucomi, .tmp0t, .tmp1t, ._, ._ },
20914 .{ ._, .f_p, .st, .tmp1t, ._, ._, ._ },
20915 } },
20916 }, .{
20917 .required_features = .{ .sahf, .x87, null, null },
20918 .src_constraints = .{ .{ .float = .qword }, .{ .float = .qword } },
20919 .patterns = &.{
20920 .{ .src = .{ .to_mem, .to_mem }, .commute = .{ 0, 1 } },
20921 },
20922 .extra_temps = .{
20923 .{ .type = .f80, .kind = .{ .reg = .st6 } },
20924 .{ .type = .f80, .kind = .{ .reg = .st7 } },
20925 .{ .type = .u8, .kind = .{ .reg = .ah } },
20926 .unused,
20927 .unused,
20928 .unused,
20929 .unused,
20930 .unused,
20931 .unused,
20932 },
20933 .dst_temps = .{.{ .cc = switch (strict) {
20934 true => .a,
20935 false => .ae,
20936 } }},
20937 .clobbers = .{ .eflags = true },
20938 .each = .{ .once = &.{
20939 .{ ._, .f_, .ld, .src1q, ._, ._, ._ },
20940 .{ ._, .f_, .ld, .src0q, ._, ._, ._ },
20941 .{ ._, .f_pp, .ucom, ._, ._, ._, ._ },
20942 .{ ._, .fn_sw, .st, .tmp2w, ._, ._, ._ },
20943 .{ ._, ._, .sahf, ._, ._, ._, ._ },
20944 } },
20945 }, .{
20946 .required_features = .{ .x87, null, null, null },
20947 .src_constraints = .{ .{ .float = .qword }, .{ .float = .qword } },
20948 .patterns = &.{
20949 .{ .src = .{ .to_mem, .to_mem }, .commute = .{ 0, 1 } },
20950 },
20951 .extra_temps = .{
20952 .{ .type = .f80, .kind = .{ .reg = .st6 } },
20953 .{ .type = .f80, .kind = .{ .reg = .st7 } },
20954 .{ .type = .u8, .kind = .{ .reg = .ah } },
20955 .unused,
20956 .unused,
20957 .unused,
20958 .unused,
20959 .unused,
20960 .unused,
20961 },
20962 .dst_temps = .{.{ .cc = switch (strict) {
20963 true => .z,
20964 false => .nc,
20965 } }},
20966 .clobbers = .{ .eflags = true },
20967 .each = .{ .once = &.{
20968 .{ ._, .f_, .ld, .src1q, ._, ._, ._ },
20969 .{ ._, .f_, .ld, .src0q, ._, ._, ._ },
20970 .{ ._, .f_pp, .ucom, ._, ._, ._, ._ },
20971 .{ ._, .fn_sw, .st, .tmp2w, ._, ._, ._ },
20972 switch (strict) {
20973 true => .{ ._, ._, .@"test", .tmp2b, .si(0b1_000_001), ._, ._ },
20974 false => .{ ._, ._r, .sh, .tmp2b, .ui(1), ._, ._ },
20975 },
20976 } },
20977 }, .{
20978 .required_features = .{ .x87, .cmov, null, null },
20979 .src_constraints = .{ .{ .float = .tbyte }, .{ .float = .tbyte } },
20980 .patterns = &.{
20981 .{ .src = .{ .to_x87, .mem }, .commute = .{ 0, 1 } },
20982 .{ .src = .{ .to_x87, .to_x87 }, .commute = .{ 0, 1 } },
20983 },
20984 .extra_temps = .{
20985 .{ .type = .f80, .kind = .{ .reg = .st7 } },
20986 .unused,
20987 .unused,
20988 .unused,
20989 .unused,
20990 .unused,
20991 .unused,
20992 .unused,
20993 .unused,
20994 },
20995 .dst_temps = .{.{ .cc = switch (strict) {
20996 true => .a,
20997 false => .ae,
20998 } }},
20999 .clobbers = .{ .eflags = true },
21000 .each = .{ .once = &.{
21001 .{ ._, .f_, .ld, .src0t, ._, ._, ._ },
21002 .{ ._, .f_p, .ucomi, .tmp0t, .src1t, ._, ._ },
21003 } },
21004 }, .{
21005 .required_features = .{ .sahf, .x87, null, null },
21006 .src_constraints = .{ .{ .float = .tbyte }, .{ .float = .tbyte } },
21007 .patterns = &.{
21008 .{ .src = .{ .mem, .mem }, .commute = .{ 0, 1 } },
21009 },
21010 .extra_temps = .{
21011 .{ .type = .f80, .kind = .{ .reg = .st6 } },
21012 .{ .type = .f80, .kind = .{ .reg = .st7 } },
21013 .{ .type = .u8, .kind = .{ .reg = .ah } },
21014 .unused,
21015 .unused,
21016 .unused,
21017 .unused,
21018 .unused,
21019 .unused,
21020 },
21021 .dst_temps = .{.{ .cc = switch (strict) {
21022 true => .a,
21023 false => .ae,
21024 } }},
21025 .clobbers = .{ .eflags = true },
21026 .each = .{ .once = &.{
21027 .{ ._, .f_, .ld, .src1t, ._, ._, ._ },
21028 .{ ._, .f_, .ld, .src0t, ._, ._, ._ },
21029 .{ ._, .f_pp, .ucom, ._, ._, ._, ._ },
21030 .{ ._, .fn_sw, .st, .tmp2w, ._, ._, ._ },
21031 .{ ._, ._, .sahf, ._, ._, ._, ._ },
21032 } },
21033 }, .{
21034 .required_features = .{ .sahf, .x87, null, null },
21035 .src_constraints = .{ .{ .float = .tbyte }, .{ .float = .tbyte } },
21036 .patterns = &.{
21037 .{ .src = .{ .to_x87, .mem }, .commute = .{ 0, 1 } },
21038 .{ .src = .{ .to_x87, .to_x87 }, .commute = .{ 0, 1 } },
21039 },
21040 .extra_temps = .{
21041 .{ .type = .f80, .kind = .{ .reg = .st6 } },
21042 .{ .type = .f80, .kind = .{ .reg = .st7 } },
21043 .{ .type = .u8, .kind = .{ .reg = .ah } },
21044 .unused,
21045 .unused,
21046 .unused,
21047 .unused,
21048 .unused,
21049 .unused,
21050 },
21051 .dst_temps = .{.{ .cc = switch (strict) {
21052 true => .a,
21053 false => .ae,
21054 } }},
21055 .clobbers = .{ .eflags = true },
21056 .each = .{ .once = &.{
21057 .{ ._, .f_, .ld, .src0t, ._, ._, ._ },
21058 .{ ._, .f_p, .ucom, .src1t, ._, ._, ._ },
21059 .{ ._, .fn_sw, .st, .tmp2w, ._, ._, ._ },
21060 .{ ._, ._, .sahf, ._, ._, ._, ._ },
21061 } },
21062 }, .{
21063 .required_features = .{ .x87, null, null, null },
21064 .src_constraints = .{ .{ .float = .tbyte }, .{ .float = .tbyte } },
21065 .patterns = &.{
21066 .{ .src = .{ .mem, .mem }, .commute = .{ 0, 1 } },
21067 },
21068 .extra_temps = .{
21069 .{ .type = .f80, .kind = .{ .reg = .st6 } },
21070 .{ .type = .f80, .kind = .{ .reg = .st7 } },
21071 .{ .type = .u8, .kind = .{ .reg = .ah } },
21072 .unused,
21073 .unused,
21074 .unused,
21075 .unused,
21076 .unused,
21077 .unused,
21078 },
21079 .dst_temps = .{.{ .cc = switch (strict) {
21080 true => .z,
21081 false => .nc,
21082 } }},
21083 .clobbers = .{ .eflags = true },
21084 .each = .{ .once = &.{
21085 .{ ._, .f_, .ld, .src1t, ._, ._, ._ },
21086 .{ ._, .f_, .ld, .src0t, ._, ._, ._ },
21087 .{ ._, .f_pp, .ucom, ._, ._, ._, ._ },
21088 .{ ._, .fn_sw, .st, .tmp2w, ._, ._, ._ },
21089 switch (strict) {
21090 true => .{ ._, ._, .@"test", .tmp2b, .si(0b1_000_001), ._, ._ },
21091 false => .{ ._, ._r, .sh, .tmp2b, .ui(1), ._, ._ },
21092 },
21093 } },
21094 }, .{
21095 .required_features = .{ .x87, null, null, null },
21096 .src_constraints = .{ .{ .float = .tbyte }, .{ .float = .tbyte } },
21097 .patterns = &.{
21098 .{ .src = .{ .to_x87, .mem }, .commute = .{ 0, 1 } },
21099 .{ .src = .{ .to_x87, .to_x87 }, .commute = .{ 0, 1 } },
21100 },
21101 .extra_temps = .{
21102 .{ .type = .f80, .kind = .{ .reg = .st6 } },
21103 .{ .type = .f80, .kind = .{ .reg = .st7 } },
21104 .{ .type = .u8, .kind = .{ .reg = .ah } },
21105 .unused,
21106 .unused,
21107 .unused,
21108 .unused,
21109 .unused,
21110 .unused,
21111 },
21112 .dst_temps = .{.{ .cc = switch (strict) {
21113 true => .z,
21114 false => .nc,
21115 } }},
21116 .clobbers = .{ .eflags = true },
21117 .each = .{ .once = &.{
21118 .{ ._, .f_, .ld, .src0t, ._, ._, ._ },
21119 .{ ._, .f_p, .ucom, .src1t, ._, ._, ._ },
21120 .{ ._, .fn_sw, .st, .tmp2w, ._, ._, ._ },
21121 switch (strict) {
21122 true => .{ ._, ._, .@"test", .tmp2b, .si(0b1_000_001), ._, ._ },
21123 false => .{ ._, ._r, .sh, .tmp2b, .ui(1), ._, ._ },
21124 },
21125 } },
21126 }, .{
21127 .required_features = .{ .sse, null, null, null },
21128 .src_constraints = .{ .{ .float = .xword }, .{ .float = .xword } },
21129 .patterns = &.{
21130 .{ .src = .{ .{ .to_reg = .xmm0 }, .{ .to_reg = .xmm1 } } },
21131 },
21132 .call_frame = .{ .alignment = .@"16" },
21133 .extra_temps = .{
21134 .{ .type = .usize, .kind = .{ .symbol = &.{ .name = "__cmptf2" } } },
21135 .{ .type = .i32, .kind = .{ .reg = .eax } },
21136 .unused,
21137 .unused,
21138 .unused,
21139 .unused,
21140 .unused,
21141 .unused,
21142 .unused,
21143 },
21144 .dst_temps = .{.{ .cc = switch (strict) {
21145 true => .l,
21146 false => .le,
21147 } }},
21148 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
21149 .each = .{ .once = &.{
21150 .{ ._, ._, .call, .tmp0d, ._, ._, ._ },
21151 .{ ._, ._, .@"test", .tmp1d, .tmp1d, ._, ._ },
2108321152 } },
21084 },
21085 .extra_temps = .{
21086 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
21087 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
21088 .unused,
21089 .unused,
21090 .unused,
21091 .unused,
21092 .unused,
21093 .unused,
21094 .unused,
21095 },
21096 .dst_temps = .{.{ .rc = .general_purpose }},
21097 .clobbers = .{ .eflags = true },
21098 .each = .{ .once = &.{
21099 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
21100 .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ },
21101 .{ .@"0:", ._r, .sh, .tmp1b, .si(1), ._, ._ },
21102 .{ ._, ._, .mov, .tmp1p, .memia(.src0p, .tmp0, .add_size), ._, ._ },
21103 .{ ._, ._, .sbb, .tmp1p, .memia(.src1p, .tmp0, .add_size), ._, ._ },
21104 .{ ._, ._c, .set, .tmp1b, ._, ._, ._ },
21105 .{ ._, .fromCond(switch (cc) {
21106 else => unreachable,
21107 .l, .ge, .b, .ae => cc,
21108 .le, .g, .be, .a => cc.commute(),
21109 }), .set, .dst0b, ._, ._, ._ },
21110 .{ ._, ._, .add, .tmp0p, .sa(.tmp1, .add_size), ._, ._ },
21111 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
2111221153 } },
21113 } },
21154 });
21155 } else err: {
21156 res[0] = ops[0].cmpInts(cmp_op, &ops[1], cg) catch |err| break :err err;
2111421157 }) catch |err| switch (err) {
2111521158 error.SelectFailed => return cg.fail("failed to select {s} {} {} {}", .{
2111621159 @tagName(air_tag),
......@@ -21126,994 +21169,502 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2112621169 .cmp_eq_optimized,
2112721170 .cmp_neq,
2112821171 .cmp_neq_optimized,
21129 => |air_tag| if (use_old) try cg.airCmp(inst, switch (air_tag) {
21130 else => unreachable,
21131 .cmp_eq, .cmp_eq_optimized => .eq,
21132 .cmp_neq, .cmp_neq_optimized => .neq,
21133 }) else fallback: {
21172 => |air_tag| if (use_old) try cg.airCmp(inst, air_tag.toCmpOp().?) else {
2113421173 const bin_op = air_datas[@intFromEnum(inst)].bin_op;
21135 const scalar_ty = cg.typeOf(bin_op.lhs).scalarType(zcu);
21136 if (cg.intInfo(scalar_ty) == null and cg.floatBits(scalar_ty) == null) break :fallback try cg.airCmp(inst, switch (air_tag) {
21137 else => unreachable,
21138 .cmp_eq, .cmp_eq_optimized => .eq,
21139 .cmp_neq, .cmp_neq_optimized => .neq,
21140 });
21174 const cmp_op = air_tag.toCmpOp().?;
2114121175 var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs });
21176 const ty = cg.typeOf(bin_op.lhs);
2114221177 var res: [1]Temp = undefined;
21143 cg.select(&res, &.{.bool}, &ops, switch (@as(Condition, switch (air_tag) {
21144 else => unreachable,
21145 .cmp_eq, .cmp_eq_optimized => .e,
21146 .cmp_neq, .cmp_neq_optimized => .ne,
21147 })) {
21148 else => unreachable,
21149 inline .e, .ne => |cc| comptime &.{ .{
21150 .src_constraints = .{ .{ .int = .byte }, .{ .int = .byte } },
21151 .patterns = &.{
21152 .{ .src = .{ .mem, .imm8 } },
21153 .{ .src = .{ .imm8, .mem }, .commute = .{ 0, 1 } },
21154 .{ .src = .{ .to_gpr, .imm8 } },
21155 .{ .src = .{ .imm8, .to_gpr }, .commute = .{ 0, 1 } },
21156 .{ .src = .{ .to_gpr, .mem } },
21157 .{ .src = .{ .mem, .to_gpr }, .commute = .{ 0, 1 } },
21158 .{ .src = .{ .to_gpr, .to_gpr } },
21159 },
21160 .dst_temps = .{.{ .cc = cc }},
21161 .clobbers = .{ .eflags = true },
21162 .each = .{ .once = &.{
21163 .{ ._, ._, .cmp, .src0b, .src1b, ._, ._ },
21164 } },
21165 }, .{
21166 .src_constraints = .{ .{ .int = .word }, .{ .int = .word } },
21167 .patterns = &.{
21168 .{ .src = .{ .mem, .imm16 } },
21169 .{ .src = .{ .imm16, .mem }, .commute = .{ 0, 1 } },
21170 .{ .src = .{ .to_gpr, .imm16 } },
21171 .{ .src = .{ .imm16, .to_gpr }, .commute = .{ 0, 1 } },
21172 .{ .src = .{ .to_gpr, .mem } },
21173 .{ .src = .{ .mem, .to_gpr }, .commute = .{ 0, 1 } },
21174 .{ .src = .{ .to_gpr, .to_gpr } },
21175 },
21176 .dst_temps = .{.{ .cc = cc }},
21177 .clobbers = .{ .eflags = true },
21178 .each = .{ .once = &.{
21179 .{ ._, ._, .cmp, .src0w, .src1w, ._, ._ },
21180 } },
21181 }, .{
21182 .src_constraints = .{ .{ .int = .dword }, .{ .int = .dword } },
21183 .patterns = &.{
21184 .{ .src = .{ .mem, .imm32 } },
21185 .{ .src = .{ .imm32, .mem }, .commute = .{ 0, 1 } },
21186 .{ .src = .{ .to_gpr, .imm32 } },
21187 .{ .src = .{ .imm32, .to_gpr }, .commute = .{ 0, 1 } },
21188 .{ .src = .{ .to_gpr, .mem } },
21189 .{ .src = .{ .mem, .to_gpr }, .commute = .{ 0, 1 } },
21190 .{ .src = .{ .to_gpr, .to_gpr } },
21191 },
21192 .dst_temps = .{.{ .cc = cc }},
21193 .clobbers = .{ .eflags = true },
21194 .each = .{ .once = &.{
21195 .{ ._, ._, .cmp, .src0d, .src1d, ._, ._ },
21196 } },
21197 }, .{
21198 .required_features = .{ .@"64bit", null, null, null },
21199 .src_constraints = .{ .{ .int = .qword }, .{ .int = .qword } },
21200 .patterns = &.{
21201 .{ .src = .{ .mem, .simm32 } },
21202 .{ .src = .{ .simm32, .mem }, .commute = .{ 0, 1 } },
21203 .{ .src = .{ .to_gpr, .simm32 } },
21204 .{ .src = .{ .simm32, .to_gpr }, .commute = .{ 0, 1 } },
21205 .{ .src = .{ .to_gpr, .mem } },
21206 .{ .src = .{ .mem, .to_gpr }, .commute = .{ 0, 1 } },
21207 .{ .src = .{ .to_gpr, .to_gpr } },
21208 },
21209 .dst_temps = .{.{ .cc = cc }},
21210 .clobbers = .{ .eflags = true },
21211 .each = .{ .once = &.{
21212 .{ ._, ._, .cmp, .src0q, .src1q, ._, ._ },
21213 } },
21214 }, .{
21215 .required_features = .{ .sse, .mmx, null, null },
21216 .src_constraints = .{ .{ .int = .qword }, .{ .int = .qword } },
21217 .patterns = &.{
21218 .{ .src = .{ .to_mut_mm, .mem } },
21219 .{ .src = .{ .mem, .to_mut_mm }, .commute = .{ 0, 1 } },
21220 .{ .src = .{ .to_mut_mm, .to_mm } },
21221 },
21222 .extra_temps = .{
21223 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
21224 .{ .kind = .{ .rc = .mmx } },
21225 .unused,
21226 .unused,
21227 .unused,
21228 .unused,
21229 .unused,
21230 .unused,
21231 .unused,
21232 },
21233 .dst_temps = .{.{ .cc = cc }},
21234 .clobbers = .{ .eflags = true },
21235 .each = .{ .once = &.{
21236 .{ ._, .p_, .xor, .tmp1q, .tmp1q, ._, ._ },
21237 .{ ._, .p_, .xor, .src0q, .src1q, ._, ._ },
21238 .{ ._, .p_b, .cmpeq, .tmp1q, .src0q, ._, ._ },
21239 .{ ._, .p_b, .movmsk, .tmp0d, .tmp1q, ._, ._ },
21240 .{ ._, ._, .xor, .tmp0d, .si(0xff), ._, ._ },
21241 } },
21242 }, .{
21243 .required_features = .{ .avx, null, null, null },
21244 .src_constraints = .{ .{ .int = .xword }, .{ .int = .xword } },
21245 .patterns = &.{
21246 .{ .src = .{ .to_xmm, .mem } },
21247 .{ .src = .{ .mem, .to_xmm }, .commute = .{ 0, 1 } },
21248 .{ .src = .{ .to_xmm, .to_xmm } },
21249 },
21250 .extra_temps = .{
21251 .{ .kind = .{ .rc = .sse } },
21252 .unused,
21253 .unused,
21254 .unused,
21255 .unused,
21256 .unused,
21257 .unused,
21258 .unused,
21259 .unused,
21260 },
21261 .dst_temps = .{.{ .cc = cc }},
21262 .clobbers = .{ .eflags = true },
21263 .each = .{ .once = &.{
21264 .{ ._, .vp_, .xor, .tmp0x, .src0x, .src1x, ._ },
21265 .{ ._, .vp_, .@"test", .tmp0x, .tmp0x, ._, ._ },
21266 } },
21267 }, .{
21268 .required_features = .{ .sse4_1, null, null, null },
21269 .src_constraints = .{ .{ .int = .xword }, .{ .int = .xword } },
21270 .patterns = &.{
21271 .{ .src = .{ .to_mut_xmm, .mem } },
21272 .{ .src = .{ .mem, .to_mut_xmm }, .commute = .{ 0, 1 } },
21273 .{ .src = .{ .to_mut_xmm, .to_xmm } },
21274 },
21275 .dst_temps = .{.{ .cc = cc }},
21276 .clobbers = .{ .eflags = true },
21277 .each = .{ .once = &.{
21278 .{ ._, .p_, .xor, .src0x, .src1x, ._, ._ },
21279 .{ ._, .p_, .@"test", .src0x, .src0x, ._, ._ },
21280 } },
21281 }, .{
21282 .required_features = .{ .sse2, null, null, null },
21283 .src_constraints = .{ .{ .int = .xword }, .{ .int = .xword } },
21284 .patterns = &.{
21285 .{ .src = .{ .to_mut_xmm, .mem } },
21286 .{ .src = .{ .mem, .to_mut_xmm }, .commute = .{ 0, 1 } },
21287 .{ .src = .{ .to_mut_xmm, .to_xmm } },
21288 },
21289 .extra_temps = .{
21290 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
21291 .{ .kind = .{ .rc = .sse } },
21292 .unused,
21293 .unused,
21294 .unused,
21295 .unused,
21296 .unused,
21297 .unused,
21298 .unused,
21299 },
21300 .dst_temps = .{.{ .cc = cc }},
21301 .clobbers = .{ .eflags = true },
21302 .each = .{ .once = &.{
21303 .{ ._, .p_, .xor, .tmp1x, .tmp1x, ._, ._ },
21304 .{ ._, .p_, .xor, .src0x, .src1x, ._, ._ },
21305 .{ ._, .p_b, .cmpeq, .tmp1x, .src0x, ._, ._ },
21306 .{ ._, .p_b, .movmsk, .tmp0d, .tmp1x, ._, ._ },
21307 .{ ._, ._, .xor, .tmp0d, .si(0xffff), ._, ._ },
21308 } },
21309 }, .{
21310 .required_features = .{ .avx2, null, null, null },
21311 .src_constraints = .{ .{ .int = .yword }, .{ .int = .yword } },
21312 .patterns = &.{
21313 .{ .src = .{ .to_ymm, .mem } },
21314 .{ .src = .{ .mem, .to_ymm }, .commute = .{ 0, 1 } },
21315 .{ .src = .{ .to_ymm, .to_ymm } },
21178 const OptInfo = struct {
21179 deaths: [2]Air.Inst.Index,
21180 res: [1]Temp,
21181 state: State,
21182 reloc: Mir.Inst.Index,
21183 };
21184 var opt_info: ?OptInfo = null;
21185 (err: switch (@as(enum { float, int }, if (cg.floatBits(ty)) |_|
21186 .float
21187 else if (cg.intInfo(ty)) |_|
21188 .int
21189 else category: {
21190 const child_ty = ty.optionalChild(zcu);
21191 const has_value_off: u31 = @intCast(child_ty.abiSize(zcu));
21192 var has_values: [2]Temp = undefined;
21193 opt_info = @as(OptInfo, undefined);
21194 for (&has_values, &ops, &opt_info.?.deaths) |*has_value, *op, *death| {
21195 has_value.* = try op.read(.bool, .{ .disp = has_value_off }, cg);
21196 const child = try op.read(child_ty, .{}, cg);
21197 try op.die(cg);
21198 op.* = child;
21199 death.* = child.index;
21200 }
21201 cg.select(
21202 &opt_info.?.res,
21203 &.{.bool},
21204 &has_values,
21205 switch (Condition.fromCompareOperatorUnsigned(cmp_op)) {
21206 else => unreachable,
21207 inline .e, .ne => |cc| comptime &.{.{
21208 .src_constraints = .{ .{ .size = .byte }, .{ .size = .byte } },
21209 .patterns = &.{
21210 .{ .src = .{ .mem, .imm8 } },
21211 .{ .src = .{ .imm8, .mem }, .commute = .{ 0, 1 } },
21212 .{ .src = .{ .to_gpr, .imm8 } },
21213 .{ .src = .{ .imm8, .to_gpr }, .commute = .{ 0, 1 } },
21214 .{ .src = .{ .mem, .to_gpr } },
21215 .{ .src = .{ .to_gpr, .mem }, .commute = .{ 0, 1 } },
21216 .{ .src = .{ .to_gpr, .to_gpr } },
21217 },
21218 .dst_temps = .{.{ .rc = .general_purpose }},
21219 .clobbers = .{ .eflags = true },
21220 .each = .{ .once = &.{
21221 .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ },
21222 .{ ._, ._, .cmp, .src0b, .src1b, ._, ._ },
21223 .{ ._, .fromCond(cc), .set, .dst0b, ._, ._, ._ },
21224 .{ ._, ._, .@"test", .src0b, .src1b, ._, ._ },
21225 } },
21226 }},
2131621227 },
21317 .extra_temps = .{
21318 .{ .kind = .{ .rc = .sse } },
21319 .unused,
21320 .unused,
21321 .unused,
21322 .unused,
21323 .unused,
21324 .unused,
21325 .unused,
21326 .unused,
21327 },
21328 .dst_temps = .{.{ .cc = cc }},
21329 .clobbers = .{ .eflags = true },
21330 .each = .{ .once = &.{
21331 .{ ._, .vp_, .xor, .tmp0y, .src0y, .src1y, ._ },
21332 .{ ._, .vp_, .@"test", .tmp0y, .tmp0y, ._, ._ },
21333 } },
21334 }, .{
21335 .required_features = .{ .avx, null, null, null },
21336 .src_constraints = .{ .{ .int = .yword }, .{ .int = .yword } },
21337 .patterns = &.{
21338 .{ .src = .{ .to_ymm, .mem } },
21339 .{ .src = .{ .mem, .to_ymm }, .commute = .{ 0, 1 } },
21340 .{ .src = .{ .to_ymm, .to_ymm } },
21341 },
21342 .extra_temps = .{
21343 .{ .kind = .{ .rc = .sse } },
21344 .unused,
21345 .unused,
21346 .unused,
21347 .unused,
21348 .unused,
21349 .unused,
21350 .unused,
21351 .unused,
21352 },
21353 .dst_temps = .{.{ .cc = cc }},
21354 .clobbers = .{ .eflags = true },
21355 .each = .{ .once = &.{
21356 .{ ._, .v_pd, .xor, .tmp0y, .src0y, .src1y, ._ },
21357 .{ ._, .vp_, .@"test", .tmp0y, .tmp0y, ._, ._ },
21358 } },
21359 }, .{
21360 .required_features = .{ .avx2, null, null, null },
21361 .src_constraints = .{
21362 .{ .remainder_int = .{ .of = .yword, .is = .xword } },
21363 .{ .remainder_int = .{ .of = .yword, .is = .xword } },
21364 },
21365 .patterns = &.{
21366 .{ .src = .{ .to_mem, .to_mem } },
21367 },
21368 .extra_temps = .{
21369 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
21370 .{ .kind = .{ .rc = .sse } },
21371 .{ .kind = .{ .rc = .sse } },
21372 .unused,
21373 .unused,
21374 .unused,
21375 .unused,
21376 .unused,
21377 .unused,
21378 },
21379 .dst_temps = .{.{ .cc = cc }},
21380 .clobbers = .{ .eflags = true },
21381 .each = .{ .once = &.{
21382 .{ ._, ._, .mov, .tmp0p, .sia(16, .src0, .sub_size), ._, ._ },
21383 .{ ._, .vp_, .xor, .tmp1y, .tmp1y, .tmp1y, ._ },
21384 .{ .@"0:", .v_dqu, .mov, .tmp2y, .memiad(.src0y, .tmp0, .add_size, -16), ._, ._ },
21385 .{ ._, .vp_, .xor, .tmp2y, .tmp2y, .memiad(.src1y, .tmp0, .add_size, -16), ._ },
21386 .{ ._, .vp_, .@"or", .tmp1y, .tmp1y, .tmp2y, ._ },
21387 .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ },
21388 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
21389 .{ ._, .v_dqa, .mov, .tmp2x, .memad(.src0x, .add_size, -16), ._, ._ },
21390 .{ ._, .vp_, .xor, .tmp2x, .tmp2x, .memad(.src1x, .add_size, -16), ._ },
21391 .{ ._, .vp_, .@"or", .tmp1y, .tmp1y, .tmp2y, ._ },
21392 .{ ._, .vp_, .@"test", .tmp1y, .tmp1y, ._, ._ },
21393 } },
21394 }, .{
21395 .required_features = .{ .avx2, null, null, null },
21396 .src_constraints = .{
21397 .{ .remainder_int = .{ .of = .yword, .is = .yword } },
21398 .{ .remainder_int = .{ .of = .yword, .is = .yword } },
21399 },
21400 .patterns = &.{
21401 .{ .src = .{ .to_mem, .to_mem } },
21402 },
21403 .extra_temps = .{
21404 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
21405 .{ .kind = .{ .rc = .sse } },
21406 .{ .kind = .{ .rc = .sse } },
21407 .unused,
21408 .unused,
21409 .unused,
21410 .unused,
21411 .unused,
21412 .unused,
21413 },
21414 .dst_temps = .{.{ .cc = cc }},
21415 .clobbers = .{ .eflags = true },
21416 .each = .{ .once = &.{
21417 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
21418 .{ ._, .vp_, .xor, .tmp1y, .tmp1y, .tmp1y, ._ },
21419 .{ .@"0:", .v_dqu, .mov, .tmp2y, .memia(.src0y, .tmp0, .add_size), ._, ._ },
21420 .{ ._, .vp_, .xor, .tmp2y, .tmp2y, .memia(.src1y, .tmp0, .add_size), ._ },
21421 .{ ._, .vp_, .@"or", .tmp1y, .tmp1y, .tmp2y, ._ },
21422 .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ },
21423 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
21424 .{ ._, .vp_, .@"test", .tmp1y, .tmp1y, ._, ._ },
21425 } },
21426 }, .{
21427 .required_features = .{ .avx, null, null, null },
21428 .src_constraints = .{
21429 .{ .remainder_int = .{ .of = .yword, .is = .xword } },
21430 .{ .remainder_int = .{ .of = .yword, .is = .xword } },
21431 },
21432 .patterns = &.{
21433 .{ .src = .{ .to_mem, .to_mem } },
21434 },
21435 .extra_temps = .{
21436 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
21437 .{ .kind = .{ .rc = .sse } },
21438 .{ .kind = .{ .rc = .sse } },
21439 .unused,
21440 .unused,
21441 .unused,
21442 .unused,
21443 .unused,
21444 .unused,
21445 },
21446 .dst_temps = .{.{ .cc = cc }},
21447 .clobbers = .{ .eflags = true },
21448 .each = .{ .once = &.{
21449 .{ ._, ._, .mov, .tmp0p, .sia(16, .src0, .sub_size), ._, ._ },
21450 .{ ._, .v_pd, .xor, .tmp1y, .tmp1y, .tmp1y, ._ },
21451 .{ .@"0:", .v_pd, .movu, .tmp2y, .memiad(.src0y, .tmp0, .add_size, -16), ._, ._ },
21452 .{ ._, .v_pd, .xor, .tmp2y, .tmp2y, .memiad(.src1y, .tmp0, .add_size, -16), ._ },
21453 .{ ._, .v_pd, .@"or", .tmp1y, .tmp1y, .tmp2y, ._ },
21454 .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ },
21455 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
21456 .{ ._, .v_pd, .mova, .tmp2x, .memad(.src0x, .add_size, -16), ._, ._ },
21457 .{ ._, .v_pd, .xor, .tmp2x, .tmp2x, .memad(.src1x, .add_size, -16), ._ },
21458 .{ ._, .v_pd, .@"or", .tmp1y, .tmp1y, .tmp2y, ._ },
21459 .{ ._, .vp_, .@"test", .tmp1y, .tmp1y, ._, ._ },
21460 } },
21461 }, .{
21462 .required_features = .{ .avx, null, null, null },
21463 .src_constraints = .{
21464 .{ .remainder_int = .{ .of = .yword, .is = .yword } },
21465 .{ .remainder_int = .{ .of = .yword, .is = .yword } },
21466 },
21467 .patterns = &.{
21468 .{ .src = .{ .to_mem, .to_mem } },
21469 },
21470 .extra_temps = .{
21471 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
21472 .{ .kind = .{ .rc = .sse } },
21473 .{ .kind = .{ .rc = .sse } },
21474 .unused,
21475 .unused,
21476 .unused,
21477 .unused,
21478 .unused,
21479 .unused,
21480 },
21481 .dst_temps = .{.{ .cc = cc }},
21482 .clobbers = .{ .eflags = true },
21483 .each = .{ .once = &.{
21484 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
21485 .{ ._, .v_pd, .xor, .tmp1y, .tmp1y, .tmp1y, ._ },
21486 .{ .@"0:", .v_pd, .movu, .tmp2y, .memia(.src0y, .tmp0, .add_size), ._, ._ },
21487 .{ ._, .v_pd, .xor, .tmp2y, .tmp2y, .memia(.src1y, .tmp0, .add_size), ._ },
21488 .{ ._, .v_pd, .@"or", .tmp1y, .tmp1y, .tmp2y, ._ },
21489 .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ },
21490 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
21491 .{ ._, .vp_, .@"test", .tmp1y, .tmp1y, ._, ._ },
21492 } },
21493 }, .{
21494 .required_features = .{ .avx, null, null, null },
21495 .src_constraints = .{
21496 .{ .remainder_int = .{ .of = .xword, .is = .xword } },
21497 .{ .remainder_int = .{ .of = .xword, .is = .xword } },
21498 },
21499 .patterns = &.{
21500 .{ .src = .{ .to_mem, .to_mem } },
21501 },
21502 .extra_temps = .{
21503 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
21504 .{ .kind = .{ .rc = .sse } },
21505 .{ .kind = .{ .rc = .sse } },
21506 .unused,
21507 .unused,
21508 .unused,
21509 .unused,
21510 .unused,
21511 .unused,
21512 },
21513 .dst_temps = .{.{ .cc = cc }},
21514 .clobbers = .{ .eflags = true },
21515 .each = .{ .once = &.{
21516 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
21517 .{ ._, .vp_, .xor, .tmp1x, .tmp1x, .tmp1x, ._ },
21518 .{ .@"0:", .v_dqu, .mov, .tmp2x, .memia(.src0x, .tmp0, .add_size), ._, ._ },
21519 .{ ._, .vp_, .xor, .tmp2x, .tmp2x, .memia(.src1x, .tmp0, .add_size), ._ },
21520 .{ ._, .vp_, .@"or", .tmp1x, .tmp1x, .tmp2x, ._ },
21521 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
21522 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
21523 .{ ._, .vp_, .@"test", .tmp1x, .tmp1x, ._, ._ },
21524 } },
21525 }, .{
21526 .required_features = .{ .sse4_1, null, null, null },
21527 .src_constraints = .{
21528 .{ .remainder_int = .{ .of = .xword, .is = .xword } },
21529 .{ .remainder_int = .{ .of = .xword, .is = .xword } },
21530 },
21531 .patterns = &.{
21532 .{ .src = .{ .to_mem, .to_mem } },
21533 },
21534 .extra_temps = .{
21535 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
21536 .{ .kind = .{ .rc = .sse } },
21537 .{ .kind = .{ .rc = .sse } },
21538 .unused,
21539 .unused,
21540 .unused,
21541 .unused,
21542 .unused,
21543 .unused,
21544 },
21545 .dst_temps = .{.{ .cc = cc }},
21546 .clobbers = .{ .eflags = true },
21547 .each = .{ .once = &.{
21548 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
21549 .{ ._, .p_, .xor, .tmp1x, .tmp1x, ._, ._ },
21550 .{ .@"0:", ._dqu, .mov, .tmp2x, .memia(.src0x, .tmp0, .add_size), ._, ._ },
21551 .{ ._, .p_, .xor, .tmp2x, .memia(.src1x, .tmp0, .add_size), ._, ._ },
21552 .{ ._, .p_, .@"or", .tmp1x, .tmp2x, ._, ._ },
21553 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
21554 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
21555 .{ ._, .p_, .@"test", .tmp1x, .tmp1x, ._, ._ },
21556 } },
21557 }, .{
21558 .required_features = .{ .sse2, null, null, null },
21559 .src_constraints = .{
21560 .{ .remainder_int = .{ .of = .xword, .is = .xword } },
21561 .{ .remainder_int = .{ .of = .xword, .is = .xword } },
21562 },
21563 .patterns = &.{
21564 .{ .src = .{ .to_mem, .to_mem } },
21565 },
21566 .extra_temps = .{
21567 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
21568 .{ .kind = .{ .rc = .sse } },
21569 .{ .kind = .{ .rc = .sse } },
21570 .unused,
21571 .unused,
21572 .unused,
21573 .unused,
21574 .unused,
21575 .unused,
21576 },
21577 .dst_temps = .{.{ .cc = cc }},
21578 .clobbers = .{ .eflags = true },
21579 .each = .{ .once = &.{
21580 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
21581 .{ ._, .p_, .xor, .tmp1x, .tmp1x, ._, ._ },
21582 .{ .@"0:", ._dqu, .mov, .tmp2x, .memia(.src0x, .tmp0, .add_size), ._, ._ },
21583 .{ ._, .p_, .xor, .tmp2x, .memia(.src1x, .tmp0, .add_size), ._, ._ },
21584 .{ ._, .p_, .@"or", .tmp1x, .tmp2x, ._, ._ },
21585 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
21586 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
21587 .{ ._, .p_, .xor, .tmp2x, .tmp2x, ._, ._ },
21588 .{ ._, .p_b, .cmpeq, .tmp1x, .tmp2x, ._, ._ },
21589 .{ ._, .p_b, .movmsk, .tmp0d, .tmp1x, ._, ._ },
21590 .{ ._, ._, .cmp, .tmp0d, .si(0xffff), ._, ._ },
21591 } },
21592 }, .{
21593 .required_features = .{ .sse, .mmx, null, null },
21594 .src_constraints = .{
21595 .{ .remainder_int = .{ .of = .qword, .is = .qword } },
21596 .{ .remainder_int = .{ .of = .qword, .is = .qword } },
21597 },
21598 .patterns = &.{
21599 .{ .src = .{ .to_mem, .to_mem } },
21600 },
21601 .extra_temps = .{
21602 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
21603 .{ .kind = .{ .rc = .mmx } },
21604 .{ .kind = .{ .rc = .mmx } },
21605 .unused,
21606 .unused,
21607 .unused,
21608 .unused,
21609 .unused,
21610 .unused,
21611 },
21612 .dst_temps = .{.{ .cc = cc }},
21613 .clobbers = .{ .eflags = true },
21614 .each = .{ .once = &.{
21615 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
21616 .{ ._, .p_, .xor, .tmp1q, .tmp1q, ._, ._ },
21617 .{ .@"0:", ._q, .mov, .tmp2q, .memia(.src0q, .tmp0, .add_size), ._, ._ },
21618 .{ ._, .p_, .xor, .tmp2q, .memia(.src1q, .tmp0, .add_size), ._, ._ },
21619 .{ ._, .p_, .@"or", .tmp1q, .tmp2q, ._, ._ },
21620 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
21621 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
21622 .{ ._, .p_, .xor, .tmp2q, .tmp2q, ._, ._ },
21623 .{ ._, .p_b, .cmpeq, .tmp1q, .tmp2q, ._, ._ },
21624 .{ ._, .p_b, .movmsk, .tmp0d, .tmp1q, ._, ._ },
21625 .{ ._, ._, .cmp, .tmp0d, .si(0xff), ._, ._ },
21626 } },
21627 }, .{
21628 .src_constraints = .{
21629 .{ .remainder_int = .{ .of = .qword, .is = .qword } },
21630 .{ .remainder_int = .{ .of = .qword, .is = .qword } },
21631 },
21632 .patterns = &.{
21633 .{ .src = .{ .to_mem, .to_mem } },
21634 },
21635 .extra_temps = .{
21636 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
21637 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
21638 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
21639 .unused,
21640 .unused,
21641 .unused,
21642 .unused,
21643 .unused,
21644 .unused,
21645 },
21646 .dst_temps = .{.{ .cc = cc }},
21647 .clobbers = .{ .eflags = true },
21648 .each = .{ .once = &.{
21649 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
21650 .{ ._, ._, .xor, .tmp1p, .tmp1p, ._, ._ },
21651 .{ .@"0:", ._, .mov, .tmp2p, .memia(.src0p, .tmp0, .add_size), ._, ._ },
21652 .{ ._, ._, .xor, .tmp2p, .memia(.src1p, .tmp0, .add_size), ._, ._ },
21653 .{ ._, ._, .@"or", .tmp1p, .tmp2p, ._, ._ },
21654 .{ ._, ._, .add, .tmp0p, .sa(.tmp2, .add_size), ._, ._ },
21655 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
21656 .{ ._, ._, .@"test", .tmp1p, .tmp1p, ._, ._ },
21657 } },
21658 }, .{
21659 .required_features = .{ .f16c, null, null, null },
21660 .src_constraints = .{ .{ .float = .word }, .{ .float = .word } },
21661 .patterns = &.{
21662 .{ .src = .{ .to_sse, .to_sse } },
21663 },
21664 .extra_temps = .{
21665 .{ .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },
21666 .{ .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } },
21667 .unused,
21668 .unused,
21669 .unused,
21670 .unused,
21671 .unused,
21672 .unused,
21673 .unused,
21674 },
21675 .dst_temps = .{.{ .cc = switch (cc) {
21676 else => unreachable,
21677 .e => .z_and_np,
21678 .ne => .nz_or_p,
21679 } }},
21680 .clobbers = .{ .eflags = true },
21681 .each = .{ .once = &.{
21682 .{ ._, .v_ps, .cvtph2, .tmp0x, .src0q, ._, ._ },
21683 .{ ._, .v_ps, .cvtph2, .tmp1x, .src1q, ._, ._ },
21684 .{ ._, .v_ss, .ucomi, .tmp0x, .tmp1x, ._, ._ },
21685 } },
21686 }, .{
21687 .required_features = .{ .sse, null, null, null },
21688 .src_constraints = .{ .{ .float = .word }, .{ .float = .word } },
21689 .patterns = &.{
21690 .{ .src = .{ .{ .to_reg = .xmm0 }, .{ .to_reg = .xmm1 } } },
21691 },
21692 .call_frame = .{ .alignment = .@"16" },
21693 .extra_temps = .{
21694 .{ .type = .usize, .kind = .{ .symbol = &.{ .name = switch (cc) {
21695 else => unreachable,
21696 .e => "__eqhf2",
21697 .ne => "__nehf2",
21698 } } } },
21699 .{ .type = .i32, .kind = .{ .reg = .eax } },
21700 .unused,
21701 .unused,
21702 .unused,
21703 .unused,
21704 .unused,
21705 .unused,
21706 .unused,
21707 },
21708 .dst_temps = .{.{ .cc = cc }},
21709 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
21710 .each = .{ .once = &.{
21711 .{ ._, ._, .call, .tmp0d, ._, ._, ._ },
21712 .{ ._, ._, .@"test", .tmp1d, .tmp1d, ._, ._ },
21713 } },
21714 }, .{
21715 .required_features = .{ .avx, null, null, null },
21716 .src_constraints = .{ .{ .float = .dword }, .{ .float = .dword } },
21717 .patterns = &.{
21718 .{ .src = .{ .to_sse, .mem } },
21719 .{ .src = .{ .mem, .to_sse }, .commute = .{ 0, 1 } },
21720 .{ .src = .{ .to_sse, .to_sse } },
21721 },
21722 .extra_temps = .{
21723 .{ .type = .f16, .kind = .{ .rc = .sse } },
21724 .{ .type = .f16, .kind = .{ .rc = .sse } },
21725 .unused,
21726 .unused,
21727 .unused,
21728 .unused,
21729 .unused,
21730 .unused,
21731 .unused,
21732 },
21733 .dst_temps = .{.{ .cc = switch (cc) {
21734 else => unreachable,
21735 .e => .z_and_np,
21736 .ne => .nz_or_p,
21737 } }},
21738 .clobbers = .{ .eflags = true },
21739 .each = .{ .once = &.{
21740 .{ ._, .v_ss, .ucomi, .src0x, .src1d, ._, ._ },
21741 } },
21742 }, .{
21743 .required_features = .{ .sse, null, null, null },
21744 .src_constraints = .{ .{ .float = .dword }, .{ .float = .dword } },
21745 .patterns = &.{
21746 .{ .src = .{ .to_sse, .mem } },
21747 .{ .src = .{ .mem, .to_sse }, .commute = .{ 0, 1 } },
21748 .{ .src = .{ .to_sse, .to_sse } },
21749 },
21750 .extra_temps = .{
21751 .{ .type = .f16, .kind = .{ .rc = .sse } },
21752 .{ .type = .f16, .kind = .{ .rc = .sse } },
21753 .unused,
21754 .unused,
21755 .unused,
21756 .unused,
21757 .unused,
21758 .unused,
21759 .unused,
21760 },
21761 .dst_temps = .{.{ .cc = switch (cc) {
21762 else => unreachable,
21763 .e => .z_and_np,
21764 .ne => .nz_or_p,
21765 } }},
21766 .clobbers = .{ .eflags = true },
21767 .each = .{ .once = &.{
21768 .{ ._, ._ss, .ucomi, .src0x, .src1d, ._, ._ },
21769 } },
21770 }, .{
21771 .required_features = .{ .sse, null, null, null },
21772 .src_constraints = .{ .{ .float = .dword }, .{ .float = .word } },
21773 .patterns = &.{
21774 .{ .src = .{ .{ .to_reg = .xmm0 }, .{ .to_reg = .xmm1 } } },
21775 },
21776 .call_frame = .{ .alignment = .@"16" },
21777 .extra_temps = .{
21778 .{ .type = .usize, .kind = .{ .symbol = &.{ .name = switch (cc) {
21779 else => unreachable,
21780 .e => "__eqsf2",
21781 .ne => "__nesf2",
21782 } } } },
21783 .{ .type = .i32, .kind = .{ .reg = .eax } },
21784 .unused,
21785 .unused,
21786 .unused,
21787 .unused,
21788 .unused,
21789 .unused,
21790 .unused,
21791 },
21792 .dst_temps = .{.{ .cc = cc }},
21793 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
21794 .each = .{ .once = &.{
21795 .{ ._, ._, .call, .tmp0d, ._, ._, ._ },
21796 .{ ._, ._, .@"test", .tmp1d, .tmp1d, ._, ._ },
21797 } },
21798 }, .{
21799 .required_features = .{ .avx, null, null, null },
21800 .src_constraints = .{ .{ .float = .qword }, .{ .float = .qword } },
21801 .patterns = &.{
21802 .{ .src = .{ .to_sse, .mem } },
21803 .{ .src = .{ .mem, .to_sse }, .commute = .{ 0, 1 } },
21804 .{ .src = .{ .to_sse, .to_sse } },
21805 },
21806 .extra_temps = .{
21807 .{ .type = .f16, .kind = .{ .rc = .sse } },
21808 .{ .type = .f16, .kind = .{ .rc = .sse } },
21809 .unused,
21810 .unused,
21811 .unused,
21812 .unused,
21813 .unused,
21814 .unused,
21815 .unused,
21816 },
21817 .dst_temps = .{.{ .cc = switch (cc) {
21818 else => unreachable,
21819 .e => .z_and_np,
21820 .ne => .nz_or_p,
21821 } }},
21822 .clobbers = .{ .eflags = true },
21823 .each = .{ .once = &.{
21824 .{ ._, .v_sd, .ucomi, .src0x, .src1q, ._, ._ },
21825 } },
21826 }, .{
21827 .required_features = .{ .sse2, null, null, null },
21828 .src_constraints = .{ .{ .float = .qword }, .{ .float = .qword } },
21829 .patterns = &.{
21830 .{ .src = .{ .to_sse, .mem } },
21831 .{ .src = .{ .mem, .to_sse }, .commute = .{ 0, 1 } },
21832 .{ .src = .{ .to_sse, .to_sse } },
21833 },
21834 .extra_temps = .{
21835 .{ .type = .f16, .kind = .{ .rc = .sse } },
21836 .{ .type = .f16, .kind = .{ .rc = .sse } },
21837 .unused,
21838 .unused,
21839 .unused,
21840 .unused,
21841 .unused,
21842 .unused,
21843 .unused,
21844 },
21845 .dst_temps = .{.{ .cc = switch (cc) {
21846 else => unreachable,
21847 .e => .z_and_np,
21848 .ne => .nz_or_p,
21849 } }},
21850 .clobbers = .{ .eflags = true },
21851 .each = .{ .once = &.{
21852 .{ ._, ._sd, .ucomi, .src0x, .src1q, ._, ._ },
21853 } },
21854 }, .{
21855 .required_features = .{ .sse, null, null, null },
21856 .src_constraints = .{ .{ .float = .qword }, .{ .float = .qword } },
21857 .patterns = &.{
21858 .{ .src = .{ .{ .to_reg = .xmm0 }, .{ .to_reg = .xmm1 } } },
21859 },
21860 .call_frame = .{ .alignment = .@"16" },
21861 .extra_temps = .{
21862 .{ .type = .usize, .kind = .{ .symbol = &.{ .name = switch (cc) {
21863 else => unreachable,
21864 .e => "__eqdf2",
21865 .ne => "__nedf2",
21866 } } } },
21867 .{ .type = .i32, .kind = .{ .reg = .eax } },
21868 .unused,
21869 .unused,
21870 .unused,
21871 .unused,
21872 .unused,
21873 .unused,
21874 .unused,
21875 },
21876 .dst_temps = .{.{ .cc = cc }},
21877 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
21878 .each = .{ .once = &.{
21879 .{ ._, ._, .call, .tmp0d, ._, ._, ._ },
21880 .{ ._, ._, .@"test", .tmp1d, .tmp1d, ._, ._ },
21881 } },
21882 }, .{
21883 .required_features = .{ .x87, .cmov, null, null },
21884 .src_constraints = .{ .{ .float = .tbyte }, .{ .float = .tbyte } },
21885 .patterns = &.{
21886 .{ .src = .{ .to_x87, .mem }, .commute = .{ 0, 1 } },
21887 .{ .src = .{ .mem, .to_x87 } },
21888 .{ .src = .{ .to_x87, .to_x87 } },
21889 },
21890 .extra_temps = .{
21891 .{ .type = .f80, .kind = .{ .reg = .st7 } },
21892 .unused,
21893 .unused,
21894 .unused,
21895 .unused,
21896 .unused,
21897 .unused,
21898 .unused,
21899 .unused,
21900 },
21901 .dst_temps = .{.{ .cc = switch (cc) {
21902 else => unreachable,
21903 .e => .z_and_np,
21904 .ne => .nz_or_p,
21905 } }},
21906 .clobbers = .{ .eflags = true },
21907 .each = .{ .once = &.{
21908 .{ ._, .f_, .ld, .src0t, ._, ._, ._ },
21909 .{ ._, .f_p, .ucomi, .tmp0t, .src1t, ._, ._ },
21910 } },
21911 }, .{
21912 .required_features = .{ .sahf, .x87, null, null },
21913 .src_constraints = .{ .{ .float = .tbyte }, .{ .float = .tbyte } },
21914 .patterns = &.{
21915 .{ .src = .{ .mem, .mem } },
21916 },
21917 .extra_temps = .{
21918 .{ .type = .f80, .kind = .{ .reg = .st6 } },
21919 .{ .type = .f80, .kind = .{ .reg = .st7 } },
21920 .{ .type = .u8, .kind = .{ .reg = .ah } },
21921 .unused,
21922 .unused,
21923 .unused,
21924 .unused,
21925 .unused,
21926 .unused,
21927 },
21928 .dst_temps = .{.{ .cc = switch (cc) {
21929 else => unreachable,
21930 .e => .z_and_np,
21931 .ne => .nz_or_p,
21932 } }},
21933 .clobbers = .{ .eflags = true },
21934 .each = .{ .once = &.{
21935 .{ ._, .f_, .ld, .src1t, ._, ._, ._ },
21936 .{ ._, .f_, .ld, .src0t, ._, ._, ._ },
21937 .{ ._, .f_pp, .ucom, ._, ._, ._, ._ },
21938 .{ ._, .fn_sw, .st, .tmp2w, ._, ._, ._ },
21939 .{ ._, ._, .sahf, ._, ._, ._, ._ },
21940 } },
21941 }, .{
21942 .required_features = .{ .@"64bit", .x87, null, null },
21943 .src_constraints = .{ .{ .float = .tbyte }, .{ .float = .tbyte } },
21944 .patterns = &.{
21945 .{ .src = .{ .mem, .mem } },
21946 },
21947 .extra_temps = .{
21948 .{ .type = .f80, .kind = .{ .reg = .st6 } },
21949 .{ .type = .f80, .kind = .{ .reg = .st7 } },
21950 .{ .type = .u8, .kind = .{ .reg = .ah } },
21951 .unused,
21952 .unused,
21953 .unused,
21954 .unused,
21955 .unused,
21956 .unused,
21957 },
21958 .dst_temps = .{.{ .cc = cc }},
21959 .clobbers = .{ .eflags = true },
21960 .each = .{ .once = &.{
21961 .{ ._, .f_, .ld, .src1t, ._, ._, ._ },
21962 .{ ._, .f_, .ld, .src0t, ._, ._, ._ },
21963 .{ ._, .f_pp, .ucom, ._, ._, ._, ._ },
21964 .{ ._, .fn_sw, .st, .tmp2w, ._, ._, ._ },
21965 .{ ._, ._, .xor, .tmp2b, .si(0b0_1_000_000), ._, ._ },
21966 .{ ._, ._, .@"test", .tmp2b, .si(0b0_1_000_100), ._, ._ },
21967 } },
21968 }, .{
21969 .required_features = .{ .x87, null, null, null },
21970 .src_constraints = .{ .{ .float = .tbyte }, .{ .float = .tbyte } },
21971 .patterns = &.{
21972 .{ .src = .{ .mem, .mem } },
21973 },
21974 .extra_temps = .{
21975 .{ .type = .f80, .kind = .{ .reg = .st6 } },
21976 .{ .type = .f80, .kind = .{ .reg = .st7 } },
21977 .{ .type = .u8, .kind = .{ .reg = .ah } },
21978 .unused,
21979 .unused,
21980 .unused,
21981 .unused,
21982 .unused,
21983 .unused,
21984 },
21985 .dst_temps = .{.{ .cc = switch (cc) {
21986 else => unreachable,
21987 .e => .z_and_np,
21988 .ne => .nz_or_p,
21989 } }},
21990 .clobbers = .{ .eflags = true },
21991 .each = .{ .once = &.{
21992 .{ ._, .f_, .ld, .src1t, ._, ._, ._ },
21993 .{ ._, .f_, .ld, .src0t, ._, ._, ._ },
21994 .{ ._, .f_pp, .ucom, ._, ._, ._, ._ },
21995 .{ ._, .fn_sw, .st, .tmp2w, ._, ._, ._ },
21996 .{ ._, ._, .sahf, ._, ._, ._, ._ },
21997 } },
21998 }, .{
21999 .required_features = .{ .sahf, .x87, null, null },
22000 .src_constraints = .{ .{ .float = .tbyte }, .{ .float = .tbyte } },
22001 .patterns = &.{
22002 .{ .src = .{ .to_x87, .mem }, .commute = .{ 0, 1 } },
22003 .{ .src = .{ .mem, .to_x87 } },
22004 .{ .src = .{ .to_x87, .to_x87 } },
22005 },
22006 .extra_temps = .{
22007 .{ .type = .f80, .kind = .{ .reg = .st6 } },
22008 .{ .type = .f80, .kind = .{ .reg = .st7 } },
22009 .{ .type = .u16, .kind = .{ .reg = .ah } },
22010 .unused,
22011 .unused,
22012 .unused,
22013 .unused,
22014 .unused,
22015 .unused,
22016 },
22017 .dst_temps = .{.{ .cc = switch (cc) {
21228 ) catch |err| switch (err) {
21229 error.SelectFailed => unreachable,
21230 else => |e| return e,
21231 };
21232 for (has_values) |has_value| for (opt_info.?.res) |opt_res| {
21233 if (has_value.index == opt_res.index) break;
21234 } else try has_value.die(cg);
21235 opt_info.?.state = cg.initRetroactiveState();
21236 opt_info.?.state.next_temp_index = cg.next_temp_index;
21237 var state = try cg.saveState();
21238 state.next_temp_index = cg.next_temp_index;
21239 for (ops) |op| try op.die(cg);
21240 try cg.saveRetroactiveState(&opt_info.?.state);
21241 opt_info.?.reloc = try cg.asmJccReloc(.z, undefined);
21242 try cg.restoreState(state, &.{}, .{
21243 .emit_instructions = false,
21244 .update_tracking = true,
21245 .resurrect = true,
21246 .close_scope = true,
21247 });
21248 break :category if (cg.floatBits(child_ty)) |_| .float else .int;
21249 })) {
21250 .float => {
21251 cg.select(&res, &.{.bool}, &ops, switch (switch (air_tag) {
2201821252 else => unreachable,
22019 .e => .z_and_np,
22020 .ne => .nz_or_p,
22021 } }},
22022 .clobbers = .{ .eflags = true },
22023 .each = .{ .once = &.{
22024 .{ ._, .f_, .ld, .src0t, ._, ._, ._ },
22025 .{ ._, .f_p, .ucom, .src1t, ._, ._, ._ },
22026 .{ ._, .fn_sw, .st, .tmp2w, ._, ._, ._ },
22027 .{ ._, ._, .sahf, ._, ._, ._, ._ },
22028 } },
22029 }, .{
22030 .required_features = .{ .@"64bit", .x87, null, null },
22031 .src_constraints = .{ .{ .float = .tbyte }, .{ .float = .tbyte } },
22032 .patterns = &.{
22033 .{ .src = .{ .to_x87, .mem }, .commute = .{ 0, 1 } },
22034 .{ .src = .{ .mem, .to_x87 } },
22035 .{ .src = .{ .to_x87, .to_x87 } },
22036 },
22037 .extra_temps = .{
22038 .{ .type = .f80, .kind = .{ .reg = .st6 } },
22039 .{ .type = .f80, .kind = .{ .reg = .st7 } },
22040 .{ .type = .u8, .kind = .{ .reg = .ah } },
22041 .unused,
22042 .unused,
22043 .unused,
22044 .unused,
22045 .unused,
22046 .unused,
22047 },
22048 .dst_temps = .{.{ .cc = cc }},
22049 .clobbers = .{ .eflags = true },
22050 .each = .{ .once = &.{
22051 .{ ._, .f_, .ld, .src0t, ._, ._, ._ },
22052 .{ ._, .f_p, .ucom, .src1t, ._, ._, ._ },
22053 .{ ._, .fn_sw, .st, .tmp2w, ._, ._, ._ },
22054 .{ ._, ._, .xor, .tmp2b, .si(0b0_1_000_000), ._, ._ },
22055 .{ ._, ._, .@"test", .tmp2b, .si(0b0_1_000_100), ._, ._ },
22056 } },
22057 }, .{
22058 .required_features = .{ .x87, null, null, null },
22059 .src_constraints = .{ .{ .float = .tbyte }, .{ .float = .tbyte } },
22060 .patterns = &.{
22061 .{ .src = .{ .to_x87, .mem }, .commute = .{ 0, 1 } },
22062 .{ .src = .{ .mem, .to_x87 } },
22063 .{ .src = .{ .to_x87, .to_x87 } },
22064 },
22065 .extra_temps = .{
22066 .{ .type = .f80, .kind = .{ .reg = .st6 } },
22067 .{ .type = .f80, .kind = .{ .reg = .st7 } },
22068 .{ .type = .u16, .kind = .{ .reg = .ah } },
22069 .unused,
22070 .unused,
22071 .unused,
22072 .unused,
22073 .unused,
22074 .unused,
22075 },
22076 .dst_temps = .{.{ .cc = switch (cc) {
21253 .cmp_eq, .cmp_neq => false,
21254 .cmp_eq_optimized, .cmp_neq_optimized => true,
21255 }) {
21256 inline false, true => |optimized| comptime &.{ .{
21257 .required_features = .{ .f16c, null, null, null },
21258 .src_constraints = .{ .{ .float = .word }, .{ .float = .word } },
21259 .patterns = &.{
21260 .{ .src = .{ .to_sse, .to_sse } },
21261 },
21262 .extra_temps = .{
21263 .{ .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },
21264 .{ .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } },
21265 .unused,
21266 .unused,
21267 .unused,
21268 .unused,
21269 .unused,
21270 .unused,
21271 .unused,
21272 },
21273 .dst_temps = .{.{ .cc = switch (optimized) {
21274 false => .z_and_np,
21275 true => .z,
21276 } }},
21277 .clobbers = .{ .eflags = true },
21278 .each = .{ .once = &.{
21279 .{ ._, .v_ps, .cvtph2, .tmp0x, .src0q, ._, ._ },
21280 .{ ._, .v_ps, .cvtph2, .tmp1x, .src1q, ._, ._ },
21281 .{ ._, .v_ss, .ucomi, .tmp0x, .tmp1x, ._, ._ },
21282 } },
21283 }, .{
21284 .required_features = .{ .sse, null, null, null },
21285 .src_constraints = .{ .{ .float = .word }, .{ .float = .word } },
21286 .patterns = &.{
21287 .{ .src = .{ .{ .to_reg = .xmm0 }, .{ .to_reg = .xmm1 } } },
21288 },
21289 .call_frame = .{ .alignment = .@"16" },
21290 .extra_temps = .{
21291 .{ .type = .usize, .kind = .{ .symbol = &.{ .name = "__cmphf2" } } },
21292 .{ .type = .i32, .kind = .{ .reg = .eax } },
21293 .unused,
21294 .unused,
21295 .unused,
21296 .unused,
21297 .unused,
21298 .unused,
21299 .unused,
21300 },
21301 .dst_temps = .{.{ .cc = .z }},
21302 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
21303 .each = .{ .once = &.{
21304 .{ ._, ._, .call, .tmp0d, ._, ._, ._ },
21305 .{ ._, ._, .@"test", .tmp1d, .tmp1d, ._, ._ },
21306 } },
21307 }, .{
21308 .required_features = .{ .avx, null, null, null },
21309 .src_constraints = .{ .{ .float = .dword }, .{ .float = .dword } },
21310 .patterns = &.{
21311 .{ .src = .{ .to_sse, .mem } },
21312 .{ .src = .{ .mem, .to_sse }, .commute = .{ 0, 1 } },
21313 .{ .src = .{ .to_sse, .to_sse } },
21314 },
21315 .dst_temps = .{.{ .cc = switch (optimized) {
21316 false => .z_and_np,
21317 true => .z,
21318 } }},
21319 .clobbers = .{ .eflags = true },
21320 .each = .{ .once = &.{
21321 .{ ._, .v_ss, .ucomi, .src0x, .src1d, ._, ._ },
21322 } },
21323 }, .{
21324 .required_features = .{ .sse, null, null, null },
21325 .src_constraints = .{ .{ .float = .dword }, .{ .float = .dword } },
21326 .patterns = &.{
21327 .{ .src = .{ .to_sse, .mem } },
21328 .{ .src = .{ .mem, .to_sse }, .commute = .{ 0, 1 } },
21329 .{ .src = .{ .to_sse, .to_sse } },
21330 },
21331 .dst_temps = .{.{ .cc = switch (optimized) {
21332 false => .z_and_np,
21333 true => .z,
21334 } }},
21335 .clobbers = .{ .eflags = true },
21336 .each = .{ .once = &.{
21337 .{ ._, ._ss, .ucomi, .src0x, .src1d, ._, ._ },
21338 } },
21339 }, .{
21340 .required_features = .{ .avx, null, null, null },
21341 .src_constraints = .{ .{ .float = .qword }, .{ .float = .qword } },
21342 .patterns = &.{
21343 .{ .src = .{ .to_sse, .mem } },
21344 .{ .src = .{ .mem, .to_sse }, .commute = .{ 0, 1 } },
21345 .{ .src = .{ .to_sse, .to_sse } },
21346 },
21347 .dst_temps = .{.{ .cc = switch (optimized) {
21348 false => .z_and_np,
21349 true => .z,
21350 } }},
21351 .clobbers = .{ .eflags = true },
21352 .each = .{ .once = &.{
21353 .{ ._, .v_sd, .ucomi, .src0x, .src1q, ._, ._ },
21354 } },
21355 }, .{
21356 .required_features = .{ .sse2, null, null, null },
21357 .src_constraints = .{ .{ .float = .qword }, .{ .float = .qword } },
21358 .patterns = &.{
21359 .{ .src = .{ .to_sse, .mem } },
21360 .{ .src = .{ .mem, .to_sse }, .commute = .{ 0, 1 } },
21361 .{ .src = .{ .to_sse, .to_sse } },
21362 },
21363 .dst_temps = .{.{ .cc = switch (optimized) {
21364 false => .z_and_np,
21365 true => .z,
21366 } }},
21367 .clobbers = .{ .eflags = true },
21368 .each = .{ .once = &.{
21369 .{ ._, ._sd, .ucomi, .src0x, .src1q, ._, ._ },
21370 } },
21371 }, .{
21372 .required_features = .{ .x87, .cmov, null, null },
21373 .src_constraints = .{ .{ .float = .qword }, .{ .float = .qword } },
21374 .patterns = &.{
21375 .{ .src = .{ .to_mem, .to_mem }, .commute = .{ 0, 1 } },
21376 },
21377 .extra_temps = .{
21378 .{ .type = .f80, .kind = .{ .reg = .st6 } },
21379 .{ .type = .f80, .kind = .{ .reg = .st7 } },
21380 .unused,
21381 .unused,
21382 .unused,
21383 .unused,
21384 .unused,
21385 .unused,
21386 .unused,
21387 },
21388 .dst_temps = .{.{ .cc = switch (optimized) {
21389 false => .z_and_np,
21390 true => .z,
21391 } }},
21392 .clobbers = .{ .eflags = true },
21393 .each = .{ .once = &.{
21394 .{ ._, .f_, .ld, .src1q, ._, ._, ._ },
21395 .{ ._, .f_, .ld, .src0q, ._, ._, ._ },
21396 .{ ._, .f_p, .ucomi, .tmp0t, .tmp1t, ._, ._ },
21397 .{ ._, .f_p, .st, .tmp1t, ._, ._, ._ },
21398 } },
21399 }, .{
21400 .required_features = .{ .sahf, .x87, null, null },
21401 .src_constraints = .{ .{ .float = .qword }, .{ .float = .qword } },
21402 .patterns = &.{
21403 .{ .src = .{ .to_mem, .to_mem }, .commute = .{ 0, 1 } },
21404 },
21405 .extra_temps = .{
21406 .{ .type = .f80, .kind = .{ .reg = .st6 } },
21407 .{ .type = .f80, .kind = .{ .reg = .st7 } },
21408 .{ .type = .u8, .kind = .{ .reg = .ah } },
21409 .unused,
21410 .unused,
21411 .unused,
21412 .unused,
21413 .unused,
21414 .unused,
21415 },
21416 .dst_temps = .{.{ .cc = switch (optimized) {
21417 false => .z_and_np,
21418 true => .z,
21419 } }},
21420 .clobbers = .{ .eflags = true },
21421 .each = .{ .once = &.{
21422 .{ ._, .f_, .ld, .src1q, ._, ._, ._ },
21423 .{ ._, .f_, .ld, .src0q, ._, ._, ._ },
21424 .{ ._, .f_pp, .ucom, ._, ._, ._, ._ },
21425 .{ ._, .fn_sw, .st, .tmp2w, ._, ._, ._ },
21426 .{ ._, ._, .sahf, ._, ._, ._, ._ },
21427 } },
21428 }, .{
21429 .required_features = .{ .x87, null, null, null },
21430 .src_constraints = .{ .{ .float = .qword }, .{ .float = .qword } },
21431 .patterns = &.{
21432 .{ .src = .{ .to_mem, .to_mem } },
21433 },
21434 .extra_temps = .{
21435 .{ .type = .f80, .kind = .{ .reg = .st6 } },
21436 .{ .type = .f80, .kind = .{ .reg = .st7 } },
21437 .{ .type = .u8, .kind = .{ .reg = .ah } },
21438 .unused,
21439 .unused,
21440 .unused,
21441 .unused,
21442 .unused,
21443 .unused,
21444 },
21445 .dst_temps = .{.{ .cc = switch (optimized) {
21446 false => .z,
21447 true => .nz,
21448 } }},
21449 .clobbers = .{ .eflags = true },
21450 .each = .{ .once = switch (optimized) {
21451 false => &.{
21452 .{ ._, .f_, .ld, .src1q, ._, ._, ._ },
21453 .{ ._, .f_, .ld, .src0q, ._, ._, ._ },
21454 .{ ._, .f_pp, .ucom, ._, ._, ._, ._ },
21455 .{ ._, .fn_sw, .st, .tmp2w, ._, ._, ._ },
21456 .{ ._, ._, .xor, .tmp2b, .si(0b1_000_000), ._, ._ },
21457 .{ ._, ._, .@"test", .tmp2b, .si(0b1_000_100), ._, ._ },
21458 },
21459 true => &.{
21460 .{ ._, .f_, .ld, .src1q, ._, ._, ._ },
21461 .{ ._, .f_, .ld, .src0q, ._, ._, ._ },
21462 .{ ._, .f_pp, .ucom, ._, ._, ._, ._ },
21463 .{ ._, .fn_sw, .st, .tmp2w, ._, ._, ._ },
21464 .{ ._, ._, .@"test", .tmp2b, .si(0b1_000_000), ._, ._ },
21465 },
21466 } },
21467 }, .{
21468 .required_features = .{ .x87, .cmov, null, null },
21469 .src_constraints = .{ .{ .float = .tbyte }, .{ .float = .tbyte } },
21470 .patterns = &.{
21471 .{ .src = .{ .to_x87, .mem }, .commute = .{ 0, 1 } },
21472 .{ .src = .{ .mem, .to_x87 } },
21473 .{ .src = .{ .to_x87, .to_x87 } },
21474 },
21475 .extra_temps = .{
21476 .{ .type = .f80, .kind = .{ .reg = .st7 } },
21477 .unused,
21478 .unused,
21479 .unused,
21480 .unused,
21481 .unused,
21482 .unused,
21483 .unused,
21484 .unused,
21485 },
21486 .dst_temps = .{.{ .cc = switch (optimized) {
21487 false => .z_and_np,
21488 true => .z,
21489 } }},
21490 .clobbers = .{ .eflags = true },
21491 .each = .{ .once = &.{
21492 .{ ._, .f_, .ld, .src0t, ._, ._, ._ },
21493 .{ ._, .f_p, .ucomi, .tmp0t, .src1t, ._, ._ },
21494 } },
21495 }, .{
21496 .required_features = .{ .sahf, .x87, null, null },
21497 .src_constraints = .{ .{ .float = .tbyte }, .{ .float = .tbyte } },
21498 .patterns = &.{
21499 .{ .src = .{ .mem, .mem } },
21500 },
21501 .extra_temps = .{
21502 .{ .type = .f80, .kind = .{ .reg = .st6 } },
21503 .{ .type = .f80, .kind = .{ .reg = .st7 } },
21504 .{ .type = .u8, .kind = .{ .reg = .ah } },
21505 .unused,
21506 .unused,
21507 .unused,
21508 .unused,
21509 .unused,
21510 .unused,
21511 },
21512 .dst_temps = .{.{ .cc = switch (optimized) {
21513 false => .z_and_np,
21514 true => .z,
21515 } }},
21516 .clobbers = .{ .eflags = true },
21517 .each = .{ .once = &.{
21518 .{ ._, .f_, .ld, .src1t, ._, ._, ._ },
21519 .{ ._, .f_, .ld, .src0t, ._, ._, ._ },
21520 .{ ._, .f_pp, .ucom, ._, ._, ._, ._ },
21521 .{ ._, .fn_sw, .st, .tmp2w, ._, ._, ._ },
21522 .{ ._, ._, .sahf, ._, ._, ._, ._ },
21523 } },
21524 }, .{
21525 .required_features = .{ .sahf, .x87, null, null },
21526 .src_constraints = .{ .{ .float = .tbyte }, .{ .float = .tbyte } },
21527 .patterns = &.{
21528 .{ .src = .{ .to_x87, .mem }, .commute = .{ 0, 1 } },
21529 .{ .src = .{ .mem, .to_x87 } },
21530 .{ .src = .{ .to_x87, .to_x87 } },
21531 },
21532 .extra_temps = .{
21533 .{ .type = .f80, .kind = .{ .reg = .st6 } },
21534 .{ .type = .f80, .kind = .{ .reg = .st7 } },
21535 .{ .type = .u8, .kind = .{ .reg = .ah } },
21536 .unused,
21537 .unused,
21538 .unused,
21539 .unused,
21540 .unused,
21541 .unused,
21542 },
21543 .dst_temps = .{.{ .cc = switch (optimized) {
21544 false => .z_and_np,
21545 true => .z,
21546 } }},
21547 .clobbers = .{ .eflags = true },
21548 .each = .{ .once = &.{
21549 .{ ._, .f_, .ld, .src0t, ._, ._, ._ },
21550 .{ ._, .f_p, .ucom, .src1t, ._, ._, ._ },
21551 .{ ._, .fn_sw, .st, .tmp2w, ._, ._, ._ },
21552 .{ ._, ._, .sahf, ._, ._, ._, ._ },
21553 } },
21554 }, .{
21555 .required_features = .{ .x87, null, null, null },
21556 .src_constraints = .{ .{ .float = .tbyte }, .{ .float = .tbyte } },
21557 .patterns = &.{
21558 .{ .src = .{ .mem, .mem } },
21559 },
21560 .extra_temps = .{
21561 .{ .type = .f80, .kind = .{ .reg = .st6 } },
21562 .{ .type = .f80, .kind = .{ .reg = .st7 } },
21563 .{ .type = .u8, .kind = .{ .reg = .ah } },
21564 .unused,
21565 .unused,
21566 .unused,
21567 .unused,
21568 .unused,
21569 .unused,
21570 },
21571 .dst_temps = .{.{ .cc = switch (optimized) {
21572 false => .z,
21573 true => .nz,
21574 } }},
21575 .clobbers = .{ .eflags = true },
21576 .each = .{ .once = switch (optimized) {
21577 false => &.{
21578 .{ ._, .f_, .ld, .src1t, ._, ._, ._ },
21579 .{ ._, .f_, .ld, .src0t, ._, ._, ._ },
21580 .{ ._, .f_pp, .ucom, ._, ._, ._, ._ },
21581 .{ ._, .fn_sw, .st, .tmp2w, ._, ._, ._ },
21582 .{ ._, ._, .xor, .tmp2b, .si(0b1_000_000), ._, ._ },
21583 .{ ._, ._, .@"test", .tmp2b, .si(0b1_000_100), ._, ._ },
21584 },
21585 true => &.{
21586 .{ ._, .f_, .ld, .src1t, ._, ._, ._ },
21587 .{ ._, .f_, .ld, .src0t, ._, ._, ._ },
21588 .{ ._, .f_pp, .ucom, ._, ._, ._, ._ },
21589 .{ ._, .fn_sw, .st, .tmp2w, ._, ._, ._ },
21590 .{ ._, ._, .@"test", .tmp2b, .si(0b1_000_000), ._, ._ },
21591 },
21592 } },
21593 }, .{
21594 .required_features = .{ .x87, null, null, null },
21595 .src_constraints = .{ .{ .float = .tbyte }, .{ .float = .tbyte } },
21596 .patterns = &.{
21597 .{ .src = .{ .to_x87, .mem }, .commute = .{ 0, 1 } },
21598 .{ .src = .{ .mem, .to_x87 } },
21599 .{ .src = .{ .to_x87, .to_x87 } },
21600 },
21601 .extra_temps = .{
21602 .{ .type = .f80, .kind = .{ .reg = .st6 } },
21603 .{ .type = .f80, .kind = .{ .reg = .st7 } },
21604 .{ .type = .u8, .kind = .{ .reg = .ah } },
21605 .unused,
21606 .unused,
21607 .unused,
21608 .unused,
21609 .unused,
21610 .unused,
21611 },
21612 .dst_temps = .{.{ .cc = switch (optimized) {
21613 false => .z,
21614 true => .nz,
21615 } }},
21616 .clobbers = .{ .eflags = true },
21617 .each = .{ .once = switch (optimized) {
21618 false => &.{
21619 .{ ._, .f_, .ld, .src0t, ._, ._, ._ },
21620 .{ ._, .f_p, .ucom, .src1t, ._, ._, ._ },
21621 .{ ._, .fn_sw, .st, .tmp2w, ._, ._, ._ },
21622 .{ ._, ._, .xor, .tmp2b, .si(0b1_000_000), ._, ._ },
21623 .{ ._, ._, .@"test", .tmp2b, .si(0b1_000_100), ._, ._ },
21624 },
21625 true => &.{
21626 .{ ._, .f_, .ld, .src0t, ._, ._, ._ },
21627 .{ ._, .f_p, .ucom, .src1t, ._, ._, ._ },
21628 .{ ._, .fn_sw, .st, .tmp2w, ._, ._, ._ },
21629 .{ ._, ._, .@"test", .tmp2b, .si(0b1_000_000), ._, ._ },
21630 },
21631 } },
21632 }, .{
21633 .required_features = .{ .sse, null, null, null },
21634 .src_constraints = .{ .{ .float = .xword }, .{ .float = .xword } },
21635 .patterns = &.{
21636 .{ .src = .{ .{ .to_reg = .xmm0 }, .{ .to_reg = .xmm1 } } },
21637 },
21638 .call_frame = .{ .alignment = .@"16" },
21639 .extra_temps = .{
21640 .{ .type = .usize, .kind = .{ .symbol = &.{ .name = "__cmptf2" } } },
21641 .{ .type = .i32, .kind = .{ .reg = .eax } },
21642 .unused,
21643 .unused,
21644 .unused,
21645 .unused,
21646 .unused,
21647 .unused,
21648 .unused,
21649 },
21650 .dst_temps = .{.{ .cc = .z }},
21651 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
21652 .each = .{ .once = &.{
21653 .{ ._, ._, .call, .tmp0d, ._, ._, ._ },
21654 .{ ._, ._, .@"test", .tmp1d, .tmp1d, ._, ._ },
21655 } },
21656 } },
21657 }) catch |err| break :err err;
21658 switch (cmp_op) {
2207721659 else => unreachable,
22078 .e => .z_and_np,
22079 .ne => .nz_or_p,
22080 } }},
22081 .clobbers = .{ .eflags = true },
22082 .each = .{ .once = &.{
22083 .{ ._, .f_, .ld, .src0t, ._, ._, ._ },
22084 .{ ._, .f_p, .ucom, .src1t, ._, ._, ._ },
22085 .{ ._, .fn_sw, .st, .tmp2w, ._, ._, ._ },
22086 .{ ._, ._, .sahf, ._, ._, ._, ._ },
22087 } },
22088 }, .{
22089 .required_features = .{ .sse, null, null, null },
22090 .src_constraints = .{ .{ .float = .xword }, .{ .float = .xword } },
22091 .patterns = &.{
22092 .{ .src = .{ .{ .to_reg = .xmm0 }, .{ .to_reg = .xmm1 } } },
22093 },
22094 .call_frame = .{ .alignment = .@"16" },
22095 .extra_temps = .{
22096 .{ .type = .usize, .kind = .{ .symbol = &.{ .name = switch (cc) {
22097 else => unreachable,
22098 .e => "__eqtf2",
22099 .ne => "__netf2",
22100 } } } },
22101 .{ .type = .i32, .kind = .{ .reg = .eax } },
22102 .unused,
22103 .unused,
22104 .unused,
22105 .unused,
22106 .unused,
22107 .unused,
22108 .unused,
22109 },
22110 .dst_temps = .{.{ .cc = cc }},
22111 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
22112 .each = .{ .once = &.{
22113 .{ ._, ._, .call, .tmp0d, ._, ._, ._ },
22114 .{ ._, ._, .@"test", .tmp1d, .tmp1d, ._, ._ },
22115 } },
22116 } },
21660 .eq => {},
21661 .neq => {
21662 const cc = &res[0].unwrap(cg).temp.tracking(cg).short.eflags;
21663 cc.* = cc.negate();
21664 },
21665 }
21666 },
21667 .int => res[0] = ops[0].cmpInts(cmp_op, &ops[1], cg) catch |err| break :err err,
2211721668 }) catch |err| switch (err) {
2211821669 error.SelectFailed => return cg.fail("failed to select {s} {} {} {}", .{
2211921670 @tagName(air_tag),
......@@ -22123,6 +21674,22 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2212321674 }),
2212421675 else => |e| return e,
2212521676 };
21677 if (opt_info) |*oi| {
21678 for (ops) |op| for (res) |r| {
21679 if (op.index == r.index) break;
21680 } else try op.die(cg);
21681 try cg.genCopy(.bool, oi.res[0].tracking(cg).short, res[0].tracking(cg).short, .{});
21682 try res[0].die(cg);
21683 res[0] = oi.res[0];
21684 try cg.restoreState(oi.state, &oi.deaths, .{
21685 .emit_instructions = true,
21686 .update_tracking = true,
21687 .resurrect = true,
21688 .close_scope = true,
21689 });
21690 cg.performReloc(oi.reloc);
21691 @memset(&ops, res[0]);
21692 }
2212621693 try res[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg);
2212721694 },
2212821695
......@@ -22169,7 +21736,10 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2216921736 .data = .{ .func = old_inline_func },
2217021737 });
2217121738 },
22172 .dbg_var_ptr, .dbg_var_val, .dbg_arg_inline => if (use_old) try cg.airDbgVar(inst) else if (cg.debug_output != .none) {
21739 .dbg_var_ptr,
21740 .dbg_var_val,
21741 .dbg_arg_inline,
21742 => if (use_old) try cg.airDbgVar(inst) else if (cg.debug_output != .none) {
2217321743 const pl_op = air_datas[@intFromEnum(inst)].pl_op;
2217421744 var ops = try cg.tempsFromOperands(inst, .{pl_op.operand});
2217521745 var mcv = ops[0].tracking(cg).short;
......@@ -22776,7 +22346,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2277622346 .work_group_size => unreachable,
2277722347 .work_group_id => unreachable,
2277822348 }
22779 cg.resetTemps();
22349 try cg.resetTemps();
2278022350 cg.checkInvariantsAfterAirInst();
2278122351 }
2278222352 verbose_tracking_log.debug("{}", .{cg.fmtTracking()});
......@@ -22798,9 +22368,6 @@ fn genLazy(self: *CodeGen, lazy_sym: link.File.LazySymbol) InnerError!void {
2279822368 const ret_reg = param_regs[0];
2279922369 const enum_mcv = MCValue{ .register = param_regs[1] };
2280022370
22801 const epilogue_relocs = try self.gpa.alloc(Mir.Inst.Index, enum_ty.enumFieldCount(zcu));
22802 defer self.gpa.free(epilogue_relocs);
22803
2280422371 const data_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
2280522372 const data_lock = self.register_manager.lockRegAssumeUnused(data_reg);
2280622373 defer self.register_manager.unlockReg(data_lock);
......@@ -22808,12 +22375,21 @@ fn genLazy(self: *CodeGen, lazy_sym: link.File.LazySymbol) InnerError!void {
2280822375
2280922376 var data_off: i32 = 0;
2281022377 const tag_names = enum_ty.enumFields(zcu);
22811 for (epilogue_relocs, 0..) |*epilogue_reloc, tag_index| {
22378 for (0..enum_ty.enumFieldCount(zcu)) |tag_index| {
22379 var arg_temp = try self.tempInit(enum_ty, enum_mcv);
22380
2281222381 const tag_name_len = tag_names.get(ip)[tag_index].length(ip);
2281322382 const tag_val = try pt.enumValueFieldIndex(enum_ty, @intCast(tag_index));
22814 const tag_mcv = try self.genTypedValue(tag_val);
22815 try self.genBinOpMir(.{ ._, .cmp }, enum_ty, enum_mcv, tag_mcv);
22816 const skip_reloc = try self.asmJccReloc(.ne, undefined);
22383 var tag_temp = try self.tempFromValue(tag_val);
22384 const cc_temp = arg_temp.cmpInts(.neq, &tag_temp, self) catch |err| switch (err) {
22385 error.SelectFailed => unreachable,
22386 else => |e| return e,
22387 };
22388 try arg_temp.die(self);
22389 try tag_temp.die(self);
22390 const skip_reloc = try self.asmJccReloc(cc_temp.tracking(self).short.eflags, undefined);
22391 try cc_temp.die(self);
22392 try self.resetTemps();
2281722393
2281822394 try self.genSetMem(
2281922395 .{ .reg = ret_reg },
......@@ -22823,17 +22399,14 @@ fn genLazy(self: *CodeGen, lazy_sym: link.File.LazySymbol) InnerError!void {
2282322399 .{},
2282422400 );
2282522401 try self.genSetMem(.{ .reg = ret_reg }, 8, .usize, .{ .immediate = tag_name_len }, .{});
22402 try self.asmOpOnly(.{ ._, .ret });
2282622403
22827 epilogue_reloc.* = try self.asmJmpReloc(undefined);
2282822404 self.performReloc(skip_reloc);
2282922405
2283022406 data_off += @intCast(tag_name_len + 1);
2283122407 }
2283222408
2283322409 try self.asmOpOnly(.{ ._2, .ud });
22834
22835 for (epilogue_relocs) |reloc| self.performReloc(reloc);
22836 try self.asmOpOnly(.{ ._, .ret });
2283722410 },
2283822411 else => return self.fail(
2283922412 "TODO implement {s} for {}",
......@@ -23103,10 +22676,12 @@ fn regClassForType(self: *CodeGen, ty: Type) Register.Class {
2310322676 const elem_ty = ty.childType(zcu);
2310422677 return if (elem_ty.toIntern() == .bool_type)
2310522678 .general_purpose
23106 else if (self.floatBits(elem_ty) == 80)
23107 .x87
22679 else if (self.floatBits(elem_ty)) |float_bits|
22680 if (float_bits == 80) .x87 else .sse
22681 else if (self.intInfo(elem_ty)) |_|
22682 .sse
2310822683 else
23109 .sse;
22684 .general_purpose;
2311022685}
2311122686
2311222687fn regSetForRegClass(rc: Register.Class) RegisterManager.RegisterBitSet {
......@@ -23149,14 +22724,19 @@ const State = struct {
2314922724 registers: RegisterManager.TrackedRegisters,
2315022725 reg_tracking: [RegisterManager.RegisterBitSet.bit_length]InstTracking,
2315122726 free_registers: RegisterManager.RegisterBitSet,
22727 next_temp_index: Temp.Index,
2315222728 inst_tracking_len: u32,
2315322729 scope_generation: u32,
2315422730};
2315522731
2315622732fn initRetroactiveState(self: *CodeGen) State {
22733 const scope_generation = self.scope_generation + 1;
22734 self.scope_generation = scope_generation;
22735
2315722736 var state: State = undefined;
22737 state.next_temp_index = @enumFromInt(0);
2315822738 state.inst_tracking_len = @intCast(self.inst_tracking.count());
23159 state.scope_generation = self.scope_generation;
22739 state.scope_generation = scope_generation;
2316022740 return state;
2316122741}
2316222742
......@@ -23185,6 +22765,11 @@ fn restoreState(self: *CodeGen, state: State, deaths: []const Air.Inst.Index, co
2318522765 close_scope: bool,
2318622766}) !void {
2318722767 if (opts.close_scope) {
22768 for (
22769 self.inst_tracking.keys()[@intFromEnum(state.next_temp_index)..@intFromEnum(self.next_temp_index)],
22770 self.inst_tracking.values()[@intFromEnum(state.next_temp_index)..@intFromEnum(self.next_temp_index)],
22771 ) |inst, *tracking| try tracking.die(self, inst);
22772 self.next_temp_index = state.next_temp_index;
2318822773 for (
2318922774 self.inst_tracking.keys()[state.inst_tracking_len..],
2319022775 self.inst_tracking.values()[state.inst_tracking_len..],
......@@ -23192,10 +22777,16 @@ fn restoreState(self: *CodeGen, state: State, deaths: []const Air.Inst.Index, co
2319222777 self.inst_tracking.shrinkRetainingCapacity(state.inst_tracking_len);
2319322778 }
2319422779
23195 if (opts.resurrect) for (
23196 self.inst_tracking.keys()[Temp.Index.max..state.inst_tracking_len],
23197 self.inst_tracking.values()[Temp.Index.max..state.inst_tracking_len],
23198 ) |inst, *tracking| try tracking.resurrect(self, inst, state.scope_generation);
22780 if (opts.resurrect) {
22781 for (
22782 self.inst_tracking.keys()[0..@intFromEnum(state.next_temp_index)],
22783 self.inst_tracking.values()[0..@intFromEnum(state.next_temp_index)],
22784 ) |inst, *tracking| try tracking.resurrect(self, inst, state.scope_generation);
22785 for (
22786 self.inst_tracking.keys()[Temp.Index.max..state.inst_tracking_len],
22787 self.inst_tracking.values()[Temp.Index.max..state.inst_tracking_len],
22788 ) |inst, *tracking| try tracking.resurrect(self, inst, state.scope_generation);
22789 }
2319922790 for (deaths) |death| try self.processDeath(death);
2320022791
2320122792 const ExpectedContents = [@typeInfo(RegisterManager.TrackedRegisters).array.len]RegisterLock;
......@@ -27359,12 +26950,12 @@ fn airBitReverse(self: *CodeGen, inst: Air.Inst.Index) !void {
2735926950 try self.asmRegisterImmediate(.{ ._, .@"and" }, tmp, imm_0000_1111);
2736026951 try self.asmRegisterImmediate(.{ ._, .@"and" }, dst, imm_0000_1111);
2736126952 }
27362 // tmp = temp1 & 0x0F...0F
27363 // dst = (temp1 >> 4) & 0x0F...0F
26953 // tmp = temp1 & 0x0f...0f
26954 // dst = (temp1 >> 4) & 0x0f...0f
2736426955 try self.asmRegisterImmediate(.{ ._l, .sh }, tmp, .u(4));
27365 // tmp = (temp1 & 0x0F...0F) << 4
26956 // tmp = (temp1 & 0x0f...0f) << 4
2736626957 try self.asmRegisterRegister(.{ ._, .@"or" }, dst, tmp);
27367 // dst = temp2 = ((temp1 >> 4) & 0x0F...0F) | ((temp1 & 0x0F...0F) << 4)
26958 // dst = temp2 = ((temp1 >> 4) & 0x0f...0f) | ((temp1 & 0x0f...0f) << 4)
2736826959 try self.asmRegisterRegister(.{ ._, .mov }, tmp, dst);
2736926960 // tmp = temp2
2737026961 try self.asmRegisterImmediate(.{ ._r, .sh }, dst, .u(2));
......@@ -33520,7 +33111,6 @@ fn genTry(
3352033111 if (operand.toIndex()) |operand_inst| try self.processDeath(operand_inst);
3352133112 }
3352233113
33523 self.scope_generation += 1;
3352433114 const state = try self.saveState();
3352533115
3352633116 for (liveness_cond_br.else_deaths) |death| try self.processDeath(death);
......@@ -33596,7 +33186,6 @@ fn airCondBr(self: *CodeGen, inst: Air.Inst.Index) !void {
3359633186 if (pl_op.operand.toIndex()) |op_inst| try self.processDeath(op_inst);
3359733187 }
3359833188
33599 self.scope_generation += 1;
3360033189 const state = try self.saveState();
3360133190 const reloc = try self.genCondBrMir(cond_ty, cond);
3360233191
......@@ -33978,7 +33567,6 @@ fn airLoop(self: *CodeGen, inst: Air.Inst.Index) !void {
3397833567 const loop = self.air.extraData(Air.Block, ty_pl.payload);
3397933568 const body: []const Air.Inst.Index = @ptrCast(self.air.extra[loop.end..][0..loop.data.body_len]);
3398033569
33981 self.scope_generation += 1;
3398233570 const state = try self.saveState();
3398333571
3398433572 try self.loops.putNoClobber(self.gpa, inst, .{
......@@ -33995,7 +33583,6 @@ fn lowerBlock(self: *CodeGen, inst: Air.Inst.Index, body: []const Air.Inst.Index
3399533583 const inst_tracking_i = self.inst_tracking.count();
3399633584 self.inst_tracking.putAssumeCapacityNoClobber(inst, .init(.unreach));
3399733585
33998 self.scope_generation += 1;
3399933586 try self.blocks.putNoClobber(self.gpa, inst, .{ .state = self.initRetroactiveState() });
3400033587 const liveness = self.liveness.getBlock(inst);
3400133588
......@@ -34033,6 +33620,8 @@ fn lowerSwitchBr(
3403333620) !void {
3403433621 const zcu = self.pt.zcu;
3403533622 const condition_ty = self.typeOf(switch_br.operand);
33623 const condition_int_info = self.intInfo(condition_ty).?;
33624 const condition_int_ty = try self.pt.intType(condition_int_info.signedness, condition_int_info.bits);
3403633625
3403733626 const ExpectedContents = extern struct {
3403833627 liveness_deaths: [1 << 8 | 1]Air.Inst.Index,
......@@ -34043,7 +33632,6 @@ fn lowerSwitchBr(
3404333632 std.heap.stackFallback(@sizeOf(ExpectedContents), self.gpa);
3404433633 const allocator = stack.get();
3404533634
34046 self.scope_generation += 1;
3404733635 const state = try self.saveState();
3404833636
3404933637 const liveness = try self.liveness.getSwitchBr(allocator, inst, switch_br.cases_len + 1);
......@@ -34104,8 +33692,17 @@ fn lowerSwitchBr(
3410433692 .{ .air_ref = Air.internedToRef(min.?.toIntern()) },
3410533693 );
3410633694 const else_reloc = if (switch_br.else_body_len > 0) else_reloc: {
34107 try self.genBinOpMir(.{ ._, .cmp }, condition_ty, condition_index, .{ .immediate = table_len - 1 });
34108 break :else_reloc try self.asmJccReloc(.a, undefined);
33695 var cond_temp = try self.tempInit(condition_ty, condition_index);
33696 var table_max_temp = try self.tempFromValue(try self.pt.intValue(condition_int_ty, table_len - 1));
33697 const cc_temp = cond_temp.cmpInts(.gt, &table_max_temp, self) catch |err| switch (err) {
33698 error.SelectFailed => unreachable,
33699 else => |e| return e,
33700 };
33701 try cond_temp.die(self);
33702 try table_max_temp.die(self);
33703 const else_reloc = try self.asmJccReloc(cc_temp.tracking(self).short.eflags, undefined);
33704 try cc_temp.die(self);
33705 break :else_reloc else_reloc;
3410933706 } else undefined;
3411033707 const table_start: u31 = @intCast(self.mir_table.items.len);
3411133708 {
......@@ -34207,7 +33804,6 @@ fn lowerSwitchBr(
3420733804 return;
3420833805 }
3420933806
34210 const signedness = if (condition_ty.isAbiInt(zcu)) condition_ty.intInfo(zcu).signedness else .unsigned;
3421133807 var cases_it = switch_br.iterateCases();
3421233808 while (cases_it.next()) |case| {
3421333809 var relocs = try allocator.alloc(Mir.Inst.Index, case.items.len + case.ranges.len);
......@@ -34223,49 +33819,70 @@ fn lowerSwitchBr(
3422333819 else => unreachable,
3422433820 },
3422533821 else => cc: {
34226 try self.genBinOpMir(.{ ._, .cmp }, condition_ty, condition, item_mcv);
34227 break :cc .e;
33822 var cond_temp = try self.tempInit(condition_ty, condition);
33823 var item_temp = try self.tempInit(condition_ty, item_mcv);
33824 const cc_temp = cond_temp.cmpInts(.eq, &item_temp, self) catch |err| switch (err) {
33825 error.SelectFailed => unreachable,
33826 else => |e| return e,
33827 };
33828 try cond_temp.die(self);
33829 try item_temp.die(self);
33830 const cc = cc_temp.tracking(self).short.eflags;
33831 try cc_temp.die(self);
33832 try self.resetTemps();
33833 break :cc cc;
3422833834 },
3422933835 };
3423033836 reloc.* = try self.asmJccReloc(cc, undefined);
3423133837 }
3423233838
3423333839 for (case.ranges, relocs[case.items.len..]) |range, *reloc| {
33840 var cond_temp = try self.tempInit(condition_ty, condition);
3423433841 const min_mcv = try self.resolveInst(range[0]);
3423533842 const max_mcv = try self.resolveInst(range[1]);
3423633843 // `null` means always false.
34237 const lt_min: ?Condition = switch (condition) {
33844 const lt_min = cc: switch (condition) {
3423833845 .eflags => |cc| switch (min_mcv.immediate) {
3423933846 0 => null, // condition never <0
3424033847 1 => cc.negate(),
3424133848 else => unreachable,
3424233849 },
34243 else => cc: {
34244 try self.genBinOpMir(.{ ._, .cmp }, condition_ty, condition, min_mcv);
34245 break :cc switch (signedness) {
34246 .unsigned => .b,
34247 .signed => .l,
33850 else => {
33851 var min_temp = try self.tempInit(condition_ty, min_mcv);
33852 const cc_temp = cond_temp.cmpInts(.lt, &min_temp, self) catch |err| switch (err) {
33853 error.SelectFailed => unreachable,
33854 else => |e| return e,
3424833855 };
33856 try min_temp.die(self);
33857 const cc = cc_temp.tracking(self).short.eflags;
33858 try cc_temp.die(self);
33859 break :cc cc;
3424933860 },
3425033861 };
3425133862 const lt_min_reloc = if (lt_min) |cc| r: {
3425233863 break :r try self.asmJccReloc(cc, undefined);
3425333864 } else null;
3425433865 // `null` means always true.
34255 const lte_max: ?Condition = switch (condition) {
33866 const lte_max = switch (condition) {
3425633867 .eflags => |cc| switch (max_mcv.immediate) {
3425733868 0 => cc.negate(),
3425833869 1 => null, // condition always >=1
3425933870 else => unreachable,
3426033871 },
3426133872 else => cc: {
34262 try self.genBinOpMir(.{ ._, .cmp }, condition_ty, condition, max_mcv);
34263 break :cc switch (signedness) {
34264 .unsigned => .be,
34265 .signed => .le,
33873 var max_temp = try self.tempInit(condition_ty, max_mcv);
33874 const cc_temp = cond_temp.cmpInts(.lte, &max_temp, self) catch |err| switch (err) {
33875 error.SelectFailed => unreachable,
33876 else => |e| return e,
3426633877 };
33878 try max_temp.die(self);
33879 const cc = cc_temp.tracking(self).short.eflags;
33880 try cc_temp.die(self);
33881 break :cc cc;
3426733882 },
3426833883 };
33884 try cond_temp.die(self);
33885 try self.resetTemps();
3426933886 // "Success" case is in `reloc`....
3427033887 if (lte_max) |cc| {
3427133888 reloc.* = try self.asmJccReloc(cc, undefined);
......@@ -34350,7 +33967,6 @@ fn airLoopSwitchBr(self: *CodeGen, inst: Air.Inst.Index) !void {
3435033967 // Ensure a register is available for dispatch.
3435133968 if (!mat_cond.isRegister()) _ = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
3435233969
34353 self.scope_generation += 1;
3435433970 const state = try self.saveState();
3435533971
3435633972 try self.loops.putNoClobber(self.gpa, inst, .{
......@@ -34403,6 +34019,8 @@ fn airSwitchDispatch(self: *CodeGen, inst: Air.Inst.Index) !void {
3440334019
3440434020 if (self.loop_switches.getPtr(br.block_inst)) |table| {
3440534021 const condition_ty = self.typeOf(br.operand);
34022 const condition_int_info = self.intInfo(condition_ty).?;
34023 const condition_int_ty = try self.pt.intType(condition_int_info.signedness, condition_int_info.bits);
3440634024 const condition_mcv = block_tracking.short;
3440734025 try self.spillEflagsIfOccupied();
3440834026 if (table.min.orderAgainstZero(self.pt.zcu).compare(.neq)) try self.genBinOpMir(
......@@ -34414,12 +34032,28 @@ fn airSwitchDispatch(self: *CodeGen, inst: Air.Inst.Index) !void {
3441434032 switch (table.else_relocs) {
3441534033 .@"unreachable" => {},
3441634034 .forward => |*else_relocs| {
34417 try self.genBinOpMir(.{ ._, .cmp }, condition_ty, condition_mcv, .{ .immediate = table.len - 1 });
34418 try else_relocs.append(self.gpa, try self.asmJccReloc(.a, undefined));
34035 var cond_temp = try self.tempInit(condition_ty, condition_mcv);
34036 var table_max_temp = try self.tempFromValue(try self.pt.intValue(condition_int_ty, table.len - 1));
34037 const cc_temp = cond_temp.cmpInts(.gt, &table_max_temp, self) catch |err| switch (err) {
34038 error.SelectFailed => unreachable,
34039 else => |e| return e,
34040 };
34041 try cond_temp.die(self);
34042 try table_max_temp.die(self);
34043 try else_relocs.append(self.gpa, try self.asmJccReloc(cc_temp.tracking(self).short.eflags, undefined));
34044 try cc_temp.die(self);
3441934045 },
3442034046 .backward => |else_reloc| {
34421 try self.genBinOpMir(.{ ._, .cmp }, condition_ty, condition_mcv, .{ .immediate = table.len - 1 });
34422 _ = try self.asmJccReloc(.a, else_reloc);
34047 var cond_temp = try self.tempInit(condition_ty, condition_mcv);
34048 var table_max_temp = try self.tempFromValue(try self.pt.intValue(condition_int_ty, table.len - 1));
34049 const cc_temp = cond_temp.cmpInts(.gt, &table_max_temp, self) catch |err| switch (err) {
34050 error.SelectFailed => unreachable,
34051 else => |e| return e,
34052 };
34053 try cond_temp.die(self);
34054 try table_max_temp.die(self);
34055 _ = try self.asmJccReloc(cc_temp.tracking(self).short.eflags, else_reloc);
34056 try cc_temp.die(self);
3442334057 },
3442434058 }
3442534059 {
......@@ -35754,21 +35388,21 @@ fn genSetReg(
3575435388 => unreachable,
3575535389 .undef => if (opts.safety) switch (dst_reg.class()) {
3575635390 .general_purpose => switch (abi_size) {
35757 1 => try self.asmRegisterImmediate(.{ ._, .mov }, dst_reg.to8(), .u(0xAA)),
35758 2 => try self.asmRegisterImmediate(.{ ._, .mov }, dst_reg.to16(), .u(0xAAAA)),
35391 1 => try self.asmRegisterImmediate(.{ ._, .mov }, dst_reg.to8(), .u(0xaa)),
35392 2 => try self.asmRegisterImmediate(.{ ._, .mov }, dst_reg.to16(), .u(0xaaaa)),
3575935393 3...4 => try self.asmRegisterImmediate(
3576035394 .{ ._, .mov },
3576135395 dst_reg.to32(),
35762 .s(@as(i32, @bitCast(@as(u32, 0xAAAAAAAA)))),
35396 .s(@as(i32, @bitCast(@as(u32, 0xaaaaaaaa)))),
3576335397 ),
3576435398 5...8 => try self.asmRegisterImmediate(
3576535399 .{ ._, .mov },
3576635400 dst_reg.to64(),
35767 .u(0xAAAAAAAAAAAAAAAA),
35401 .u(0xaaaaaaaaaaaaaaaa),
3576835402 ),
3576935403 else => unreachable,
3577035404 },
35771 .segment, .x87, .mmx, .sse => {
35405 .segment, .mmx, .sse => {
3577235406 const full_ty = try pt.vectorType(.{
3577335407 .len = self.vectorSize(.float),
3577435408 .child = .u8_type,
......@@ -35780,6 +35414,9 @@ fn genSetReg(
3578035414 } })),
3578135415 ), opts);
3578235416 },
35417 .x87 => try self.genSetReg(dst_reg, .f80, try self.genTypedValue(
35418 try pt.floatValue(.f80, @as(f80, @bitCast(@as(u80, 0xaaaaaaaaaaaaaaaaaaaa)))),
35419 ), opts),
3578335420 .ip, .cr, .dr => unreachable,
3578435421 },
3578535422 .eflags => |cc| try self.asmSetccRegister(cc, dst_reg.to8()),
......@@ -40161,8 +39798,16 @@ fn resolveCallingConventionValues(
4016139798 const zcu = pt.zcu;
4016239799 const ip = &zcu.intern_pool;
4016339800 const cc = fn_info.cc;
40164 const param_types = try self.gpa.alloc(Type, fn_info.param_types.len + var_args.len);
40165 defer self.gpa.free(param_types);
39801
39802 const ExpectedContents = extern struct {
39803 param_types: [32][@sizeOf(Type)]u8 align(@alignOf(Type)),
39804 };
39805 var stack align(@max(@alignOf(ExpectedContents), @alignOf(std.heap.StackFallbackAllocator(0)))) =
39806 std.heap.stackFallback(@sizeOf(ExpectedContents), self.gpa);
39807 const allocator = stack.get();
39808
39809 const param_types = try allocator.alloc(Type, fn_info.param_types.len + var_args.len);
39810 defer allocator.free(param_types);
4016639811
4016739812 for (param_types[0..fn_info.param_types.len], fn_info.param_types.get(ip)) |*dest, src|
4016839813 dest.* = .fromInterned(src);
......@@ -40735,6 +40380,11 @@ fn hasFeature(cg: *CodeGen, feature: std.Target.x86.Feature) bool {
4073540380 .ReleaseSmall => true,
4073640381 },
4073740382 .mmx => false,
40383 .sahf => switch (cg.target.cpu.arch) {
40384 else => unreachable,
40385 .x86 => true,
40386 .x86_64 => null,
40387 },
4073840388 else => null,
4073940389 } orelse std.Target.x86.featureSetHas(cg.target.cpu.features, feature);
4074040390}
......@@ -40746,8 +40396,7 @@ fn typeOf(self: *CodeGen, inst: Air.Inst.Ref) Type {
4074640396}
4074740397
4074840398fn typeOfIndex(self: *CodeGen, inst: Air.Inst.Index) Type {
40749 const temp: Temp = .{ .index = inst };
40750 return temp.typeOf(self);
40399 return Temp.typeOf(.{ .index = inst }, self);
4075140400}
4075240401
4075340402fn intCompilerRtAbiName(int_bits: u32) u8 {
......@@ -40777,980 +40426,1922 @@ fn floatCompilerRtAbiType(self: *CodeGen, ty: Type, other_ty: Type) Type {
4077740426 return ty;
4077840427}
4077940428
40780fn floatLibcAbiPrefix(ty: Type) []const u8 {
40781 return switch (ty.toIntern()) {
40782 .f16_type, .f80_type => "__",
40783 .f32_type, .f64_type, .f128_type, .c_longdouble_type => "",
40784 else => unreachable,
40785 };
40786}
40429fn floatLibcAbiPrefix(ty: Type) []const u8 {
40430 return switch (ty.toIntern()) {
40431 .f16_type, .f80_type => "__",
40432 .f32_type, .f64_type, .f128_type, .c_longdouble_type => "",
40433 else => unreachable,
40434 };
40435}
40436
40437fn floatLibcAbiSuffix(ty: Type) []const u8 {
40438 return switch (ty.toIntern()) {
40439 .f16_type => "h",
40440 .f32_type => "f",
40441 .f64_type => "",
40442 .f80_type => "x",
40443 .f128_type => "q",
40444 .c_longdouble_type => "l",
40445 else => unreachable,
40446 };
40447}
40448
40449fn promoteInt(self: *CodeGen, ty: Type) Type {
40450 const pt = self.pt;
40451 const zcu = pt.zcu;
40452 const int_info: InternPool.Key.IntType = switch (ty.toIntern()) {
40453 .bool_type => .{ .signedness = .unsigned, .bits = 1 },
40454 else => if (ty.isAbiInt(zcu)) ty.intInfo(zcu) else return ty,
40455 };
40456 for ([_]Type{
40457 .c_int, .c_uint,
40458 .c_long, .c_ulong,
40459 .c_longlong, .c_ulonglong,
40460 }) |promote_ty| {
40461 const promote_info = promote_ty.intInfo(zcu);
40462 if (int_info.signedness == .signed and promote_info.signedness == .unsigned) continue;
40463 if (int_info.bits + @intFromBool(int_info.signedness == .unsigned and
40464 promote_info.signedness == .signed) <= promote_info.bits) return promote_ty;
40465 }
40466 return ty;
40467}
40468
40469fn promoteVarArg(self: *CodeGen, ty: Type) Type {
40470 if (!ty.isRuntimeFloat()) return self.promoteInt(ty);
40471 switch (ty.floatBits(self.target.*)) {
40472 32, 64 => return .f64,
40473 else => |float_bits| {
40474 assert(float_bits == self.target.cTypeBitSize(.longdouble));
40475 return .c_longdouble;
40476 },
40477 }
40478}
40479
40480fn unalignedSize(cg: *CodeGen, ty: Type) u64 {
40481 const zcu = cg.pt.zcu;
40482 return switch (zcu.intern_pool.indexToKey(ty.toIntern())) {
40483 .vector_type => |vector_type| Type.fromInterned(vector_type.child).abiSize(zcu) * vector_type.len,
40484 else => ty.abiSize(zcu),
40485 };
40486}
40487
40488fn intInfo(cg: *CodeGen, ty: Type) ?std.builtin.Type.Int {
40489 const zcu = cg.pt.zcu;
40490 const ip = &zcu.intern_pool;
40491 var ty_index = ty.ip_index;
40492 while (true) switch (ip.indexToKey(ty_index)) {
40493 .int_type => |int_type| return int_type,
40494 .ptr_type => |ptr_type| return switch (ptr_type.flags.size) {
40495 .one, .many, .c => .{ .signedness = .unsigned, .bits = cg.target.ptrBitWidth() },
40496 .slice => null,
40497 },
40498 .opt_type => |opt_child| return if (!Type.fromInterned(opt_child).hasRuntimeBitsIgnoreComptime(zcu))
40499 .{ .signedness = .unsigned, .bits = 1 }
40500 else switch (ip.indexToKey(opt_child)) {
40501 .ptr_type => |ptr_type| switch (ptr_type.flags.size) {
40502 .one, .many => switch (ptr_type.flags.is_allowzero) {
40503 false => .{ .signedness = .unsigned, .bits = cg.target.ptrBitWidth() },
40504 true => null,
40505 },
40506 .slice, .c => null,
40507 },
40508 else => null,
40509 },
40510 .error_union_type => |error_union_type| return if (!Type.fromInterned(error_union_type.payload_type)
40511 .hasRuntimeBitsIgnoreComptime(zcu)) .{ .signedness = .unsigned, .bits = zcu.errorSetBits() } else null,
40512 .simple_type => |simple_type| return switch (simple_type) {
40513 .bool => .{ .signedness = .unsigned, .bits = 1 },
40514 .anyerror => .{ .signedness = .unsigned, .bits = zcu.errorSetBits() },
40515 .isize => .{ .signedness = .signed, .bits = cg.target.ptrBitWidth() },
40516 .usize => .{ .signedness = .unsigned, .bits = cg.target.ptrBitWidth() },
40517 .c_char => .{ .signedness = cg.target.charSignedness(), .bits = cg.target.cTypeBitSize(.char) },
40518 .c_short => .{ .signedness = .signed, .bits = cg.target.cTypeBitSize(.short) },
40519 .c_ushort => .{ .signedness = .unsigned, .bits = cg.target.cTypeBitSize(.short) },
40520 .c_int => .{ .signedness = .signed, .bits = cg.target.cTypeBitSize(.int) },
40521 .c_uint => .{ .signedness = .unsigned, .bits = cg.target.cTypeBitSize(.int) },
40522 .c_long => .{ .signedness = .signed, .bits = cg.target.cTypeBitSize(.long) },
40523 .c_ulong => .{ .signedness = .unsigned, .bits = cg.target.cTypeBitSize(.long) },
40524 .c_longlong => .{ .signedness = .signed, .bits = cg.target.cTypeBitSize(.longlong) },
40525 .c_ulonglong => .{ .signedness = .unsigned, .bits = cg.target.cTypeBitSize(.longlong) },
40526 .f16, .f32, .f64, .f80, .f128, .c_longdouble => null,
40527 .anyopaque,
40528 .void,
40529 .type,
40530 .comptime_int,
40531 .comptime_float,
40532 .noreturn,
40533 .null,
40534 .undefined,
40535 .enum_literal,
40536 .adhoc_inferred_error_set,
40537 .generic_poison,
40538 => unreachable,
40539 },
40540 .struct_type => {
40541 const loaded_struct = ip.loadStructType(ty_index);
40542 switch (loaded_struct.layout) {
40543 .auto, .@"extern" => return null,
40544 .@"packed" => ty_index = loaded_struct.backingIntTypeUnordered(ip),
40545 }
40546 },
40547 .union_type => return switch (ip.loadUnionType(ty_index).flagsUnordered(ip).layout) {
40548 .auto, .@"extern" => null,
40549 .@"packed" => .{ .signedness = .unsigned, .bits = @intCast(ty.bitSize(zcu)) },
40550 },
40551 .enum_type => ty_index = ip.loadEnumType(ty_index).tag_ty,
40552 .error_set_type, .inferred_error_set_type => return .{ .signedness = .unsigned, .bits = zcu.errorSetBits() },
40553 else => return null,
40554 };
40555}
40556
40557fn floatBits(cg: *CodeGen, ty: Type) ?u16 {
40558 return if (ty.isRuntimeFloat()) ty.floatBits(cg.target.*) else null;
40559}
40560
40561const Temp = struct {
40562 index: Air.Inst.Index,
40563
40564 fn unwrap(temp: Temp, cg: *CodeGen) union(enum) {
40565 ref: Air.Inst.Ref,
40566 temp: Index,
40567 err_ret_trace,
40568 } {
40569 switch (temp.index.unwrap()) {
40570 .ref => |ref| return .{ .ref = ref },
40571 .target => |target_index| {
40572 if (temp.index == err_ret_trace_index) return .err_ret_trace;
40573 const temp_index: Index = @enumFromInt(target_index);
40574 assert(temp_index.isValid(cg));
40575 return .{ .temp = temp_index };
40576 },
40577 }
40578 }
40579
40580 fn typeOf(temp: Temp, cg: *CodeGen) Type {
40581 return switch (temp.unwrap(cg)) {
40582 .ref => switch (cg.air.instructions.items(.tag)[@intFromEnum(temp.index)]) {
40583 .loop_switch_br => cg.typeOf(cg.air.unwrapSwitch(temp.index).operand),
40584 else => cg.air.typeOfIndex(temp.index, &cg.pt.zcu.intern_pool),
40585 },
40586 .temp => |temp_index| temp_index.typeOf(cg),
40587 .err_ret_trace => .usize,
40588 };
40589 }
40590
40591 fn isMut(temp: Temp, cg: *CodeGen) bool {
40592 return switch (temp.unwrap(cg)) {
40593 .ref, .err_ret_trace => false,
40594 .temp => |temp_index| switch (temp_index.tracking(cg).short) {
40595 .none,
40596 .unreach,
40597 .dead,
40598 .undef,
40599 .immediate,
40600 .eflags,
40601 .register_offset,
40602 .register_mask,
40603 .memory,
40604 .load_symbol,
40605 .lea_symbol,
40606 .indirect,
40607 .load_direct,
40608 .lea_direct,
40609 .load_got,
40610 .lea_got,
40611 .load_tlv,
40612 .lea_tlv,
40613 .lea_frame,
40614 .elementwise_regs_then_frame,
40615 .reserved_frame,
40616 .air_ref,
40617 => false,
40618 .register,
40619 .register_pair,
40620 .register_triple,
40621 .register_quadruple,
40622 .register_overflow,
40623 => true,
40624 .load_frame => |frame_addr| !frame_addr.index.isNamed(),
40625 },
40626 };
40627 }
40628
40629 fn tracking(temp: Temp, cg: *CodeGen) InstTracking {
40630 return cg.inst_tracking.get(temp.index).?;
40631 }
40632
40633 fn getOffset(temp: Temp, off: i32, cg: *CodeGen) !Temp {
40634 const new_temp_index = cg.next_temp_index;
40635 cg.temp_type[@intFromEnum(new_temp_index)] = .usize;
40636 cg.next_temp_index = @enumFromInt(@intFromEnum(new_temp_index) + 1);
40637 switch (temp.tracking(cg).short) {
40638 else => |mcv| std.debug.panic("{s}: {}\n", .{ @src().fn_name, mcv }),
40639 .register => |reg| {
40640 const new_reg =
40641 try cg.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);
40642 new_temp_index.tracking(cg).* = .init(.{ .register = new_reg });
40643 try cg.asmRegisterMemory(.{ ._, .lea }, new_reg.to64(), .{
40644 .base = .{ .reg = reg.to64() },
40645 .mod = .{ .rm = .{
40646 .size = .qword,
40647 .disp = off,
40648 } },
40649 });
40650 },
40651 .register_offset => |reg_off| {
40652 const new_reg =
40653 try cg.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);
40654 new_temp_index.tracking(cg).* = .init(.{ .register = new_reg });
40655 try cg.asmRegisterMemory(.{ ._, .lea }, new_reg.to64(), .{
40656 .base = .{ .reg = reg_off.reg.to64() },
40657 .mod = .{ .rm = .{
40658 .size = .qword,
40659 .disp = reg_off.off + off,
40660 } },
40661 });
40662 },
40663 .lea_symbol => |sym_off| new_temp_index.tracking(cg).* = .init(.{ .lea_symbol = .{
40664 .sym_index = sym_off.sym_index,
40665 .off = sym_off.off + off,
40666 } }),
40667 .load_frame => |frame_addr| {
40668 const new_reg =
40669 try cg.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);
40670 new_temp_index.tracking(cg).* = .init(.{ .register_offset = .{
40671 .reg = new_reg,
40672 .off = off,
40673 } });
40674 try cg.asmRegisterMemory(.{ ._, .mov }, new_reg.to64(), .{
40675 .base = .{ .frame = frame_addr.index },
40676 .mod = .{ .rm = .{
40677 .size = .qword,
40678 .disp = frame_addr.off,
40679 } },
40680 });
40681 },
40682 .lea_frame => |frame_addr| new_temp_index.tracking(cg).* = .init(.{ .lea_frame = .{
40683 .index = frame_addr.index,
40684 .off = frame_addr.off + off,
40685 } }),
40686 }
40687 return .{ .index = new_temp_index.toIndex() };
40688 }
40689
40690 fn toOffset(temp: *Temp, off: i32, cg: *CodeGen) !void {
40691 if (off == 0) return;
40692 switch (temp.unwrap(cg)) {
40693 .ref, .err_ret_trace => {},
40694 .temp => |temp_index| {
40695 const temp_tracking = temp_index.tracking(cg);
40696 switch (temp_tracking.short) {
40697 else => {},
40698 .register => |reg| {
40699 try cg.freeValue(temp_tracking.long);
40700 temp_tracking.* = .init(.{ .register_offset = .{
40701 .reg = reg,
40702 .off = off,
40703 } });
40704 return;
40705 },
40706 .register_offset => |reg_off| {
40707 try cg.freeValue(temp_tracking.long);
40708 temp_tracking.* = .init(.{ .register_offset = .{
40709 .reg = reg_off.reg,
40710 .off = reg_off.off + off,
40711 } });
40712 return;
40713 },
40714 .lea_symbol => |sym_off| {
40715 assert(std.meta.eql(temp_tracking.long.lea_symbol, sym_off));
40716 temp_tracking.* = .init(.{ .lea_symbol = .{
40717 .sym_index = sym_off.sym_index,
40718 .off = sym_off.off + off,
40719 } });
40720 return;
40721 },
40722 .lea_frame => |frame_addr| {
40723 assert(std.meta.eql(temp_tracking.long.lea_frame, frame_addr));
40724 temp_tracking.* = .init(.{ .lea_frame = .{
40725 .index = frame_addr.index,
40726 .off = frame_addr.off + off,
40727 } });
40728 return;
40729 },
40730 }
40731 },
40732 }
40733 const new_temp = try temp.getOffset(off, cg);
40734 try temp.die(cg);
40735 temp.* = new_temp;
40736 }
40737
40738 fn getLimb(temp: Temp, limb_ty: Type, limb_index: u28, cg: *CodeGen) !Temp {
40739 const new_temp_index = cg.next_temp_index;
40740 cg.temp_type[@intFromEnum(new_temp_index)] = limb_ty;
40741 switch (temp.tracking(cg).short) {
40742 else => |mcv| std.debug.panic("{s}: {}\n", .{ @src().fn_name, mcv }),
40743 .immediate => |imm| {
40744 assert(limb_index == 0);
40745 new_temp_index.tracking(cg).* = .init(.{ .immediate = imm });
40746 },
40747 .register => |reg| {
40748 assert(limb_index == 0);
40749 const new_reg =
40750 try cg.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);
40751 new_temp_index.tracking(cg).* = .init(.{ .register = new_reg });
40752 try cg.asmRegisterRegister(.{ ._, .mov }, new_reg.to64(), reg.to64());
40753 },
40754 .register_pair => |regs| {
40755 const new_reg =
40756 try cg.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);
40757 new_temp_index.tracking(cg).* = .init(.{ .register = new_reg });
40758 try cg.asmRegisterRegister(.{ ._, .mov }, new_reg.to64(), regs[limb_index].to64());
40759 },
40760 .register_offset => |reg_off| {
40761 assert(limb_index == 0);
40762 const new_reg =
40763 try cg.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);
40764 new_temp_index.tracking(cg).* = .init(.{ .register = new_reg });
40765 try cg.asmRegisterMemory(.{ ._, .lea }, new_reg.to64(), .{
40766 .base = .{ .reg = reg_off.reg.to64() },
40767 .mod = .{ .rm = .{
40768 .size = .qword,
40769 .disp = reg_off.off + @as(u31, limb_index) * 8,
40770 } },
40771 });
40772 },
40773 .load_symbol => |sym_off| {
40774 const new_reg =
40775 try cg.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);
40776 new_temp_index.tracking(cg).* = .init(.{ .register = new_reg });
40777 try cg.asmRegisterMemory(.{ ._, .mov }, new_reg.to64(), .{
40778 .base = .{ .reloc = sym_off.sym_index },
40779 .mod = .{ .rm = .{
40780 .size = .qword,
40781 .disp = sym_off.off + @as(u31, limb_index) * 8,
40782 } },
40783 });
40784 },
40785 .lea_symbol => |sym_off| {
40786 assert(limb_index == 0);
40787 new_temp_index.tracking(cg).* = .init(.{ .lea_symbol = sym_off });
40788 },
40789 .load_frame => |frame_addr| {
40790 const new_reg =
40791 try cg.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);
40792 new_temp_index.tracking(cg).* = .init(.{ .register = new_reg });
40793 try cg.asmRegisterMemory(.{ ._, .mov }, new_reg.to64(), .{
40794 .base = .{ .frame = frame_addr.index },
40795 .mod = .{ .rm = .{
40796 .size = .qword,
40797 .disp = frame_addr.off + @as(u31, limb_index) * 8,
40798 } },
40799 });
40800 },
40801 .lea_frame => |frame_addr| {
40802 assert(limb_index == 0);
40803 new_temp_index.tracking(cg).* = .init(.{ .lea_frame = frame_addr });
40804 },
40805 }
40806 cg.next_temp_index = @enumFromInt(@intFromEnum(new_temp_index) + 1);
40807 return .{ .index = new_temp_index.toIndex() };
40808 }
40809
40810 fn toLimb(temp: *Temp, limb_ty: Type, limb_index: u28, cg: *CodeGen) !void {
40811 switch (temp.unwrap(cg)) {
40812 .ref => {},
40813 .temp => |temp_index| {
40814 const temp_tracking = temp_index.tracking(cg);
40815 switch (temp_tracking.short) {
40816 else => {},
40817 .register, .lea_symbol, .lea_frame => {
40818 assert(limb_index == 0);
40819 cg.temp_type[@intFromEnum(temp_index)] = limb_ty;
40820 return;
40821 },
40822 .register_pair => |regs| {
40823 switch (temp_tracking.long) {
40824 .none, .reserved_frame => {},
40825 else => temp_tracking.long =
40826 temp_tracking.long.address().offset(@as(u31, limb_index) * 8).deref(),
40827 }
40828 for (regs, 0..) |reg, reg_index| if (reg_index != limb_index)
40829 cg.register_manager.freeReg(reg);
40830 temp_tracking.* = .init(.{ .register = regs[limb_index] });
40831 cg.temp_type[@intFromEnum(temp_index)] = limb_ty;
40832 return;
40833 },
40834 .load_symbol => |sym_off| {
40835 assert(std.meta.eql(temp_tracking.long.load_symbol, sym_off));
40836 temp_tracking.* = .init(.{ .load_symbol = .{
40837 .sym_index = sym_off.sym_index,
40838 .off = sym_off.off + @as(u31, limb_index) * 8,
40839 } });
40840 cg.temp_type[@intFromEnum(temp_index)] = limb_ty;
40841 return;
40842 },
40843 .load_frame => |frame_addr| if (!frame_addr.index.isNamed()) {
40844 assert(std.meta.eql(temp_tracking.long.load_frame, frame_addr));
40845 temp_tracking.* = .init(.{ .load_frame = .{
40846 .index = frame_addr.index,
40847 .off = frame_addr.off + @as(u31, limb_index) * 8,
40848 } });
40849 cg.temp_type[@intFromEnum(temp_index)] = limb_ty;
40850 return;
40851 },
40852 }
40853 },
40854 .err_ret_trace => unreachable,
40855 }
40856 const new_temp = try temp.getLimb(limb_ty, limb_index, cg);
40857 try temp.die(cg);
40858 temp.* = new_temp;
40859 }
40860
40861 fn toSlicePtr(temp: *Temp, cg: *CodeGen) !void {
40862 const temp_ty = temp.typeOf(cg);
40863 if (temp_ty.isSlice(cg.pt.zcu)) try temp.toLimb(temp_ty.slicePtrFieldType(cg.pt.zcu), 0, cg);
40864 }
40865
40866 fn toSliceLen(temp: *Temp, cg: *CodeGen) !void {
40867 try temp.toLimb(.usize, 1, cg);
40868 }
40869
40870 fn toReg(temp: *Temp, new_reg: Register, cg: *CodeGen) !bool {
40871 const val, const ty: Type = val_ty: switch (temp.unwrap(cg)) {
40872 .ref => |ref| .{ temp.tracking(cg).short, cg.typeOf(ref) },
40873 .temp => |temp_index| {
40874 const temp_tracking = temp_index.tracking(cg);
40875 if (temp_tracking.short == .register and
40876 temp_tracking.short.register == new_reg) return false;
40877 break :val_ty .{ temp_tracking.short, temp_index.typeOf(cg) };
40878 },
40879 .err_ret_trace => .{ temp.tracking(cg).short, .usize },
40880 };
40881 const new_temp_index = cg.next_temp_index;
40882 try cg.register_manager.getReg(new_reg, new_temp_index.toIndex());
40883 cg.temp_type[@intFromEnum(new_temp_index)] = ty;
40884 try cg.genSetReg(new_reg, ty, val, .{});
40885 new_temp_index.tracking(cg).* = .init(.{ .register = new_reg });
40886 try temp.die(cg);
40887 cg.next_temp_index = @enumFromInt(@intFromEnum(new_temp_index) + 1);
40888 temp.* = .{ .index = new_temp_index.toIndex() };
40889 return true;
40890 }
40891
40892 fn toRegClass(temp: *Temp, mut: bool, rc: Register.Class, cg: *CodeGen) !bool {
40893 const val = temp.tracking(cg).short;
40894 if (!mut or temp.isMut(cg)) switch (val) {
40895 else => {},
40896 .register => |reg| if (reg.class() == rc) return false,
40897 .register_offset => |reg_off| if (reg_off.reg.class() == rc and reg_off.off == 0) return false,
40898 };
40899 const ty = temp.typeOf(cg);
40900 const new_temp_index = cg.next_temp_index;
40901 cg.temp_type[@intFromEnum(new_temp_index)] = ty;
40902 const new_reg = try cg.register_manager.allocReg(new_temp_index.toIndex(), regSetForRegClass(rc));
40903 try cg.genSetReg(new_reg, ty, val, .{});
40904 new_temp_index.tracking(cg).* = .init(.{ .register = new_reg });
40905 try temp.die(cg);
40906 cg.next_temp_index = @enumFromInt(@intFromEnum(new_temp_index) + 1);
40907 temp.* = .{ .index = new_temp_index.toIndex() };
40908 return true;
40909 }
40910
40911 fn toPair(first_temp: *Temp, second_temp: *Temp, cg: *CodeGen) !void {
40912 while (true) for ([_]*Temp{ first_temp, second_temp }) |part_temp| {
40913 if (try part_temp.toRegClass(true, .general_purpose, cg)) break;
40914 } else break;
40915 const first_temp_tracking = first_temp.unwrap(cg).temp.tracking(cg);
40916 const second_temp_tracking = second_temp.unwrap(cg).temp.tracking(cg);
40917 const result: MCValue = .{ .register_pair = .{
40918 first_temp_tracking.short.register,
40919 second_temp_tracking.short.register,
40920 } };
40921 const result_temp_index = cg.next_temp_index;
40922 const result_temp: Temp = .{ .index = result_temp_index.toIndex() };
40923 assert(cg.reuseTemp(result_temp.index, first_temp.index, first_temp_tracking));
40924 assert(cg.reuseTemp(result_temp.index, second_temp.index, second_temp_tracking));
40925 cg.temp_type[@intFromEnum(result_temp_index)] = .slice_const_u8;
40926 result_temp_index.tracking(cg).* = .init(result);
40927 first_temp.* = result_temp;
40928 second_temp.* = result_temp;
40929 }
40930
40931 fn asMask(temp: Temp, info: MaskInfo, cg: *CodeGen) void {
40932 assert(info.scalar != .none);
40933 const mcv = &temp.unwrap(cg).temp.tracking(cg).short;
40934 const reg = mcv.register;
40935 mcv.* = .{ .register_mask = .{ .reg = reg, .info = info } };
40936 }
4078740937
40788fn floatLibcAbiSuffix(ty: Type) []const u8 {
40789 return switch (ty.toIntern()) {
40790 .f16_type => "h",
40791 .f32_type => "f",
40792 .f64_type => "",
40793 .f80_type => "x",
40794 .f128_type => "q",
40795 .c_longdouble_type => "l",
40796 else => unreachable,
40797 };
40798}
40938 fn toLea(temp: *Temp, cg: *CodeGen) !bool {
40939 switch (temp.tracking(cg).short) {
40940 .none,
40941 .unreach,
40942 .dead,
40943 .undef,
40944 .eflags,
40945 .register_pair,
40946 .register_triple,
40947 .register_quadruple,
40948 .register_overflow,
40949 .register_mask,
40950 .elementwise_regs_then_frame,
40951 .reserved_frame,
40952 .air_ref,
40953 => unreachable, // not a valid pointer
40954 .immediate,
40955 .register,
40956 .register_offset,
40957 .lea_direct,
40958 .lea_got,
40959 .lea_tlv,
40960 .lea_frame,
40961 => return false,
40962 .memory,
40963 .indirect,
40964 .load_symbol,
40965 .load_direct,
40966 .load_got,
40967 .load_tlv,
40968 .load_frame,
40969 => return temp.toRegClass(true, .general_purpose, cg),
40970 .lea_symbol => |sym_off| {
40971 const off = sym_off.off;
40972 if (off == 0) return false;
40973 try temp.toOffset(-off, cg);
40974 while (try temp.toRegClass(true, .general_purpose, cg)) {}
40975 try temp.toOffset(off, cg);
40976 return true;
40977 },
40978 }
40979 }
4079940980
40800fn promoteInt(self: *CodeGen, ty: Type) Type {
40801 const pt = self.pt;
40802 const zcu = pt.zcu;
40803 const int_info: InternPool.Key.IntType = switch (ty.toIntern()) {
40804 .bool_type => .{ .signedness = .unsigned, .bits = 1 },
40805 else => if (ty.isAbiInt(zcu)) ty.intInfo(zcu) else return ty,
40806 };
40807 for ([_]Type{
40808 .c_int, .c_uint,
40809 .c_long, .c_ulong,
40810 .c_longlong, .c_ulonglong,
40811 }) |promote_ty| {
40812 const promote_info = promote_ty.intInfo(zcu);
40813 if (int_info.signedness == .signed and promote_info.signedness == .unsigned) continue;
40814 if (int_info.bits + @intFromBool(int_info.signedness == .unsigned and
40815 promote_info.signedness == .signed) <= promote_info.bits) return promote_ty;
40981 fn toMemory(temp: *Temp, cg: *CodeGen) !bool {
40982 const temp_tracking = temp.tracking(cg);
40983 if (temp_tracking.short.isMemory()) return false;
40984 const new_temp_index = cg.next_temp_index;
40985 const ty = temp.typeOf(cg);
40986 cg.temp_type[@intFromEnum(new_temp_index)] = ty;
40987 const new_frame_index = try cg.allocFrameIndex(.initSpill(ty, cg.pt.zcu));
40988 try cg.genSetMem(.{ .frame = new_frame_index }, 0, ty, temp_tracking.short, .{});
40989 new_temp_index.tracking(cg).* = .init(.{ .load_frame = .{ .index = new_frame_index } });
40990 try temp.die(cg);
40991 cg.next_temp_index = @enumFromInt(@intFromEnum(new_temp_index) + 1);
40992 temp.* = .{ .index = new_temp_index.toIndex() };
40993 return true;
4081640994 }
40817 return ty;
40818}
4081940995
40820fn promoteVarArg(self: *CodeGen, ty: Type) Type {
40821 if (!ty.isRuntimeFloat()) return self.promoteInt(ty);
40822 switch (ty.floatBits(self.target.*)) {
40823 32, 64 => return .f64,
40824 else => |float_bits| {
40825 assert(float_bits == self.target.cTypeBitSize(.longdouble));
40826 return .c_longdouble;
40827 },
40996 // hack around linker relocation bugs
40997 fn toBase(temp: *Temp, cg: *CodeGen) !bool {
40998 const temp_tracking = temp.tracking(cg);
40999 if (temp_tracking.short.isBase()) return false;
41000 if (try temp.toMemory(cg)) return true;
41001 const new_temp_index = cg.next_temp_index;
41002 cg.temp_type[@intFromEnum(new_temp_index)] = temp.typeOf(cg);
41003 const new_reg =
41004 try cg.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);
41005 try cg.genSetReg(new_reg, .usize, temp_tracking.short.address(), .{});
41006 new_temp_index.tracking(cg).* = .init(.{ .indirect = .{ .reg = new_reg } });
41007 try temp.die(cg);
41008 cg.next_temp_index = @enumFromInt(@intFromEnum(new_temp_index) + 1);
41009 temp.* = .{ .index = new_temp_index.toIndex() };
41010 return true;
4082841011 }
40829}
4083041012
40831fn unalignedSize(cg: *CodeGen, ty: Type) u64 {
40832 const zcu = cg.pt.zcu;
40833 return switch (zcu.intern_pool.indexToKey(ty.toIntern())) {
40834 .vector_type => |vector_type| Type.fromInterned(vector_type.child).abiSize(zcu) * vector_type.len,
40835 else => ty.abiSize(zcu),
41013 const AccessOptions = struct {
41014 disp: i32 = 0,
41015 safe: bool = false,
4083641016 };
40837}
4083841017
40839fn intInfo(cg: *CodeGen, ty: Type) ?std.builtin.Type.Int {
40840 const zcu = cg.pt.zcu;
40841 const ip = &zcu.intern_pool;
40842 var ty_index = ty.ip_index;
40843 while (true) switch (ip.indexToKey(ty_index)) {
40844 .int_type => |int_type| return int_type,
40845 .ptr_type => |ptr_type| return switch (ptr_type.flags.size) {
40846 .one, .many, .c => .{ .signedness = .unsigned, .bits = cg.target.ptrBitWidth() },
40847 .slice => null,
40848 },
40849 .opt_type => |opt_child| return if (!Type.fromInterned(opt_child).hasRuntimeBitsIgnoreComptime(zcu))
40850 .{ .signedness = .unsigned, .bits = 1 }
40851 else switch (ip.indexToKey(opt_child)) {
40852 .ptr_type => |ptr_type| switch (ptr_type.flags.size) {
40853 .one, .many => switch (ptr_type.flags.is_allowzero) {
40854 false => .{ .signedness = .unsigned, .bits = cg.target.ptrBitWidth() },
40855 true => null,
40856 },
40857 .slice, .c => null,
41018 fn load(ptr: *Temp, val_ty: Type, opts: AccessOptions, cg: *CodeGen) !Temp {
41019 const val = try cg.tempAlloc(val_ty);
41020 try ptr.toOffset(opts.disp, cg);
41021 while (try ptr.toLea(cg)) {}
41022 const val_mcv = val.tracking(cg).short;
41023 switch (val_mcv) {
41024 else => |mcv| std.debug.panic("{s}: {}\n", .{ @src().fn_name, mcv }),
41025 .register => |val_reg| try ptr.loadReg(val_ty, registerAlias(
41026 val_reg,
41027 @intCast(val_ty.abiSize(cg.pt.zcu)),
41028 ), cg),
41029 inline .register_pair,
41030 .register_triple,
41031 .register_quadruple,
41032 => |val_regs| for (val_regs) |val_reg| {
41033 try ptr.loadReg(val_ty, val_reg, cg);
41034 try ptr.toOffset(@divExact(val_reg.bitSize(), 8), cg);
41035 while (try ptr.toLea(cg)) {}
4085841036 },
40859 else => null,
40860 },
40861 .error_union_type => |error_union_type| return if (!Type.fromInterned(error_union_type.payload_type)
40862 .hasRuntimeBitsIgnoreComptime(zcu)) .{ .signedness = .unsigned, .bits = zcu.errorSetBits() } else null,
40863 .simple_type => |simple_type| return switch (simple_type) {
40864 .bool => .{ .signedness = .unsigned, .bits = 1 },
40865 .anyerror => .{ .signedness = .unsigned, .bits = zcu.errorSetBits() },
40866 .isize => .{ .signedness = .signed, .bits = cg.target.ptrBitWidth() },
40867 .usize => .{ .signedness = .unsigned, .bits = cg.target.ptrBitWidth() },
40868 .c_char => .{ .signedness = cg.target.charSignedness(), .bits = cg.target.cTypeBitSize(.char) },
40869 .c_short => .{ .signedness = .signed, .bits = cg.target.cTypeBitSize(.short) },
40870 .c_ushort => .{ .signedness = .unsigned, .bits = cg.target.cTypeBitSize(.short) },
40871 .c_int => .{ .signedness = .signed, .bits = cg.target.cTypeBitSize(.int) },
40872 .c_uint => .{ .signedness = .unsigned, .bits = cg.target.cTypeBitSize(.int) },
40873 .c_long => .{ .signedness = .signed, .bits = cg.target.cTypeBitSize(.long) },
40874 .c_ulong => .{ .signedness = .unsigned, .bits = cg.target.cTypeBitSize(.long) },
40875 .c_longlong => .{ .signedness = .signed, .bits = cg.target.cTypeBitSize(.longlong) },
40876 .c_ulonglong => .{ .signedness = .unsigned, .bits = cg.target.cTypeBitSize(.longlong) },
40877 .f16, .f32, .f64, .f80, .f128, .c_longdouble => null,
40878 .anyopaque,
40879 .void,
40880 .type,
40881 .comptime_int,
40882 .comptime_float,
40883 .noreturn,
40884 .null,
40885 .undefined,
40886 .enum_literal,
40887 .adhoc_inferred_error_set,
40888 .generic_poison,
40889 => unreachable,
40890 },
40891 .struct_type => {
40892 const loaded_struct = ip.loadStructType(ty_index);
40893 switch (loaded_struct.layout) {
40894 .auto, .@"extern" => return null,
40895 .@"packed" => ty_index = loaded_struct.backingIntTypeUnordered(ip),
41037 .register_offset => |val_reg_off| switch (val_reg_off.off) {
41038 0 => try ptr.loadReg(val_ty, registerAlias(
41039 val_reg_off.reg,
41040 @intCast(val_ty.abiSize(cg.pt.zcu)),
41041 ), cg),
41042 else => unreachable,
41043 },
41044 .memory, .indirect, .load_frame, .load_symbol => {
41045 var val_ptr = try cg.tempInit(.usize, val_mcv.address());
41046 var len = try cg.tempInit(.usize, .{ .immediate = val_ty.abiSize(cg.pt.zcu) });
41047 try val_ptr.memcpy(ptr, &len, cg);
41048 try val_ptr.die(cg);
41049 try len.die(cg);
41050 },
41051 }
41052 return val;
41053 }
41054
41055 fn store(ptr: *Temp, val: *Temp, opts: AccessOptions, cg: *CodeGen) !void {
41056 const val_ty = val.typeOf(cg);
41057 try ptr.toOffset(opts.disp, cg);
41058 while (try ptr.toLea(cg)) {}
41059 val_to_gpr: while (true) : (while (try ptr.toLea(cg) or
41060 try val.toRegClass(false, .general_purpose, cg))
41061 {}) {
41062 const val_mcv = val.tracking(cg).short;
41063 switch (val_mcv) {
41064 else => |mcv| std.debug.panic("{s}: {}\n", .{ @src().fn_name, mcv }),
41065 .undef => if (opts.safe) {
41066 var pat = try cg.tempInit(.u8, .{ .immediate = 0xaa });
41067 var len = try cg.tempInit(.usize, .{ .immediate = val_ty.abiSize(cg.pt.zcu) });
41068 try ptr.memset(&pat, &len, cg);
41069 try pat.die(cg);
41070 try len.die(cg);
41071 },
41072 .immediate => |val_imm| {
41073 const val_op: Immediate = if (std.math.cast(u31, val_imm)) |val_uimm31|
41074 .u(val_uimm31)
41075 else if (std.math.cast(i32, @as(i64, @bitCast(val_imm)))) |val_simm32|
41076 .s(val_simm32)
41077 else
41078 continue :val_to_gpr;
41079 // hack around linker relocation bugs
41080 switch (ptr.tracking(cg).short) {
41081 else => {},
41082 .lea_symbol => while (try ptr.toRegClass(false, .general_purpose, cg)) {},
41083 }
41084 try cg.asmMemoryImmediate(
41085 .{ ._, .mov },
41086 try ptr.tracking(cg).short.deref().mem(cg, .{
41087 .size = cg.memSize(val_ty),
41088 }),
41089 val_op,
41090 );
41091 },
41092 .eflags => |cc| {
41093 // hack around linker relocation bugs
41094 switch (ptr.tracking(cg).short) {
41095 else => {},
41096 .lea_symbol => while (try ptr.toRegClass(false, .general_purpose, cg)) {},
41097 }
41098 try cg.asmSetccMemory(
41099 cc,
41100 try ptr.tracking(cg).short.deref().mem(cg, .{ .size = .byte }),
41101 );
41102 },
41103 .register => |val_reg| try ptr.storeRegs(val_ty, &.{registerAlias(
41104 val_reg,
41105 @intCast(val_ty.abiSize(cg.pt.zcu)),
41106 )}, cg),
41107 inline .register_pair,
41108 .register_triple,
41109 .register_quadruple,
41110 => |val_regs| try ptr.storeRegs(val_ty, &val_regs, cg),
41111 .register_offset => |val_reg_off| switch (val_reg_off.off) {
41112 0 => try ptr.storeRegs(val_ty, &.{registerAlias(
41113 val_reg_off.reg,
41114 @intCast(val_ty.abiSize(cg.pt.zcu)),
41115 )}, cg),
41116 else => continue :val_to_gpr,
41117 },
41118 .register_overflow => |val_reg_ov| {
41119 const ip = &cg.pt.zcu.intern_pool;
41120 const first_ty: Type = .fromInterned(first_ty: switch (ip.indexToKey(val_ty.toIntern())) {
41121 .tuple_type => |tuple_type| {
41122 const tuple_field_types = tuple_type.types.get(ip);
41123 assert(tuple_field_types.len == 2 and tuple_field_types[1] == .u1_type);
41124 break :first_ty tuple_field_types[0];
41125 },
41126 .opt_type => |opt_child| {
41127 assert(!val_ty.optionalReprIsPayload(cg.pt.zcu));
41128 break :first_ty opt_child;
41129 },
41130 else => std.debug.panic("{s}: {}\n", .{ @src().fn_name, val_ty.fmt(cg.pt) }),
41131 });
41132 const first_size: u31 = @intCast(first_ty.abiSize(cg.pt.zcu));
41133 try ptr.storeRegs(first_ty, &.{registerAlias(val_reg_ov.reg, first_size)}, cg);
41134 try ptr.toOffset(first_size, cg);
41135 try cg.asmSetccMemory(
41136 val_reg_ov.eflags,
41137 try ptr.tracking(cg).short.deref().mem(cg, .{ .size = .byte }),
41138 );
41139 },
41140 .lea_frame, .lea_symbol => continue :val_to_gpr,
41141 .memory, .indirect, .load_frame, .load_symbol => {
41142 var val_ptr = try cg.tempInit(.usize, val_mcv.address());
41143 var len = try cg.tempInit(.usize, .{ .immediate = val_ty.abiSize(cg.pt.zcu) });
41144 try ptr.memcpy(&val_ptr, &len, cg);
41145 try val_ptr.die(cg);
41146 try len.die(cg);
41147 },
4089641148 }
40897 },
40898 .union_type => return switch (ip.loadUnionType(ty_index).flagsUnordered(ip).layout) {
40899 .auto, .@"extern" => null,
40900 .@"packed" => .{ .signedness = .unsigned, .bits = @intCast(ty.bitSize(zcu)) },
40901 },
40902 .enum_type => ty_index = ip.loadEnumType(ty_index).tag_ty,
40903 .error_set_type, .inferred_error_set_type => return .{ .signedness = .unsigned, .bits = zcu.errorSetBits() },
40904 else => return null,
40905 };
40906}
41149 break;
41150 }
41151 }
4090741152
40908fn floatBits(cg: *CodeGen, ty: Type) ?u16 {
40909 return if (ty.isRuntimeFloat()) ty.floatBits(cg.target.*) else null;
40910}
41153 fn read(src: *Temp, val_ty: Type, opts: AccessOptions, cg: *CodeGen) !Temp {
41154 var val = try cg.tempAlloc(val_ty);
41155 while (try src.toBase(cg)) {}
41156 const val_mcv = val.tracking(cg).short;
41157 switch (val_mcv) {
41158 else => |mcv| std.debug.panic("{s}: {}\n", .{ @src().fn_name, mcv }),
41159 .register => |val_reg| try src.readReg(opts.disp, val_ty, registerAlias(
41160 val_reg,
41161 @intCast(val_ty.abiSize(cg.pt.zcu)),
41162 ), cg),
41163 inline .register_pair, .register_triple, .register_quadruple => |val_regs| {
41164 var disp = opts.disp;
41165 for (val_regs) |val_reg| {
41166 try src.readReg(disp, val_ty, val_reg, cg);
41167 disp += @divExact(val_reg.bitSize(), 8);
41168 }
41169 },
41170 .register_offset => |val_reg_off| switch (val_reg_off.off) {
41171 0 => try src.readReg(opts.disp, val_ty, registerAlias(
41172 val_reg_off.reg,
41173 @intCast(val_ty.abiSize(cg.pt.zcu)),
41174 ), cg),
41175 else => unreachable,
41176 },
41177 .memory, .indirect, .load_frame, .load_symbol => {
41178 var val_ptr = try cg.tempInit(.usize, val_mcv.address());
41179 var src_ptr =
41180 try cg.tempInit(.usize, src.tracking(cg).short.address().offset(opts.disp));
41181 var len = try cg.tempInit(.usize, .{ .immediate = val_ty.abiSize(cg.pt.zcu) });
41182 try val_ptr.memcpy(&src_ptr, &len, cg);
41183 try val_ptr.die(cg);
41184 try src_ptr.die(cg);
41185 try len.die(cg);
41186 },
41187 }
41188 return val;
41189 }
4091141190
40912const Temp = struct {
40913 index: Air.Inst.Index,
41191 fn write(dst: *Temp, val: *Temp, opts: AccessOptions, cg: *CodeGen) !void {
41192 const val_ty = val.typeOf(cg);
41193 while (try dst.toBase(cg)) {}
41194 val_to_gpr: while (true) : (while (try dst.toBase(cg) or
41195 try val.toRegClass(false, .general_purpose, cg))
41196 {}) {
41197 const val_mcv = val.tracking(cg).short;
41198 switch (val_mcv) {
41199 else => |mcv| std.debug.panic("{s}: {}\n", .{ @src().fn_name, mcv }),
41200 .undef => if (opts.safe) {
41201 var dst_ptr = try cg.tempInit(.usize, dst.tracking(cg).short.address().offset(opts.disp));
41202 var pat = try cg.tempInit(.u8, .{ .immediate = 0xaa });
41203 var len = try cg.tempInit(.usize, .{ .immediate = val_ty.abiSize(cg.pt.zcu) });
41204 try dst_ptr.memset(&pat, &len, cg);
41205 try dst_ptr.die(cg);
41206 try pat.die(cg);
41207 try len.die(cg);
41208 },
41209 .immediate => |val_imm| {
41210 const val_op: Immediate = if (std.math.cast(u31, val_imm)) |val_uimm31|
41211 .u(val_uimm31)
41212 else if (std.math.cast(i32, @as(i64, @bitCast(val_imm)))) |val_simm32|
41213 .s(val_simm32)
41214 else
41215 continue :val_to_gpr;
41216 try cg.asmMemoryImmediate(
41217 .{ ._, .mov },
41218 try dst.tracking(cg).short.mem(cg, .{
41219 .size = cg.memSize(val_ty),
41220 .disp = opts.disp,
41221 }),
41222 val_op,
41223 );
41224 },
41225 .eflags => |cc| try cg.asmSetccMemory(
41226 cc,
41227 try dst.tracking(cg).short.mem(cg, .{
41228 .size = .byte,
41229 .disp = opts.disp,
41230 }),
41231 ),
41232 .register => |val_reg| try dst.writeRegs(opts.disp, val_ty, &.{registerAlias(
41233 val_reg,
41234 @intCast(val_ty.abiSize(cg.pt.zcu)),
41235 )}, cg),
41236 inline .register_pair,
41237 .register_triple,
41238 .register_quadruple,
41239 => |val_regs| try dst.writeRegs(opts.disp, val_ty, &val_regs, cg),
41240 .register_offset => |val_reg_off| switch (val_reg_off.off) {
41241 0 => try dst.writeRegs(opts.disp, val_ty, &.{registerAlias(
41242 val_reg_off.reg,
41243 @intCast(val_ty.abiSize(cg.pt.zcu)),
41244 )}, cg),
41245 else => continue :val_to_gpr,
41246 },
41247 .register_overflow => |val_reg_ov| {
41248 const ip = &cg.pt.zcu.intern_pool;
41249 const first_ty: Type = .fromInterned(first_ty: switch (ip.indexToKey(val_ty.toIntern())) {
41250 .tuple_type => |tuple_type| {
41251 const tuple_field_types = tuple_type.types.get(ip);
41252 assert(tuple_field_types.len == 2 and tuple_field_types[1] == .u1_type);
41253 break :first_ty tuple_field_types[0];
41254 },
41255 .opt_type => |opt_child| {
41256 assert(!val_ty.optionalReprIsPayload(cg.pt.zcu));
41257 break :first_ty opt_child;
41258 },
41259 else => std.debug.panic("{s}: {}\n", .{ @src().fn_name, val_ty.fmt(cg.pt) }),
41260 });
41261 const first_size: u31 = @intCast(first_ty.abiSize(cg.pt.zcu));
41262 try dst.writeRegs(opts.disp, first_ty, &.{registerAlias(val_reg_ov.reg, first_size)}, cg);
41263 try cg.asmSetccMemory(
41264 val_reg_ov.eflags,
41265 try dst.tracking(cg).short.mem(cg, .{
41266 .size = .byte,
41267 .disp = opts.disp + first_size,
41268 }),
41269 );
41270 },
41271 .lea_frame, .lea_symbol => continue :val_to_gpr,
41272 .memory, .indirect, .load_frame, .load_symbol => {
41273 var dst_ptr =
41274 try cg.tempInit(.usize, dst.tracking(cg).short.address().offset(opts.disp));
41275 var val_ptr = try cg.tempInit(.usize, val_mcv.address());
41276 var len = try cg.tempInit(.usize, .{ .immediate = val_ty.abiSize(cg.pt.zcu) });
41277 try dst_ptr.memcpy(&val_ptr, &len, cg);
41278 try dst_ptr.die(cg);
41279 try val_ptr.die(cg);
41280 try len.die(cg);
41281 },
41282 }
41283 break;
41284 }
41285 }
4091441286
40915 fn unwrap(temp: Temp, cg: *CodeGen) union(enum) {
40916 ref: Air.Inst.Ref,
40917 temp: Index,
40918 err_ret_trace,
40919 } {
40920 switch (temp.index.unwrap()) {
40921 .ref => |ref| return .{ .ref = ref },
40922 .target => |target_index| {
40923 if (temp.index == err_ret_trace_index) return .err_ret_trace;
40924 const temp_index: Index = @enumFromInt(target_index);
40925 assert(temp_index.isValid(cg));
40926 return .{ .temp = temp_index };
40927 },
41287 fn loadReg(ptr: *Temp, dst_ty: Type, dst_reg: Register, cg: *CodeGen) !void {
41288 const dst_rc = dst_reg.class();
41289 const strat = try cg.moveStrategy(dst_ty, dst_rc, false);
41290 // hack around linker relocation bugs
41291 switch (ptr.tracking(cg).short) {
41292 else => {},
41293 .lea_symbol => |sym_off| if (dst_rc != .general_purpose or sym_off.off != 0)
41294 while (try ptr.toRegClass(false, .general_purpose, cg)) {},
41295 }
41296 try strat.read(cg, dst_reg, try ptr.tracking(cg).short.deref().mem(cg, .{
41297 .size = .fromBitSize(@min(8 * dst_ty.abiSize(cg.pt.zcu), dst_reg.bitSize())),
41298 }));
41299 }
41300
41301 fn storeRegs(ptr: *Temp, src_ty: Type, src_regs: []const Register, cg: *CodeGen) !void {
41302 var part_disp: u31 = 0;
41303 var deferred_disp: u31 = 0;
41304 var src_abi_size: u32 = @intCast(src_ty.abiSize(cg.pt.zcu));
41305 for (src_regs) |src_reg| {
41306 const src_rc = src_reg.class();
41307 const part_bit_size = @min(8 * src_abi_size, src_reg.bitSize());
41308 const part_size = @divExact(part_bit_size, 8);
41309 if (src_rc == .x87 or std.math.isPowerOfTwo(part_size)) {
41310 // hack around linker relocation bugs
41311 switch (ptr.tracking(cg).short) {
41312 else => {},
41313 .lea_symbol => while (try ptr.toRegClass(false, .general_purpose, cg)) {},
41314 }
41315 const strat = try cg.moveStrategy(src_ty, src_rc, false);
41316 try strat.write(cg, try ptr.tracking(cg).short.deref().mem(cg, .{
41317 .size = .fromBitSize(part_bit_size),
41318 .disp = part_disp,
41319 }), registerAlias(src_reg, part_size));
41320 } else {
41321 const frame_size = std.math.ceilPowerOfTwoAssert(u32, part_size);
41322 const frame_index = try cg.allocFrameIndex(.init(.{
41323 .size = frame_size,
41324 .alignment = .fromNonzeroByteUnits(frame_size),
41325 }));
41326 const strat = try cg.moveStrategy(src_ty, src_rc, true);
41327 try strat.write(cg, .{
41328 .base = .{ .frame = frame_index },
41329 .mod = .{ .rm = .{ .size = .fromSize(frame_size) } },
41330 }, registerAlias(src_reg, frame_size));
41331 try ptr.toOffset(deferred_disp, cg);
41332 deferred_disp = 0;
41333 var src_ptr = try cg.tempInit(.usize, .{ .lea_frame = .{ .index = frame_index } });
41334 var len = try cg.tempInit(.usize, .{ .immediate = src_abi_size });
41335 try ptr.memcpy(&src_ptr, &len, cg);
41336 try src_ptr.die(cg);
41337 try len.die(cg);
41338 }
41339 part_disp += part_size;
41340 deferred_disp += part_size;
41341 src_abi_size -= part_size;
4092841342 }
4092941343 }
4093041344
40931 fn typeOf(temp: Temp, cg: *CodeGen) Type {
40932 return switch (temp.unwrap(cg)) {
40933 .ref => switch (cg.air.instructions.items(.tag)[@intFromEnum(temp.index)]) {
40934 .loop_switch_br => cg.typeOf(cg.air.unwrapSwitch(temp.index).operand),
40935 else => cg.air.typeOfIndex(temp.index, &cg.pt.zcu.intern_pool),
40936 },
40937 .temp => |temp_index| temp_index.typeOf(cg),
40938 .err_ret_trace => .usize,
40939 };
41345 fn readReg(src: Temp, disp: i32, dst_ty: Type, dst_reg: Register, cg: *CodeGen) !void {
41346 const strat = try cg.moveStrategy(dst_ty, dst_reg.class(), false);
41347 try strat.read(cg, dst_reg, try src.tracking(cg).short.mem(cg, .{
41348 .size = .fromBitSize(@min(8 * dst_ty.abiSize(cg.pt.zcu), dst_reg.bitSize())),
41349 .disp = disp,
41350 }));
4094041351 }
4094141352
40942 fn isMut(temp: Temp, cg: *CodeGen) bool {
40943 return switch (temp.unwrap(cg)) {
40944 .ref, .err_ret_trace => false,
40945 .temp => |temp_index| switch (temp_index.tracking(cg).short) {
40946 .none,
40947 .unreach,
40948 .dead,
40949 .undef,
40950 .immediate,
40951 .eflags,
40952 .register_offset,
40953 .register_mask,
40954 .memory,
40955 .load_symbol,
40956 .lea_symbol,
40957 .indirect,
40958 .load_direct,
40959 .lea_direct,
40960 .load_got,
40961 .lea_got,
40962 .load_tlv,
40963 .lea_tlv,
40964 .lea_frame,
40965 .elementwise_regs_then_frame,
40966 .reserved_frame,
40967 .air_ref,
40968 => false,
40969 .register,
40970 .register_pair,
40971 .register_triple,
40972 .register_quadruple,
40973 .register_overflow,
40974 => true,
40975 .load_frame => |frame_addr| !frame_addr.index.isNamed(),
40976 },
40977 };
41353 fn writeRegs(dst: Temp, disp: i32, src_ty: Type, src_regs: []const Register, cg: *CodeGen) !void {
41354 var part_disp = disp;
41355 var src_abi_size: u32 = @intCast(src_ty.abiSize(cg.pt.zcu));
41356 for (src_regs) |src_reg| {
41357 const src_rc = src_reg.class();
41358 const part_bit_size = @min(8 * src_abi_size, src_reg.bitSize());
41359 const part_size = @divExact(part_bit_size, 8);
41360 if (src_rc == .x87 or std.math.isPowerOfTwo(part_size)) {
41361 const strat = try cg.moveStrategy(src_ty, src_rc, false);
41362 try strat.write(cg, try dst.tracking(cg).short.mem(cg, .{
41363 .size = .fromBitSize(part_bit_size),
41364 .disp = part_disp,
41365 }), registerAlias(src_reg, part_size));
41366 } else {
41367 const frame_size = std.math.ceilPowerOfTwoAssert(u32, part_size);
41368 const frame_index = try cg.allocFrameIndex(.init(.{
41369 .size = frame_size,
41370 .alignment = .fromNonzeroByteUnits(frame_size),
41371 }));
41372 const strat = try cg.moveStrategy(src_ty, src_rc, true);
41373 try strat.write(cg, .{
41374 .base = .{ .frame = frame_index },
41375 .mod = .{ .rm = .{ .size = .fromSize(frame_size) } },
41376 }, registerAlias(src_reg, frame_size));
41377 var dst_ptr = try cg.tempInit(.usize, dst.tracking(cg).short.address());
41378 try dst_ptr.toOffset(part_disp, cg);
41379 var src_ptr = try cg.tempInit(.usize, .{ .lea_frame = .{ .index = frame_index } });
41380 var len = try cg.tempInit(.usize, .{ .immediate = src_abi_size });
41381 try dst_ptr.memcpy(&src_ptr, &len, cg);
41382 try dst_ptr.die(cg);
41383 try src_ptr.die(cg);
41384 try len.die(cg);
41385 }
41386 part_disp += part_size;
41387 src_abi_size -= part_size;
41388 }
4097841389 }
4097941390
40980 fn tracking(temp: Temp, cg: *CodeGen) InstTracking {
40981 return cg.inst_tracking.get(temp.index).?;
41391 fn memcpy(dst: *Temp, src: *Temp, len: *Temp, cg: *CodeGen) !void {
41392 while (true) for ([_]*Temp{ dst, src, len }, [_]Register{ .rdi, .rsi, .rcx }) |temp, reg| {
41393 if (try temp.toReg(reg, cg)) break;
41394 } else break;
41395 try cg.asmOpOnly(.{ .@"rep _sb", .mov });
4098241396 }
4098341397
40984 fn getOffset(temp: Temp, off: i32, cg: *CodeGen) !Temp {
40985 const new_temp_index = cg.next_temp_index;
40986 cg.temp_type[@intFromEnum(new_temp_index)] = .usize;
40987 cg.next_temp_index = @enumFromInt(@intFromEnum(new_temp_index) + 1);
40988 switch (temp.tracking(cg).short) {
40989 else => |mcv| std.debug.panic("{s}: {}\n", .{ @src().fn_name, mcv }),
40990 .register => |reg| {
40991 const new_reg =
40992 try cg.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);
40993 new_temp_index.tracking(cg).* = .init(.{ .register = new_reg });
40994 try cg.asmRegisterMemory(.{ ._, .lea }, new_reg.to64(), .{
40995 .base = .{ .reg = reg.to64() },
40996 .mod = .{ .rm = .{
40997 .size = .qword,
40998 .disp = off,
41398 fn memset(dst: *Temp, val: *Temp, len: *Temp, cg: *CodeGen) !void {
41399 while (true) for ([_]*Temp{ dst, val, len }, [_]Register{ .rdi, .rax, .rcx }) |temp, reg| {
41400 if (try temp.toReg(reg, cg)) break;
41401 } else break;
41402 try cg.asmOpOnly(.{ .@"rep _sb", .sto });
41403 }
41404
41405 /// Supports any `op` using `cg.intInfo(lhs.typeOf(cg)).?.signedness` as the signedness.
41406 /// Returns `error.SelectFailed` when `cg.intInfo(lhs.typeOf(cg)) == null`.
41407 fn cmpInts(lhs: *Temp, op: std.math.CompareOperator, rhs: *Temp, cg: *CodeGen) !Temp {
41408 var ops: [2]Temp = .{ lhs.*, rhs.* };
41409 var res: [1]Temp = undefined;
41410 switch (op) {
41411 .lt, .lte, .gte, .gt => {
41412 const commute = switch (op) {
41413 .lt, .gte => false,
41414 .lte, .gt => true,
41415 else => unreachable,
41416 };
41417 if (commute) std.mem.swap(Temp, &ops[0], &ops[1]);
41418 try cg.select(&res, &.{.bool}, &ops, comptime &.{ .{
41419 .src_constraints = .{ .{ .signed_int = .byte }, .{ .signed_int = .byte } },
41420 .patterns = &.{
41421 .{ .src = .{ .imm8, .mem }, .commute = .{ 0, 1 } },
41422 .{ .src = .{ .imm8, .to_gpr }, .commute = .{ 0, 1 } },
41423 .{ .src = .{ .mem, .to_gpr }, .commute = .{ 0, 1 } },
41424 },
41425 .dst_temps = .{.{ .cc = .g }},
41426 .clobbers = .{ .eflags = true },
41427 .each = .{ .once = &.{
41428 .{ ._, ._, .cmp, .src0b, .src1b, ._, ._ },
4099941429 } },
41000 });
41001 },
41002 .register_offset => |reg_off| {
41003 const new_reg =
41004 try cg.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);
41005 new_temp_index.tracking(cg).* = .init(.{ .register = new_reg });
41006 try cg.asmRegisterMemory(.{ ._, .lea }, new_reg.to64(), .{
41007 .base = .{ .reg = reg_off.reg.to64() },
41008 .mod = .{ .rm = .{
41009 .size = .qword,
41010 .disp = reg_off.off + off,
41430 }, .{
41431 .src_constraints = .{ .{ .signed_int = .byte }, .{ .signed_int = .byte } },
41432 .patterns = &.{
41433 .{ .src = .{ .mem, .imm8 } },
41434 .{ .src = .{ .to_gpr, .imm8 } },
41435 .{ .src = .{ .to_gpr, .mem } },
41436 .{ .src = .{ .to_gpr, .to_gpr } },
41437 },
41438 .dst_temps = .{.{ .cc = .l }},
41439 .clobbers = .{ .eflags = true },
41440 .each = .{ .once = &.{
41441 .{ ._, ._, .cmp, .src0b, .src1b, ._, ._ },
41442 } },
41443 }, .{
41444 .src_constraints = .{ .{ .unsigned_int = .byte }, .{ .unsigned_int = .byte } },
41445 .patterns = &.{
41446 .{ .src = .{ .imm8, .mem }, .commute = .{ 0, 1 } },
41447 .{ .src = .{ .imm8, .to_gpr }, .commute = .{ 0, 1 } },
41448 .{ .src = .{ .mem, .to_gpr }, .commute = .{ 0, 1 } },
41449 },
41450 .dst_temps = .{.{ .cc = .a }},
41451 .clobbers = .{ .eflags = true },
41452 .each = .{ .once = &.{
41453 .{ ._, ._, .cmp, .src0b, .src1b, ._, ._ },
41454 } },
41455 }, .{
41456 .src_constraints = .{ .{ .unsigned_int = .byte }, .{ .unsigned_int = .byte } },
41457 .patterns = &.{
41458 .{ .src = .{ .mem, .imm8 } },
41459 .{ .src = .{ .to_gpr, .imm8 } },
41460 .{ .src = .{ .to_gpr, .mem } },
41461 .{ .src = .{ .to_gpr, .to_gpr } },
41462 },
41463 .dst_temps = .{.{ .cc = .b }},
41464 .clobbers = .{ .eflags = true },
41465 .each = .{ .once = &.{
41466 .{ ._, ._, .cmp, .src0b, .src1b, ._, ._ },
41467 } },
41468 }, .{
41469 .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word } },
41470 .patterns = &.{
41471 .{ .src = .{ .imm16, .mem }, .commute = .{ 0, 1 } },
41472 .{ .src = .{ .imm16, .to_gpr }, .commute = .{ 0, 1 } },
41473 .{ .src = .{ .mem, .to_gpr }, .commute = .{ 0, 1 } },
41474 },
41475 .dst_temps = .{.{ .cc = .g }},
41476 .clobbers = .{ .eflags = true },
41477 .each = .{ .once = &.{
41478 .{ ._, ._, .cmp, .src0w, .src1w, ._, ._ },
41479 } },
41480 }, .{
41481 .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word } },
41482 .patterns = &.{
41483 .{ .src = .{ .mem, .imm16 } },
41484 .{ .src = .{ .to_gpr, .imm16 } },
41485 .{ .src = .{ .to_gpr, .mem } },
41486 .{ .src = .{ .to_gpr, .to_gpr } },
41487 },
41488 .dst_temps = .{.{ .cc = .l }},
41489 .clobbers = .{ .eflags = true },
41490 .each = .{ .once = &.{
41491 .{ ._, ._, .cmp, .src0w, .src1w, ._, ._ },
41492 } },
41493 }, .{
41494 .src_constraints = .{ .{ .unsigned_int = .word }, .{ .unsigned_int = .word } },
41495 .patterns = &.{
41496 .{ .src = .{ .imm16, .mem }, .commute = .{ 0, 1 } },
41497 .{ .src = .{ .imm16, .to_gpr }, .commute = .{ 0, 1 } },
41498 .{ .src = .{ .mem, .to_gpr }, .commute = .{ 0, 1 } },
41499 },
41500 .dst_temps = .{.{ .cc = .a }},
41501 .clobbers = .{ .eflags = true },
41502 .each = .{ .once = &.{
41503 .{ ._, ._, .cmp, .src0w, .src1w, ._, ._ },
41504 } },
41505 }, .{
41506 .src_constraints = .{ .{ .unsigned_int = .word }, .{ .unsigned_int = .word } },
41507 .patterns = &.{
41508 .{ .src = .{ .mem, .imm16 } },
41509 .{ .src = .{ .to_gpr, .imm16 } },
41510 .{ .src = .{ .to_gpr, .mem } },
41511 .{ .src = .{ .to_gpr, .to_gpr } },
41512 },
41513 .dst_temps = .{.{ .cc = .b }},
41514 .clobbers = .{ .eflags = true },
41515 .each = .{ .once = &.{
41516 .{ ._, ._, .cmp, .src0w, .src1w, ._, ._ },
41517 } },
41518 }, .{
41519 .src_constraints = .{ .{ .signed_int = .dword }, .{ .signed_int = .dword } },
41520 .patterns = &.{
41521 .{ .src = .{ .imm32, .mem }, .commute = .{ 0, 1 } },
41522 .{ .src = .{ .imm32, .to_gpr }, .commute = .{ 0, 1 } },
41523 .{ .src = .{ .mem, .to_gpr }, .commute = .{ 0, 1 } },
41524 },
41525 .dst_temps = .{.{ .cc = .g }},
41526 .clobbers = .{ .eflags = true },
41527 .each = .{ .once = &.{
41528 .{ ._, ._, .cmp, .src0d, .src1d, ._, ._ },
41529 } },
41530 }, .{
41531 .src_constraints = .{ .{ .signed_int = .dword }, .{ .signed_int = .dword } },
41532 .patterns = &.{
41533 .{ .src = .{ .mem, .imm32 } },
41534 .{ .src = .{ .to_gpr, .imm32 } },
41535 .{ .src = .{ .to_gpr, .mem } },
41536 .{ .src = .{ .to_gpr, .to_gpr } },
41537 },
41538 .dst_temps = .{.{ .cc = .l }},
41539 .clobbers = .{ .eflags = true },
41540 .each = .{ .once = &.{
41541 .{ ._, ._, .cmp, .src0d, .src1d, ._, ._ },
41542 } },
41543 }, .{
41544 .src_constraints = .{ .{ .unsigned_int = .dword }, .{ .unsigned_int = .dword } },
41545 .patterns = &.{
41546 .{ .src = .{ .imm32, .mem }, .commute = .{ 0, 1 } },
41547 .{ .src = .{ .imm32, .to_gpr }, .commute = .{ 0, 1 } },
41548 .{ .src = .{ .mem, .to_gpr }, .commute = .{ 0, 1 } },
41549 },
41550 .dst_temps = .{.{ .cc = .a }},
41551 .clobbers = .{ .eflags = true },
41552 .each = .{ .once = &.{
41553 .{ ._, ._, .cmp, .src0d, .src1d, ._, ._ },
41554 } },
41555 }, .{
41556 .src_constraints = .{ .{ .unsigned_int = .dword }, .{ .unsigned_int = .dword } },
41557 .patterns = &.{
41558 .{ .src = .{ .mem, .imm32 } },
41559 .{ .src = .{ .to_gpr, .imm32 } },
41560 .{ .src = .{ .to_gpr, .mem } },
41561 .{ .src = .{ .to_gpr, .to_gpr } },
41562 },
41563 .dst_temps = .{.{ .cc = .b }},
41564 .clobbers = .{ .eflags = true },
41565 .each = .{ .once = &.{
41566 .{ ._, ._, .cmp, .src0d, .src1d, ._, ._ },
41567 } },
41568 }, .{
41569 .required_features = .{ .@"64bit", null, null, null },
41570 .src_constraints = .{ .{ .signed_int = .qword }, .{ .signed_int = .qword } },
41571 .patterns = &.{
41572 .{ .src = .{ .simm32, .mem }, .commute = .{ 0, 1 } },
41573 .{ .src = .{ .simm32, .to_gpr }, .commute = .{ 0, 1 } },
41574 .{ .src = .{ .mem, .to_gpr }, .commute = .{ 0, 1 } },
41575 },
41576 .dst_temps = .{.{ .cc = .g }},
41577 .clobbers = .{ .eflags = true },
41578 .each = .{ .once = &.{
41579 .{ ._, ._, .cmp, .src0q, .src1q, ._, ._ },
41580 } },
41581 }, .{
41582 .required_features = .{ .@"64bit", null, null, null },
41583 .src_constraints = .{ .{ .signed_int = .qword }, .{ .signed_int = .qword } },
41584 .patterns = &.{
41585 .{ .src = .{ .mem, .simm32 } },
41586 .{ .src = .{ .to_gpr, .simm32 } },
41587 .{ .src = .{ .to_gpr, .mem } },
41588 .{ .src = .{ .to_gpr, .to_gpr } },
41589 },
41590 .dst_temps = .{.{ .cc = .l }},
41591 .clobbers = .{ .eflags = true },
41592 .each = .{ .once = &.{
41593 .{ ._, ._, .cmp, .src0q, .src1q, ._, ._ },
41594 } },
41595 }, .{
41596 .required_features = .{ .@"64bit", null, null, null },
41597 .src_constraints = .{ .{ .unsigned_int = .qword }, .{ .unsigned_int = .qword } },
41598 .patterns = &.{
41599 .{ .src = .{ .simm32, .mem }, .commute = .{ 0, 1 } },
41600 .{ .src = .{ .simm32, .to_gpr }, .commute = .{ 0, 1 } },
41601 .{ .src = .{ .mem, .to_gpr }, .commute = .{ 0, 1 } },
41602 },
41603 .dst_temps = .{.{ .cc = .a }},
41604 .clobbers = .{ .eflags = true },
41605 .each = .{ .once = &.{
41606 .{ ._, ._, .cmp, .src0q, .src1q, ._, ._ },
41607 } },
41608 }, .{
41609 .required_features = .{ .@"64bit", null, null, null },
41610 .src_constraints = .{ .{ .unsigned_int = .qword }, .{ .unsigned_int = .qword } },
41611 .patterns = &.{
41612 .{ .src = .{ .mem, .simm32 } },
41613 .{ .src = .{ .to_gpr, .simm32 } },
41614 .{ .src = .{ .to_gpr, .mem } },
41615 .{ .src = .{ .to_gpr, .to_gpr } },
41616 },
41617 .dst_temps = .{.{ .cc = .b }},
41618 .clobbers = .{ .eflags = true },
41619 .each = .{ .once = &.{
41620 .{ ._, ._, .cmp, .src0q, .src1q, ._, ._ },
41621 } },
41622 }, .{
41623 .required_features = .{ .@"64bit", null, null, null },
41624 .src_constraints = .{
41625 .{ .signed_remainder_int = .{ .of = .qword, .is = .qword } },
41626 .{ .signed_remainder_int = .{ .of = .qword, .is = .qword } },
41627 },
41628 .patterns = &.{
41629 .{ .src = .{ .to_mem, .to_mem } },
41630 },
41631 .extra_temps = .{
41632 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
41633 .{ .type = .i64, .kind = .{ .rc = .general_purpose } },
41634 .unused,
41635 .unused,
41636 .unused,
41637 .unused,
41638 .unused,
41639 .unused,
41640 .unused,
41641 },
41642 .dst_temps = .{.{ .cc = .l }},
41643 .clobbers = .{ .eflags = true },
41644 .each = .{ .once = &.{
41645 .{ ._, ._, .mov, .tmp0p, .sia(1, .src0, .sub_size_div_8), ._, ._ },
41646 .{ ._, ._c, .cl, ._, ._, ._, ._ },
41647 .{ .@"0:", ._, .mov, .tmp1q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ },
41648 .{ ._, ._, .sbb, .tmp1q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ },
41649 .{ ._, ._c, .in, .tmp0p, ._, ._, ._ },
41650 .{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
41651 .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ },
41652 .{ ._, ._, .sbb, .tmp1q, .memad(.src1q, .add_size, -8), ._, ._ },
41653 } },
41654 }, .{
41655 .required_features = .{ .@"64bit", null, null, null },
41656 .src_constraints = .{
41657 .{ .unsigned_remainder_int = .{ .of = .qword, .is = .qword } },
41658 .{ .unsigned_remainder_int = .{ .of = .qword, .is = .qword } },
41659 },
41660 .patterns = &.{
41661 .{ .src = .{ .to_mem, .to_mem } },
41662 },
41663 .extra_temps = .{
41664 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
41665 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
41666 .unused,
41667 .unused,
41668 .unused,
41669 .unused,
41670 .unused,
41671 .unused,
41672 .unused,
41673 },
41674 .dst_temps = .{.{ .cc = .b }},
41675 .clobbers = .{ .eflags = true },
41676 .each = .{ .once = &.{
41677 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size_div_8), ._, ._ },
41678 .{ ._, ._c, .cl, ._, ._, ._, ._ },
41679 .{ .@"0:", ._, .mov, .tmp1q, .memsia(.src0q, .@"8", .tmp0, .add_size), ._, ._ },
41680 .{ ._, ._, .sbb, .tmp1q, .memsia(.src1q, .@"8", .tmp0, .add_size), ._, ._ },
41681 .{ ._, ._c, .in, .tmp0p, ._, ._, ._ },
41682 .{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
41683 } },
41684 }, .{
41685 .src_constraints = .{
41686 .{ .signed_remainder_int = .{ .of = .dword, .is = .dword } },
41687 .{ .signed_remainder_int = .{ .of = .dword, .is = .dword } },
41688 },
41689 .patterns = &.{
41690 .{ .src = .{ .to_mem, .to_mem } },
41691 },
41692 .extra_temps = .{
41693 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
41694 .{ .type = .i32, .kind = .{ .rc = .general_purpose } },
41695 .unused,
41696 .unused,
41697 .unused,
41698 .unused,
41699 .unused,
41700 .unused,
41701 .unused,
41702 },
41703 .dst_temps = .{.{ .cc = .l }},
41704 .clobbers = .{ .eflags = true },
41705 .each = .{ .once = &.{
41706 .{ ._, ._, .mov, .tmp0p, .sia(1, .src0, .sub_size_div_4), ._, ._ },
41707 .{ ._, ._c, .cl, ._, ._, ._, ._ },
41708 .{ .@"0:", ._, .mov, .tmp1q, .memsiad(.src0q, .@"4", .tmp0, .add_size, -4), ._, ._ },
41709 .{ ._, ._, .sbb, .tmp1q, .memsiad(.src1q, .@"4", .tmp0, .add_size, -4), ._, ._ },
41710 .{ ._, ._c, .in, .tmp0p, ._, ._, ._ },
41711 .{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
41712 .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -4), ._, ._ },
41713 .{ ._, ._, .sbb, .tmp1q, .memad(.src1q, .add_size, -4), ._, ._ },
41714 } },
41715 }, .{
41716 .src_constraints = .{
41717 .{ .unsigned_remainder_int = .{ .of = .dword, .is = .dword } },
41718 .{ .unsigned_remainder_int = .{ .of = .dword, .is = .dword } },
41719 },
41720 .patterns = &.{
41721 .{ .src = .{ .to_mem, .to_mem } },
41722 },
41723 .extra_temps = .{
41724 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
41725 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
41726 .unused,
41727 .unused,
41728 .unused,
41729 .unused,
41730 .unused,
41731 .unused,
41732 .unused,
41733 },
41734 .dst_temps = .{.{ .cc = .b }},
41735 .clobbers = .{ .eflags = true },
41736 .each = .{ .once = &.{
41737 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size_div_4), ._, ._ },
41738 .{ ._, ._c, .cl, ._, ._, ._, ._ },
41739 .{ .@"0:", ._, .mov, .tmp1q, .memsia(.src0q, .@"4", .tmp0, .add_size), ._, ._ },
41740 .{ ._, ._, .sbb, .tmp1q, .memsia(.src1q, .@"4", .tmp0, .add_size), ._, ._ },
41741 .{ ._, ._c, .in, .tmp0p, ._, ._, ._ },
41742 .{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
4101141743 } },
41012 });
41013 },
41014 .lea_symbol => |sym_off| new_temp_index.tracking(cg).* = .init(.{ .lea_symbol = .{
41015 .sym_index = sym_off.sym_index,
41016 .off = sym_off.off + off,
41017 } }),
41018 .load_frame => |frame_addr| {
41019 const new_reg =
41020 try cg.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);
41021 new_temp_index.tracking(cg).* = .init(.{ .register_offset = .{
41022 .reg = new_reg,
41023 .off = off,
4102441744 } });
41025 try cg.asmRegisterMemory(.{ ._, .mov }, new_reg.to64(), .{
41026 .base = .{ .frame = frame_addr.index },
41027 .mod = .{ .rm = .{
41028 .size = .qword,
41029 .disp = frame_addr.off,
41745 if (commute) std.mem.swap(Temp, &ops[0], &ops[1]);
41746 },
41747 .eq, .neq => {
41748 try cg.select(&res, &.{.bool}, &ops, comptime &.{ .{
41749 .src_constraints = .{ .{ .int = .byte }, .{ .int = .byte } },
41750 .patterns = &.{
41751 .{ .src = .{ .mem, .imm8 } },
41752 .{ .src = .{ .imm8, .mem }, .commute = .{ 0, 1 } },
41753 .{ .src = .{ .to_gpr, .imm8 } },
41754 .{ .src = .{ .imm8, .to_gpr }, .commute = .{ 0, 1 } },
41755 .{ .src = .{ .to_gpr, .mem } },
41756 .{ .src = .{ .mem, .to_gpr }, .commute = .{ 0, 1 } },
41757 .{ .src = .{ .to_gpr, .to_gpr } },
41758 },
41759 .dst_temps = .{.{ .cc = .e }},
41760 .clobbers = .{ .eflags = true },
41761 .each = .{ .once = &.{
41762 .{ ._, ._, .cmp, .src0b, .src1b, ._, ._ },
4103041763 } },
41031 });
41032 },
41033 .lea_frame => |frame_addr| new_temp_index.tracking(cg).* = .init(.{ .lea_frame = .{
41034 .index = frame_addr.index,
41035 .off = frame_addr.off + off,
41036 } }),
41037 }
41038 return .{ .index = new_temp_index.toIndex() };
41039 }
41040
41041 fn toOffset(temp: *Temp, off: i32, cg: *CodeGen) !void {
41042 if (off == 0) return;
41043 switch (temp.unwrap(cg)) {
41044 .ref, .err_ret_trace => {},
41045 .temp => |temp_index| {
41046 const temp_tracking = temp_index.tracking(cg);
41047 switch (temp_tracking.short) {
41048 else => {},
41049 .register => |reg| {
41050 try cg.freeValue(temp_tracking.long);
41051 temp_tracking.* = .init(.{ .register_offset = .{
41052 .reg = reg,
41053 .off = off,
41054 } });
41055 return;
41764 }, .{
41765 .src_constraints = .{ .{ .int = .word }, .{ .int = .word } },
41766 .patterns = &.{
41767 .{ .src = .{ .mem, .imm16 } },
41768 .{ .src = .{ .imm16, .mem }, .commute = .{ 0, 1 } },
41769 .{ .src = .{ .to_gpr, .imm16 } },
41770 .{ .src = .{ .imm16, .to_gpr }, .commute = .{ 0, 1 } },
41771 .{ .src = .{ .to_gpr, .mem } },
41772 .{ .src = .{ .mem, .to_gpr }, .commute = .{ 0, 1 } },
41773 .{ .src = .{ .to_gpr, .to_gpr } },
4105641774 },
41057 .register_offset => |reg_off| {
41058 try cg.freeValue(temp_tracking.long);
41059 temp_tracking.* = .init(.{ .register_offset = .{
41060 .reg = reg_off.reg,
41061 .off = reg_off.off + off,
41062 } });
41063 return;
41775 .dst_temps = .{.{ .cc = .e }},
41776 .clobbers = .{ .eflags = true },
41777 .each = .{ .once = &.{
41778 .{ ._, ._, .cmp, .src0w, .src1w, ._, ._ },
41779 } },
41780 }, .{
41781 .src_constraints = .{ .{ .int = .dword }, .{ .int = .dword } },
41782 .patterns = &.{
41783 .{ .src = .{ .mem, .imm32 } },
41784 .{ .src = .{ .imm32, .mem }, .commute = .{ 0, 1 } },
41785 .{ .src = .{ .to_gpr, .imm32 } },
41786 .{ .src = .{ .imm32, .to_gpr }, .commute = .{ 0, 1 } },
41787 .{ .src = .{ .to_gpr, .mem } },
41788 .{ .src = .{ .mem, .to_gpr }, .commute = .{ 0, 1 } },
41789 .{ .src = .{ .to_gpr, .to_gpr } },
4106441790 },
41065 .lea_symbol => |sym_off| {
41066 assert(std.meta.eql(temp_tracking.long.lea_symbol, sym_off));
41067 temp_tracking.* = .init(.{ .lea_symbol = .{
41068 .sym_index = sym_off.sym_index,
41069 .off = sym_off.off + off,
41070 } });
41071 return;
41791 .dst_temps = .{.{ .cc = .e }},
41792 .clobbers = .{ .eflags = true },
41793 .each = .{ .once = &.{
41794 .{ ._, ._, .cmp, .src0d, .src1d, ._, ._ },
41795 } },
41796 }, .{
41797 .required_features = .{ .@"64bit", null, null, null },
41798 .src_constraints = .{ .{ .int = .qword }, .{ .int = .qword } },
41799 .patterns = &.{
41800 .{ .src = .{ .mem, .simm32 } },
41801 .{ .src = .{ .simm32, .mem }, .commute = .{ 0, 1 } },
41802 .{ .src = .{ .to_gpr, .simm32 } },
41803 .{ .src = .{ .simm32, .to_gpr }, .commute = .{ 0, 1 } },
41804 .{ .src = .{ .to_gpr, .mem } },
41805 .{ .src = .{ .mem, .to_gpr }, .commute = .{ 0, 1 } },
41806 .{ .src = .{ .to_gpr, .to_gpr } },
4107241807 },
41073 .lea_frame => |frame_addr| {
41074 assert(std.meta.eql(temp_tracking.long.lea_frame, frame_addr));
41075 temp_tracking.* = .init(.{ .lea_frame = .{
41076 .index = frame_addr.index,
41077 .off = frame_addr.off + off,
41078 } });
41079 return;
41808 .dst_temps = .{.{ .cc = .e }},
41809 .clobbers = .{ .eflags = true },
41810 .each = .{ .once = &.{
41811 .{ ._, ._, .cmp, .src0q, .src1q, ._, ._ },
41812 } },
41813 }, .{
41814 .required_features = .{ .sse, .mmx, null, null },
41815 .src_constraints = .{ .{ .int = .qword }, .{ .int = .qword } },
41816 .patterns = &.{
41817 .{ .src = .{ .to_mut_mm, .mem } },
41818 .{ .src = .{ .mem, .to_mut_mm }, .commute = .{ 0, 1 } },
41819 .{ .src = .{ .to_mut_mm, .to_mm } },
41820 },
41821 .extra_temps = .{
41822 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
41823 .{ .kind = .{ .rc = .mmx } },
41824 .unused,
41825 .unused,
41826 .unused,
41827 .unused,
41828 .unused,
41829 .unused,
41830 .unused,
41831 },
41832 .dst_temps = .{.{ .cc = .z }},
41833 .clobbers = .{ .eflags = true },
41834 .each = .{ .once = &.{
41835 .{ ._, .p_, .xor, .tmp1q, .tmp1q, ._, ._ },
41836 .{ ._, .p_, .xor, .src0q, .src1q, ._, ._ },
41837 .{ ._, .p_b, .cmpeq, .tmp1q, .src0q, ._, ._ },
41838 .{ ._, .p_b, .movmsk, .tmp0d, .tmp1q, ._, ._ },
41839 .{ ._, ._, .cmp, .tmp0b, .si(-1), ._, ._ },
41840 } },
41841 }, .{
41842 .required_features = .{ .avx, null, null, null },
41843 .src_constraints = .{ .{ .int = .xword }, .{ .int = .xword } },
41844 .patterns = &.{
41845 .{ .src = .{ .to_xmm, .mem } },
41846 .{ .src = .{ .mem, .to_xmm }, .commute = .{ 0, 1 } },
41847 .{ .src = .{ .to_xmm, .to_xmm } },
41848 },
41849 .extra_temps = .{
41850 .{ .kind = .{ .rc = .sse } },
41851 .unused,
41852 .unused,
41853 .unused,
41854 .unused,
41855 .unused,
41856 .unused,
41857 .unused,
41858 .unused,
4108041859 },
41081 }
41082 },
41083 }
41084 const new_temp = try temp.getOffset(off, cg);
41085 try temp.die(cg);
41086 temp.* = new_temp;
41087 }
41088
41089 fn getLimb(temp: Temp, limb_ty: Type, limb_index: u28, cg: *CodeGen) !Temp {
41090 const new_temp_index = cg.next_temp_index;
41091 cg.temp_type[@intFromEnum(new_temp_index)] = limb_ty;
41092 switch (temp.tracking(cg).short) {
41093 else => |mcv| std.debug.panic("{s}: {}\n", .{ @src().fn_name, mcv }),
41094 .immediate => |imm| {
41095 assert(limb_index == 0);
41096 new_temp_index.tracking(cg).* = .init(.{ .immediate = imm });
41097 },
41098 .register => |reg| {
41099 assert(limb_index == 0);
41100 const new_reg =
41101 try cg.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);
41102 new_temp_index.tracking(cg).* = .init(.{ .register = new_reg });
41103 try cg.asmRegisterRegister(.{ ._, .mov }, new_reg.to64(), reg.to64());
41104 },
41105 .register_pair => |regs| {
41106 const new_reg =
41107 try cg.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);
41108 new_temp_index.tracking(cg).* = .init(.{ .register = new_reg });
41109 try cg.asmRegisterRegister(.{ ._, .mov }, new_reg.to64(), regs[limb_index].to64());
41110 },
41111 .register_offset => |reg_off| {
41112 assert(limb_index == 0);
41113 const new_reg =
41114 try cg.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);
41115 new_temp_index.tracking(cg).* = .init(.{ .register = new_reg });
41116 try cg.asmRegisterMemory(.{ ._, .lea }, new_reg.to64(), .{
41117 .base = .{ .reg = reg_off.reg.to64() },
41118 .mod = .{ .rm = .{
41119 .size = .qword,
41120 .disp = reg_off.off + @as(u31, limb_index) * 8,
41860 .dst_temps = .{.{ .cc = .z }},
41861 .clobbers = .{ .eflags = true },
41862 .each = .{ .once = &.{
41863 .{ ._, .vp_, .xor, .tmp0x, .src0x, .src1x, ._ },
41864 .{ ._, .vp_, .@"test", .tmp0x, .tmp0x, ._, ._ },
4112141865 } },
41122 });
41123 },
41124 .load_symbol => |sym_off| {
41125 const new_reg =
41126 try cg.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);
41127 new_temp_index.tracking(cg).* = .init(.{ .register = new_reg });
41128 try cg.asmRegisterMemory(.{ ._, .mov }, new_reg.to64(), .{
41129 .base = .{ .reloc = sym_off.sym_index },
41130 .mod = .{ .rm = .{
41131 .size = .qword,
41132 .disp = sym_off.off + @as(u31, limb_index) * 8,
41866 }, .{
41867 .required_features = .{ .sse4_1, null, null, null },
41868 .src_constraints = .{ .{ .int = .xword }, .{ .int = .xword } },
41869 .patterns = &.{
41870 .{ .src = .{ .to_mut_xmm, .mem } },
41871 .{ .src = .{ .mem, .to_mut_xmm }, .commute = .{ 0, 1 } },
41872 .{ .src = .{ .to_mut_xmm, .to_xmm } },
41873 },
41874 .dst_temps = .{.{ .cc = .z }},
41875 .clobbers = .{ .eflags = true },
41876 .each = .{ .once = &.{
41877 .{ ._, .p_, .xor, .src0x, .src1x, ._, ._ },
41878 .{ ._, .p_, .@"test", .src0x, .src0x, ._, ._ },
4113341879 } },
41134 });
41135 },
41136 .lea_symbol => |sym_off| {
41137 assert(limb_index == 0);
41138 new_temp_index.tracking(cg).* = .init(.{ .lea_symbol = sym_off });
41139 },
41140 .load_frame => |frame_addr| {
41141 const new_reg =
41142 try cg.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);
41143 new_temp_index.tracking(cg).* = .init(.{ .register = new_reg });
41144 try cg.asmRegisterMemory(.{ ._, .mov }, new_reg.to64(), .{
41145 .base = .{ .frame = frame_addr.index },
41146 .mod = .{ .rm = .{
41147 .size = .qword,
41148 .disp = frame_addr.off + @as(u31, limb_index) * 8,
41880 }, .{
41881 .required_features = .{ .sse2, .fast_imm16, null, null },
41882 .src_constraints = .{ .{ .int = .xword }, .{ .int = .xword } },
41883 .patterns = &.{
41884 .{ .src = .{ .to_mut_xmm, .mem } },
41885 .{ .src = .{ .mem, .to_mut_xmm }, .commute = .{ 0, 1 } },
41886 .{ .src = .{ .to_mut_xmm, .to_xmm } },
41887 },
41888 .extra_temps = .{
41889 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
41890 .{ .kind = .{ .rc = .sse } },
41891 .unused,
41892 .unused,
41893 .unused,
41894 .unused,
41895 .unused,
41896 .unused,
41897 .unused,
41898 },
41899 .dst_temps = .{.{ .cc = .z }},
41900 .clobbers = .{ .eflags = true },
41901 .each = .{ .once = &.{
41902 .{ ._, .p_, .xor, .tmp1x, .tmp1x, ._, ._ },
41903 .{ ._, .p_, .xor, .src0x, .src1x, ._, ._ },
41904 .{ ._, .p_b, .cmpeq, .tmp1x, .src0x, ._, ._ },
41905 .{ ._, .p_b, .movmsk, .tmp0d, .tmp1x, ._, ._ },
41906 .{ ._, ._, .cmp, .tmp0w, .si(-1), ._, ._ },
4114941907 } },
41150 });
41151 },
41152 .lea_frame => |frame_addr| {
41153 assert(limb_index == 0);
41154 new_temp_index.tracking(cg).* = .init(.{ .lea_frame = frame_addr });
41155 },
41156 }
41157 cg.next_temp_index = @enumFromInt(@intFromEnum(new_temp_index) + 1);
41158 return .{ .index = new_temp_index.toIndex() };
41159 }
41160
41161 fn toLimb(temp: *Temp, limb_ty: Type, limb_index: u28, cg: *CodeGen) !void {
41162 switch (temp.unwrap(cg)) {
41163 .ref => {},
41164 .temp => |temp_index| {
41165 const temp_tracking = temp_index.tracking(cg);
41166 switch (temp_tracking.short) {
41167 else => {},
41168 .register, .lea_symbol, .lea_frame => {
41169 assert(limb_index == 0);
41170 cg.temp_type[@intFromEnum(temp_index)] = limb_ty;
41171 return;
41908 }, .{
41909 .required_features = .{ .sse2, null, null, null },
41910 .src_constraints = .{ .{ .int = .xword }, .{ .int = .xword } },
41911 .patterns = &.{
41912 .{ .src = .{ .to_mut_xmm, .mem } },
41913 .{ .src = .{ .mem, .to_mut_xmm }, .commute = .{ 0, 1 } },
41914 .{ .src = .{ .to_mut_xmm, .to_xmm } },
4117241915 },
41173 .register_pair => |regs| {
41174 switch (temp_tracking.long) {
41175 .none, .reserved_frame => {},
41176 else => temp_tracking.long =
41177 temp_tracking.long.address().offset(@as(u31, limb_index) * 8).deref(),
41178 }
41179 for (regs, 0..) |reg, reg_index| if (reg_index != limb_index)
41180 cg.register_manager.freeReg(reg);
41181 temp_tracking.* = .init(.{ .register = regs[limb_index] });
41182 cg.temp_type[@intFromEnum(temp_index)] = limb_ty;
41183 return;
41916 .extra_temps = .{
41917 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
41918 .{ .kind = .{ .rc = .sse } },
41919 .unused,
41920 .unused,
41921 .unused,
41922 .unused,
41923 .unused,
41924 .unused,
41925 .unused,
4118441926 },
41185 .load_symbol => |sym_off| {
41186 assert(std.meta.eql(temp_tracking.long.load_symbol, sym_off));
41187 temp_tracking.* = .init(.{ .load_symbol = .{
41188 .sym_index = sym_off.sym_index,
41189 .off = sym_off.off + @as(u31, limb_index) * 8,
41190 } });
41191 cg.temp_type[@intFromEnum(temp_index)] = limb_ty;
41192 return;
41927 .dst_temps = .{.{ .cc = .z }},
41928 .clobbers = .{ .eflags = true },
41929 .each = .{ .once = &.{
41930 .{ ._, .p_, .xor, .tmp1x, .tmp1x, ._, ._ },
41931 .{ ._, .p_, .xor, .src0x, .src1x, ._, ._ },
41932 .{ ._, .p_b, .cmpeq, .tmp1x, .src0x, ._, ._ },
41933 .{ ._, .p_b, .movmsk, .tmp0d, .tmp1x, ._, ._ },
41934 .{ ._, ._, .xor, .tmp0d, .si(0xffff), ._, ._ },
41935 } },
41936 }, .{
41937 .required_features = .{ .avx2, null, null, null },
41938 .src_constraints = .{ .{ .int = .yword }, .{ .int = .yword } },
41939 .patterns = &.{
41940 .{ .src = .{ .to_ymm, .mem } },
41941 .{ .src = .{ .mem, .to_ymm }, .commute = .{ 0, 1 } },
41942 .{ .src = .{ .to_ymm, .to_ymm } },
4119341943 },
41194 .load_frame => |frame_addr| if (!frame_addr.index.isNamed()) {
41195 assert(std.meta.eql(temp_tracking.long.load_frame, frame_addr));
41196 temp_tracking.* = .init(.{ .load_frame = .{
41197 .index = frame_addr.index,
41198 .off = frame_addr.off + @as(u31, limb_index) * 8,
41199 } });
41200 cg.temp_type[@intFromEnum(temp_index)] = limb_ty;
41201 return;
41944 .extra_temps = .{
41945 .{ .kind = .{ .rc = .sse } },
41946 .unused,
41947 .unused,
41948 .unused,
41949 .unused,
41950 .unused,
41951 .unused,
41952 .unused,
41953 .unused,
4120241954 },
41203 }
41204 },
41205 .err_ret_trace => unreachable,
41206 }
41207 const new_temp = try temp.getLimb(limb_ty, limb_index, cg);
41208 try temp.die(cg);
41209 temp.* = new_temp;
41210 }
41211
41212 fn toSlicePtr(temp: *Temp, cg: *CodeGen) !void {
41213 const temp_ty = temp.typeOf(cg);
41214 if (temp_ty.isSlice(cg.pt.zcu)) try temp.toLimb(temp_ty.slicePtrFieldType(cg.pt.zcu), 0, cg);
41215 }
41216
41217 fn toSliceLen(temp: *Temp, cg: *CodeGen) !void {
41218 try temp.toLimb(.usize, 1, cg);
41219 }
41220
41221 fn toReg(temp: *Temp, new_reg: Register, cg: *CodeGen) !bool {
41222 const val, const ty: Type = val_ty: switch (temp.unwrap(cg)) {
41223 .ref => |ref| .{ temp.tracking(cg).short, cg.typeOf(ref) },
41224 .temp => |temp_index| {
41225 const temp_tracking = temp_index.tracking(cg);
41226 if (temp_tracking.short == .register and
41227 temp_tracking.short.register == new_reg) return false;
41228 break :val_ty .{ temp_tracking.short, temp_index.typeOf(cg) };
41229 },
41230 .err_ret_trace => .{ temp.tracking(cg).short, .usize },
41231 };
41232 const new_temp_index = cg.next_temp_index;
41233 try cg.register_manager.getReg(new_reg, new_temp_index.toIndex());
41234 cg.temp_type[@intFromEnum(new_temp_index)] = ty;
41235 try cg.genSetReg(new_reg, ty, val, .{});
41236 new_temp_index.tracking(cg).* = .init(.{ .register = new_reg });
41237 try temp.die(cg);
41238 cg.next_temp_index = @enumFromInt(@intFromEnum(new_temp_index) + 1);
41239 temp.* = .{ .index = new_temp_index.toIndex() };
41240 return true;
41241 }
41242
41243 fn toRegClass(temp: *Temp, mut: bool, rc: Register.Class, cg: *CodeGen) !bool {
41244 const val = temp.tracking(cg).short;
41245 if (!mut or temp.isMut(cg)) switch (val) {
41246 else => {},
41247 .register => |reg| if (reg.class() == rc) return false,
41248 .register_offset => |reg_off| if (reg_off.reg.class() == rc and reg_off.off == 0) return false,
41249 };
41250 const ty = temp.typeOf(cg);
41251 const new_temp_index = cg.next_temp_index;
41252 cg.temp_type[@intFromEnum(new_temp_index)] = ty;
41253 const new_reg = try cg.register_manager.allocReg(new_temp_index.toIndex(), regSetForRegClass(rc));
41254 try cg.genSetReg(new_reg, ty, val, .{});
41255 new_temp_index.tracking(cg).* = .init(.{ .register = new_reg });
41256 try temp.die(cg);
41257 cg.next_temp_index = @enumFromInt(@intFromEnum(new_temp_index) + 1);
41258 temp.* = .{ .index = new_temp_index.toIndex() };
41259 return true;
41260 }
41261
41262 fn toPair(first_temp: *Temp, second_temp: *Temp, cg: *CodeGen) !void {
41263 while (true) for ([_]*Temp{ first_temp, second_temp }) |part_temp| {
41264 if (try part_temp.toRegClass(true, .general_purpose, cg)) break;
41265 } else break;
41266 const first_temp_tracking = first_temp.unwrap(cg).temp.tracking(cg);
41267 const second_temp_tracking = second_temp.unwrap(cg).temp.tracking(cg);
41268 const result: MCValue = .{ .register_pair = .{
41269 first_temp_tracking.short.register,
41270 second_temp_tracking.short.register,
41271 } };
41272 const result_temp_index = cg.next_temp_index;
41273 const result_temp: Temp = .{ .index = result_temp_index.toIndex() };
41274 assert(cg.reuseTemp(result_temp.index, first_temp.index, first_temp_tracking));
41275 assert(cg.reuseTemp(result_temp.index, second_temp.index, second_temp_tracking));
41276 cg.temp_type[@intFromEnum(result_temp_index)] = .slice_const_u8;
41277 result_temp_index.tracking(cg).* = .init(result);
41278 first_temp.* = result_temp;
41279 second_temp.* = result_temp;
41280 }
41281
41282 fn asMask(temp: Temp, info: MaskInfo, cg: *CodeGen) void {
41283 assert(info.scalar != .none);
41284 const mcv = &temp.unwrap(cg).temp.tracking(cg).short;
41285 const reg = mcv.register;
41286 mcv.* = .{ .register_mask = .{ .reg = reg, .info = info } };
41287 }
41288
41289 fn toLea(temp: *Temp, cg: *CodeGen) !bool {
41290 switch (temp.tracking(cg).short) {
41291 .none,
41292 .unreach,
41293 .dead,
41294 .undef,
41295 .eflags,
41296 .register_pair,
41297 .register_triple,
41298 .register_quadruple,
41299 .register_overflow,
41300 .register_mask,
41301 .elementwise_regs_then_frame,
41302 .reserved_frame,
41303 .air_ref,
41304 => unreachable, // not a valid pointer
41305 .immediate,
41306 .register,
41307 .register_offset,
41308 .lea_direct,
41309 .lea_got,
41310 .lea_tlv,
41311 .lea_frame,
41312 => return false,
41313 .memory,
41314 .indirect,
41315 .load_symbol,
41316 .load_direct,
41317 .load_got,
41318 .load_tlv,
41319 .load_frame,
41320 => return temp.toRegClass(true, .general_purpose, cg),
41321 .lea_symbol => |sym_off| {
41322 const off = sym_off.off;
41323 if (off == 0) return false;
41324 try temp.toOffset(-off, cg);
41325 while (try temp.toRegClass(true, .general_purpose, cg)) {}
41326 try temp.toOffset(off, cg);
41327 return true;
41328 },
41329 }
41330 }
41331
41332 fn toMemory(temp: *Temp, cg: *CodeGen) !bool {
41333 const temp_tracking = temp.tracking(cg);
41334 if (temp_tracking.short.isMemory()) return false;
41335 const new_temp_index = cg.next_temp_index;
41336 const ty = temp.typeOf(cg);
41337 cg.temp_type[@intFromEnum(new_temp_index)] = ty;
41338 const new_frame_index = try cg.allocFrameIndex(.initSpill(ty, cg.pt.zcu));
41339 try cg.genSetMem(.{ .frame = new_frame_index }, 0, ty, temp_tracking.short, .{});
41340 new_temp_index.tracking(cg).* = .init(.{ .load_frame = .{ .index = new_frame_index } });
41341 try temp.die(cg);
41342 cg.next_temp_index = @enumFromInt(@intFromEnum(new_temp_index) + 1);
41343 temp.* = .{ .index = new_temp_index.toIndex() };
41344 return true;
41345 }
41346
41347 // hack around linker relocation bugs
41348 fn toBase(temp: *Temp, cg: *CodeGen) !bool {
41349 const temp_tracking = temp.tracking(cg);
41350 if (temp_tracking.short.isBase()) return false;
41351 if (try temp.toMemory(cg)) return true;
41352 const new_temp_index = cg.next_temp_index;
41353 cg.temp_type[@intFromEnum(new_temp_index)] = temp.typeOf(cg);
41354 const new_reg =
41355 try cg.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);
41356 try cg.genSetReg(new_reg, .usize, temp_tracking.short.address(), .{});
41357 new_temp_index.tracking(cg).* = .init(.{ .indirect = .{ .reg = new_reg } });
41358 try temp.die(cg);
41359 cg.next_temp_index = @enumFromInt(@intFromEnum(new_temp_index) + 1);
41360 temp.* = .{ .index = new_temp_index.toIndex() };
41361 return true;
41362 }
41363
41364 const AccessOptions = struct {
41365 disp: i32 = 0,
41366 safe: bool = false,
41367 };
41368
41369 fn load(ptr: *Temp, val_ty: Type, opts: AccessOptions, cg: *CodeGen) !Temp {
41370 const val = try cg.tempAlloc(val_ty);
41371 try ptr.toOffset(opts.disp, cg);
41372 while (try ptr.toLea(cg)) {}
41373 const val_mcv = val.tracking(cg).short;
41374 switch (val_mcv) {
41375 else => |mcv| std.debug.panic("{s}: {}\n", .{ @src().fn_name, mcv }),
41376 .register => |val_reg| try ptr.loadReg(val_ty, registerAlias(
41377 val_reg,
41378 @intCast(val_ty.abiSize(cg.pt.zcu)),
41379 ), cg),
41380 inline .register_pair,
41381 .register_triple,
41382 .register_quadruple,
41383 => |val_regs| for (val_regs) |val_reg| {
41384 try ptr.loadReg(val_ty, val_reg, cg);
41385 try ptr.toOffset(@divExact(val_reg.bitSize(), 8), cg);
41386 while (try ptr.toLea(cg)) {}
41387 },
41388 .register_offset => |val_reg_off| switch (val_reg_off.off) {
41389 0 => try ptr.loadReg(val_ty, registerAlias(
41390 val_reg_off.reg,
41391 @intCast(val_ty.abiSize(cg.pt.zcu)),
41392 ), cg),
41393 else => unreachable,
41394 },
41395 .memory, .indirect, .load_frame, .load_symbol => {
41396 var val_ptr = try cg.tempInit(.usize, val_mcv.address());
41397 var len = try cg.tempInit(.usize, .{ .immediate = val_ty.abiSize(cg.pt.zcu) });
41398 try val_ptr.memcpy(ptr, &len, cg);
41399 try val_ptr.die(cg);
41400 try len.die(cg);
41401 },
41402 }
41403 return val;
41404 }
41405
41406 fn store(ptr: *Temp, val: *Temp, opts: AccessOptions, cg: *CodeGen) !void {
41407 const val_ty = val.typeOf(cg);
41408 try ptr.toOffset(opts.disp, cg);
41409 while (try ptr.toLea(cg)) {}
41410 val_to_gpr: while (true) : (while (try ptr.toLea(cg) or
41411 try val.toRegClass(false, .general_purpose, cg))
41412 {}) {
41413 const val_mcv = val.tracking(cg).short;
41414 switch (val_mcv) {
41415 else => |mcv| std.debug.panic("{s}: {}\n", .{ @src().fn_name, mcv }),
41416 .undef => if (opts.safe) {
41417 var pat = try cg.tempInit(.u8, .{ .immediate = 0xaa });
41418 var len = try cg.tempInit(.usize, .{ .immediate = val_ty.abiSize(cg.pt.zcu) });
41419 try ptr.memset(&pat, &len, cg);
41420 try pat.die(cg);
41421 try len.die(cg);
41422 },
41423 .immediate => |val_imm| {
41424 const val_op: Immediate = if (std.math.cast(u31, val_imm)) |val_uimm31|
41425 .u(val_uimm31)
41426 else if (std.math.cast(i32, @as(i64, @bitCast(val_imm)))) |val_simm32|
41427 .s(val_simm32)
41428 else
41429 continue :val_to_gpr;
41430 // hack around linker relocation bugs
41431 switch (ptr.tracking(cg).short) {
41432 else => {},
41433 .lea_symbol => while (try ptr.toRegClass(false, .general_purpose, cg)) {},
41434 }
41435 try cg.asmMemoryImmediate(
41436 .{ ._, .mov },
41437 try ptr.tracking(cg).short.deref().mem(cg, .{
41438 .size = cg.memSize(val_ty),
41439 }),
41440 val_op,
41441 );
41442 },
41443 .eflags => |cc| {
41444 // hack around linker relocation bugs
41445 switch (ptr.tracking(cg).short) {
41446 else => {},
41447 .lea_symbol => while (try ptr.toRegClass(false, .general_purpose, cg)) {},
41448 }
41449 try cg.asmSetccMemory(
41450 cc,
41451 try ptr.tracking(cg).short.deref().mem(cg, .{ .size = .byte }),
41452 );
41453 },
41454 .register => |val_reg| try ptr.storeRegs(val_ty, &.{registerAlias(
41455 val_reg,
41456 @intCast(val_ty.abiSize(cg.pt.zcu)),
41457 )}, cg),
41458 inline .register_pair,
41459 .register_triple,
41460 .register_quadruple,
41461 => |val_regs| try ptr.storeRegs(val_ty, &val_regs, cg),
41462 .register_offset => |val_reg_off| switch (val_reg_off.off) {
41463 0 => try ptr.storeRegs(val_ty, &.{registerAlias(
41464 val_reg_off.reg,
41465 @intCast(val_ty.abiSize(cg.pt.zcu)),
41466 )}, cg),
41467 else => continue :val_to_gpr,
41468 },
41469 .register_overflow => |val_reg_ov| {
41470 const ip = &cg.pt.zcu.intern_pool;
41471 const first_ty: Type = .fromInterned(first_ty: switch (ip.indexToKey(val_ty.toIntern())) {
41472 .tuple_type => |tuple_type| {
41473 const tuple_field_types = tuple_type.types.get(ip);
41474 assert(tuple_field_types.len == 2 and tuple_field_types[1] == .u1_type);
41475 break :first_ty tuple_field_types[0];
41476 },
41477 .opt_type => |opt_child| {
41478 assert(!val_ty.optionalReprIsPayload(cg.pt.zcu));
41479 break :first_ty opt_child;
41480 },
41481 else => std.debug.panic("{s}: {}\n", .{ @src().fn_name, val_ty.fmt(cg.pt) }),
41482 });
41483 const first_size: u31 = @intCast(first_ty.abiSize(cg.pt.zcu));
41484 try ptr.storeRegs(first_ty, &.{registerAlias(val_reg_ov.reg, first_size)}, cg);
41485 try ptr.toOffset(first_size, cg);
41486 try cg.asmSetccMemory(
41487 val_reg_ov.eflags,
41488 try ptr.tracking(cg).short.deref().mem(cg, .{ .size = .byte }),
41489 );
41490 },
41491 .lea_frame, .lea_symbol => continue :val_to_gpr,
41492 .memory, .indirect, .load_frame, .load_symbol => {
41493 var val_ptr = try cg.tempInit(.usize, val_mcv.address());
41494 var len = try cg.tempInit(.usize, .{ .immediate = val_ty.abiSize(cg.pt.zcu) });
41495 try ptr.memcpy(&val_ptr, &len, cg);
41496 try val_ptr.die(cg);
41497 try len.die(cg);
41498 },
41499 }
41500 break;
41501 }
41502 }
41503
41504 fn read(src: *Temp, val_ty: Type, opts: AccessOptions, cg: *CodeGen) !Temp {
41505 var val = try cg.tempAlloc(val_ty);
41506 while (try src.toBase(cg)) {}
41507 const val_mcv = val.tracking(cg).short;
41508 switch (val_mcv) {
41509 else => |mcv| std.debug.panic("{s}: {}\n", .{ @src().fn_name, mcv }),
41510 .register => |val_reg| try src.readReg(opts.disp, val_ty, registerAlias(
41511 val_reg,
41512 @intCast(val_ty.abiSize(cg.pt.zcu)),
41513 ), cg),
41514 inline .register_pair, .register_triple, .register_quadruple => |val_regs| {
41515 var disp = opts.disp;
41516 for (val_regs) |val_reg| {
41517 try src.readReg(disp, val_ty, val_reg, cg);
41518 disp += @divExact(val_reg.bitSize(), 8);
41519 }
41520 },
41521 .register_offset => |val_reg_off| switch (val_reg_off.off) {
41522 0 => try src.readReg(opts.disp, val_ty, registerAlias(
41523 val_reg_off.reg,
41524 @intCast(val_ty.abiSize(cg.pt.zcu)),
41525 ), cg),
41526 else => unreachable,
41527 },
41528 .memory, .indirect, .load_frame, .load_symbol => {
41529 var val_ptr = try cg.tempInit(.usize, val_mcv.address());
41530 var src_ptr =
41531 try cg.tempInit(.usize, src.tracking(cg).short.address().offset(opts.disp));
41532 var len = try cg.tempInit(.usize, .{ .immediate = val_ty.abiSize(cg.pt.zcu) });
41533 try val_ptr.memcpy(&src_ptr, &len, cg);
41534 try val_ptr.die(cg);
41535 try src_ptr.die(cg);
41536 try len.die(cg);
41955 .dst_temps = .{.{ .cc = .z }},
41956 .clobbers = .{ .eflags = true },
41957 .each = .{ .once = &.{
41958 .{ ._, .vp_, .xor, .tmp0y, .src0y, .src1y, ._ },
41959 .{ ._, .vp_, .@"test", .tmp0y, .tmp0y, ._, ._ },
41960 } },
41961 }, .{
41962 .required_features = .{ .avx, null, null, null },
41963 .src_constraints = .{ .{ .int = .yword }, .{ .int = .yword } },
41964 .patterns = &.{
41965 .{ .src = .{ .to_ymm, .mem } },
41966 .{ .src = .{ .mem, .to_ymm }, .commute = .{ 0, 1 } },
41967 .{ .src = .{ .to_ymm, .to_ymm } },
41968 },
41969 .extra_temps = .{
41970 .{ .kind = .{ .rc = .sse } },
41971 .unused,
41972 .unused,
41973 .unused,
41974 .unused,
41975 .unused,
41976 .unused,
41977 .unused,
41978 .unused,
41979 },
41980 .dst_temps = .{.{ .cc = .z }},
41981 .clobbers = .{ .eflags = true },
41982 .each = .{ .once = &.{
41983 .{ ._, .v_pd, .xor, .tmp0y, .src0y, .src1y, ._ },
41984 .{ ._, .vp_, .@"test", .tmp0y, .tmp0y, ._, ._ },
41985 } },
41986 }, .{
41987 .required_features = .{ .avx2, null, null, null },
41988 .src_constraints = .{
41989 .{ .remainder_int = .{ .of = .yword, .is = .xword } },
41990 .{ .remainder_int = .{ .of = .yword, .is = .xword } },
41991 },
41992 .patterns = &.{
41993 .{ .src = .{ .to_mem, .to_mem } },
41994 },
41995 .extra_temps = .{
41996 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
41997 .{ .kind = .{ .rc = .sse } },
41998 .unused,
41999 .unused,
42000 .unused,
42001 .unused,
42002 .unused,
42003 .unused,
42004 .unused,
42005 },
42006 .dst_temps = .{.{ .cc = .z }},
42007 .clobbers = .{ .eflags = true },
42008 .each = .{ .once = &.{
42009 .{ ._, ._, .mov, .tmp0p, .sia(16, .src0, .sub_size), ._, ._ },
42010 .{ .@"0:", .v_dqu, .mov, .tmp1y, .memiad(.src0y, .tmp0, .add_size, -16), ._, ._ },
42011 .{ ._, .vp_, .xor, .tmp1y, .tmp1y, .memiad(.src1y, .tmp0, .add_size, -16), ._ },
42012 .{ ._, .vp_, .@"test", .tmp1y, .tmp1y, ._, ._ },
42013 .{ ._, ._nz, .j, .@"0f", ._, ._, ._ },
42014 .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ },
42015 .{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
42016 .{ ._, .v_dqa, .mov, .tmp1x, .memad(.src0x, .add_size, -16), ._, ._ },
42017 .{ ._, .vp_, .xor, .tmp1x, .tmp1x, .memad(.src1x, .add_size, -16), ._ },
42018 .{ ._, .vp_, .@"test", .tmp1x, .tmp1x, ._, ._ },
42019 } },
42020 }, .{
42021 .required_features = .{ .avx, null, null, null },
42022 .src_constraints = .{
42023 .{ .remainder_int = .{ .of = .yword, .is = .xword } },
42024 .{ .remainder_int = .{ .of = .yword, .is = .xword } },
42025 },
42026 .patterns = &.{
42027 .{ .src = .{ .to_mem, .to_mem } },
42028 },
42029 .extra_temps = .{
42030 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
42031 .{ .kind = .{ .rc = .sse } },
42032 .unused,
42033 .unused,
42034 .unused,
42035 .unused,
42036 .unused,
42037 .unused,
42038 .unused,
42039 },
42040 .dst_temps = .{.{ .cc = .z }},
42041 .clobbers = .{ .eflags = true },
42042 .each = .{ .once = &.{
42043 .{ ._, ._, .mov, .tmp0p, .sia(16, .src0, .sub_size), ._, ._ },
42044 .{ .@"0:", .v_pd, .movu, .tmp1y, .memiad(.src0y, .tmp0, .add_size, -16), ._, ._ },
42045 .{ ._, .v_pd, .xor, .tmp1y, .tmp1y, .memiad(.src1y, .tmp0, .add_size, -16), ._ },
42046 .{ ._, .vp_, .@"test", .tmp1y, .tmp1y, ._, ._ },
42047 .{ ._, ._nz, .j, .@"0f", ._, ._, ._ },
42048 .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ },
42049 .{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
42050 .{ ._, .v_pd, .mova, .tmp1x, .memad(.src0x, .add_size, -16), ._, ._ },
42051 .{ ._, .v_pd, .xor, .tmp1x, .tmp1x, .memad(.src1x, .add_size, -16), ._ },
42052 .{ ._, .vp_, .@"test", .tmp1x, .tmp1x, ._, ._ },
42053 } },
42054 }, .{
42055 .required_features = .{ .avx2, null, null, null },
42056 .src_constraints = .{
42057 .{ .remainder_int = .{ .of = .yword, .is = .yword } },
42058 .{ .remainder_int = .{ .of = .yword, .is = .yword } },
42059 },
42060 .patterns = &.{
42061 .{ .src = .{ .to_mem, .to_mem } },
42062 },
42063 .extra_temps = .{
42064 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
42065 .{ .kind = .{ .rc = .sse } },
42066 .unused,
42067 .unused,
42068 .unused,
42069 .unused,
42070 .unused,
42071 .unused,
42072 .unused,
42073 },
42074 .dst_temps = .{.{ .cc = .z }},
42075 .clobbers = .{ .eflags = true },
42076 .each = .{ .once = &.{
42077 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
42078 .{ .@"0:", .v_dqu, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_size), ._, ._ },
42079 .{ ._, .vp_, .xor, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_size), ._ },
42080 .{ ._, .vp_, .@"test", .tmp1y, .tmp1y, ._, ._ },
42081 .{ ._, ._nz, .j, .@"0f", ._, ._, ._ },
42082 .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ },
42083 .{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
42084 } },
42085 }, .{
42086 .required_features = .{ .avx, null, null, null },
42087 .src_constraints = .{
42088 .{ .remainder_int = .{ .of = .yword, .is = .yword } },
42089 .{ .remainder_int = .{ .of = .yword, .is = .yword } },
42090 },
42091 .patterns = &.{
42092 .{ .src = .{ .to_mem, .to_mem } },
42093 },
42094 .extra_temps = .{
42095 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
42096 .{ .kind = .{ .rc = .sse } },
42097 .unused,
42098 .unused,
42099 .unused,
42100 .unused,
42101 .unused,
42102 .unused,
42103 .unused,
42104 },
42105 .dst_temps = .{.{ .cc = .z }},
42106 .clobbers = .{ .eflags = true },
42107 .each = .{ .once = &.{
42108 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
42109 .{ .@"0:", .v_pd, .movu, .tmp1y, .memia(.src0y, .tmp0, .add_size), ._, ._ },
42110 .{ ._, .v_pd, .xor, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_size), ._ },
42111 .{ ._, .vp_, .@"test", .tmp1y, .tmp1y, ._, ._ },
42112 .{ ._, ._nz, .j, .@"0f", ._, ._, ._ },
42113 .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ },
42114 .{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
42115 } },
42116 }, .{
42117 .required_features = .{ .avx, null, null, null },
42118 .src_constraints = .{
42119 .{ .remainder_int = .{ .of = .xword, .is = .xword } },
42120 .{ .remainder_int = .{ .of = .xword, .is = .xword } },
42121 },
42122 .patterns = &.{
42123 .{ .src = .{ .to_mem, .to_mem } },
42124 },
42125 .extra_temps = .{
42126 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
42127 .{ .kind = .{ .rc = .sse } },
42128 .unused,
42129 .unused,
42130 .unused,
42131 .unused,
42132 .unused,
42133 .unused,
42134 .unused,
42135 },
42136 .dst_temps = .{.{ .cc = .z }},
42137 .clobbers = .{ .eflags = true },
42138 .each = .{ .once = &.{
42139 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
42140 .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ },
42141 .{ ._, .vp_, .xor, .tmp1x, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._ },
42142 .{ ._, .vp_, .@"test", .tmp1x, .tmp1x, ._, ._ },
42143 .{ ._, ._nz, .j, .@"0f", ._, ._, ._ },
42144 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
42145 .{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
42146 } },
42147 }, .{
42148 .required_features = .{ .sse4_1, null, null, null },
42149 .src_constraints = .{
42150 .{ .remainder_int = .{ .of = .xword, .is = .xword } },
42151 .{ .remainder_int = .{ .of = .xword, .is = .xword } },
42152 },
42153 .patterns = &.{
42154 .{ .src = .{ .to_mem, .to_mem } },
42155 },
42156 .extra_temps = .{
42157 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
42158 .{ .kind = .{ .rc = .sse } },
42159 .unused,
42160 .unused,
42161 .unused,
42162 .unused,
42163 .unused,
42164 .unused,
42165 .unused,
42166 },
42167 .dst_temps = .{.{ .cc = .z }},
42168 .clobbers = .{ .eflags = true },
42169 .each = .{ .once = &.{
42170 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
42171 .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ },
42172 .{ ._, .p_, .xor, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._, ._ },
42173 .{ ._, .p_, .@"test", .tmp1x, .tmp1x, ._, ._ },
42174 .{ ._, ._nz, .j, .@"0f", ._, ._, ._ },
42175 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
42176 .{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
42177 } },
42178 }, .{
42179 .required_features = .{ .sse2, .fast_imm16, null, null },
42180 .src_constraints = .{
42181 .{ .remainder_int = .{ .of = .xword, .is = .xword } },
42182 .{ .remainder_int = .{ .of = .xword, .is = .xword } },
42183 },
42184 .patterns = &.{
42185 .{ .src = .{ .to_mem, .to_mem } },
42186 },
42187 .extra_temps = .{
42188 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
42189 .{ .kind = .{ .rc = .sse } },
42190 .{ .type = .u16, .kind = .{ .rc = .general_purpose } },
42191 .unused,
42192 .unused,
42193 .unused,
42194 .unused,
42195 .unused,
42196 .unused,
42197 },
42198 .dst_temps = .{.{ .cc = .z }},
42199 .clobbers = .{ .eflags = true },
42200 .each = .{ .once = &.{
42201 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
42202 .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ },
42203 .{ ._, .p_b, .cmpeq, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._, ._ },
42204 .{ ._, .p_b, .movmsk, .tmp2d, .tmp1x, ._, ._ },
42205 .{ ._, ._, .cmp, .tmp2w, .si(-1), ._, ._ },
42206 .{ ._, ._nz, .j, .@"0f", ._, ._, ._ },
42207 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
42208 .{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
42209 } },
42210 }, .{
42211 .required_features = .{ .sse2, null, null, null },
42212 .src_constraints = .{
42213 .{ .remainder_int = .{ .of = .xword, .is = .xword } },
42214 .{ .remainder_int = .{ .of = .xword, .is = .xword } },
42215 },
42216 .patterns = &.{
42217 .{ .src = .{ .to_mem, .to_mem } },
42218 },
42219 .extra_temps = .{
42220 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
42221 .{ .kind = .{ .rc = .sse } },
42222 .{ .type = .u16, .kind = .{ .rc = .general_purpose } },
42223 .unused,
42224 .unused,
42225 .unused,
42226 .unused,
42227 .unused,
42228 .unused,
42229 },
42230 .dst_temps = .{.{ .cc = .z }},
42231 .clobbers = .{ .eflags = true },
42232 .each = .{ .once = &.{
42233 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
42234 .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ },
42235 .{ ._, .p_b, .cmpeq, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._, ._ },
42236 .{ ._, .p_b, .movmsk, .tmp2d, .tmp1x, ._, ._ },
42237 .{ ._, ._, .xor, .tmp2d, .ui(0xffff), ._, ._ },
42238 .{ ._, ._nz, .j, .@"0f", ._, ._, ._ },
42239 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
42240 .{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
42241 } },
42242 }, .{
42243 .required_features = .{ .sse, .mmx, null, null },
42244 .src_constraints = .{
42245 .{ .remainder_int = .{ .of = .qword, .is = .qword } },
42246 .{ .remainder_int = .{ .of = .qword, .is = .qword } },
42247 },
42248 .patterns = &.{
42249 .{ .src = .{ .to_mem, .to_mem } },
42250 },
42251 .extra_temps = .{
42252 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
42253 .{ .kind = .{ .rc = .mmx } },
42254 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
42255 .unused,
42256 .unused,
42257 .unused,
42258 .unused,
42259 .unused,
42260 .unused,
42261 },
42262 .dst_temps = .{.{ .cc = .z }},
42263 .clobbers = .{ .eflags = true },
42264 .each = .{ .once = &.{
42265 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
42266 .{ .@"0:", ._q, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_size), ._, ._ },
42267 .{ ._, .p_b, .cmpeq, .tmp1q, .memia(.src1q, .tmp0, .add_size), ._, ._ },
42268 .{ ._, .p_b, .movmsk, .tmp2d, .tmp1q, ._, ._ },
42269 .{ ._, ._, .cmp, .tmp2b, .si(-1), ._, ._ },
42270 .{ ._, ._nz, .j, .@"0f", ._, ._, ._ },
42271 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
42272 .{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
42273 } },
42274 }, .{
42275 .required_features = .{ .@"64bit", null, null, null },
42276 .src_constraints = .{
42277 .{ .remainder_int = .{ .of = .qword, .is = .qword } },
42278 .{ .remainder_int = .{ .of = .qword, .is = .qword } },
42279 },
42280 .patterns = &.{
42281 .{ .src = .{ .to_mem, .to_mem } },
42282 },
42283 .extra_temps = .{
42284 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
42285 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
42286 .unused,
42287 .unused,
42288 .unused,
42289 .unused,
42290 .unused,
42291 .unused,
42292 .unused,
42293 },
42294 .dst_temps = .{.{ .cc = .z }},
42295 .clobbers = .{ .eflags = true },
42296 .each = .{ .once = &.{
42297 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
42298 .{ .@"0:", ._, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_size), ._, ._ },
42299 .{ ._, ._, .xor, .tmp1q, .memia(.src1q, .tmp0, .add_size), ._, ._ },
42300 .{ ._, ._nz, .j, .@"0f", ._, ._, ._ },
42301 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
42302 .{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
42303 } },
42304 }, .{
42305 .src_constraints = .{
42306 .{ .remainder_int = .{ .of = .dword, .is = .dword } },
42307 .{ .remainder_int = .{ .of = .dword, .is = .dword } },
42308 },
42309 .patterns = &.{
42310 .{ .src = .{ .to_mem, .to_mem } },
42311 },
42312 .extra_temps = .{
42313 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
42314 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
42315 .unused,
42316 .unused,
42317 .unused,
42318 .unused,
42319 .unused,
42320 .unused,
42321 .unused,
42322 },
42323 .dst_temps = .{.{ .cc = .z }},
42324 .clobbers = .{ .eflags = true },
42325 .each = .{ .once = &.{
42326 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
42327 .{ .@"0:", ._, .mov, .tmp1d, .memia(.src0d, .tmp0, .add_size), ._, ._ },
42328 .{ ._, ._, .xor, .tmp1d, .memia(.src1d, .tmp0, .add_size), ._, ._ },
42329 .{ ._, ._nz, .j, .@"0f", ._, ._, ._ },
42330 .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ },
42331 .{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
42332 } },
42333 } });
4153742334 },
4153842335 }
41539 return val;
41540 }
41541
41542 fn write(dst: *Temp, val: *Temp, opts: AccessOptions, cg: *CodeGen) !void {
41543 const val_ty = val.typeOf(cg);
41544 while (try dst.toBase(cg)) {}
41545 val_to_gpr: while (true) : (while (try dst.toBase(cg) or
41546 try val.toRegClass(false, .general_purpose, cg))
41547 {}) {
41548 const val_mcv = val.tracking(cg).short;
41549 switch (val_mcv) {
41550 else => |mcv| std.debug.panic("{s}: {}\n", .{ @src().fn_name, mcv }),
41551 .undef => if (opts.safe) {
41552 var dst_ptr = try cg.tempInit(.usize, dst.tracking(cg).short.address().offset(opts.disp));
41553 var pat = try cg.tempInit(.u8, .{ .immediate = 0xaa });
41554 var len = try cg.tempInit(.usize, .{ .immediate = val_ty.abiSize(cg.pt.zcu) });
41555 try dst_ptr.memset(&pat, &len, cg);
41556 try dst_ptr.die(cg);
41557 try pat.die(cg);
41558 try len.die(cg);
41559 },
41560 .immediate => |val_imm| {
41561 const val_op: Immediate = if (std.math.cast(u31, val_imm)) |val_uimm31|
41562 .u(val_uimm31)
41563 else if (std.math.cast(i32, @as(i64, @bitCast(val_imm)))) |val_simm32|
41564 .s(val_simm32)
41565 else
41566 continue :val_to_gpr;
41567 try cg.asmMemoryImmediate(
41568 .{ ._, .mov },
41569 try dst.tracking(cg).short.mem(cg, .{
41570 .size = cg.memSize(val_ty),
41571 .disp = opts.disp,
41572 }),
41573 val_op,
41574 );
41575 },
41576 .eflags => |cc| try cg.asmSetccMemory(
41577 cc,
41578 try dst.tracking(cg).short.mem(cg, .{
41579 .size = .byte,
41580 .disp = opts.disp,
41581 }),
41582 ),
41583 .register => |val_reg| try dst.writeRegs(opts.disp, val_ty, &.{registerAlias(
41584 val_reg,
41585 @intCast(val_ty.abiSize(cg.pt.zcu)),
41586 )}, cg),
41587 inline .register_pair,
41588 .register_triple,
41589 .register_quadruple,
41590 => |val_regs| try dst.writeRegs(opts.disp, val_ty, &val_regs, cg),
41591 .register_offset => |val_reg_off| switch (val_reg_off.off) {
41592 0 => try dst.writeRegs(opts.disp, val_ty, &.{registerAlias(
41593 val_reg_off.reg,
41594 @intCast(val_ty.abiSize(cg.pt.zcu)),
41595 )}, cg),
41596 else => continue :val_to_gpr,
41597 },
41598 .register_overflow => |val_reg_ov| {
41599 const ip = &cg.pt.zcu.intern_pool;
41600 const first_ty: Type = .fromInterned(first_ty: switch (ip.indexToKey(val_ty.toIntern())) {
41601 .tuple_type => |tuple_type| {
41602 const tuple_field_types = tuple_type.types.get(ip);
41603 assert(tuple_field_types.len == 2 and tuple_field_types[1] == .u1_type);
41604 break :first_ty tuple_field_types[0];
41605 },
41606 .opt_type => |opt_child| {
41607 assert(!val_ty.optionalReprIsPayload(cg.pt.zcu));
41608 break :first_ty opt_child;
41609 },
41610 else => std.debug.panic("{s}: {}\n", .{ @src().fn_name, val_ty.fmt(cg.pt) }),
41611 });
41612 const first_size: u31 = @intCast(first_ty.abiSize(cg.pt.zcu));
41613 try dst.writeRegs(opts.disp, first_ty, &.{registerAlias(val_reg_ov.reg, first_size)}, cg);
41614 try cg.asmSetccMemory(
41615 val_reg_ov.eflags,
41616 try dst.tracking(cg).short.mem(cg, .{
41617 .size = .byte,
41618 .disp = opts.disp + first_size,
41619 }),
41620 );
41621 },
41622 .lea_frame, .lea_symbol => continue :val_to_gpr,
41623 .memory, .indirect, .load_frame, .load_symbol => {
41624 var dst_ptr =
41625 try cg.tempInit(.usize, dst.tracking(cg).short.address().offset(opts.disp));
41626 var val_ptr = try cg.tempInit(.usize, val_mcv.address());
41627 var len = try cg.tempInit(.usize, .{ .immediate = val_ty.abiSize(cg.pt.zcu) });
41628 try dst_ptr.memcpy(&val_ptr, &len, cg);
41629 try dst_ptr.die(cg);
41630 try val_ptr.die(cg);
41631 try len.die(cg);
41632 },
41633 }
41634 break;
41635 }
41636 }
41637
41638 fn loadReg(ptr: *Temp, dst_ty: Type, dst_reg: Register, cg: *CodeGen) !void {
41639 const dst_rc = dst_reg.class();
41640 const strat = try cg.moveStrategy(dst_ty, dst_rc, false);
41641 // hack around linker relocation bugs
41642 switch (ptr.tracking(cg).short) {
41643 else => {},
41644 .lea_symbol => |sym_off| if (dst_rc != .general_purpose or sym_off.off != 0)
41645 while (try ptr.toRegClass(false, .general_purpose, cg)) {},
41646 }
41647 try strat.read(cg, dst_reg, try ptr.tracking(cg).short.deref().mem(cg, .{
41648 .size = .fromBitSize(@min(8 * dst_ty.abiSize(cg.pt.zcu), dst_reg.bitSize())),
41649 }));
41650 }
41651
41652 fn storeRegs(ptr: *Temp, src_ty: Type, src_regs: []const Register, cg: *CodeGen) !void {
41653 var part_disp: u31 = 0;
41654 var deferred_disp: u31 = 0;
41655 var src_abi_size: u32 = @intCast(src_ty.abiSize(cg.pt.zcu));
41656 for (src_regs) |src_reg| {
41657 const src_rc = src_reg.class();
41658 const part_bit_size = @min(8 * src_abi_size, src_reg.bitSize());
41659 const part_size = @divExact(part_bit_size, 8);
41660 if (src_rc == .x87 or std.math.isPowerOfTwo(part_size)) {
41661 // hack around linker relocation bugs
41662 switch (ptr.tracking(cg).short) {
41663 else => {},
41664 .lea_symbol => while (try ptr.toRegClass(false, .general_purpose, cg)) {},
41665 }
41666 const strat = try cg.moveStrategy(src_ty, src_rc, false);
41667 try strat.write(cg, try ptr.tracking(cg).short.deref().mem(cg, .{
41668 .size = .fromBitSize(part_bit_size),
41669 .disp = part_disp,
41670 }), registerAlias(src_reg, part_size));
41671 } else {
41672 const frame_size = std.math.ceilPowerOfTwoAssert(u32, part_size);
41673 const frame_index = try cg.allocFrameIndex(.init(.{
41674 .size = frame_size,
41675 .alignment = .fromNonzeroByteUnits(frame_size),
41676 }));
41677 const strat = try cg.moveStrategy(src_ty, src_rc, true);
41678 try strat.write(cg, .{
41679 .base = .{ .frame = frame_index },
41680 .mod = .{ .rm = .{ .size = .fromSize(frame_size) } },
41681 }, registerAlias(src_reg, frame_size));
41682 try ptr.toOffset(deferred_disp, cg);
41683 deferred_disp = 0;
41684 var src_ptr = try cg.tempInit(.usize, .{ .lea_frame = .{ .index = frame_index } });
41685 var len = try cg.tempInit(.usize, .{ .immediate = src_abi_size });
41686 try ptr.memcpy(&src_ptr, &len, cg);
41687 try src_ptr.die(cg);
41688 try len.die(cg);
41689 }
41690 part_disp += part_size;
41691 deferred_disp += part_size;
41692 src_abi_size -= part_size;
41693 }
41694 }
41695
41696 fn readReg(src: Temp, disp: i32, dst_ty: Type, dst_reg: Register, cg: *CodeGen) !void {
41697 const strat = try cg.moveStrategy(dst_ty, dst_reg.class(), false);
41698 try strat.read(cg, dst_reg, try src.tracking(cg).short.mem(cg, .{
41699 .size = .fromBitSize(@min(8 * dst_ty.abiSize(cg.pt.zcu), dst_reg.bitSize())),
41700 .disp = disp,
41701 }));
41702 }
41703
41704 fn writeRegs(dst: Temp, disp: i32, src_ty: Type, src_regs: []const Register, cg: *CodeGen) !void {
41705 var part_disp = disp;
41706 var src_abi_size: u32 = @intCast(src_ty.abiSize(cg.pt.zcu));
41707 for (src_regs) |src_reg| {
41708 const src_rc = src_reg.class();
41709 const part_bit_size = @min(8 * src_abi_size, src_reg.bitSize());
41710 const part_size = @divExact(part_bit_size, 8);
41711 if (src_rc == .x87 or std.math.isPowerOfTwo(part_size)) {
41712 const strat = try cg.moveStrategy(src_ty, src_rc, false);
41713 try strat.write(cg, try dst.tracking(cg).short.mem(cg, .{
41714 .size = .fromBitSize(part_bit_size),
41715 .disp = part_disp,
41716 }), registerAlias(src_reg, part_size));
41717 } else {
41718 const frame_size = std.math.ceilPowerOfTwoAssert(u32, part_size);
41719 const frame_index = try cg.allocFrameIndex(.init(.{
41720 .size = frame_size,
41721 .alignment = .fromNonzeroByteUnits(frame_size),
41722 }));
41723 const strat = try cg.moveStrategy(src_ty, src_rc, true);
41724 try strat.write(cg, .{
41725 .base = .{ .frame = frame_index },
41726 .mod = .{ .rm = .{ .size = .fromSize(frame_size) } },
41727 }, registerAlias(src_reg, frame_size));
41728 var dst_ptr = try cg.tempInit(.usize, dst.tracking(cg).short.address());
41729 try dst_ptr.toOffset(part_disp, cg);
41730 var src_ptr = try cg.tempInit(.usize, .{ .lea_frame = .{ .index = frame_index } });
41731 var len = try cg.tempInit(.usize, .{ .immediate = src_abi_size });
41732 try dst_ptr.memcpy(&src_ptr, &len, cg);
41733 try dst_ptr.die(cg);
41734 try src_ptr.die(cg);
41735 try len.die(cg);
41736 }
41737 part_disp += part_size;
41738 src_abi_size -= part_size;
42336 if (switch (op) {
42337 .lt, .gt, .eq => false,
42338 .lte, .gte, .neq => true,
42339 }) {
42340 const cc = &res[0].unwrap(cg).temp.tracking(cg).short.eflags;
42341 cc.* = cc.negate();
4173942342 }
41740 }
41741
41742 fn memcpy(dst: *Temp, src: *Temp, len: *Temp, cg: *CodeGen) !void {
41743 while (true) for ([_]*Temp{ dst, src, len }, [_]Register{ .rdi, .rsi, .rcx }) |temp, reg| {
41744 if (try temp.toReg(reg, cg)) break;
41745 } else break;
41746 try cg.asmOpOnly(.{ .@"rep _sb", .mov });
41747 }
41748
41749 fn memset(dst: *Temp, val: *Temp, len: *Temp, cg: *CodeGen) !void {
41750 while (true) for ([_]*Temp{ dst, val, len }, [_]Register{ .rdi, .rax, .rcx }) |temp, reg| {
41751 if (try temp.toReg(reg, cg)) break;
41752 } else break;
41753 try cg.asmOpOnly(.{ .@"rep _sb", .sto });
42343 lhs.*, rhs.* = ops;
42344 return res[0];
4175442345 }
4175542346
4175642347 fn finish(
......@@ -41790,7 +42381,7 @@ const Temp = struct {
4179042381 }
4179142382 }
4179242383
41793 const Index = enum(u4) {
42384 const Index = enum(u5) {
4179442385 _,
4179542386
4179642387 fn toIndex(index: Index) Air.Inst.Index {
......@@ -41837,12 +42428,20 @@ const Temp = struct {
4183742428 };
4183842429};
4183942430
41840fn resetTemps(cg: *CodeGen) void {
42431fn resetTemps(cg: *CodeGen) !void {
42432 var any_valid = false;
4184142433 for (0..@intFromEnum(cg.next_temp_index)) |temp_index| {
4184242434 const temp: Temp.Index = @enumFromInt(temp_index);
41843 assert(!temp.isValid(cg));
42435 if (temp.isValid(cg)) {
42436 any_valid = true;
42437 tracking_log.err("failed to kill {}: {}", .{
42438 temp.toIndex(),
42439 cg.temp_type[temp_index].fmt(cg.pt),
42440 });
42441 }
4184442442 cg.temp_type[temp_index] = undefined;
4184542443 }
42444 if (any_valid) return cg.fail("failed to kill all temps", .{});
4184642445 cg.next_temp_index = @enumFromInt(0);
4184742446}
4184842447
......@@ -42228,6 +42827,8 @@ const Select = struct {
4222842827 signed_po2_int: Memory.Size,
4222942828 unsigned_po2_or_exact_int: Memory.Size,
4223042829 remainder_int: OfIsSizes,
42830 signed_remainder_int: OfIsSizes,
42831 unsigned_remainder_int: OfIsSizes,
4223142832 exact_remainder_int: OfIsSizes,
4223242833 signed_or_exact_remainder_int: OfIsSizes,
4223342834 unsigned_or_exact_remainder_int: OfIsSizes,
......@@ -42342,6 +42943,10 @@ const Select = struct {
4234242943 of_is.is.bitSize(cg.target) >= (int_info.bits - 1) % of_is.of.bitSize(cg.target) + 1
4234342944 else
4234442945 false,
42946 .signed_remainder_int => |of_is| if (cg.intInfo(ty)) |int_info| int_info.signedness == .signed and
42947 of_is.is.bitSize(cg.target) >= (int_info.bits - 1) % of_is.of.bitSize(cg.target) + 1 else false,
42948 .unsigned_remainder_int => |of_is| if (cg.intInfo(ty)) |int_info| int_info.signedness == .unsigned and
42949 of_is.is.bitSize(cg.target) >= (int_info.bits - 1) % of_is.of.bitSize(cg.target) + 1 else false,
4234542950 .exact_remainder_int => |of_is| if (cg.intInfo(ty)) |int_info|
4234642951 of_is.is.bitSize(cg.target) == (int_info.bits - 1) % of_is.of.bitSize(cg.target) + 1
4234742952 else
......@@ -42749,6 +43354,7 @@ const Select = struct {
4274943354 const add_size: Adjust = .{ .sign = .pos, .lhs = .size, .op = .mul, .rhs = .@"1" };
4275043355 const add_size_div_8: Adjust = .{ .sign = .pos, .lhs = .size, .op = .div, .rhs = .@"8" };
4275143356 const sub_size_div_8: Adjust = .{ .sign = .neg, .lhs = .size, .op = .div, .rhs = .@"8" };
43357 const sub_size_div_4: Adjust = .{ .sign = .neg, .lhs = .size, .op = .div, .rhs = .@"4" };
4275243358 const sub_size: Adjust = .{ .sign = .neg, .lhs = .size, .op = .mul, .rhs = .@"1" };
4275343359 const add_size_sub_elem_size: Adjust = .{ .sign = .pos, .lhs = .size_sub_elem_size, .op = .mul, .rhs = .@"1" };
4275443360 const add_src0_unaligned_size: Adjust = .{ .sign = .pos, .lhs = .src0_unaligned_size, .op = .mul, .rhs = .@"1" };
src/arch/x86_64/bits.zig-2
......@@ -159,7 +159,6 @@ pub const Condition = enum(u5) {
159159 .ae => .be,
160160 .b => .a,
161161 .be => .ae,
162 .c => .a,
163162 .g => .l,
164163 .ge => .le,
165164 .l => .g,
......@@ -168,7 +167,6 @@ pub const Condition = enum(u5) {
168167 .nae => .nbe,
169168 .nb => .na,
170169 .nbe => .nae,
171 .nc => .na,
172170 .ng => .nl,
173171 .nge => .nle,
174172 .nl => .ng,
test/behavior/x86_64/math.zig+487-77
......@@ -17,7 +17,14 @@ const Sse = if (std.Target.x86.featureSetHas(builtin.cpu.features, .avx))
1717else
1818 @Vector(16, u8);
1919
20inline fn sign(rhs: anytype) switch (@typeInfo(@TypeOf(rhs))) {
20inline fn runtime(comptime Type: type, comptime value: Type) Type {
21 if (@inComptime()) return value;
22 return struct {
23 var variable: Type = value;
24 }.variable;
25}
26
27fn sign(rhs: anytype) switch (@typeInfo(@TypeOf(rhs))) {
2128 else => bool,
2229 .vector => |vector| @Vector(vector.len, bool),
2330} {
......@@ -39,7 +46,7 @@ inline fn sign(rhs: anytype) switch (@typeInfo(@TypeOf(rhs))) {
3946 },
4047 }
4148}
42inline fn boolAnd(lhs: anytype, rhs: @TypeOf(lhs)) @TypeOf(lhs) {
49fn boolAnd(lhs: anytype, rhs: @TypeOf(lhs)) @TypeOf(lhs) {
4350 switch (@typeInfo(@TypeOf(lhs))) {
4451 .bool => return lhs and rhs,
4552 .vector => |vector| switch (vector.child) {
......@@ -55,7 +62,7 @@ inline fn boolAnd(lhs: anytype, rhs: @TypeOf(lhs)) @TypeOf(lhs) {
5562 }
5663 @compileError("unsupported boolAnd type: " ++ @typeName(@TypeOf(lhs)));
5764}
58inline fn boolOr(lhs: anytype, rhs: @TypeOf(lhs)) @TypeOf(lhs) {
65fn boolOr(lhs: anytype, rhs: @TypeOf(lhs)) @TypeOf(lhs) {
5966 switch (@typeInfo(@TypeOf(lhs))) {
6067 .bool => return lhs or rhs,
6168 .vector => |vector| switch (vector.child) {
......@@ -186,6 +193,90 @@ fn unary(comptime op: anytype, comptime opts: struct { strict: bool = false }) t
186193 );
187194 }
188195 fn testIntTypes() !void {
196 try testArgs(i1, undefined);
197 try testArgs(u1, undefined);
198 try testArgs(i2, undefined);
199 try testArgs(u2, undefined);
200 try testArgs(i3, undefined);
201 try testArgs(u3, undefined);
202 try testArgs(i4, undefined);
203 try testArgs(u4, undefined);
204 try testArgs(i5, undefined);
205 try testArgs(u5, undefined);
206 try testArgs(i7, undefined);
207 try testArgs(u7, undefined);
208 try testArgs(i8, undefined);
209 try testArgs(u8, undefined);
210 try testArgs(i9, undefined);
211 try testArgs(u9, undefined);
212 try testArgs(i15, undefined);
213 try testArgs(u15, undefined);
214 try testArgs(i16, undefined);
215 try testArgs(u16, undefined);
216 try testArgs(i17, undefined);
217 try testArgs(u17, undefined);
218 try testArgs(i31, undefined);
219 try testArgs(u31, undefined);
220 try testArgs(i32, undefined);
221 try testArgs(u32, undefined);
222 try testArgs(i33, undefined);
223 try testArgs(u33, undefined);
224 try testArgs(i63, undefined);
225 try testArgs(u63, undefined);
226 try testArgs(i64, undefined);
227 try testArgs(u64, undefined);
228 try testArgs(i65, undefined);
229 try testArgs(u65, undefined);
230 try testArgs(i95, undefined);
231 try testArgs(u95, undefined);
232 try testArgs(i96, undefined);
233 try testArgs(u96, undefined);
234 try testArgs(i97, undefined);
235 try testArgs(u97, undefined);
236 try testArgs(i127, undefined);
237 try testArgs(u127, undefined);
238 try testArgs(i128, undefined);
239 try testArgs(u128, undefined);
240 try testArgs(i129, undefined);
241 try testArgs(u129, undefined);
242 try testArgs(i159, undefined);
243 try testArgs(u159, undefined);
244 try testArgs(i160, undefined);
245 try testArgs(u160, undefined);
246 try testArgs(i161, undefined);
247 try testArgs(u161, undefined);
248 try testArgs(i191, undefined);
249 try testArgs(u191, undefined);
250 try testArgs(i192, undefined);
251 try testArgs(u192, undefined);
252 try testArgs(i193, undefined);
253 try testArgs(u193, undefined);
254 try testArgs(i223, undefined);
255 try testArgs(u223, undefined);
256 try testArgs(i224, undefined);
257 try testArgs(u224, undefined);
258 try testArgs(i225, undefined);
259 try testArgs(u225, undefined);
260 try testArgs(i255, undefined);
261 try testArgs(u255, undefined);
262 try testArgs(i256, undefined);
263 try testArgs(u256, undefined);
264 try testArgs(i257, undefined);
265 try testArgs(u257, undefined);
266 try testArgs(i511, undefined);
267 try testArgs(u511, undefined);
268 try testArgs(i512, undefined);
269 try testArgs(u512, undefined);
270 try testArgs(i513, undefined);
271 try testArgs(u513, undefined);
272 try testArgs(i1023, undefined);
273 try testArgs(u1023, undefined);
274 try testArgs(i1024, undefined);
275 try testArgs(u1024, undefined);
276 try testArgs(i1025, undefined);
277 try testArgs(u1025, undefined);
278 }
279 fn testInts() !void {
189280 try testArgs(i1, -1);
190281 try testArgs(i1, 0);
191282 try testArgs(u1, 0);
......@@ -549,6 +640,13 @@ fn unary(comptime op: anytype, comptime opts: struct { strict: bool = false }) t
549640 try testArgs(u1025, 1 << 1024);
550641 }
551642 fn testFloatTypes() !void {
643 try testArgs(f16, undefined);
644 try testArgs(f32, undefined);
645 try testArgs(f64, undefined);
646 try testArgs(f80, undefined);
647 try testArgs(f128, undefined);
648 }
649 fn testFloats() !void {
552650 try testArgs(f16, -nan(f16));
553651 try testArgs(f16, -inf(f16));
554652 try testArgs(f16, -fmax(f16));
......@@ -645,6 +743,168 @@ fn unary(comptime op: anytype, comptime opts: struct { strict: bool = false }) t
645743 try testArgs(f128, nan(f128));
646744 }
647745 fn testIntVectorTypes() !void {
746 try testArgs(@Vector(3, i1), undefined);
747 try testArgs(@Vector(3, u1), undefined);
748 try testArgs(@Vector(3, i2), undefined);
749 try testArgs(@Vector(3, u2), undefined);
750 try testArgs(@Vector(3, i3), undefined);
751 try testArgs(@Vector(3, u3), undefined);
752 try testArgs(@Vector(3, i4), undefined);
753 try testArgs(@Vector(1, i4), undefined);
754 try testArgs(@Vector(2, i4), undefined);
755 try testArgs(@Vector(4, i4), undefined);
756 try testArgs(@Vector(8, i4), undefined);
757 try testArgs(@Vector(16, i4), undefined);
758 try testArgs(@Vector(32, i4), undefined);
759 try testArgs(@Vector(64, i4), undefined);
760 try testArgs(@Vector(128, i4), undefined);
761 try testArgs(@Vector(256, i4), undefined);
762 try testArgs(@Vector(3, u4), undefined);
763 try testArgs(@Vector(1, u4), undefined);
764 try testArgs(@Vector(2, u4), undefined);
765 try testArgs(@Vector(4, u4), undefined);
766 try testArgs(@Vector(8, u4), undefined);
767 try testArgs(@Vector(16, u4), undefined);
768 try testArgs(@Vector(32, u4), undefined);
769 try testArgs(@Vector(64, u4), undefined);
770 try testArgs(@Vector(128, u4), undefined);
771 try testArgs(@Vector(256, u4), undefined);
772 try testArgs(@Vector(3, i5), undefined);
773 try testArgs(@Vector(3, u5), undefined);
774 try testArgs(@Vector(3, i7), undefined);
775 try testArgs(@Vector(3, u7), undefined);
776 try testArgs(@Vector(3, i8), undefined);
777 try testArgs(@Vector(1, i8), undefined);
778 try testArgs(@Vector(2, i8), undefined);
779 try testArgs(@Vector(4, i8), undefined);
780 try testArgs(@Vector(8, i8), undefined);
781 try testArgs(@Vector(16, i8), undefined);
782 try testArgs(@Vector(32, i8), undefined);
783 try testArgs(@Vector(64, i8), undefined);
784 try testArgs(@Vector(128, i8), undefined);
785 try testArgs(@Vector(3, u8), undefined);
786 try testArgs(@Vector(1, u8), undefined);
787 try testArgs(@Vector(2, u8), undefined);
788 try testArgs(@Vector(4, u8), undefined);
789 try testArgs(@Vector(8, u8), undefined);
790 try testArgs(@Vector(16, u8), undefined);
791 try testArgs(@Vector(32, u8), undefined);
792 try testArgs(@Vector(64, u8), undefined);
793 try testArgs(@Vector(128, u8), undefined);
794 try testArgs(@Vector(3, i9), undefined);
795 try testArgs(@Vector(3, u9), undefined);
796 try testArgs(@Vector(3, i15), undefined);
797 try testArgs(@Vector(3, u15), undefined);
798 try testArgs(@Vector(3, i16), undefined);
799 try testArgs(@Vector(1, i16), undefined);
800 try testArgs(@Vector(2, i16), undefined);
801 try testArgs(@Vector(4, i16), undefined);
802 try testArgs(@Vector(8, i16), undefined);
803 try testArgs(@Vector(16, i16), undefined);
804 try testArgs(@Vector(32, i16), undefined);
805 try testArgs(@Vector(64, i16), undefined);
806 try testArgs(@Vector(3, u16), undefined);
807 try testArgs(@Vector(1, u16), undefined);
808 try testArgs(@Vector(2, u16), undefined);
809 try testArgs(@Vector(4, u16), undefined);
810 try testArgs(@Vector(8, u16), undefined);
811 try testArgs(@Vector(16, u16), undefined);
812 try testArgs(@Vector(32, u16), undefined);
813 try testArgs(@Vector(64, u16), undefined);
814 try testArgs(@Vector(3, i17), undefined);
815 try testArgs(@Vector(3, u17), undefined);
816 try testArgs(@Vector(3, i31), undefined);
817 try testArgs(@Vector(3, u31), undefined);
818 try testArgs(@Vector(3, i32), undefined);
819 try testArgs(@Vector(1, i32), undefined);
820 try testArgs(@Vector(2, i32), undefined);
821 try testArgs(@Vector(4, i32), undefined);
822 try testArgs(@Vector(8, i32), undefined);
823 try testArgs(@Vector(16, i32), undefined);
824 try testArgs(@Vector(32, i32), undefined);
825 try testArgs(@Vector(3, u32), undefined);
826 try testArgs(@Vector(1, u32), undefined);
827 try testArgs(@Vector(2, u32), undefined);
828 try testArgs(@Vector(4, u32), undefined);
829 try testArgs(@Vector(8, u32), undefined);
830 try testArgs(@Vector(16, u32), undefined);
831 try testArgs(@Vector(32, u32), undefined);
832 try testArgs(@Vector(3, i33), undefined);
833 try testArgs(@Vector(3, u33), undefined);
834 try testArgs(@Vector(3, i63), undefined);
835 try testArgs(@Vector(3, u63), undefined);
836 try testArgs(@Vector(3, i64), undefined);
837 try testArgs(@Vector(1, i64), undefined);
838 try testArgs(@Vector(2, i64), undefined);
839 try testArgs(@Vector(4, i64), undefined);
840 try testArgs(@Vector(8, i64), undefined);
841 try testArgs(@Vector(16, i64), undefined);
842 try testArgs(@Vector(3, u64), undefined);
843 try testArgs(@Vector(1, u64), undefined);
844 try testArgs(@Vector(2, u64), undefined);
845 try testArgs(@Vector(4, u64), undefined);
846 try testArgs(@Vector(8, u64), undefined);
847 try testArgs(@Vector(16, u64), undefined);
848 try testArgs(@Vector(3, i65), undefined);
849 try testArgs(@Vector(3, u65), undefined);
850 try testArgs(@Vector(3, i127), undefined);
851 try testArgs(@Vector(3, u127), undefined);
852 try testArgs(@Vector(3, i128), undefined);
853 try testArgs(@Vector(1, i128), undefined);
854 try testArgs(@Vector(2, i128), undefined);
855 try testArgs(@Vector(4, i128), undefined);
856 try testArgs(@Vector(8, i128), undefined);
857 try testArgs(@Vector(3, u128), undefined);
858 try testArgs(@Vector(1, u128), undefined);
859 try testArgs(@Vector(2, u128), undefined);
860 try testArgs(@Vector(4, u128), undefined);
861 try testArgs(@Vector(8, u128), undefined);
862 try testArgs(@Vector(3, i129), undefined);
863 try testArgs(@Vector(3, u129), undefined);
864 try testArgs(@Vector(3, i191), undefined);
865 try testArgs(@Vector(3, u191), undefined);
866 try testArgs(@Vector(3, i192), undefined);
867 try testArgs(@Vector(1, i192), undefined);
868 try testArgs(@Vector(2, i192), undefined);
869 try testArgs(@Vector(4, i192), undefined);
870 try testArgs(@Vector(3, u192), undefined);
871 try testArgs(@Vector(1, u192), undefined);
872 try testArgs(@Vector(2, u192), undefined);
873 try testArgs(@Vector(4, u192), undefined);
874 try testArgs(@Vector(3, i193), undefined);
875 try testArgs(@Vector(3, u193), undefined);
876 try testArgs(@Vector(3, i255), undefined);
877 try testArgs(@Vector(3, u255), undefined);
878 try testArgs(@Vector(3, i256), undefined);
879 try testArgs(@Vector(1, i256), undefined);
880 try testArgs(@Vector(2, i256), undefined);
881 try testArgs(@Vector(4, i256), undefined);
882 try testArgs(@Vector(3, u256), undefined);
883 try testArgs(@Vector(1, u256), undefined);
884 try testArgs(@Vector(2, u256), undefined);
885 try testArgs(@Vector(4, u256), undefined);
886 try testArgs(@Vector(3, i257), undefined);
887 try testArgs(@Vector(3, u257), undefined);
888 try testArgs(@Vector(3, i511), undefined);
889 try testArgs(@Vector(3, u511), undefined);
890 try testArgs(@Vector(3, i512), undefined);
891 try testArgs(@Vector(1, i512), undefined);
892 try testArgs(@Vector(2, i512), undefined);
893 try testArgs(@Vector(3, u512), undefined);
894 try testArgs(@Vector(1, u512), undefined);
895 try testArgs(@Vector(2, u512), undefined);
896 try testArgs(@Vector(3, i513), undefined);
897 try testArgs(@Vector(3, u513), undefined);
898 try testArgs(@Vector(3, i1023), undefined);
899 try testArgs(@Vector(3, u1023), undefined);
900 try testArgs(@Vector(3, i1024), undefined);
901 try testArgs(@Vector(1, i1024), undefined);
902 try testArgs(@Vector(3, u1024), undefined);
903 try testArgs(@Vector(1, u1024), undefined);
904 try testArgs(@Vector(3, i1025), undefined);
905 try testArgs(@Vector(3, u1025), undefined);
906 }
907 fn testIntVectors() !void {
648908 try testArgs(@Vector(3, i1), .{ -1 << 0, -1, 0 });
649909 try testArgs(@Vector(3, u1), .{ 0, 1, 1 << 0 });
650910
......@@ -1190,6 +1450,38 @@ fn unary(comptime op: anytype, comptime opts: struct { strict: bool = false }) t
11901450 try testArgs(@Vector(3, u1025), .{ 0, 1, 1 << 1024 });
11911451 }
11921452 fn testFloatVectorTypes() !void {
1453 try testArgs(@Vector(1, f16), undefined);
1454 try testArgs(@Vector(2, f16), undefined);
1455 try testArgs(@Vector(4, f16), undefined);
1456 try testArgs(@Vector(8, f16), undefined);
1457 try testArgs(@Vector(16, f16), undefined);
1458 try testArgs(@Vector(32, f16), undefined);
1459 try testArgs(@Vector(64, f16), undefined);
1460
1461 try testArgs(@Vector(1, f32), undefined);
1462 try testArgs(@Vector(2, f32), undefined);
1463 try testArgs(@Vector(4, f32), undefined);
1464 try testArgs(@Vector(8, f32), undefined);
1465 try testArgs(@Vector(16, f32), undefined);
1466 try testArgs(@Vector(32, f32), undefined);
1467
1468 try testArgs(@Vector(1, f64), undefined);
1469 try testArgs(@Vector(2, f64), undefined);
1470 try testArgs(@Vector(4, f64), undefined);
1471 try testArgs(@Vector(8, f64), undefined);
1472 try testArgs(@Vector(16, f64), undefined);
1473
1474 try testArgs(@Vector(1, f80), undefined);
1475 try testArgs(@Vector(2, f80), undefined);
1476 try testArgs(@Vector(4, f80), undefined);
1477 try testArgs(@Vector(8, f80), undefined);
1478
1479 try testArgs(@Vector(1, f128), undefined);
1480 try testArgs(@Vector(2, f128), undefined);
1481 try testArgs(@Vector(4, f128), undefined);
1482 try testArgs(@Vector(8, f128), undefined);
1483 }
1484 fn testFloatVectors() !void {
11931485 try testArgs(@Vector(1, f16), .{
11941486 -0x1.17cp-12,
11951487 });
......@@ -1396,7 +1688,7 @@ fn binary(comptime op: anytype, comptime opts: struct { strict: bool = false })
13961688 imm_rhs,
13971689 );
13981690 }
1399 fn testIntTypes() !void {
1691 fn testInts() !void {
14001692 try testArgs(i8, 0x48, 0x6c);
14011693 try testArgs(u8, 0xbb, 0x43);
14021694 try testArgs(i16, -0x0fdf, 0x302e);
......@@ -1407,48 +1699,16 @@ fn binary(comptime op: anytype, comptime opts: struct { strict: bool = false })
14071699 try testArgs(u64, 0x71138bc6b4a38898, 0x1bc4043de9438c7b);
14081700 try testArgs(i128, 0x76d428c46cdeaa2ac43de8abffb22f6d, 0x427f7545abe434a12544fdbe2a012889);
14091701 try testArgs(u128, 0xe05fc132ef2cd8affee00a907f0a851f, 0x29f912a72cfc6a7c6973426a9636da9a);
1410 try testArgs(
1411 i256,
1412 -0x53d4148cee74ea43477a65b3daa7b8fdadcbf4508e793f4af113b8d8da5a7eb6,
1413 -0x30dcbaf7b9b7a3df033694e6795444d842fb0b8f79bc18b3ea8a6b7ccad3ea91,
1414 );
1415 try testArgs(
1416 u256,
1417 0xb7935f5c2f3b1ae7a422c0a7c446884294b7d5370bada307d2fe5a4c4284a999,
1418 0x310e6e196ba4f143b8d285ca6addf7f3bb3344224aff221b27607a31e148be08,
1419 );
1420 try testArgs(
1421 i258,
1422 -0x0eee283365108dbeea0bec82f5147418d8ffe86f9eed00e414b4eccd65c21239a,
1423 -0x122c730073fc29a24cd6e3e6263566879bc5325d8566b8db31fcb4a76f7ab95eb,
1424 );
1425 try testArgs(
1426 u258,
1427 0x186d5ddaab8cb8cb04e5b41e36f812e039d008baf49f12894c39e29a07796d800,
1428 0x2072daba6ffad168826163eb136f6d28ca4360c8e7e5e41e29755e19e4753a4f5,
1429 );
1430 try testArgs(
1431 i495,
1432 0x2fe6bc5448c55ce18252e2c9d44777505dfe63ff249a8027a6626c7d8dd9893fd5731e51474727be556f757facb586a4e04bbc0148c6c7ad692302f46fbd,
1433 -0x016a358821ef8240172f3a08e8830c06e6bcf2225f5f4d41ed42b44d249385f55cc594e1278ecac31c73faed890e5054af1a561483bb1bb6fb1f753514cf,
1434 );
1435 try testArgs(
1436 u495,
1437 0x6eaf4e252b3bf74b75bac59e0b43ca5326bad2a25b3fdb74a67ef132ac5e47d72eebc3316fb2351ee66c50dc5afb92a75cea9b0e35160652c7db39eeb158,
1438 0x49fbed744a92b549d8c05bb3512c617d24dd824f3f69bdf3923bc326a75674b85f5b828d2566fab9c86f571d12c2a63c9164feb0d191d27905533d09622a,
1439 );
1440 try testArgs(
1441 i512,
1442 -0x3a6876ca92775286c6e1504a64a9b8d56985bebf4a1b66539d404e0e96f24b226f70c4bcff295fdc2043b82513b2052dc45fd78f7e9e80e5b3e101757289f054,
1443 0x5080c516a819bd32a0a5f0976441bbfbcf89e77684f1f10eb326aeb28e1f8d593278cff60fc99b8ffc87d8696882c64728dd3c322b7142803f4341f85a03bc10,
1444 );
1445 try testArgs(
1446 u512,
1447 0xe5b1fedca3c77db765e517aabd05ffc524a3a8aff1784bbf67c45b894447ede32b65b9940e78173c591e56e078932d465f235aece7ad47b7f229df7ba8f12295,
1448 0x8b4bb7c2969e3b121cc1082c442f8b4330f0a50058438fed56447175bb10178607ecfe425cb54dacc25ef26810f3e04681de1844f1aa8d029aca75d658634806,
1449 );
1702 try testArgs(i256, -0x53d4148cee74ea43477a65b3daa7b8fdadcbf4508e793f4af113b8d8da5a7eb6, -0x30dcbaf7b9b7a3df033694e6795444d842fb0b8f79bc18b3ea8a6b7ccad3ea91);
1703 try testArgs(u256, 0xb7935f5c2f3b1ae7a422c0a7c446884294b7d5370bada307d2fe5a4c4284a999, 0x310e6e196ba4f143b8d285ca6addf7f3bb3344224aff221b27607a31e148be08);
1704 try testArgs(i258, -0x0eee283365108dbeea0bec82f5147418d8ffe86f9eed00e414b4eccd65c21239a, -0x122c730073fc29a24cd6e3e6263566879bc5325d8566b8db31fcb4a76f7ab95eb);
1705 try testArgs(u258, 0x186d5ddaab8cb8cb04e5b41e36f812e039d008baf49f12894c39e29a07796d800, 0x2072daba6ffad168826163eb136f6d28ca4360c8e7e5e41e29755e19e4753a4f5);
1706 try testArgs(i495, 0x2fe6bc5448c55ce18252e2c9d44777505dfe63ff249a8027a6626c7d8dd9893fd5731e51474727be556f757facb586a4e04bbc0148c6c7ad692302f46fbd, -0x016a358821ef8240172f3a08e8830c06e6bcf2225f5f4d41ed42b44d249385f55cc594e1278ecac31c73faed890e5054af1a561483bb1bb6fb1f753514cf);
1707 try testArgs(u495, 0x6eaf4e252b3bf74b75bac59e0b43ca5326bad2a25b3fdb74a67ef132ac5e47d72eebc3316fb2351ee66c50dc5afb92a75cea9b0e35160652c7db39eeb158, 0x49fbed744a92b549d8c05bb3512c617d24dd824f3f69bdf3923bc326a75674b85f5b828d2566fab9c86f571d12c2a63c9164feb0d191d27905533d09622a);
1708 try testArgs(i512, -0x3a6876ca92775286c6e1504a64a9b8d56985bebf4a1b66539d404e0e96f24b226f70c4bcff295fdc2043b82513b2052dc45fd78f7e9e80e5b3e101757289f054, 0x5080c516a819bd32a0a5f0976441bbfbcf89e77684f1f10eb326aeb28e1f8d593278cff60fc99b8ffc87d8696882c64728dd3c322b7142803f4341f85a03bc10);
1709 try testArgs(u512, 0xe5b1fedca3c77db765e517aabd05ffc524a3a8aff1784bbf67c45b894447ede32b65b9940e78173c591e56e078932d465f235aece7ad47b7f229df7ba8f12295, 0x8b4bb7c2969e3b121cc1082c442f8b4330f0a50058438fed56447175bb10178607ecfe425cb54dacc25ef26810f3e04681de1844f1aa8d029aca75d658634806);
14501710 }
1451 fn testFloatTypes() !void {
1711 fn testFloats() !void {
14521712 @setEvalBranchQuota(21_700);
14531713
14541714 try testArgs(f16, -nan(f16), -nan(f16));
......@@ -3161,7 +3421,7 @@ fn binary(comptime op: anytype, comptime opts: struct { strict: bool = false })
31613421 try testArgs(f128, nan(f128), inf(f128));
31623422 try testArgs(f128, nan(f128), nan(f128));
31633423 }
3164 fn testIntVectorTypes() !void {
3424 fn testIntVectors() !void {
31653425 try testArgs(@Vector(1, i8), .{
31663426 -0x54,
31673427 }, .{
......@@ -3781,7 +4041,7 @@ fn binary(comptime op: anytype, comptime opts: struct { strict: bool = false })
37814041 0xf1e3bbe031d59351770a7a501b6e969b2c00d144f17648db3f944b69dfeb7be72e5ff933a061eba4eaa422f8ca09e5a97d0b0dd740fd4076eba8c72d7a278523f399202dc2d043c4e0eb58a2bcd4066e2146e321810b1ee4d3afdddb4f026bcc7905ce17e033a7727b4e08f33b53c63d8c9f763fc6c31d0523eb38c30d5e40bc,
37824042 });
37834043 }
3784 fn testFloatVectorTypes() !void {
4044 fn testFloatVectors() !void {
37854045 @setEvalBranchQuota(21_700);
37864046
37874047 try testArgs(@Vector(1, f16), .{
......@@ -4178,76 +4438,226 @@ inline fn bitNot(comptime Type: type, rhs: Type) @TypeOf(~rhs) {
41784438 return ~rhs;
41794439}
41804440test bitNot {
4181 const t = unary(bitNot, .{});
4182 try t.testIntTypes();
4183 try t.testIntVectorTypes();
4441 const test_bit_not = unary(bitNot, .{});
4442 try test_bit_not.testInts();
4443 try test_bit_not.testIntVectors();
41844444}
41854445
41864446inline fn abs(comptime Type: type, rhs: Type) @TypeOf(@abs(rhs)) {
41874447 return @abs(rhs);
41884448}
41894449test abs {
4190 const t = unary(abs, .{ .strict = true });
4191 try t.testIntTypes();
4192 try t.testIntVectorTypes();
4193 try t.testFloatTypes();
4194 try t.testFloatVectorTypes();
4450 const test_abs = unary(abs, .{ .strict = true });
4451 try test_abs.testInts();
4452 try test_abs.testIntVectors();
4453 try test_abs.testFloats();
4454 try test_abs.testFloatVectors();
41954455}
41964456
41974457inline fn clz(comptime Type: type, rhs: Type) @TypeOf(@clz(rhs)) {
41984458 return @clz(rhs);
41994459}
42004460test clz {
4201 const t = unary(clz, .{});
4202 try t.testIntTypes();
4203 try t.testIntVectorTypes();
4461 const test_clz = unary(clz, .{});
4462 try test_clz.testInts();
4463 try test_clz.testIntVectors();
4464}
4465
4466inline fn equal(comptime Type: type, lhs: Type, rhs: Type) @TypeOf(lhs == rhs) {
4467 return lhs == rhs;
4468}
4469test equal {
4470 const test_equal = binary(equal, .{});
4471 try test_equal.testInts();
4472 try test_equal.testFloats();
4473}
4474
4475inline fn notEqual(comptime Type: type, lhs: Type, rhs: Type) @TypeOf(lhs != rhs) {
4476 return lhs != rhs;
4477}
4478test notEqual {
4479 const test_not_equal = binary(notEqual, .{});
4480 try test_not_equal.testInts();
4481 try test_not_equal.testFloats();
4482}
4483
4484inline fn lessThan(comptime Type: type, lhs: Type, rhs: Type) @TypeOf(lhs < rhs) {
4485 return lhs < rhs;
4486}
4487test lessThan {
4488 const test_less_than = binary(lessThan, .{});
4489 try test_less_than.testInts();
4490 try test_less_than.testFloats();
4491}
4492
4493inline fn lessThanOrEqual(comptime Type: type, lhs: Type, rhs: Type) @TypeOf(lhs <= rhs) {
4494 return lhs <= rhs;
4495}
4496test lessThanOrEqual {
4497 const test_less_than_or_equal = binary(lessThanOrEqual, .{});
4498 try test_less_than_or_equal.testInts();
4499 try test_less_than_or_equal.testFloats();
4500}
4501
4502inline fn greaterThan(comptime Type: type, lhs: Type, rhs: Type) @TypeOf(lhs > rhs) {
4503 return lhs > rhs;
4504}
4505test greaterThan {
4506 const test_greater_than = binary(greaterThan, .{});
4507 try test_greater_than.testInts();
4508 try test_greater_than.testFloats();
4509}
4510
4511inline fn greaterThanOrEqual(comptime Type: type, lhs: Type, rhs: Type) @TypeOf(lhs >= rhs) {
4512 return lhs >= rhs;
4513}
4514test greaterThanOrEqual {
4515 const test_greater_than_or_equal = binary(greaterThanOrEqual, .{});
4516 try test_greater_than_or_equal.testInts();
4517 try test_greater_than_or_equal.testFloats();
42044518}
42054519
42064520inline fn bitAnd(comptime Type: type, lhs: Type, rhs: Type) @TypeOf(lhs & rhs) {
42074521 return lhs & rhs;
42084522}
42094523test bitAnd {
4210 const t = binary(bitAnd, .{});
4211 try t.testIntTypes();
4212 try t.testIntVectorTypes();
4524 const test_bit_and = binary(bitAnd, .{});
4525 try test_bit_and.testInts();
4526 try test_bit_and.testIntVectors();
42134527}
42144528
42154529inline fn bitOr(comptime Type: type, lhs: Type, rhs: Type) @TypeOf(lhs | rhs) {
42164530 return lhs | rhs;
42174531}
42184532test bitOr {
4219 const t = binary(bitOr, .{});
4220 try t.testIntTypes();
4221 try t.testIntVectorTypes();
4533 const test_bit_or = binary(bitOr, .{});
4534 try test_bit_or.testInts();
4535 try test_bit_or.testIntVectors();
42224536}
42234537
42244538inline fn bitXor(comptime Type: type, lhs: Type, rhs: Type) @TypeOf(lhs ^ rhs) {
42254539 return lhs ^ rhs;
42264540}
42274541test bitXor {
4228 const t = binary(bitXor, .{});
4229 try t.testIntTypes();
4230 try t.testIntVectorTypes();
4542 const test_bit_xor = binary(bitXor, .{});
4543 try test_bit_xor.testInts();
4544 try test_bit_xor.testIntVectors();
42314545}
42324546
42334547inline fn min(comptime Type: type, lhs: Type, rhs: Type) Type {
42344548 return @min(lhs, rhs);
42354549}
42364550test min {
4237 const t = binary(min, .{});
4238 try t.testIntTypes();
4239 try t.testIntVectorTypes();
4240 try t.testFloatTypes();
4241 try t.testFloatVectorTypes();
4551 const test_min = binary(min, .{});
4552 try test_min.testInts();
4553 try test_min.testIntVectors();
4554 try test_min.testFloats();
4555 try test_min.testFloatVectors();
42424556}
42434557
42444558inline fn max(comptime Type: type, lhs: Type, rhs: Type) Type {
42454559 return @max(lhs, rhs);
42464560}
42474561test max {
4248 const t = binary(max, .{});
4249 try t.testIntTypes();
4250 try t.testIntVectorTypes();
4251 try t.testFloatTypes();
4252 try t.testFloatVectorTypes();
4562 const test_max = binary(max, .{});
4563 try test_max.testInts();
4564 try test_max.testIntVectors();
4565 try test_max.testFloats();
4566 try test_max.testFloatVectors();
4567}
4568
4569inline fn nullIsNull(comptime Type: type, _: Type) bool {
4570 return runtime(?Type, null) == null;
4571}
4572test nullIsNull {
4573 const test_null_is_null = unary(nullIsNull, .{});
4574 try test_null_is_null.testIntTypes();
4575 try test_null_is_null.testIntVectorTypes();
4576 try test_null_is_null.testFloatTypes();
4577 try test_null_is_null.testFloatVectorTypes();
4578}
4579
4580inline fn nullIsNotNull(comptime Type: type, _: Type) bool {
4581 return runtime(?Type, null) != null;
4582}
4583test nullIsNotNull {
4584 const test_null_is_not_null = unary(nullIsNotNull, .{});
4585 try test_null_is_not_null.testIntTypes();
4586 try test_null_is_not_null.testIntVectorTypes();
4587 try test_null_is_not_null.testFloatTypes();
4588 try test_null_is_not_null.testFloatVectorTypes();
4589}
4590
4591inline fn optionalIsNull(comptime Type: type, lhs: Type) bool {
4592 return @as(?Type, lhs) == null;
4593}
4594test optionalIsNull {
4595 const test_optional_is_null = unary(optionalIsNull, .{});
4596 try test_optional_is_null.testInts();
4597 try test_optional_is_null.testFloats();
4598}
4599
4600inline fn optionalIsNotNull(comptime Type: type, lhs: Type) bool {
4601 return @as(?Type, lhs) != null;
4602}
4603test optionalIsNotNull {
4604 const test_optional_is_not_null = unary(optionalIsNotNull, .{});
4605 try test_optional_is_not_null.testInts();
4606 try test_optional_is_not_null.testFloats();
4607}
4608
4609inline fn nullEqualNull(comptime Type: type, _: Type) bool {
4610 return runtime(?Type, null) == runtime(?Type, null);
4611}
4612test nullEqualNull {
4613 const test_null_equal_null = unary(nullEqualNull, .{});
4614 try test_null_equal_null.testIntTypes();
4615 try test_null_equal_null.testFloatTypes();
4616}
4617
4618inline fn nullNotEqualNull(comptime Type: type, _: Type) bool {
4619 return runtime(?Type, null) != runtime(?Type, null);
4620}
4621test nullNotEqualNull {
4622 const test_null_not_equal_null = unary(nullNotEqualNull, .{});
4623 try test_null_not_equal_null.testIntTypes();
4624 try test_null_not_equal_null.testFloatTypes();
4625}
4626
4627inline fn optionalEqualNull(comptime Type: type, lhs: Type) bool {
4628 return lhs == runtime(?Type, null);
4629}
4630test optionalEqualNull {
4631 const test_optional_equal_null = unary(optionalEqualNull, .{});
4632 try test_optional_equal_null.testInts();
4633 try test_optional_equal_null.testFloats();
4634}
4635
4636inline fn optionalNotEqualNull(comptime Type: type, lhs: Type) bool {
4637 return lhs != runtime(?Type, null);
4638}
4639test optionalNotEqualNull {
4640 const test_optional_not_equal_null = unary(optionalIsNotNull, .{});
4641 try test_optional_not_equal_null.testInts();
4642 try test_optional_not_equal_null.testFloats();
4643}
4644
4645inline fn optionalsEqual(comptime Type: type, lhs: Type, rhs: Type) bool {
4646 if (@inComptime()) return lhs == rhs; // workaround https://github.com/ziglang/zig/issues/22636
4647 return @as(?Type, lhs) == rhs;
4648}
4649test optionalsEqual {
4650 const test_optionals_equal = binary(optionalsEqual, .{});
4651 try test_optionals_equal.testInts();
4652 try test_optionals_equal.testFloats();
4653}
4654
4655inline fn optionalsNotEqual(comptime Type: type, lhs: Type, rhs: Type) bool {
4656 if (@inComptime()) return lhs != rhs; // workaround https://github.com/ziglang/zig/issues/22636
4657 return lhs != @as(?Type, rhs);
4658}
4659test optionalsNotEqual {
4660 const test_optionals_not_equal = binary(optionalsNotEqual, .{});
4661 try test_optionals_not_equal.testInts();
4662 try test_optionals_not_equal.testFloats();
42534663}