| ... | ... | @@ -210,6 +210,8 @@ pub const MCValue = union(enum) { |
| 210 | 210 | register_offset: bits.RegisterOffset, |
| 211 | 211 | /// The value is a tuple { wrapped, overflow } where wrapped value is stored in the GP register. |
| 212 | 212 | register_overflow: struct { reg: Register, eflags: Condition }, |
| 213 | /// The value is a tuple { wrapped, overflow } where wrapped value is stored across two registers. |
| 214 | register_pair_overflow: struct { pair: [2]Register, eflags: Condition }, |
| 213 | 215 | /// The value is a bool vector stored in a vector register with a different scalar type. |
| 214 | 216 | register_mask: Mask, |
| 215 | 217 | /// The value is in memory at a hard-coded address. |
| ... | ... | @@ -265,6 +267,7 @@ pub const MCValue = union(enum) { |
| 265 | 267 | .register_mask, |
| 266 | 268 | .eflags, |
| 267 | 269 | .register_overflow, |
| 270 | .register_pair_overflow, |
| 268 | 271 | .indirect_load_frame, |
| 269 | 272 | .lea_frame, |
| 270 | 273 | .lea_nav, |
| ... | ... | @@ -363,6 +366,7 @@ pub const MCValue = union(enum) { |
| 363 | 366 | .indirect, |
| 364 | 367 | .indirect_mask, |
| 365 | 368 | => |*pl| (&pl.reg)[0..1], |
| 369 | .register_pair_overflow => |*rpo| &rpo.pair, |
| 366 | 370 | else => &.{}, |
| 367 | 371 | }; |
| 368 | 372 | } |
| ... | ... | @@ -371,6 +375,7 @@ pub const MCValue = union(enum) { |
| 371 | 375 | return switch (mcv) { |
| 372 | 376 | .eflags => |cc| cc, |
| 373 | 377 | .register_overflow => |reg_ov| reg_ov.eflags, |
| 378 | .register_pair_overflow => |rpo| rpo.eflags, |
| 374 | 379 | else => null, |
| 375 | 380 | }; |
| 376 | 381 | } |
| ... | ... | @@ -396,6 +401,7 @@ pub const MCValue = union(enum) { |
| 396 | 401 | .register_quadruple, |
| 397 | 402 | .register_offset, |
| 398 | 403 | .register_overflow, |
| 404 | .register_pair_overflow, |
| 399 | 405 | .register_mask, |
| 400 | 406 | .indirect_mask, |
| 401 | 407 | .lea_frame, |
| ... | ... | @@ -437,6 +443,7 @@ pub const MCValue = union(enum) { |
| 437 | 443 | .register_triple, |
| 438 | 444 | .register_quadruple, |
| 439 | 445 | .register_overflow, |
| 446 | .register_pair_overflow, |
| 440 | 447 | .register_mask, |
| 441 | 448 | .memory, |
| 442 | 449 | .indirect, |
| ... | ... | @@ -487,6 +494,7 @@ pub const MCValue = union(enum) { |
| 487 | 494 | .register_triple, |
| 488 | 495 | .register_quadruple, |
| 489 | 496 | .register_overflow, |
| 497 | .register_pair_overflow, |
| 490 | 498 | .register_mask, |
| 491 | 499 | .memory, |
| 492 | 500 | .indirect, |
| ... | ... | @@ -530,6 +538,7 @@ pub const MCValue = union(enum) { |
| 530 | 538 | .register_quadruple, |
| 531 | 539 | .register_offset, |
| 532 | 540 | .register_overflow, |
| 541 | .register_pair_overflow, |
| 533 | 542 | .register_mask, |
| 534 | 543 | .indirect_load_frame, |
| 535 | 544 | .lea_frame, |
| ... | ... | @@ -604,6 +613,11 @@ pub const MCValue = union(enum) { |
| 604 | 613 | @tagName(pl.eflags), |
| 605 | 614 | @tagName(pl.reg), |
| 606 | 615 | }), |
| 616 | .register_pair_overflow => |pl| try w.print("{t}:({t}:{t})", .{ |
| 617 | pl.eflags, |
| 618 | pl.pair[0], |
| 619 | pl.pair[1], |
| 620 | }), |
| 607 | 621 | .register_mask => |pl| try w.print("mask({s},{f}):{s}{s}", .{ |
| 608 | 622 | @tagName(pl.info.kind), |
| 609 | 623 | pl.info.scalar, |
| ... | ... | @@ -693,6 +707,7 @@ const InstTracking = struct { |
| 693 | 707 | .register_quadruple, |
| 694 | 708 | .register_offset, |
| 695 | 709 | .register_overflow, |
| 710 | .register_pair_overflow, |
| 696 | 711 | .register_mask, |
| 697 | 712 | .indirect, |
| 698 | 713 | .indirect_mask, |
| ... | ... | @@ -750,6 +765,10 @@ const InstTracking = struct { |
| 750 | 765 | function.register_manager.freeReg(reg_ov.reg); |
| 751 | 766 | function.eflags_inst = null; |
| 752 | 767 | }, |
| 768 | .register_pair_overflow => |rpo| { |
| 769 | for (rpo.pair) |r| function.register_manager.freeReg(r); |
| 770 | function.eflags_inst = null; |
| 771 | }, |
| 753 | 772 | .register_mask => |reg_mask| function.register_manager.freeReg(reg_mask.reg), |
| 754 | 773 | .eflags => function.eflags_inst = null, |
| 755 | 774 | else => {}, // TODO process stack allocation death |
| ... | ... | @@ -793,6 +812,7 @@ const InstTracking = struct { |
| 793 | 812 | .register_quadruple, |
| 794 | 813 | .register_offset, |
| 795 | 814 | .register_overflow, |
| 815 | .register_pair_overflow, |
| 796 | 816 | .register_mask, |
| 797 | 817 | .indirect, |
| 798 | 818 | .indirect_mask, |
| ... | ... | @@ -53047,7 +53067,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 53047 | 53067 | }), |
| 53048 | 53068 | else => |e| return e, |
| 53049 | 53069 | }; |
| 53050 | | try res[0].withOverflow(&res[1], cg); |
| 53070 | try res[0].withOverflow(&res[1], ty_pl.ty, cg); |
| 53051 | 53071 | try res[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg); |
| 53052 | 53072 | }, |
| 53053 | 53073 | .sub_with_overflow => |air_tag| { |
| ... | ... | @@ -53952,7 +53972,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 53952 | 53972 | }), |
| 53953 | 53973 | else => |e| return e, |
| 53954 | 53974 | }; |
| 53955 | | try res[0].withOverflow(&res[1], cg); |
| 53975 | try res[0].withOverflow(&res[1], ty_pl.ty, cg); |
| 53956 | 53976 | try res[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg); |
| 53957 | 53977 | }, |
| 53958 | 53978 | .mul_with_overflow => |air_tag| { |
| ... | ... | @@ -57549,7 +57569,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 57549 | 57569 | }), |
| 57550 | 57570 | else => |e| return e, |
| 57551 | 57571 | }; |
| 57552 | | try res[0].withOverflow(&res[1], cg); |
| 57572 | try res[0].withOverflow(&res[1], ty_pl.ty, cg); |
| 57553 | 57573 | try res[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg); |
| 57554 | 57574 | }, |
| 57555 | 57575 | .shl_with_overflow => |air_tag| { |
| ... | ... | @@ -60894,7 +60914,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 60894 | 60914 | }), |
| 60895 | 60915 | else => |e| return e, |
| 60896 | 60916 | }; |
| 60897 | | try res[0].withOverflow(&res[1], cg); |
| 60917 | try res[0].withOverflow(&res[1], ty_pl.ty, cg); |
| 60898 | 60918 | try res[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg); |
| 60899 | 60919 | }, |
| 60900 | 60920 | .alloc => { |
| ... | ... | @@ -104070,7 +104090,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 104070 | 104090 | const ty_pl = air_datas[@backingInt(inst)].ty_pl; |
| 104071 | 104091 | const bin_op = cg.air.extraData(Air.Bin, ty_pl.payload).data; |
| 104072 | 104092 | var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs }); |
| 104073 | | try ops[0].toPair(&ops[1], cg); |
| 104093 | try ops[0].toPair(&ops[1], ty_pl.ty, cg); |
| 104074 | 104094 | try ops[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg); |
| 104075 | 104095 | }, |
| 104076 | 104096 | .slice_len => { |
| ... | ... | @@ -104487,7 +104507,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 104487 | 104507 | var len = try cg.tempInit(.usize, .{ |
| 104488 | 104508 | .immediate = cg.typeOf(ty_op.operand).childType(zcu).arrayLen(zcu), |
| 104489 | 104509 | }); |
| 104490 | | try ops[0].toPair(&len, cg); |
| 104510 | try ops[0].toPair(&len, ty_op.ty, cg); |
| 104491 | 104511 | try ops[0].finish(inst, &.{ty_op.operand}, &ops, cg); |
| 104492 | 104512 | }, |
| 104493 | 104513 | .int_from_float, .int_from_float_optimized => |air_tag| { |
| ... | ... | @@ -171079,7 +171099,113 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 171079 | 171099 | }; |
| 171080 | 171100 | for (ops) |op| try op.die(cg); |
| 171081 | 171101 | }, |
| 171082 | | .cmpxchg_weak, .cmpxchg_strong => try cg.airCmpxchg(inst), |
| 171102 | .cmpxchg_weak, .cmpxchg_strong => |air_tag| { |
| 171103 | const ty_pl = air_datas[@backingInt(inst)].ty_pl; |
| 171104 | const extra = cg.air.extraData(Air.Cmpxchg, ty_pl.payload).data; |
| 171105 | var ops = try cg.tempsFromOperands(inst, .{ extra.ptr, extra.expected_value, extra.new_value }); |
| 171106 | var res: [2]Temp = undefined; |
| 171107 | (if (ty_pl.ty.optionalReprIsPayload(zcu)) |
| 171108 | cg.select(res[0..1], &.{ty_pl.ty}, &ops, comptime &.{.{ |
| 171109 | .required_features = .{ .@"64bit", .cmov, null, null }, |
| 171110 | .src_constraints = .{ .any, .{ .size = .qword }, .{ .size = .qword } }, |
| 171111 | .patterns = &.{ |
| 171112 | .{ .src = .{ .to_gpr, .{ .to_reg = .rax }, .to_gpr } }, |
| 171113 | }, |
| 171114 | .extra_temps = .{ |
| 171115 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| 171116 | .unused, |
| 171117 | .unused, |
| 171118 | .unused, |
| 171119 | .unused, |
| 171120 | .unused, |
| 171121 | .unused, |
| 171122 | .unused, |
| 171123 | .unused, |
| 171124 | .unused, |
| 171125 | .unused, |
| 171126 | }, |
| 171127 | .dst_temps = .{ .{ .ref = .src1 }, .unused }, |
| 171128 | .clobbers = .{ .eflags = true }, |
| 171129 | .each = .{ .once = &.{ |
| 171130 | .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, |
| 171131 | .{ ._, .@"lock _", .cmpxchg, .lea(.src0q), .src2q, ._, ._ }, |
| 171132 | .{ ._, ._e, .cmov, .dst0q, .tmp0q, ._, ._ }, |
| 171133 | } }, |
| 171134 | }}) |
| 171135 | else |
| 171136 | cg.select(&res, &.{ ty_pl.ty.childType(zcu), .u1 }, &ops, comptime &.{ .{ |
| 171137 | .required_features = .{ .@"64bit", null, null, null }, |
| 171138 | .src_constraints = .{ .any, .{ .int = .byte }, .{ .int = .byte } }, |
| 171139 | .patterns = &.{ |
| 171140 | .{ .src = .{ .to_gpr, .{ .to_reg = .rax }, .to_gpr } }, |
| 171141 | }, |
| 171142 | .dst_temps = .{ .{ .ref = .src1 }, .{ .cc = .ne } }, |
| 171143 | .clobbers = .{ .eflags = true }, |
| 171144 | .each = .{ .once = &.{ |
| 171145 | .{ ._, .@"lock _", .cmpxchg, .lea(.src0b), .src2b, ._, ._ }, |
| 171146 | } }, |
| 171147 | }, .{ |
| 171148 | .required_features = .{ .@"64bit", null, null, null }, |
| 171149 | .src_constraints = .{ .any, .{ .int = .word }, .{ .int = .word } }, |
| 171150 | .patterns = &.{ |
| 171151 | .{ .src = .{ .to_gpr, .{ .to_reg = .rax }, .to_gpr } }, |
| 171152 | }, |
| 171153 | .dst_temps = .{ .{ .ref = .src1 }, .{ .cc = .ne } }, |
| 171154 | .clobbers = .{ .eflags = true }, |
| 171155 | .each = .{ .once = &.{ |
| 171156 | .{ ._, .@"lock _", .cmpxchg, .lea(.src0w), .src2w, ._, ._ }, |
| 171157 | } }, |
| 171158 | }, .{ |
| 171159 | .required_features = .{ .@"64bit", null, null, null }, |
| 171160 | .src_constraints = .{ .any, .{ .int = .dword }, .{ .int = .dword } }, |
| 171161 | .patterns = &.{ |
| 171162 | .{ .src = .{ .to_gpr, .{ .to_reg = .rax }, .to_gpr } }, |
| 171163 | }, |
| 171164 | .dst_temps = .{ .{ .ref = .src1 }, .{ .cc = .ne } }, |
| 171165 | .clobbers = .{ .eflags = true }, |
| 171166 | .each = .{ .once = &.{ |
| 171167 | .{ ._, .@"lock _", .cmpxchg, .lea(.src0d), .src2d, ._, ._ }, |
| 171168 | } }, |
| 171169 | }, .{ |
| 171170 | .required_features = .{ .@"64bit", null, null, null }, |
| 171171 | .src_constraints = .{ .any, .{ .int = .qword }, .{ .int = .qword } }, |
| 171172 | .patterns = &.{ |
| 171173 | .{ .src = .{ .to_gpr, .{ .to_reg = .rax }, .to_gpr } }, |
| 171174 | }, |
| 171175 | .dst_temps = .{ .{ .ref = .src1 }, .{ .cc = .ne } }, |
| 171176 | .clobbers = .{ .eflags = true }, |
| 171177 | .each = .{ .once = &.{ |
| 171178 | .{ ._, .@"lock _", .cmpxchg, .lea(.src0q), .src2q, ._, ._ }, |
| 171179 | } }, |
| 171180 | }, .{ |
| 171181 | .required_features = .{ .@"64bit", .cx16, null, null }, |
| 171182 | .src_constraints = .{ .any, .{ .int = .xword }, .{ .int = .xword } }, |
| 171183 | .patterns = &.{.{ |
| 171184 | .src = .{ |
| 171185 | .to_gpr, |
| 171186 | .{ .to_reg_pair = .{ .rax, .rdx } }, |
| 171187 | .{ .to_reg_pair = .{ .rbx, .rcx } }, |
| 171188 | }, |
| 171189 | }}, |
| 171190 | .dst_temps = .{ .{ .ref = .src1 }, .{ .cc = .ne } }, |
| 171191 | .clobbers = .{ .eflags = true }, |
| 171192 | .each = .{ .once = &.{ |
| 171193 | .{ ._, .@"lock _16b", .cmpxchg, .lea(.src0x), ._, ._, ._ }, |
| 171194 | } }, |
| 171195 | } })) catch |err| switch (err) { |
| 171196 | error.SelectFailed => return cg.fail("failed to select {s} {f} {f} {f} {f}", .{ |
| 171197 | @tagName(air_tag), |
| 171198 | ty_pl.ty.fmt(zcu), |
| 171199 | ops[0].tracking(cg), |
| 171200 | ops[1].tracking(cg), |
| 171201 | ops[2].tracking(cg), |
| 171202 | }), |
| 171203 | else => |e| return e, |
| 171204 | }; |
| 171205 | if (!ty_pl.ty.optionalReprIsPayload(zcu)) |
| 171206 | try res[0].withOverflow(&res[1], ty_pl.ty, cg); |
| 171207 | try res[0].finish(inst, &.{ extra.ptr, extra.expected_value, extra.new_value }, &ops, cg); |
| 171208 | }, |
| 171083 | 171209 | .atomic_load => try cg.airAtomicLoad(inst), |
| 171084 | 171210 | .atomic_store_unordered => try cg.airAtomicStore(inst, .unordered), |
| 171085 | 171211 | .atomic_store_monotonic => try cg.airAtomicStore(inst, .monotonic), |
| ... | ... | @@ -171421,7 +171547,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 171421 | 171547 | }), |
| 171422 | 171548 | else => |e| return e, |
| 171423 | 171549 | }; |
| 171424 | | if (res[0].tracking(cg).short != .register_pair) try ops[0].toPair(&res[0], cg); |
| 171550 | if (res[0].tracking(cg).short != .register_pair) try ops[0].toPair(&res[0], .slice_const_u8_sentinel_0, cg); |
| 171425 | 171551 | try res[0].finish(inst, &.{un_op}, &ops, cg); |
| 171426 | 171552 | }, |
| 171427 | 171553 | .error_name => |air_tag| { |
| ... | ... | @@ -171524,7 +171650,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 171524 | 171650 | else => |e| return e, |
| 171525 | 171651 | }; |
| 171526 | 171652 | const orig_res = res; |
| 171527 | | try res[0].toPair(&res[1], cg); |
| 171653 | try res[0].toPair(&res[1], .slice_const_u8_sentinel_0, cg); |
| 171528 | 171654 | for (&ops) |*op| for (orig_res) |orig_r| { |
| 171529 | 171655 | if (op.index != orig_r.index) continue; |
| 171530 | 171656 | op.* = res[0]; |
| ... | ... | @@ -174108,7 +174234,10 @@ fn getValue(self: *CodeGen, value: MCValue, inst: ?Air.Inst.Index) !void { |
| 174108 | 174234 | for (value.getRegs()) |reg| try self.register_manager.getReg(reg, inst); |
| 174109 | 174235 | switch (value) { |
| 174110 | 174236 | else => {}, |
| 174111 | | .eflags, .register_overflow => self.eflags_inst = inst, |
| 174237 | .eflags, |
| 174238 | .register_overflow, |
| 174239 | .register_pair_overflow, |
| 174240 | => self.eflags_inst = inst, |
| 174112 | 174241 | } |
| 174113 | 174242 | } |
| 174114 | 174243 | |
| ... | ... | @@ -174136,6 +174265,10 @@ fn freeValue(self: *CodeGen, value: MCValue, comptime opts: FreeOptions) !void { |
| 174136 | 174265 | try self.freeReg(reg_ov.reg, opts); |
| 174137 | 174266 | self.eflags_inst = null; |
| 174138 | 174267 | }, |
| 174268 | .register_pair_overflow => |rpo| { |
| 174269 | for (rpo.pair) |r| try self.freeReg(r, opts); |
| 174270 | self.eflags_inst = null; |
| 174271 | }, |
| 174139 | 174272 | .register_mask => |reg_mask| try self.freeReg(reg_mask.reg, opts), |
| 174140 | 174273 | .eflags => self.eflags_inst = null, |
| 174141 | 174274 | else => {}, // TODO process stack allocation death |
| ... | ... | @@ -174678,7 +174811,12 @@ fn reuseOperandAdvanced( |
| 174678 | 174811 | return false; |
| 174679 | 174812 | |
| 174680 | 174813 | switch (mcv) { |
| 174681 | | .register, .register_pair, .register_overflow, .register_mask => for (mcv.getRegs()) |reg| { |
| 174814 | .register, |
| 174815 | .register_pair, |
| 174816 | .register_overflow, |
| 174817 | .register_pair_overflow, |
| 174818 | .register_mask, |
| 174819 | => for (mcv.getRegs()) |reg| { |
| 174682 | 174820 | // If it's in the registers table, need to associate the register(s) with the |
| 174683 | 174821 | // new instruction. |
| 174684 | 174822 | if (maybe_tracked_inst) |tracked_inst| { |
| ... | ... | @@ -174693,7 +174831,10 @@ fn reuseOperandAdvanced( |
| 174693 | 174831 | else => return false, |
| 174694 | 174832 | } |
| 174695 | 174833 | switch (mcv) { |
| 174696 | | .eflags, .register_overflow => self.eflags_inst = maybe_tracked_inst, |
| 174834 | .eflags, |
| 174835 | .register_overflow, |
| 174836 | .register_pair_overflow, |
| 174837 | => self.eflags_inst = maybe_tracked_inst, |
| 174697 | 174838 | else => {}, |
| 174698 | 174839 | } |
| 174699 | 174840 | |
| ... | ... | @@ -174720,6 +174861,7 @@ fn load(self: *CodeGen, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) InnerE |
| 174720 | 174861 | .register_triple, |
| 174721 | 174862 | .register_quadruple, |
| 174722 | 174863 | .register_overflow, |
| 174864 | .register_pair_overflow, |
| 174723 | 174865 | .register_mask, |
| 174724 | 174866 | .indirect_load_frame, |
| 174725 | 174867 | .indirect_mask, |
| ... | ... | @@ -174779,6 +174921,7 @@ fn store( |
| 174779 | 174921 | .register_triple, |
| 174780 | 174922 | .register_quadruple, |
| 174781 | 174923 | .register_overflow, |
| 174924 | .register_pair_overflow, |
| 174782 | 174925 | .register_mask, |
| 174783 | 174926 | .indirect_load_frame, |
| 174784 | 174927 | .indirect_mask, |
| ... | ... | @@ -174830,6 +174973,7 @@ fn genUnOpMir(self: *CodeGen, mir_tag: Mir.Inst.FixedTag, dst_ty: Type, dst_mcv: |
| 174830 | 174973 | .register_offset, |
| 174831 | 174974 | .eflags, |
| 174832 | 174975 | .register_overflow, |
| 174976 | .register_pair_overflow, |
| 174833 | 174977 | .register_mask, |
| 174834 | 174978 | .indirect_load_frame, |
| 174835 | 174979 | .indirect_mask, |
| ... | ... | @@ -175526,6 +175670,7 @@ fn genBinOpMir( |
| 175526 | 175670 | .immediate, |
| 175527 | 175671 | .eflags, |
| 175528 | 175672 | .register_overflow, |
| 175673 | .register_pair_overflow, |
| 175529 | 175674 | .register_mask, |
| 175530 | 175675 | .lea_frame, |
| 175531 | 175676 | .lea_nav, |
| ... | ... | @@ -175574,6 +175719,7 @@ fn genBinOpMir( |
| 175574 | 175719 | .dead, |
| 175575 | 175720 | .undef, |
| 175576 | 175721 | .register_overflow, |
| 175722 | .register_pair_overflow, |
| 175577 | 175723 | .register_mask, |
| 175578 | 175724 | .indirect_load_frame, |
| 175579 | 175725 | .indirect_mask, |
| ... | ... | @@ -175752,6 +175898,7 @@ fn genBinOpMir( |
| 175752 | 175898 | .dead, |
| 175753 | 175899 | .undef, |
| 175754 | 175900 | .register_overflow, |
| 175901 | .register_pair_overflow, |
| 175755 | 175902 | .register_mask, |
| 175756 | 175903 | .indirect_load_frame, |
| 175757 | 175904 | .indirect_mask, |
| ... | ... | @@ -175862,6 +176009,7 @@ fn genBinOpMir( |
| 175862 | 176009 | .dead, |
| 175863 | 176010 | .undef, |
| 175864 | 176011 | .register_overflow, |
| 176012 | .register_pair_overflow, |
| 175865 | 176013 | .register_mask, |
| 175866 | 176014 | .indirect_load_frame, |
| 175867 | 176015 | .indirect_mask, |
| ... | ... | @@ -178971,6 +179119,7 @@ fn genCopy(self: *CodeGen, ty: Type, dst_mcv: MCValue, src_mcv: MCValue, opts: C |
| 178971 | 179119 | .immediate, |
| 178972 | 179120 | .eflags, |
| 178973 | 179121 | .register_overflow, |
| 179122 | .register_pair_overflow, |
| 178974 | 179123 | .register_mask, |
| 178975 | 179124 | .indirect_load_frame, |
| 178976 | 179125 | .indirect_mask, |
| ... | ... | @@ -178995,6 +179144,7 @@ fn genCopy(self: *CodeGen, ty: Type, dst_mcv: MCValue, src_mcv: MCValue, opts: C |
| 178995 | 179144 | .dead, |
| 178996 | 179145 | .undef, |
| 178997 | 179146 | .register_overflow, |
| 179147 | .register_pair_overflow, |
| 178998 | 179148 | .register_tee, |
| 178999 | 179149 | .elementwise_gpr, |
| 179000 | 179150 | .elementwise_sse, |
| ... | ... | @@ -179198,6 +179348,7 @@ fn genSetReg( |
| 179198 | 179348 | .ywordwise_sse, |
| 179199 | 179349 | .zwordwise_sse, |
| 179200 | 179350 | .reserved_frame, |
| 179351 | .register_pair_overflow, |
| 179201 | 179352 | => unreachable, |
| 179202 | 179353 | .undef => if (opts.safety) switch (dst_reg.class()) { |
| 179203 | 179354 | .general_purpose, .gphi => switch (abi_size) { |
| ... | ... | @@ -180001,6 +180152,23 @@ fn genSetMem( |
| 180001 | 180152 | @tagName(src_mcv), ty.fmt(zcu), |
| 180002 | 180153 | }), |
| 180003 | 180154 | }, |
| 180155 | .register_pair_overflow => |rpo| switch (ty.zigTypeTag(zcu)) { |
| 180156 | .optional => { |
| 180157 | assert(!ty.optionalReprIsPayload(zcu)); |
| 180158 | const child_ty = ty.optionalChild(zcu); |
| 180159 | try cg.genSetMem(base, disp, child_ty, .{ .register_pair = rpo.pair }, opts); |
| 180160 | try cg.genSetMem( |
| 180161 | base, |
| 180162 | disp + @as(i32, @intCast(child_ty.abiSize(zcu))), |
| 180163 | .bool, |
| 180164 | .{ .eflags = rpo.eflags }, |
| 180165 | opts, |
| 180166 | ); |
| 180167 | }, |
| 180168 | else => return cg.fail("TODO implement genSetMem for {s} of {f}", .{ |
| 180169 | @tagName(src_mcv), ty.fmt(zcu), |
| 180170 | }), |
| 180171 | }, |
| 180004 | 180172 | .register_offset => |reg_off| { |
| 180005 | 180173 | const src_reg = cg.copyToTmpRegister(ty, src_mcv) catch |err| switch (err) { |
| 180006 | 180174 | error.OutOfRegisters => { |
| ... | ... | @@ -180335,111 +180503,6 @@ fn airBitCast(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 180335 | 180503 | return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); |
| 180336 | 180504 | } |
| 180337 | 180505 | |
| 180338 | | fn airCmpxchg(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 180339 | | const pt = self.pt; |
| 180340 | | const zcu = pt.zcu; |
| 180341 | | const ty_pl = self.air.instructions.items(.data)[@backingInt(inst)].ty_pl; |
| 180342 | | const extra = self.air.extraData(Air.Cmpxchg, ty_pl.payload).data; |
| 180343 | | |
| 180344 | | const dst_ty = self.typeOfIndex(inst); |
| 180345 | | const ptr_ty = self.typeOf(extra.ptr); |
| 180346 | | const val_ty = self.typeOf(extra.expected_value); |
| 180347 | | const val_abi_size: u32 = @intCast(val_ty.abiSize(zcu)); |
| 180348 | | |
| 180349 | | try self.spillRegisters(&.{ .rax, .rdx, .rbx, .rcx }); |
| 180350 | | const regs_lock = self.register_manager.lockRegsAssumeUnused(4, .{ .rax, .rdx, .rbx, .rcx }); |
| 180351 | | defer for (regs_lock) |lock| self.register_manager.unlockReg(lock); |
| 180352 | | |
| 180353 | | const exp_mcv = try self.resolveInst(extra.expected_value); |
| 180354 | | if (val_abi_size > 8) { |
| 180355 | | const exp_addr_mcv: MCValue = switch (exp_mcv) { |
| 180356 | | .memory, .indirect, .load_frame => exp_mcv.address(), |
| 180357 | | else => .{ .register = try self.copyToTmpRegister(.usize, exp_mcv.address()) }, |
| 180358 | | }; |
| 180359 | | const exp_addr_lock = |
| 180360 | | if (exp_addr_mcv.getReg()) |reg| self.register_manager.lockReg(reg) else null; |
| 180361 | | defer if (exp_addr_lock) |lock| self.register_manager.unlockReg(lock); |
| 180362 | | |
| 180363 | | try self.genSetReg(.rax, .usize, exp_addr_mcv.deref(), .{}); |
| 180364 | | try self.genSetReg(.rdx, .usize, exp_addr_mcv.offset(8).deref(), .{}); |
| 180365 | | } else try self.genSetReg(.rax, val_ty, exp_mcv, .{}); |
| 180366 | | |
| 180367 | | const new_mcv = try self.resolveInst(extra.new_value); |
| 180368 | | const new_reg = if (val_abi_size > 8) new: { |
| 180369 | | const new_addr_mcv: MCValue = switch (new_mcv) { |
| 180370 | | .memory, .indirect, .load_frame => new_mcv.address(), |
| 180371 | | else => .{ .register = try self.copyToTmpRegister(.usize, new_mcv.address()) }, |
| 180372 | | }; |
| 180373 | | const new_addr_lock = |
| 180374 | | if (new_addr_mcv.getReg()) |reg| self.register_manager.lockReg(reg) else null; |
| 180375 | | defer if (new_addr_lock) |lock| self.register_manager.unlockReg(lock); |
| 180376 | | |
| 180377 | | try self.genSetReg(.rbx, .usize, new_addr_mcv.deref(), .{}); |
| 180378 | | try self.genSetReg(.rcx, .usize, new_addr_mcv.offset(8).deref(), .{}); |
| 180379 | | break :new null; |
| 180380 | | } else try self.copyToTmpRegister(val_ty, new_mcv); |
| 180381 | | const new_lock = if (new_reg) |reg| self.register_manager.lockRegAssumeUnused(reg) else null; |
| 180382 | | defer if (new_lock) |lock| self.register_manager.unlockReg(lock); |
| 180383 | | |
| 180384 | | const ptr_mcv = try self.resolveInst(extra.ptr); |
| 180385 | | const mem_size: Memory.Size = .fromSize(val_abi_size); |
| 180386 | | const ptr_mem: Memory = switch (ptr_mcv) { |
| 180387 | | .immediate, .register, .register_offset, .lea_frame => try ptr_mcv.deref().mem(self, .{ .size = mem_size }), |
| 180388 | | else => .{ |
| 180389 | | .base = .{ .reg = try self.copyToTmpRegister(ptr_ty, ptr_mcv) }, |
| 180390 | | .mod = .{ .rm = .{ .size = mem_size } }, |
| 180391 | | }, |
| 180392 | | }; |
| 180393 | | switch (ptr_mem.mod) { |
| 180394 | | .rm => {}, |
| 180395 | | .off => return self.fail("TODO airCmpxchg with {s}", .{@tagName(ptr_mcv)}), |
| 180396 | | } |
| 180397 | | const ptr_lock = switch (ptr_mem.base) { |
| 180398 | | .none, .frame, .nav, .uav => null, |
| 180399 | | .reg => |reg| self.register_manager.lockReg(reg), |
| 180400 | | .table, .rip_inst, .lazy_sym, .extern_func => unreachable, |
| 180401 | | }; |
| 180402 | | defer if (ptr_lock) |lock| self.register_manager.unlockReg(lock); |
| 180403 | | |
| 180404 | | try self.spillEflagsIfOccupied(); |
| 180405 | | const null_reg = if (dst_ty.optionalReprIsPayload(zcu) and !self.liveness.isUnused(inst)) null_reg: { |
| 180406 | | const null_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); |
| 180407 | | try self.asmRegisterRegister(.{ ._, .xor }, null_reg.to32(), null_reg.to32()); |
| 180408 | | break :null_reg null_reg; |
| 180409 | | } else null; |
| 180410 | | const null_lock = if (null_reg) |reg| self.register_manager.lockRegAssumeUnused(reg) else null; |
| 180411 | | defer if (null_lock) |lock| self.register_manager.unlockReg(lock); |
| 180412 | | |
| 180413 | | if (val_abi_size <= 8) try self.asmMemoryRegister( |
| 180414 | | .{ .@"lock _", .cmpxchg }, |
| 180415 | | ptr_mem, |
| 180416 | | registerAlias(new_reg.?, val_abi_size), |
| 180417 | | ) else try self.asmMemory(.{ .@"lock _16b", .cmpxchg }, ptr_mem); |
| 180418 | | |
| 180419 | | const result: MCValue = result: { |
| 180420 | | if (self.liveness.isUnused(inst)) break :result .unreach; |
| 180421 | | |
| 180422 | | if (null_reg) |reg| try self.asmCmovccRegisterRegister( |
| 180423 | | .e, |
| 180424 | | registerAlias(.rax, val_abi_size), |
| 180425 | | registerAlias(reg, val_abi_size), |
| 180426 | | ); |
| 180427 | | |
| 180428 | | if (val_abi_size <= 8) { |
| 180429 | | if (null_reg) |_| break :result .{ .register = .rax }; |
| 180430 | | self.eflags_inst = inst; |
| 180431 | | break :result .{ .register_overflow = .{ .reg = .rax, .eflags = .ne } }; |
| 180432 | | } |
| 180433 | | |
| 180434 | | const dst_mcv = try self.allocRegOrMem(inst, false); |
| 180435 | | try self.genCopy(.usize, dst_mcv, .{ .register = .rax }, .{}); |
| 180436 | | try self.genCopy(.usize, dst_mcv.address().offset(8).deref(), .{ .register = .rdx }, .{}); |
| 180437 | | if (null_reg == null) try self.genCopy(.bool, dst_mcv.address().offset(16).deref(), .{ .eflags = .ne }, .{}); |
| 180438 | | break :result dst_mcv; |
| 180439 | | }; |
| 180440 | | return self.finishAir(inst, result, .{ extra.ptr, extra.expected_value, extra.new_value }); |
| 180441 | | } |
| 180442 | | |
| 180443 | 180506 | fn atomicOp( |
| 180444 | 180507 | self: *CodeGen, |
| 180445 | 180508 | ptr_mcv: MCValue, |
| ... | ... | @@ -180475,7 +180538,7 @@ fn atomicOp( |
| 180475 | 180538 | }; |
| 180476 | 180539 | switch (ptr_mem.mod) { |
| 180477 | 180540 | .rm => {}, |
| 180478 | | .off => return self.fail("TODO airCmpxchg with {s}", .{@tagName(ptr_mcv)}), |
| 180541 | .off => return self.fail("TODO atomicOp with {s}", .{@tagName(ptr_mcv)}), |
| 180479 | 180542 | } |
| 180480 | 180543 | const mem_lock = switch (ptr_mem.base) { |
| 180481 | 180544 | .none, .frame, .nav, .uav => null, |
| ... | ... | @@ -182855,6 +182918,7 @@ const Temp = struct { |
| 182855 | 182918 | .register_triple, |
| 182856 | 182919 | .register_quadruple, |
| 182857 | 182920 | .register_overflow, |
| 182921 | .register_pair_overflow, |
| 182858 | 182922 | => true, |
| 182859 | 182923 | .load_frame => |frame_addr| !frame_addr.index.isNamed(), |
| 182860 | 182924 | }, |
| ... | ... | @@ -183235,7 +183299,7 @@ const Temp = struct { |
| 183235 | 183299 | return true; |
| 183236 | 183300 | } |
| 183237 | 183301 | |
| 183238 | | fn toPair(first_temp: *Temp, second_temp: *Temp, cg: *CodeGen) InnerError!void { |
| 183302 | fn toPair(first_temp: *Temp, second_temp: *Temp, result_ty: Type, cg: *CodeGen) InnerError!void { |
| 183239 | 183303 | while (true) for ([_]*Temp{ first_temp, second_temp }) |part_temp| { |
| 183240 | 183304 | if (try part_temp.toRegClass(true, .general_purpose, cg)) break; |
| 183241 | 183305 | } else break; |
| ... | ... | @@ -183250,13 +183314,13 @@ const Temp = struct { |
| 183250 | 183314 | assert(cg.reuseTemp(result_temp.index, first_temp.index, first_temp_tracking)); |
| 183251 | 183315 | assert(cg.reuseTemp(result_temp.index, second_temp.index, second_temp_tracking)); |
| 183252 | 183316 | result_temp_index.tracking(cg).* = .init(result); |
| 183253 | | cg.temp_type[@backingInt(result_temp_index)] = .slice_const_u8; |
| 183317 | cg.temp_type[@backingInt(result_temp_index)] = result_ty; |
| 183254 | 183318 | cg.next_temp_index = @fromBackingInt(@intCast(@backingInt(result_temp_index) + 1)); |
| 183255 | 183319 | first_temp.* = result_temp; |
| 183256 | 183320 | second_temp.* = result_temp; |
| 183257 | 183321 | } |
| 183258 | 183322 | |
| 183259 | | fn withOverflow(temp: *Temp, overflow_temp: *Temp, cg: *CodeGen) InnerError!void { |
| 183323 | fn withOverflow(temp: *Temp, overflow_temp: *Temp, result_ty: Type, cg: *CodeGen) InnerError!void { |
| 183260 | 183324 | const temp_index = temp.unwrap(cg).temp; |
| 183261 | 183325 | const temp_tracking = temp_index.tracking(cg); |
| 183262 | 183326 | const overflow_temp_tracking = overflow_temp.unwrap(cg).temp.tracking(cg); |
| ... | ... | @@ -183268,7 +183332,7 @@ const Temp = struct { |
| 183268 | 183332 | assert(cg.reuseTemp(result_temp.index, temp.index, temp_tracking)); |
| 183269 | 183333 | assert(cg.reuseTemp(result_temp.index, overflow_temp.index, overflow_temp_tracking)); |
| 183270 | 183334 | result_temp_index.tracking(cg).* = .init(.{ .register_overflow = .{ .reg = reg, .eflags = overflow_cc } }); |
| 183271 | | cg.temp_type[@backingInt(result_temp_index)] = .slice_const_u8; |
| 183335 | cg.temp_type[@backingInt(result_temp_index)] = result_ty; |
| 183272 | 183336 | cg.next_temp_index = @fromBackingInt(@intCast(@backingInt(result_temp_index) + 1)); |
| 183273 | 183337 | temp.* = result_temp; |
| 183274 | 183338 | overflow_temp.* = result_temp; |
| ... | ... | @@ -183280,7 +183344,22 @@ const Temp = struct { |
| 183280 | 183344 | assert(cg.reuseTemp(result_temp.index, temp.index, temp_tracking)); |
| 183281 | 183345 | assert(cg.reuseTemp(result_temp.index, overflow_temp.index, overflow_temp_tracking)); |
| 183282 | 183346 | result_temp_index.tracking(cg).* = .init(.{ .register_pair = .{ reg, overflow_reg } }); |
| 183283 | | cg.temp_type[@backingInt(result_temp_index)] = .slice_const_u8; |
| 183347 | cg.temp_type[@backingInt(result_temp_index)] = result_ty; |
| 183348 | cg.next_temp_index = @fromBackingInt(@intCast(@backingInt(result_temp_index) + 1)); |
| 183349 | temp.* = result_temp; |
| 183350 | overflow_temp.* = result_temp; |
| 183351 | return; |
| 183352 | }, |
| 183353 | else => {}, |
| 183354 | }, |
| 183355 | .register_pair => |pair| switch (overflow_temp_tracking.short) { |
| 183356 | .eflags => |overflow_cc| { |
| 183357 | const result_temp_index = cg.next_temp_index; |
| 183358 | const result_temp: Temp = .{ .index = result_temp_index.toIndex() }; |
| 183359 | assert(cg.reuseTemp(result_temp.index, temp.index, temp_tracking)); |
| 183360 | assert(cg.reuseTemp(result_temp.index, overflow_temp.index, overflow_temp_tracking)); |
| 183361 | result_temp_index.tracking(cg).* = .init(.{ .register_pair_overflow = .{ .pair = pair, .eflags = overflow_cc } }); |
| 183362 | cg.temp_type[@backingInt(result_temp_index)] = result_ty; |
| 183284 | 183363 | cg.next_temp_index = @fromBackingInt(@intCast(@backingInt(result_temp_index) + 1)); |
| 183285 | 183364 | temp.* = result_temp; |
| 183286 | 183365 | overflow_temp.* = result_temp; |
| ... | ... | @@ -183320,6 +183399,7 @@ const Temp = struct { |
| 183320 | 183399 | .register_triple, |
| 183321 | 183400 | .register_quadruple, |
| 183322 | 183401 | .register_overflow, |
| 183402 | .register_pair_overflow, |
| 183323 | 183403 | .register_mask, |
| 183324 | 183404 | .indirect_load_frame, |
| 183325 | 183405 | .indirect_mask, |
| ... | ... | @@ -189802,6 +189882,7 @@ const Temp = struct { |
| 189802 | 189882 | .register_quadruple, |
| 189803 | 189883 | .register_offset, |
| 189804 | 189884 | .register_overflow, |
| 189885 | .register_pair_overflow, |
| 189805 | 189886 | .register_mask, |
| 189806 | 189887 | .memory, |
| 189807 | 189888 | .indirect, |
| ... | ... | @@ -189920,6 +190001,7 @@ fn reuseTemp( |
| 189920 | 190001 | .register_pair, |
| 189921 | 190002 | .register_offset, |
| 189922 | 190003 | .register_overflow, |
| 190004 | .register_pair_overflow, |
| 189923 | 190005 | .register_mask, |
| 189924 | 190006 | .indirect, |
| 189925 | 190007 | => for (tracking.short.getRegs()) |tracked_reg| { |
| ... | ... | @@ -189931,7 +190013,10 @@ fn reuseTemp( |
| 189931 | 190013 | else => {}, |
| 189932 | 190014 | } |
| 189933 | 190015 | switch (tracking.short) { |
| 189934 | | .eflags, .register_overflow => cg.eflags_inst = new_inst, |
| 190016 | .eflags, |
| 190017 | .register_overflow, |
| 190018 | .register_pair_overflow, |
| 190019 | => cg.eflags_inst = new_inst, |
| 189935 | 190020 | else => {}, |
| 189936 | 190021 | } |
| 189937 | 190022 | tracking.reuse(cg, new_inst, old_inst); |
| ... | ... | @@ -192514,7 +192599,13 @@ const Select = struct { |
| 192514 | 192599 | } }, |
| 192515 | 192600 | .eflags => unreachable, |
| 192516 | 192601 | .register => |reg| .{ .reg = s.lowerReg(reg.toSize(op.flags.base.size, s.cg.target)) }, |
| 192517 | | .register_pair, .register_triple, .register_quadruple, .register_offset, .register_overflow => unreachable, |
| 192602 | .register_pair, |
| 192603 | .register_triple, |
| 192604 | .register_quadruple, |
| 192605 | .register_offset, |
| 192606 | .register_overflow, |
| 192607 | .register_pair_overflow, |
| 192608 | => unreachable, |
| 192518 | 192609 | .register_mask => |reg_mask| .{ .reg = s.lowerReg(reg_mask.reg.toSize(op.flags.base.size, s.cg.target)) }, |
| 192519 | 192610 | .lea_nav => |nav| .{ .imm = .{ .nav = .{ .index = nav } } }, |
| 192520 | 192611 | .lea_uav => |uav| .{ .imm = .{ .uav = uav } }, |
David Rubin <david@vortan.dev> 2026-10-23 17:04:46-05:00
Alex Rønne Petersen <alex@alexrp.com> 2026-10-25 01:47:06+02:00