| ... | ... | @@ -150,7 +150,7 @@ const Owner = union(enum) { |
| 150 | 150 | }; |
| 151 | 151 | |
| 152 | 152 | const MaskKind = enum(u1) { sign, all }; |
| 153 | | const MaskInfo = packed struct { kind: MaskKind, inverted: bool, scalar: Memory.Size }; |
| 153 | const MaskInfo = packed struct { kind: MaskKind, inverted: bool = false, scalar: Memory.Size }; |
| 154 | 154 | |
| 155 | 155 | pub const MCValue = union(enum) { |
| 156 | 156 | /// No runtime bits. `void` types, empty structs, u0, enums with 1 tag, etc. |
| ... | ... | @@ -2657,10 +2657,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 2657 | 2657 | } else { // hack around Sema OPV bugs |
| 2658 | 2658 | res[0] = ops[0]; |
| 2659 | 2659 | } |
| 2660 | | for (ops) |op| for (res) |r| { |
| 2661 | | if (op.index == r.index) break; |
| 2662 | | } else try op.die(cg); |
| 2663 | | try res[0].moveTo(inst, cg); |
| 2660 | try res[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg); |
| 2664 | 2661 | }, |
| 2665 | 2662 | .ptr_sub => |air_tag| if (use_old) try cg.airPtrArithmetic(inst, air_tag) else { |
| 2666 | 2663 | const ty_pl = air_datas[@intFromEnum(inst)].ty_pl; |
| ... | ... | @@ -2792,17 +2789,14 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 2792 | 2789 | // hack around Sema OPV bugs |
| 2793 | 2790 | res[0] = ops[0]; |
| 2794 | 2791 | } |
| 2795 | | for (ops) |op| for (res) |r| { |
| 2796 | | if (op.index == r.index) break; |
| 2797 | | } else try op.die(cg); |
| 2798 | | try res[0].moveTo(inst, cg); |
| 2792 | try res[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg); |
| 2799 | 2793 | }, |
| 2800 | 2794 | .alloc => if (use_old) try cg.airAlloc(inst) else { |
| 2801 | 2795 | const ty = air_datas[@intFromEnum(inst)].ty; |
| 2802 | | var slot = try cg.tempInit(ty, .{ .lea_frame = .{ |
| 2796 | const slot = try cg.tempInit(ty, .{ .lea_frame = .{ |
| 2803 | 2797 | .index = try cg.allocMemPtr(inst), |
| 2804 | 2798 | } }); |
| 2805 | | try slot.moveTo(inst, cg); |
| 2799 | try slot.finish(inst, &.{}, &.{}, cg); |
| 2806 | 2800 | }, |
| 2807 | 2801 | .inferred_alloc, .inferred_alloc_comptime => unreachable, |
| 2808 | 2802 | .ret_ptr => if (use_old) try cg.airRetPtr(inst) else { |
| ... | ... | @@ -2818,7 +2812,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 2818 | 2812 | break :slot slot; |
| 2819 | 2813 | }, |
| 2820 | 2814 | }; |
| 2821 | | try slot.moveTo(inst, cg); |
| 2815 | try slot.finish(inst, &.{}, &.{}, cg); |
| 2822 | 2816 | }, |
| 2823 | 2817 | .assembly => try cg.airAsm(inst), |
| 2824 | 2818 | .bit_and, .bit_or, .xor, .bool_and, .bool_or => |air_tag| if (use_old) try cg.airBinOp(inst, air_tag) else { |
| ... | ... | @@ -3154,10 +3148,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 3154 | 3148 | }), |
| 3155 | 3149 | else => |e| return e, |
| 3156 | 3150 | }; |
| 3157 | | for (ops) |op| for (res) |r| { |
| 3158 | | if (op.index == r.index) break; |
| 3159 | | } else try op.die(cg); |
| 3160 | | try res[0].moveTo(inst, cg); |
| 3151 | try res[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg); |
| 3161 | 3152 | }, |
| 3162 | 3153 | .not => |air_tag| if (use_old) try cg.airUnOp(inst, air_tag) else { |
| 3163 | 3154 | const ty_op = air_datas[@intFromEnum(inst)].ty_op; |
| ... | ... | @@ -4207,10 +4198,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 4207 | 4198 | }), |
| 4208 | 4199 | else => |e| return e, |
| 4209 | 4200 | }; |
| 4210 | | for (ops) |op| for (res) |r| { |
| 4211 | | if (op.index == r.index) break; |
| 4212 | | } else try op.die(cg); |
| 4213 | | try res[0].moveTo(inst, cg); |
| 4201 | try res[0].finish(inst, &.{ty_op.operand}, &ops, cg); |
| 4214 | 4202 | }, |
| 4215 | 4203 | |
| 4216 | 4204 | .block => if (use_old) try cg.airBlock(inst) else { |
| ... | ... | @@ -4250,13 +4238,13 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 4250 | 4238 | .index = .ret_addr, |
| 4251 | 4239 | } }); |
| 4252 | 4240 | while (try slot.toRegClass(true, .general_purpose, cg)) {} |
| 4253 | | try slot.moveTo(inst, cg); |
| 4241 | try slot.finish(inst, &.{}, &.{}, cg); |
| 4254 | 4242 | }, |
| 4255 | 4243 | .frame_addr => if (use_old) try cg.airFrameAddress(inst) else { |
| 4256 | | var slot = try cg.tempInit(.usize, .{ .lea_frame = .{ |
| 4244 | const slot = try cg.tempInit(.usize, .{ .lea_frame = .{ |
| 4257 | 4245 | .index = .base_ptr, |
| 4258 | 4246 | } }); |
| 4259 | | try slot.moveTo(inst, cg); |
| 4247 | try slot.finish(inst, &.{}, &.{}, cg); |
| 4260 | 4248 | }, |
| 4261 | 4249 | .call => try cg.airCall(inst, .auto, .{ .safety = true }), |
| 4262 | 4250 | .call_always_tail => try cg.airCall(inst, .always_tail, .{ .safety = true }), |
| ... | ... | @@ -6986,10 +6974,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 6986 | 6974 | }), |
| 6987 | 6975 | else => |e| return e, |
| 6988 | 6976 | }; |
| 6989 | | for (ops) |op| for (res) |r| { |
| 6990 | | if (op.index == r.index) break; |
| 6991 | | } else try op.die(cg); |
| 6992 | | try res[0].moveTo(inst, cg); |
| 6977 | try res[0].finish(inst, &.{ty_op.operand}, &ops, cg); |
| 6993 | 6978 | }, |
| 6994 | 6979 | |
| 6995 | 6980 | .cmp_vector, .cmp_vector_optimized => |air_tag| if (use_old) try cg.airCmpVector(inst) else fallback: { |
| ... | ... | @@ -8394,6 +8379,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 8394 | 8379 | } }, |
| 8395 | 8380 | }, .{ |
| 8396 | 8381 | .dst_constraints = .{.{ .bool_vec = .byte }}, |
| 8382 | .src_constraints = .{ .any_scalar_int, .any_scalar_int }, |
| 8397 | 8383 | .patterns = &.{ |
| 8398 | 8384 | .{ .src = .{ .to_mem, .to_mem } }, |
| 8399 | 8385 | }, |
| ... | ... | @@ -8550,6 +8536,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 8550 | 8536 | } }, |
| 8551 | 8537 | }, .{ |
| 8552 | 8538 | .dst_constraints = .{.{ .bool_vec = .dword }}, |
| 8539 | .src_constraints = .{ .any_scalar_int, .any_scalar_int }, |
| 8553 | 8540 | .patterns = &.{ |
| 8554 | 8541 | .{ .src = .{ .to_mem, .to_mem } }, |
| 8555 | 8542 | }, |
| ... | ... | @@ -8710,6 +8697,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 8710 | 8697 | }, .{ |
| 8711 | 8698 | .required_features = .{ .@"64bit", null, null, null }, |
| 8712 | 8699 | .dst_constraints = .{.{ .bool_vec = .qword }}, |
| 8700 | .src_constraints = .{ .any_scalar_int, .any_scalar_int }, |
| 8713 | 8701 | .patterns = &.{ |
| 8714 | 8702 | .{ .src = .{ .to_mem, .to_mem } }, |
| 8715 | 8703 | }, |
| ... | ... | @@ -8784,6 +8772,72 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 8784 | 8772 | .{ ._, ._r, .sh, .tmp3d, .si(3), ._, ._ }, |
| 8785 | 8773 | .{ ._, ._, .mov, .memi(.dst0p, .tmp3), .tmp2p, ._, ._ }, |
| 8786 | 8774 | } }, |
| 8775 | }, .{ |
| 8776 | .required_features = .{ .f16c, null, null, null }, |
| 8777 | .src_constraints = .{ |
| 8778 | .{ .scalar_exact_float = .{ .of = .qword, .is = .word } }, |
| 8779 | .{ .scalar_exact_float = .{ .of = .qword, .is = .word } }, |
| 8780 | }, |
| 8781 | .patterns = &.{ |
| 8782 | .{ .src = .{ .mem, .mem } }, |
| 8783 | .{ .src = .{ .sse, .mem } }, |
| 8784 | .{ .src = .{ .mem, .sse } }, |
| 8785 | .{ .src = .{ .to_sse, .to_sse } }, |
| 8786 | }, |
| 8787 | .extra_temps = .{ |
| 8788 | .{ .kind = .{ .rc = .sse } }, |
| 8789 | .unused, |
| 8790 | .unused, |
| 8791 | .unused, |
| 8792 | .unused, |
| 8793 | .unused, |
| 8794 | }, |
| 8795 | .dst_temps = .{.{ .rc_mask = .{ .rc = .sse, .info = .{ |
| 8796 | .kind = .all, |
| 8797 | .scalar = .dword, |
| 8798 | } } }}, |
| 8799 | .each = .{ .once = &.{ |
| 8800 | .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, |
| 8801 | .{ ._, .v_ps, .cvtph2, .tmp0x, .src1q, ._, ._ }, |
| 8802 | .{ ._, .v_ps, .cmp, .dst0x, .dst0x, .tmp0x, .si(switch (cc) { |
| 8803 | else => unreachable, |
| 8804 | .e => 0b00000, |
| 8805 | .ne => 0b00100, |
| 8806 | }) }, |
| 8807 | } }, |
| 8808 | }, .{ |
| 8809 | .required_features = .{ .f16c, null, null, null }, |
| 8810 | .src_constraints = .{ |
| 8811 | .{ .scalar_exact_float = .{ .of = .xword, .is = .word } }, |
| 8812 | .{ .scalar_exact_float = .{ .of = .xword, .is = .word } }, |
| 8813 | }, |
| 8814 | .patterns = &.{ |
| 8815 | .{ .src = .{ .mem, .mem } }, |
| 8816 | .{ .src = .{ .to_sse, .mem } }, |
| 8817 | .{ .src = .{ .mem, .to_sse } }, |
| 8818 | .{ .src = .{ .to_sse, .to_sse } }, |
| 8819 | }, |
| 8820 | .extra_temps = .{ |
| 8821 | .{ .kind = .{ .rc = .sse } }, |
| 8822 | .unused, |
| 8823 | .unused, |
| 8824 | .unused, |
| 8825 | .unused, |
| 8826 | .unused, |
| 8827 | }, |
| 8828 | .dst_temps = .{.{ .rc_mask = .{ .rc = .sse, .info = .{ |
| 8829 | .kind = .all, |
| 8830 | .scalar = .dword, |
| 8831 | } } }}, |
| 8832 | .each = .{ .once = &.{ |
| 8833 | .{ ._, .v_ps, .cvtph2, .dst0y, .src0x, ._, ._ }, |
| 8834 | .{ ._, .v_ps, .cvtph2, .tmp0y, .src1x, ._, ._ }, |
| 8835 | .{ ._, .v_ps, .cmp, .dst0y, .dst0y, .tmp0y, .si(switch (cc) { |
| 8836 | else => unreachable, |
| 8837 | .e => 0b00000, |
| 8838 | .ne => 0b00100, |
| 8839 | }) }, |
| 8840 | } }, |
| 8787 | 8841 | } }, |
| 8788 | 8842 | }) catch |err| switch (err) { |
| 8789 | 8843 | error.SelectFailed => return cg.fail("failed to select {s} {} {} {}", .{ |
| ... | ... | @@ -8797,10 +8851,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 8797 | 8851 | .gte => unreachable, |
| 8798 | 8852 | .gt => unreachable, |
| 8799 | 8853 | } |
| 8800 | | for (ops) |op| for (res) |r| { |
| 8801 | | if (op.index == r.index) break; |
| 8802 | | } else try op.die(cg); |
| 8803 | | try res[0].moveTo(inst, cg); |
| 8854 | try res[0].finish(inst, &.{ extra.lhs, extra.rhs }, &ops, cg); |
| 8804 | 8855 | }, |
| 8805 | 8856 | |
| 8806 | 8857 | .abs => |air_tag| if (use_old) try cg.airAbs(inst) else fallback: { |
| ... | ... | @@ -9836,10 +9887,18 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 9836 | 9887 | .required_features = .{ .x87, null, null, null }, |
| 9837 | 9888 | .src_constraints = .{ .{ .scalar_exact_float = .{ .of = .xword, .is = .tbyte } }, .any }, |
| 9838 | 9889 | .patterns = &.{ |
| 9890 | .{ .src = .{ .mem, .none } }, |
| 9839 | 9891 | .{ .src = .{ .to_x87, .none } }, |
| 9840 | 9892 | }, |
| 9893 | .extra_temps = .{ |
| 9894 | .{ .type = .f80, .kind = .{ .reg = .st7 } }, |
| 9895 | .unused, |
| 9896 | .unused, |
| 9897 | .unused, |
| 9898 | .unused, |
| 9899 | .unused, |
| 9900 | }, |
| 9841 | 9901 | .dst_temps = .{.{ .mut_reg = .{ .ref = .src0, .rc = .x87 } }}, |
| 9842 | | .clobbers = .{ .st = 1 }, |
| 9843 | 9902 | .each = .{ .once = &.{ |
| 9844 | 9903 | .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, |
| 9845 | 9904 | .{ ._, .f_, .abs, ._, ._, ._, ._ }, |
| ... | ... | @@ -9967,10 +10026,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 9967 | 10026 | }), |
| 9968 | 10027 | else => |e| return e, |
| 9969 | 10028 | }; |
| 9970 | | for (ops) |op| for (res) |r| { |
| 9971 | | if (op.index == r.index) break; |
| 9972 | | } else try op.die(cg); |
| 9973 | | try res[0].moveTo(inst, cg); |
| 10029 | try res[0].finish(inst, &.{ty_op.operand}, &ops, cg); |
| 9974 | 10030 | }, |
| 9975 | 10031 | |
| 9976 | 10032 | .cmp_lt, |
| ... | ... | @@ -10166,10 +10222,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 10166 | 10222 | }), |
| 10167 | 10223 | else => |e| return e, |
| 10168 | 10224 | }; |
| 10169 | | for (ops) |op| for (res) |r| { |
| 10170 | | if (op.index == r.index) break; |
| 10171 | | } else try op.die(cg); |
| 10172 | | try res[0].moveTo(inst, cg); |
| 10225 | try res[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg); |
| 10173 | 10226 | }, |
| 10174 | 10227 | .cmp_eq, |
| 10175 | 10228 | .cmp_eq_optimized, |
| ... | ... | @@ -10182,7 +10235,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 10182 | 10235 | }) else fallback: { |
| 10183 | 10236 | const bin_op = air_datas[@intFromEnum(inst)].bin_op; |
| 10184 | 10237 | const scalar_ty = cg.typeOf(bin_op.lhs).scalarType(zcu); |
| 10185 | | if (cg.intInfo(scalar_ty) == null) break :fallback try cg.airCmp(inst, switch (air_tag) { |
| 10238 | if (cg.intInfo(scalar_ty) == null and cg.floatBits(scalar_ty) == null) break :fallback try cg.airCmp(inst, switch (air_tag) { |
| 10186 | 10239 | else => unreachable, |
| 10187 | 10240 | .cmp_eq, .cmp_eq_optimized => .eq, |
| 10188 | 10241 | .cmp_neq, .cmp_neq_optimized => .neq, |
| ... | ... | @@ -10424,6 +10477,10 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 10424 | 10477 | } }, |
| 10425 | 10478 | }, .{ |
| 10426 | 10479 | .required_features = .{ .avx2, null, null, null }, |
| 10480 | .src_constraints = .{ |
| 10481 | .{ .remainder_int = .{ .of = .yword, .is = .yword } }, |
| 10482 | .{ .remainder_int = .{ .of = .yword, .is = .yword } }, |
| 10483 | }, |
| 10427 | 10484 | .patterns = &.{ |
| 10428 | 10485 | .{ .src = .{ .to_mem, .to_mem } }, |
| 10429 | 10486 | }, |
| ... | ... | @@ -10481,6 +10538,10 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 10481 | 10538 | } }, |
| 10482 | 10539 | }, .{ |
| 10483 | 10540 | .required_features = .{ .avx, null, null, null }, |
| 10541 | .src_constraints = .{ |
| 10542 | .{ .remainder_int = .{ .of = .yword, .is = .yword } }, |
| 10543 | .{ .remainder_int = .{ .of = .yword, .is = .yword } }, |
| 10544 | }, |
| 10484 | 10545 | .patterns = &.{ |
| 10485 | 10546 | .{ .src = .{ .to_mem, .to_mem } }, |
| 10486 | 10547 | }, |
| ... | ... | @@ -10506,6 +10567,10 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 10506 | 10567 | } }, |
| 10507 | 10568 | }, .{ |
| 10508 | 10569 | .required_features = .{ .avx, null, null, null }, |
| 10570 | .src_constraints = .{ |
| 10571 | .{ .remainder_int = .{ .of = .xword, .is = .xword } }, |
| 10572 | .{ .remainder_int = .{ .of = .xword, .is = .xword } }, |
| 10573 | }, |
| 10509 | 10574 | .patterns = &.{ |
| 10510 | 10575 | .{ .src = .{ .to_mem, .to_mem } }, |
| 10511 | 10576 | }, |
| ... | ... | @@ -10531,6 +10596,10 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 10531 | 10596 | } }, |
| 10532 | 10597 | }, .{ |
| 10533 | 10598 | .required_features = .{ .sse4_1, null, null, null }, |
| 10599 | .src_constraints = .{ |
| 10600 | .{ .remainder_int = .{ .of = .xword, .is = .xword } }, |
| 10601 | .{ .remainder_int = .{ .of = .xword, .is = .xword } }, |
| 10602 | }, |
| 10534 | 10603 | .patterns = &.{ |
| 10535 | 10604 | .{ .src = .{ .to_mem, .to_mem } }, |
| 10536 | 10605 | }, |
| ... | ... | @@ -10556,6 +10625,10 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 10556 | 10625 | } }, |
| 10557 | 10626 | }, .{ |
| 10558 | 10627 | .required_features = .{ .sse2, null, null, null }, |
| 10628 | .src_constraints = .{ |
| 10629 | .{ .remainder_int = .{ .of = .xword, .is = .xword } }, |
| 10630 | .{ .remainder_int = .{ .of = .xword, .is = .xword } }, |
| 10631 | }, |
| 10559 | 10632 | .patterns = &.{ |
| 10560 | 10633 | .{ .src = .{ .to_mem, .to_mem } }, |
| 10561 | 10634 | }, |
| ... | ... | @@ -10584,6 +10657,10 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 10584 | 10657 | } }, |
| 10585 | 10658 | }, .{ |
| 10586 | 10659 | .required_features = .{ .sse, .mmx, null, null }, |
| 10660 | .src_constraints = .{ |
| 10661 | .{ .remainder_int = .{ .of = .qword, .is = .qword } }, |
| 10662 | .{ .remainder_int = .{ .of = .qword, .is = .qword } }, |
| 10663 | }, |
| 10587 | 10664 | .patterns = &.{ |
| 10588 | 10665 | .{ .src = .{ .to_mem, .to_mem } }, |
| 10589 | 10666 | }, |
| ... | ... | @@ -10611,6 +10688,10 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 10611 | 10688 | .{ ._, ._, .cmp, .tmp0d, .si(0xff), ._, ._ }, |
| 10612 | 10689 | } }, |
| 10613 | 10690 | }, .{ |
| 10691 | .src_constraints = .{ |
| 10692 | .{ .remainder_int = .{ .of = .qword, .is = .qword } }, |
| 10693 | .{ .remainder_int = .{ .of = .qword, .is = .qword } }, |
| 10694 | }, |
| 10614 | 10695 | .patterns = &.{ |
| 10615 | 10696 | .{ .src = .{ .to_mem, .to_mem } }, |
| 10616 | 10697 | }, |
| ... | ... | @@ -10634,6 +10715,443 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 10634 | 10715 | .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, |
| 10635 | 10716 | .{ ._, ._, .@"test", .tmp1p, .tmp1p, ._, ._ }, |
| 10636 | 10717 | } }, |
| 10718 | }, .{ |
| 10719 | .required_features = .{ .f16c, null, null, null }, |
| 10720 | .src_constraints = .{ .{ .float = .word }, .{ .float = .word } }, |
| 10721 | .patterns = &.{ |
| 10722 | .{ .src = .{ .to_sse, .to_sse } }, |
| 10723 | }, |
| 10724 | .extra_temps = .{ |
| 10725 | .{ .kind = .{ .rc = .sse } }, |
| 10726 | .{ .kind = .{ .rc = .sse } }, |
| 10727 | .unused, |
| 10728 | .unused, |
| 10729 | .unused, |
| 10730 | .unused, |
| 10731 | }, |
| 10732 | .dst_temps = .{.{ .cc = switch (cc) { |
| 10733 | else => unreachable, |
| 10734 | .e => .z_and_np, |
| 10735 | .ne => .nz_or_p, |
| 10736 | } }}, |
| 10737 | .clobbers = .{ .eflags = true }, |
| 10738 | .each = .{ .once = &.{ |
| 10739 | .{ ._, .vp_, .unpcklwd, .tmp0x, .src0x, .src1x, ._ }, |
| 10740 | .{ ._, .v_ps, .cvtph2, .tmp0x, .tmp0x, ._, ._ }, |
| 10741 | .{ ._, .v_, .movshdup, .tmp1x, .tmp0x, ._, ._ }, |
| 10742 | .{ ._, .v_ss, .ucomi, .tmp0x, .tmp1x, ._, ._ }, |
| 10743 | } }, |
| 10744 | }, .{ |
| 10745 | .required_features = .{ .sse, null, null, null }, |
| 10746 | .src_constraints = .{ .{ .float = .word }, .{ .float = .word } }, |
| 10747 | .patterns = &.{ |
| 10748 | .{ .src = .{ .{ .to_reg = .xmm0 }, .{ .to_reg = .xmm1 } } }, |
| 10749 | }, |
| 10750 | .call_frame = .{ .size = 0, .alignment = .@"16" }, |
| 10751 | .extra_temps = .{ |
| 10752 | .{ .type = .usize, .kind = .{ .symbol = &.{ .name = switch (cc) { |
| 10753 | else => unreachable, |
| 10754 | .e => "__eqhf2", |
| 10755 | .ne => "__nehf2", |
| 10756 | } } } }, |
| 10757 | .{ .type = .i32, .kind = .{ .reg = .eax } }, |
| 10758 | .unused, |
| 10759 | .unused, |
| 10760 | .unused, |
| 10761 | .unused, |
| 10762 | }, |
| 10763 | .dst_temps = .{.{ .cc = cc }}, |
| 10764 | .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, |
| 10765 | .each = .{ .once = &.{ |
| 10766 | .{ ._, ._, .call, .tmp0p, ._, ._, ._ }, |
| 10767 | .{ ._, ._, .@"test", .tmp1d, .tmp1d, ._, ._ }, |
| 10768 | } }, |
| 10769 | }, .{ |
| 10770 | .required_features = .{ .avx, null, null, null }, |
| 10771 | .src_constraints = .{ .{ .float = .dword }, .{ .float = .dword } }, |
| 10772 | .patterns = &.{ |
| 10773 | .{ .src = .{ .to_sse, .mem } }, |
| 10774 | .{ .src = .{ .mem, .to_sse }, .commute = .{ 0, 1 } }, |
| 10775 | .{ .src = .{ .to_sse, .to_sse } }, |
| 10776 | }, |
| 10777 | .extra_temps = .{ |
| 10778 | .{ .type = .f16, .kind = .{ .rc = .sse } }, |
| 10779 | .{ .type = .f16, .kind = .{ .rc = .sse } }, |
| 10780 | .unused, |
| 10781 | .unused, |
| 10782 | .unused, |
| 10783 | .unused, |
| 10784 | }, |
| 10785 | .dst_temps = .{.{ .cc = switch (cc) { |
| 10786 | else => unreachable, |
| 10787 | .e => .z_and_np, |
| 10788 | .ne => .nz_or_p, |
| 10789 | } }}, |
| 10790 | .clobbers = .{ .eflags = true }, |
| 10791 | .each = .{ .once = &.{ |
| 10792 | .{ ._, .v_ss, .ucomi, .src0x, .src1d, ._, ._ }, |
| 10793 | } }, |
| 10794 | }, .{ |
| 10795 | .required_features = .{ .sse, null, null, null }, |
| 10796 | .src_constraints = .{ .{ .float = .dword }, .{ .float = .dword } }, |
| 10797 | .patterns = &.{ |
| 10798 | .{ .src = .{ .to_sse, .mem } }, |
| 10799 | .{ .src = .{ .mem, .to_sse }, .commute = .{ 0, 1 } }, |
| 10800 | .{ .src = .{ .to_sse, .to_sse } }, |
| 10801 | }, |
| 10802 | .extra_temps = .{ |
| 10803 | .{ .type = .f16, .kind = .{ .rc = .sse } }, |
| 10804 | .{ .type = .f16, .kind = .{ .rc = .sse } }, |
| 10805 | .unused, |
| 10806 | .unused, |
| 10807 | .unused, |
| 10808 | .unused, |
| 10809 | }, |
| 10810 | .dst_temps = .{.{ .cc = switch (cc) { |
| 10811 | else => unreachable, |
| 10812 | .e => .z_and_np, |
| 10813 | .ne => .nz_or_p, |
| 10814 | } }}, |
| 10815 | .clobbers = .{ .eflags = true }, |
| 10816 | .each = .{ .once = &.{ |
| 10817 | .{ ._, ._ss, .ucomi, .src0x, .src1d, ._, ._ }, |
| 10818 | } }, |
| 10819 | }, .{ |
| 10820 | .required_features = .{ .sse, null, null, null }, |
| 10821 | .src_constraints = .{ .{ .float = .dword }, .{ .float = .word } }, |
| 10822 | .patterns = &.{ |
| 10823 | .{ .src = .{ .{ .to_reg = .xmm0 }, .{ .to_reg = .xmm1 } } }, |
| 10824 | }, |
| 10825 | .call_frame = .{ .size = 0, .alignment = .@"16" }, |
| 10826 | .extra_temps = .{ |
| 10827 | .{ .type = .usize, .kind = .{ .symbol = &.{ .name = switch (cc) { |
| 10828 | else => unreachable, |
| 10829 | .e => "__eqsf2", |
| 10830 | .ne => "__nesf2", |
| 10831 | } } } }, |
| 10832 | .{ .type = .i32, .kind = .{ .reg = .eax } }, |
| 10833 | .unused, |
| 10834 | .unused, |
| 10835 | .unused, |
| 10836 | .unused, |
| 10837 | }, |
| 10838 | .dst_temps = .{.{ .cc = cc }}, |
| 10839 | .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, |
| 10840 | .each = .{ .once = &.{ |
| 10841 | .{ ._, ._, .call, .tmp0p, ._, ._, ._ }, |
| 10842 | .{ ._, ._, .@"test", .tmp1d, .tmp1d, ._, ._ }, |
| 10843 | } }, |
| 10844 | }, .{ |
| 10845 | .required_features = .{ .avx, null, null, null }, |
| 10846 | .src_constraints = .{ .{ .float = .qword }, .{ .float = .qword } }, |
| 10847 | .patterns = &.{ |
| 10848 | .{ .src = .{ .to_sse, .mem } }, |
| 10849 | .{ .src = .{ .mem, .to_sse }, .commute = .{ 0, 1 } }, |
| 10850 | .{ .src = .{ .to_sse, .to_sse } }, |
| 10851 | }, |
| 10852 | .extra_temps = .{ |
| 10853 | .{ .type = .f16, .kind = .{ .rc = .sse } }, |
| 10854 | .{ .type = .f16, .kind = .{ .rc = .sse } }, |
| 10855 | .unused, |
| 10856 | .unused, |
| 10857 | .unused, |
| 10858 | .unused, |
| 10859 | }, |
| 10860 | .dst_temps = .{.{ .cc = switch (cc) { |
| 10861 | else => unreachable, |
| 10862 | .e => .z_and_np, |
| 10863 | .ne => .nz_or_p, |
| 10864 | } }}, |
| 10865 | .clobbers = .{ .eflags = true }, |
| 10866 | .each = .{ .once = &.{ |
| 10867 | .{ ._, .v_sd, .ucomi, .src0x, .src1q, ._, ._ }, |
| 10868 | } }, |
| 10869 | }, .{ |
| 10870 | .required_features = .{ .sse2, null, null, null }, |
| 10871 | .src_constraints = .{ .{ .float = .qword }, .{ .float = .qword } }, |
| 10872 | .patterns = &.{ |
| 10873 | .{ .src = .{ .to_sse, .mem } }, |
| 10874 | .{ .src = .{ .mem, .to_sse }, .commute = .{ 0, 1 } }, |
| 10875 | .{ .src = .{ .to_sse, .to_sse } }, |
| 10876 | }, |
| 10877 | .extra_temps = .{ |
| 10878 | .{ .type = .f16, .kind = .{ .rc = .sse } }, |
| 10879 | .{ .type = .f16, .kind = .{ .rc = .sse } }, |
| 10880 | .unused, |
| 10881 | .unused, |
| 10882 | .unused, |
| 10883 | .unused, |
| 10884 | }, |
| 10885 | .dst_temps = .{.{ .cc = switch (cc) { |
| 10886 | else => unreachable, |
| 10887 | .e => .z_and_np, |
| 10888 | .ne => .nz_or_p, |
| 10889 | } }}, |
| 10890 | .clobbers = .{ .eflags = true }, |
| 10891 | .each = .{ .once = &.{ |
| 10892 | .{ ._, ._sd, .ucomi, .src0x, .src1q, ._, ._ }, |
| 10893 | } }, |
| 10894 | }, .{ |
| 10895 | .required_features = .{ .sse, null, null, null }, |
| 10896 | .src_constraints = .{ .{ .float = .qword }, .{ .float = .qword } }, |
| 10897 | .patterns = &.{ |
| 10898 | .{ .src = .{ .{ .to_reg = .xmm0 }, .{ .to_reg = .xmm1 } } }, |
| 10899 | }, |
| 10900 | .call_frame = .{ .size = 0, .alignment = .@"16" }, |
| 10901 | .extra_temps = .{ |
| 10902 | .{ .type = .usize, .kind = .{ .symbol = &.{ .name = switch (cc) { |
| 10903 | else => unreachable, |
| 10904 | .e => "__eqdf2", |
| 10905 | .ne => "__nedf2", |
| 10906 | } } } }, |
| 10907 | .{ .type = .i32, .kind = .{ .reg = .eax } }, |
| 10908 | .unused, |
| 10909 | .unused, |
| 10910 | .unused, |
| 10911 | .unused, |
| 10912 | }, |
| 10913 | .dst_temps = .{.{ .cc = cc }}, |
| 10914 | .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, |
| 10915 | .each = .{ .once = &.{ |
| 10916 | .{ ._, ._, .call, .tmp0p, ._, ._, ._ }, |
| 10917 | .{ ._, ._, .@"test", .tmp1d, .tmp1d, ._, ._ }, |
| 10918 | } }, |
| 10919 | }, .{ |
| 10920 | .required_features = .{ .x87, .cmov, null, null }, |
| 10921 | .src_constraints = .{ .{ .float = .tbyte }, .{ .float = .tbyte } }, |
| 10922 | .patterns = &.{ |
| 10923 | .{ .src = .{ .mem, .mem } }, |
| 10924 | }, |
| 10925 | .extra_temps = .{ |
| 10926 | .{ .type = .f80, .kind = .{ .reg = .st6 } }, |
| 10927 | .{ .type = .f80, .kind = .{ .reg = .st7 } }, |
| 10928 | .unused, |
| 10929 | .unused, |
| 10930 | .unused, |
| 10931 | .unused, |
| 10932 | }, |
| 10933 | .dst_temps = .{.{ .cc = switch (cc) { |
| 10934 | else => unreachable, |
| 10935 | .e => .z_and_np, |
| 10936 | .ne => .nz_or_p, |
| 10937 | } }}, |
| 10938 | .clobbers = .{ .eflags = true }, |
| 10939 | .each = .{ .once = &.{ |
| 10940 | .{ ._, .f_, .ld, .src1t, ._, ._, ._ }, |
| 10941 | .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, |
| 10942 | .{ ._, .f_p, .ucomi, .tmp0t, .tmp1t, ._, ._ }, |
| 10943 | .{ ._, .f_p, .st, .tmp1t, ._, ._, ._ }, |
| 10944 | } }, |
| 10945 | }, .{ |
| 10946 | .required_features = .{ .x87, .cmov, null, null }, |
| 10947 | .src_constraints = .{ .{ .float = .tbyte }, .{ .float = .tbyte } }, |
| 10948 | .patterns = &.{ |
| 10949 | .{ .src = .{ .to_x87, .mem }, .commute = .{ 0, 1 } }, |
| 10950 | .{ .src = .{ .mem, .to_x87 } }, |
| 10951 | .{ .src = .{ .to_x87, .to_x87 } }, |
| 10952 | }, |
| 10953 | .extra_temps = .{ |
| 10954 | .{ .type = .f80, .kind = .{ .reg = .st7 } }, |
| 10955 | .unused, |
| 10956 | .unused, |
| 10957 | .unused, |
| 10958 | .unused, |
| 10959 | .unused, |
| 10960 | }, |
| 10961 | .dst_temps = .{.{ .cc = switch (cc) { |
| 10962 | else => unreachable, |
| 10963 | .e => .z_and_np, |
| 10964 | .ne => .nz_or_p, |
| 10965 | } }}, |
| 10966 | .clobbers = .{ .eflags = true }, |
| 10967 | .each = .{ .once = &.{ |
| 10968 | .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, |
| 10969 | .{ ._, .f_p, .ucomi, .tmp0t, .src1t, ._, ._ }, |
| 10970 | } }, |
| 10971 | }, .{ |
| 10972 | .required_features = .{ .sahf, .x87, null, null }, |
| 10973 | .src_constraints = .{ .{ .float = .tbyte }, .{ .float = .tbyte } }, |
| 10974 | .patterns = &.{ |
| 10975 | .{ .src = .{ .mem, .mem } }, |
| 10976 | }, |
| 10977 | .extra_temps = .{ |
| 10978 | .{ .type = .f80, .kind = .{ .reg = .st6 } }, |
| 10979 | .{ .type = .f80, .kind = .{ .reg = .st7 } }, |
| 10980 | .{ .type = .u16, .kind = .{ .reg = .ax } }, |
| 10981 | .unused, |
| 10982 | .unused, |
| 10983 | .unused, |
| 10984 | }, |
| 10985 | .dst_temps = .{.{ .cc = switch (cc) { |
| 10986 | else => unreachable, |
| 10987 | .e => .z_and_np, |
| 10988 | .ne => .nz_or_p, |
| 10989 | } }}, |
| 10990 | .clobbers = .{ .eflags = true }, |
| 10991 | .each = .{ .once = &.{ |
| 10992 | .{ ._, .f_, .ld, .src1t, ._, ._, ._ }, |
| 10993 | .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, |
| 10994 | .{ ._, .f_pp, .ucom, ._, ._, ._, ._ }, |
| 10995 | .{ ._, .fn_sw, .st, .tmp2w, ._, ._, ._ }, |
| 10996 | .{ ._, ._, .sahf, ._, ._, ._, ._ }, |
| 10997 | } }, |
| 10998 | }, .{ |
| 10999 | .required_features = .{ .@"64bit", .x87, null, null }, |
| 11000 | .src_constraints = .{ .{ .float = .tbyte }, .{ .float = .tbyte } }, |
| 11001 | .patterns = &.{ |
| 11002 | .{ .src = .{ .mem, .mem } }, |
| 11003 | }, |
| 11004 | .extra_temps = .{ |
| 11005 | .{ .type = .f80, .kind = .{ .reg = .st6 } }, |
| 11006 | .{ .type = .f80, .kind = .{ .reg = .st7 } }, |
| 11007 | .{ .type = .u16, .kind = .{ .reg = .ax } }, |
| 11008 | .{ .type = .u8, .kind = .{ .reg = .ah } }, |
| 11009 | .unused, |
| 11010 | .unused, |
| 11011 | }, |
| 11012 | .dst_temps = .{.{ .cc = cc }}, |
| 11013 | .clobbers = .{ .eflags = true }, |
| 11014 | .each = .{ .once = &.{ |
| 11015 | .{ ._, .f_, .ld, .src1t, ._, ._, ._ }, |
| 11016 | .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, |
| 11017 | .{ ._, .f_pp, .ucom, ._, ._, ._, ._ }, |
| 11018 | .{ ._, .fn_sw, .st, .tmp2w, ._, ._, ._ }, |
| 11019 | .{ ._, ._, .xor, .tmp3b, .si(0b0_1_000_000), ._, ._ }, |
| 11020 | .{ ._, ._, .@"test", .tmp3b, .si(0b0_1_000_100), ._, ._ }, |
| 11021 | } }, |
| 11022 | }, .{ |
| 11023 | .required_features = .{ .x87, null, null, null }, |
| 11024 | .src_constraints = .{ .{ .float = .tbyte }, .{ .float = .tbyte } }, |
| 11025 | .patterns = &.{ |
| 11026 | .{ .src = .{ .mem, .mem } }, |
| 11027 | }, |
| 11028 | .extra_temps = .{ |
| 11029 | .{ .type = .f80, .kind = .{ .reg = .st6 } }, |
| 11030 | .{ .type = .f80, .kind = .{ .reg = .st7 } }, |
| 11031 | .{ .type = .u16, .kind = .{ .reg = .ax } }, |
| 11032 | .unused, |
| 11033 | .unused, |
| 11034 | .unused, |
| 11035 | }, |
| 11036 | .dst_temps = .{.{ .cc = switch (cc) { |
| 11037 | else => unreachable, |
| 11038 | .e => .z_and_np, |
| 11039 | .ne => .nz_or_p, |
| 11040 | } }}, |
| 11041 | .clobbers = .{ .eflags = true }, |
| 11042 | .each = .{ .once = &.{ |
| 11043 | .{ ._, .f_, .ld, .src1t, ._, ._, ._ }, |
| 11044 | .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, |
| 11045 | .{ ._, .f_pp, .ucom, ._, ._, ._, ._ }, |
| 11046 | .{ ._, .fn_sw, .st, .tmp2w, ._, ._, ._ }, |
| 11047 | .{ ._, ._, .sahf, ._, ._, ._, ._ }, |
| 11048 | } }, |
| 11049 | }, .{ |
| 11050 | .required_features = .{ .sahf, .x87, null, null }, |
| 11051 | .src_constraints = .{ .{ .float = .tbyte }, .{ .float = .tbyte } }, |
| 11052 | .patterns = &.{ |
| 11053 | .{ .src = .{ .to_x87, .mem }, .commute = .{ 0, 1 } }, |
| 11054 | .{ .src = .{ .mem, .to_x87 } }, |
| 11055 | .{ .src = .{ .to_x87, .to_x87 } }, |
| 11056 | }, |
| 11057 | .extra_temps = .{ |
| 11058 | .{ .type = .f80, .kind = .{ .reg = .st6 } }, |
| 11059 | .{ .type = .f80, .kind = .{ .reg = .st7 } }, |
| 11060 | .{ .type = .u16, .kind = .{ .reg = .ax } }, |
| 11061 | .unused, |
| 11062 | .unused, |
| 11063 | .unused, |
| 11064 | }, |
| 11065 | .dst_temps = .{.{ .cc = switch (cc) { |
| 11066 | else => unreachable, |
| 11067 | .e => .z_and_np, |
| 11068 | .ne => .nz_or_p, |
| 11069 | } }}, |
| 11070 | .clobbers = .{ .eflags = true }, |
| 11071 | .each = .{ .once = &.{ |
| 11072 | .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, |
| 11073 | .{ ._, .f_p, .ucom, .src1t, ._, ._, ._ }, |
| 11074 | .{ ._, .fn_sw, .st, .tmp2w, ._, ._, ._ }, |
| 11075 | .{ ._, ._, .sahf, ._, ._, ._, ._ }, |
| 11076 | } }, |
| 11077 | }, .{ |
| 11078 | .required_features = .{ .@"64bit", .x87, null, null }, |
| 11079 | .src_constraints = .{ .{ .float = .tbyte }, .{ .float = .tbyte } }, |
| 11080 | .patterns = &.{ |
| 11081 | .{ .src = .{ .to_x87, .mem }, .commute = .{ 0, 1 } }, |
| 11082 | .{ .src = .{ .mem, .to_x87 } }, |
| 11083 | .{ .src = .{ .to_x87, .to_x87 } }, |
| 11084 | }, |
| 11085 | .extra_temps = .{ |
| 11086 | .{ .type = .f80, .kind = .{ .reg = .st6 } }, |
| 11087 | .{ .type = .f80, .kind = .{ .reg = .st7 } }, |
| 11088 | .{ .type = .u16, .kind = .{ .reg = .ax } }, |
| 11089 | .{ .type = .u8, .kind = .{ .reg = .ah } }, |
| 11090 | .unused, |
| 11091 | .unused, |
| 11092 | }, |
| 11093 | .dst_temps = .{.{ .cc = cc }}, |
| 11094 | .clobbers = .{ .eflags = true }, |
| 11095 | .each = .{ .once = &.{ |
| 11096 | .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, |
| 11097 | .{ ._, .f_p, .ucom, .src1t, ._, ._, ._ }, |
| 11098 | .{ ._, .fn_sw, .st, .tmp2w, ._, ._, ._ }, |
| 11099 | .{ ._, ._, .xor, .tmp3b, .si(0b0_1_000_000), ._, ._ }, |
| 11100 | .{ ._, ._, .@"test", .tmp3b, .si(0b0_1_000_100), ._, ._ }, |
| 11101 | } }, |
| 11102 | }, .{ |
| 11103 | .required_features = .{ .x87, null, null, null }, |
| 11104 | .src_constraints = .{ .{ .float = .tbyte }, .{ .float = .tbyte } }, |
| 11105 | .patterns = &.{ |
| 11106 | .{ .src = .{ .to_x87, .mem }, .commute = .{ 0, 1 } }, |
| 11107 | .{ .src = .{ .mem, .to_x87 } }, |
| 11108 | .{ .src = .{ .to_x87, .to_x87 } }, |
| 11109 | }, |
| 11110 | .extra_temps = .{ |
| 11111 | .{ .type = .f80, .kind = .{ .reg = .st6 } }, |
| 11112 | .{ .type = .f80, .kind = .{ .reg = .st7 } }, |
| 11113 | .{ .type = .u16, .kind = .{ .reg = .ax } }, |
| 11114 | .unused, |
| 11115 | .unused, |
| 11116 | .unused, |
| 11117 | }, |
| 11118 | .dst_temps = .{.{ .cc = switch (cc) { |
| 11119 | else => unreachable, |
| 11120 | .e => .z_and_np, |
| 11121 | .ne => .nz_or_p, |
| 11122 | } }}, |
| 11123 | .clobbers = .{ .eflags = true }, |
| 11124 | .each = .{ .once = &.{ |
| 11125 | .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, |
| 11126 | .{ ._, .f_p, .ucom, .src1t, ._, ._, ._ }, |
| 11127 | .{ ._, .fn_sw, .st, .tmp2w, ._, ._, ._ }, |
| 11128 | .{ ._, ._, .sahf, ._, ._, ._, ._ }, |
| 11129 | } }, |
| 11130 | }, .{ |
| 11131 | .required_features = .{ .sse, null, null, null }, |
| 11132 | .src_constraints = .{ .{ .float = .xword }, .{ .float = .xword } }, |
| 11133 | .patterns = &.{ |
| 11134 | .{ .src = .{ .{ .to_reg = .xmm0 }, .{ .to_reg = .xmm1 } } }, |
| 11135 | }, |
| 11136 | .call_frame = .{ .size = 0, .alignment = .@"16" }, |
| 11137 | .extra_temps = .{ |
| 11138 | .{ .type = .usize, .kind = .{ .symbol = &.{ .name = switch (cc) { |
| 11139 | else => unreachable, |
| 11140 | .e => "__eqtf2", |
| 11141 | .ne => "__netf2", |
| 11142 | } } } }, |
| 11143 | .{ .type = .i32, .kind = .{ .reg = .eax } }, |
| 11144 | .unused, |
| 11145 | .unused, |
| 11146 | .unused, |
| 11147 | .unused, |
| 11148 | }, |
| 11149 | .dst_temps = .{.{ .cc = cc }}, |
| 11150 | .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, |
| 11151 | .each = .{ .once = &.{ |
| 11152 | .{ ._, ._, .call, .tmp0p, ._, ._, ._ }, |
| 11153 | .{ ._, ._, .@"test", .tmp1d, .tmp1d, ._, ._ }, |
| 11154 | } }, |
| 10637 | 11155 | } }, |
| 10638 | 11156 | }) catch |err| switch (err) { |
| 10639 | 11157 | error.SelectFailed => return cg.fail("failed to select {s} {} {} {}", .{ |
| ... | ... | @@ -10644,10 +11162,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 10644 | 11162 | }), |
| 10645 | 11163 | else => |e| return e, |
| 10646 | 11164 | }; |
| 10647 | | for (ops) |op| for (res) |r| { |
| 10648 | | if (op.index == r.index) break; |
| 10649 | | } else try op.die(cg); |
| 10650 | | try res[0].moveTo(inst, cg); |
| 11165 | try res[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg); |
| 10651 | 11166 | }, |
| 10652 | 11167 | |
| 10653 | 11168 | .cond_br => try cg.airCondBr(inst), |
| ... | ... | @@ -10718,9 +11233,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 10718 | 11233 | .fromSize(opt_child_abi_size) }), |
| 10719 | 11234 | .u(0), |
| 10720 | 11235 | ); |
| 10721 | | var is_null = try cg.tempInit(.bool, .{ .eflags = .e }); |
| 10722 | | try ops[0].die(cg); |
| 10723 | | try is_null.moveTo(inst, cg); |
| 11236 | const is_null = try cg.tempInit(.bool, .{ .eflags = .e }); |
| 11237 | try is_null.finish(inst, &.{un_op}, &ops, cg); |
| 10724 | 11238 | }, |
| 10725 | 11239 | .is_non_null_ptr => if (use_old) try cg.airIsNonNullPtr(inst) else { |
| 10726 | 11240 | const un_op = air_datas[@intFromEnum(inst)].un_op; |
| ... | ... | @@ -10741,9 +11255,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 10741 | 11255 | .fromSize(opt_child_abi_size) }), |
| 10742 | 11256 | .u(0), |
| 10743 | 11257 | ); |
| 10744 | | var is_non_null = try cg.tempInit(.bool, .{ .eflags = .ne }); |
| 10745 | | try ops[0].die(cg); |
| 10746 | | try is_non_null.moveTo(inst, cg); |
| 11258 | const is_non_null = try cg.tempInit(.bool, .{ .eflags = .ne }); |
| 11259 | try is_non_null.finish(inst, &.{un_op}, &ops, cg); |
| 10747 | 11260 | }, |
| 10748 | 11261 | .is_err_ptr => if (use_old) try cg.airIsErrPtr(inst) else { |
| 10749 | 11262 | const un_op = air_datas[@intFromEnum(inst)].un_op; |
| ... | ... | @@ -10759,9 +11272,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 10759 | 11272 | try ops[0].tracking(cg).short.deref().mem(cg, .{ .size = cg.memSize(eu_err_ty) }), |
| 10760 | 11273 | .u(0), |
| 10761 | 11274 | ); |
| 10762 | | var is_err = try cg.tempInit(.bool, .{ .eflags = .ne }); |
| 10763 | | try ops[0].die(cg); |
| 10764 | | try is_err.moveTo(inst, cg); |
| 11275 | const is_err = try cg.tempInit(.bool, .{ .eflags = .ne }); |
| 11276 | try is_err.finish(inst, &.{un_op}, &ops, cg); |
| 10765 | 11277 | }, |
| 10766 | 11278 | .is_non_err_ptr => if (use_old) try cg.airIsNonErrPtr(inst) else { |
| 10767 | 11279 | const un_op = air_datas[@intFromEnum(inst)].un_op; |
| ... | ... | @@ -10777,9 +11289,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 10777 | 11289 | try ops[0].tracking(cg).short.deref().mem(cg, .{ .size = cg.memSize(eu_err_ty) }), |
| 10778 | 11290 | .u(0), |
| 10779 | 11291 | ); |
| 10780 | | var is_non_err = try cg.tempInit(.bool, .{ .eflags = .e }); |
| 10781 | | try ops[0].die(cg); |
| 10782 | | try is_non_err.moveTo(inst, cg); |
| 11292 | const is_non_err = try cg.tempInit(.bool, .{ .eflags = .e }); |
| 11293 | try is_non_err.finish(inst, &.{un_op}, &ops, cg); |
| 10783 | 11294 | }, |
| 10784 | 11295 | .load => if (use_old) try cg.airLoad(inst) else fallback: { |
| 10785 | 11296 | const ty_op = air_datas[@intFromEnum(inst)].ty_op; |
| ... | ... | @@ -10790,26 +11301,25 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 10790 | 11301 | (ptr_info.flags.vector_index == .none or val_ty.toIntern() == .bool_type)) |
| 10791 | 11302 | break :fallback try cg.airLoad(inst); |
| 10792 | 11303 | var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); |
| 10793 | | var res = try ops[0].load(val_ty, .{ |
| 11304 | const res = try ops[0].load(val_ty, .{ |
| 10794 | 11305 | .disp = switch (ptr_info.flags.vector_index) { |
| 10795 | 11306 | .none => 0, |
| 10796 | 11307 | .runtime => unreachable, |
| 10797 | 11308 | else => |vector_index| @intCast(val_ty.abiSize(zcu) * @intFromEnum(vector_index)), |
| 10798 | 11309 | }, |
| 10799 | 11310 | }, cg); |
| 10800 | | for (ops) |op| if (op.index != res.index) try op.die(cg); |
| 10801 | | try res.moveTo(inst, cg); |
| 11311 | try res.finish(inst, &.{ty_op.operand}, &ops, cg); |
| 10802 | 11312 | }, |
| 10803 | 11313 | .int_from_ptr => if (use_old) try cg.airIntFromPtr(inst) else { |
| 10804 | 11314 | const un_op = air_datas[@intFromEnum(inst)].un_op; |
| 10805 | 11315 | var ops = try cg.tempsFromOperands(inst, .{un_op}); |
| 10806 | 11316 | try ops[0].toSlicePtr(cg); |
| 10807 | | try ops[0].moveTo(inst, cg); |
| 11317 | try ops[0].finish(inst, &.{un_op}, &ops, cg); |
| 10808 | 11318 | }, |
| 10809 | 11319 | .int_from_bool => if (use_old) try cg.airIntFromBool(inst) else { |
| 10810 | 11320 | const un_op = air_datas[@intFromEnum(inst)].un_op; |
| 10811 | | var ops = try cg.tempsFromOperands(inst, .{un_op}); |
| 10812 | | try ops[0].moveTo(inst, cg); |
| 11321 | const ops = try cg.tempsFromOperands(inst, .{un_op}); |
| 11322 | try ops[0].finish(inst, &.{un_op}, &ops, cg); |
| 10813 | 11323 | }, |
| 10814 | 11324 | .ret => try cg.airRet(inst, false), |
| 10815 | 11325 | .ret_safe => try cg.airRet(inst, true), |
| ... | ... | @@ -10848,8 +11358,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 10848 | 11358 | .unreach => {}, |
| 10849 | 11359 | .optional_payload_ptr => if (use_old) try cg.airOptionalPayloadPtr(inst) else { |
| 10850 | 11360 | const ty_op = air_datas[@intFromEnum(inst)].ty_op; |
| 10851 | | var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); |
| 10852 | | try ops[0].moveTo(inst, cg); |
| 11361 | const ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); |
| 11362 | try ops[0].finish(inst, &.{ty_op.operand}, &ops, cg); |
| 10853 | 11363 | }, |
| 10854 | 11364 | .optional_payload_ptr_set => if (use_old) try cg.airOptionalPayloadPtrSet(inst) else { |
| 10855 | 11365 | const ty_op = air_datas[@intFromEnum(inst)].ty_op; |
| ... | ... | @@ -10864,7 +11374,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 10864 | 11374 | try has_value.die(cg); |
| 10865 | 11375 | try ops[0].toOffset(-opt_child_abi_size, cg); |
| 10866 | 11376 | } |
| 10867 | | try ops[0].moveTo(inst, cg); |
| 11377 | try ops[0].finish(inst, &.{ty_op.operand}, &ops, cg); |
| 10868 | 11378 | }, |
| 10869 | 11379 | .unwrap_errunion_payload_ptr => if (use_old) try cg.airUnwrapErrUnionPayloadPtr(inst) else { |
| 10870 | 11380 | const ty_op = air_datas[@intFromEnum(inst)].ty_op; |
| ... | ... | @@ -10873,7 +11383,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 10873 | 11383 | const eu_pl_off: i32 = @intCast(codegen.errUnionPayloadOffset(eu_pl_ty, zcu)); |
| 10874 | 11384 | var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); |
| 10875 | 11385 | try ops[0].toOffset(eu_pl_off, cg); |
| 10876 | | try ops[0].moveTo(inst, cg); |
| 11386 | try ops[0].finish(inst, &.{ty_op.operand}, &ops, cg); |
| 10877 | 11387 | }, |
| 10878 | 11388 | .unwrap_errunion_err_ptr => if (use_old) try cg.airUnwrapErrUnionErrPtr(inst) else { |
| 10879 | 11389 | const ty_op = air_datas[@intFromEnum(inst)].ty_op; |
| ... | ... | @@ -10882,9 +11392,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 10882 | 11392 | const eu_err_off: i32 = @intCast(codegen.errUnionErrorOffset(eu_pl_ty, zcu)); |
| 10883 | 11393 | var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); |
| 10884 | 11394 | try ops[0].toOffset(eu_err_off, cg); |
| 10885 | | var err = try ops[0].load(eu_ty.errorUnionSet(zcu), .{}, cg); |
| 10886 | | try ops[0].die(cg); |
| 10887 | | try err.moveTo(inst, cg); |
| 11395 | const err = try ops[0].load(eu_ty.errorUnionSet(zcu), .{}, cg); |
| 11396 | try err.finish(inst, &.{ty_op.operand}, &ops, cg); |
| 10888 | 11397 | }, |
| 10889 | 11398 | .errunion_payload_ptr_set => if (use_old) try cg.airErrUnionPayloadPtrSet(inst) else { |
| 10890 | 11399 | const ty_op = air_datas[@intFromEnum(inst)].ty_op; |
| ... | ... | @@ -10899,7 +11408,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 10899 | 11408 | try ops[0].store(&no_err, .{}, cg); |
| 10900 | 11409 | try no_err.die(cg); |
| 10901 | 11410 | try ops[0].toOffset(eu_pl_off - eu_err_off, cg); |
| 10902 | | try ops[0].moveTo(inst, cg); |
| 11411 | try ops[0].finish(inst, &.{ty_op.operand}, &ops, cg); |
| 10903 | 11412 | }, |
| 10904 | 11413 | .struct_field_ptr => if (use_old) try cg.airStructFieldPtr(inst) else { |
| 10905 | 11414 | const ty_pl = air_datas[@intFromEnum(inst)].ty_pl; |
| ... | ... | @@ -10910,7 +11419,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 10910 | 11419 | ty_pl.ty.toType(), |
| 10911 | 11420 | extra.field_index, |
| 10912 | 11421 | ), cg); |
| 10913 | | try ops[0].moveTo(inst, cg); |
| 11422 | try ops[0].finish(inst, &.{extra.struct_operand}, &ops, cg); |
| 10914 | 11423 | }, |
| 10915 | 11424 | .struct_field_ptr_index_0, |
| 10916 | 11425 | .struct_field_ptr_index_1, |
| ... | ... | @@ -10936,7 +11445,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 10936 | 11445 | .struct_field_ptr_index_3 => 3, |
| 10937 | 11446 | }, |
| 10938 | 11447 | ), cg); |
| 10939 | | try ops[0].moveTo(inst, cg); |
| 11448 | try ops[0].finish(inst, &.{ty_op.operand}, &ops, cg); |
| 10940 | 11449 | }, |
| 10941 | 11450 | .struct_field_val => if (use_old) try cg.airStructFieldVal(inst) else fallback: { |
| 10942 | 11451 | const ty_pl = air_datas[@intFromEnum(inst)].ty_pl; |
| ... | ... | @@ -10953,8 +11462,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 10953 | 11462 | try ops[0].read(field_ty, .{ .disp = field_off }, cg) |
| 10954 | 11463 | else |
| 10955 | 11464 | try cg.tempInit(field_ty, .none); |
| 10956 | | for (ops) |op| if (op.index != res.index) try op.die(cg); |
| 10957 | | try res.moveTo(inst, cg); |
| 11465 | try res.finish(inst, &.{extra.struct_operand}, &ops, cg); |
| 10958 | 11466 | }, |
| 10959 | 11467 | .set_union_tag => if (use_old) try cg.airSetUnionTag(inst) else { |
| 10960 | 11468 | const bin_op = air_datas[@intFromEnum(inst)].bin_op; |
| ... | ... | @@ -10965,7 +11473,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 10965 | 11473 | if (union_layout.tag_size > 0) try ops[0].store(&ops[1], .{ |
| 10966 | 11474 | .disp = @intCast(union_layout.tagOffset()), |
| 10967 | 11475 | }, cg); |
| 10968 | | for (ops) |op| try op.die(cg); |
| 11476 | const res = try cg.tempInit(.void, .none); |
| 11477 | try res.finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg); |
| 10969 | 11478 | }, |
| 10970 | 11479 | .get_union_tag => if (use_old) try cg.airGetUnionTag(inst) else { |
| 10971 | 11480 | const ty_op = air_datas[@intFromEnum(inst)].ty_op; |
| ... | ... | @@ -10973,42 +11482,41 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 10973 | 11482 | var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); |
| 10974 | 11483 | const union_layout = union_ty.unionGetLayout(zcu); |
| 10975 | 11484 | assert(union_layout.tag_size > 0); |
| 10976 | | var res = try ops[0].read(ty_op.ty.toType(), .{ |
| 11485 | const res = try ops[0].read(ty_op.ty.toType(), .{ |
| 10977 | 11486 | .disp = @intCast(union_layout.tagOffset()), |
| 10978 | 11487 | }, cg); |
| 10979 | | for (ops) |op| if (op.index != res.index) try op.die(cg); |
| 10980 | | try res.moveTo(inst, cg); |
| 11488 | try res.finish(inst, &.{ty_op.operand}, &ops, cg); |
| 10981 | 11489 | }, |
| 10982 | 11490 | .slice => if (use_old) try cg.airSlice(inst) else { |
| 10983 | 11491 | const ty_pl = air_datas[@intFromEnum(inst)].ty_pl; |
| 10984 | 11492 | const bin_op = cg.air.extraData(Air.Bin, ty_pl.payload).data; |
| 10985 | 11493 | var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs }); |
| 10986 | 11494 | try ops[0].toPair(&ops[1], cg); |
| 10987 | | try ops[0].moveTo(inst, cg); |
| 11495 | try ops[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg); |
| 10988 | 11496 | }, |
| 10989 | 11497 | .slice_len => if (use_old) try cg.airSliceLen(inst) else { |
| 10990 | 11498 | const ty_op = air_datas[@intFromEnum(inst)].ty_op; |
| 10991 | 11499 | var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); |
| 10992 | 11500 | try ops[0].toSliceLen(cg); |
| 10993 | | try ops[0].moveTo(inst, cg); |
| 11501 | try ops[0].finish(inst, &.{ty_op.operand}, &ops, cg); |
| 10994 | 11502 | }, |
| 10995 | 11503 | .slice_ptr => if (use_old) try cg.airSlicePtr(inst) else { |
| 10996 | 11504 | const ty_op = air_datas[@intFromEnum(inst)].ty_op; |
| 10997 | 11505 | var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); |
| 10998 | 11506 | try ops[0].toSlicePtr(cg); |
| 10999 | | try ops[0].moveTo(inst, cg); |
| 11507 | try ops[0].finish(inst, &.{ty_op.operand}, &ops, cg); |
| 11000 | 11508 | }, |
| 11001 | 11509 | .ptr_slice_len_ptr => if (use_old) try cg.airPtrSliceLenPtr(inst) else { |
| 11002 | 11510 | const ty_op = air_datas[@intFromEnum(inst)].ty_op; |
| 11003 | 11511 | var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); |
| 11004 | 11512 | try ops[0].toOffset(8, cg); |
| 11005 | | try ops[0].moveTo(inst, cg); |
| 11513 | try ops[0].finish(inst, &.{ty_op.operand}, &ops, cg); |
| 11006 | 11514 | }, |
| 11007 | 11515 | .ptr_slice_ptr_ptr => if (use_old) try cg.airPtrSlicePtrPtr(inst) else { |
| 11008 | 11516 | const ty_op = air_datas[@intFromEnum(inst)].ty_op; |
| 11009 | 11517 | var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); |
| 11010 | 11518 | try ops[0].toOffset(0, cg); |
| 11011 | | try ops[0].moveTo(inst, cg); |
| 11519 | try ops[0].finish(inst, &.{ty_op.operand}, &ops, cg); |
| 11012 | 11520 | }, |
| 11013 | 11521 | .slice_elem_val, .ptr_elem_val => |air_tag| if (use_old) switch (air_tag) { |
| 11014 | 11522 | else => unreachable, |
| ... | ... | @@ -11139,10 +11647,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 11139 | 11647 | // hack around Sema OPV bugs |
| 11140 | 11648 | res[0] = try cg.tempInit(res_ty, .none); |
| 11141 | 11649 | } |
| 11142 | | for (ops) |op| for (res) |r| { |
| 11143 | | if (op.index == r.index) break; |
| 11144 | | } else try op.die(cg); |
| 11145 | | try res[0].moveTo(inst, cg); |
| 11650 | try res[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg); |
| 11146 | 11651 | }, |
| 11147 | 11652 | .slice_elem_ptr, .ptr_elem_ptr => |air_tag| if (use_old) switch (air_tag) { |
| 11148 | 11653 | else => unreachable, |
| ... | ... | @@ -11195,8 +11700,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 11195 | 11700 | } }, |
| 11196 | 11701 | }); |
| 11197 | 11702 | } |
| 11198 | | try ops[1].die(cg); |
| 11199 | | try ops[0].moveTo(inst, cg); |
| 11703 | try ops[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg); |
| 11200 | 11704 | }, |
| 11201 | 11705 | .array_to_slice => if (use_old) try cg.airArrayToSlice(inst) else { |
| 11202 | 11706 | const ty_op = air_datas[@intFromEnum(inst)].ty_op; |
| ... | ... | @@ -11205,7 +11709,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 11205 | 11709 | .immediate = cg.typeOf(ty_op.operand).childType(zcu).arrayLen(zcu), |
| 11206 | 11710 | }); |
| 11207 | 11711 | try ops[0].toPair(&len, cg); |
| 11208 | | try ops[0].moveTo(inst, cg); |
| 11712 | try ops[0].finish(inst, &.{ty_op.operand}, &ops, cg); |
| 11209 | 11713 | }, |
| 11210 | 11714 | .error_set_has_value => return cg.fail("TODO implement error_set_has_value", .{}), |
| 11211 | 11715 | .union_init => if (use_old) try cg.airUnionInit(inst) else { |
| ... | ... | @@ -11228,8 +11732,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 11228 | 11732 | try res.write(&ops[0], .{ |
| 11229 | 11733 | .disp = @intCast(union_layout.payloadOffset()), |
| 11230 | 11734 | }, cg); |
| 11231 | | try ops[0].die(cg); |
| 11232 | | try res.moveTo(inst, cg); |
| 11735 | try res.finish(inst, &.{extra.init}, &ops, cg); |
| 11233 | 11736 | }, |
| 11234 | 11737 | .field_parent_ptr => if (use_old) try cg.airFieldParentPtr(inst) else { |
| 11235 | 11738 | const ty_pl = air_datas[@intFromEnum(inst)].ty_pl; |
| ... | ... | @@ -11240,7 +11743,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 11240 | 11743 | cg.typeOf(extra.field_ptr), |
| 11241 | 11744 | extra.field_index, |
| 11242 | 11745 | ), cg); |
| 11243 | | try ops[0].moveTo(inst, cg); |
| 11746 | try ops[0].finish(inst, &.{extra.field_ptr}, &ops, cg); |
| 11244 | 11747 | }, |
| 11245 | 11748 | |
| 11246 | 11749 | .is_named_enum_value => return cg.fail("TODO implement is_named_enum_value", .{}), |
| ... | ... | @@ -11274,8 +11777,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 11274 | 11777 | |
| 11275 | 11778 | .addrspace_cast => { |
| 11276 | 11779 | const ty_op = air_datas[@intFromEnum(inst)].ty_op; |
| 11277 | | var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); |
| 11278 | | try ops[0].moveTo(inst, cg); |
| 11780 | const ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); |
| 11781 | try ops[0].finish(inst, &.{ty_op.operand}, &ops, cg); |
| 11279 | 11782 | }, |
| 11280 | 11783 | |
| 11281 | 11784 | .save_err_return_trace_index => { |
| ... | ... | @@ -21474,11 +21977,16 @@ fn airRet(self: *CodeGen, inst: Air.Inst.Index, safety: bool) !void { |
| 21474 | 21977 | const ret_ty = self.fn_type.fnReturnType(zcu); |
| 21475 | 21978 | switch (self.ret_mcv.short) { |
| 21476 | 21979 | .none => {}, |
| 21477 | | .register, |
| 21478 | | .register_pair, |
| 21479 | | .register_triple, |
| 21480 | | .register_quadruple, |
| 21481 | | => try self.genCopy(ret_ty, self.ret_mcv.short, .{ .air_ref = un_op }, .{ .safety = safety }), |
| 21980 | .register => |reg| { |
| 21981 | const reg_lock = self.register_manager.lockRegAssumeUnused(reg); |
| 21982 | defer self.register_manager.unlockReg(reg_lock); |
| 21983 | try self.genCopy(ret_ty, self.ret_mcv.short, .{ .air_ref = un_op }, .{ .safety = safety }); |
| 21984 | }, |
| 21985 | inline .register_pair, .register_triple, .register_quadruple => |regs| { |
| 21986 | const reg_locks = self.register_manager.lockRegsAssumeUnused(regs.len, regs); |
| 21987 | defer for (reg_locks) |reg_lock| self.register_manager.unlockReg(reg_lock); |
| 21988 | try self.genCopy(ret_ty, self.ret_mcv.short, .{ .air_ref = un_op }, .{ .safety = safety }); |
| 21989 | }, |
| 21482 | 21990 | .indirect => |reg_off| { |
| 21483 | 21991 | try self.register_manager.getReg(reg_off.reg, null); |
| 21484 | 21992 | const lock = self.register_manager.lockRegAssumeUnused(reg_off.reg); |
| ... | ... | @@ -29772,6 +30280,7 @@ const Temp = struct { |
| 29772 | 30280 | cg.temp_type[@intFromEnum(result_temp_index)] = .slice_const_u8; |
| 29773 | 30281 | result_temp_index.tracking(cg).* = .init(result); |
| 29774 | 30282 | first_temp.* = result_temp; |
| 30283 | second_temp.* = result_temp; |
| 29775 | 30284 | } |
| 29776 | 30285 | |
| 29777 | 30286 | fn asMask(temp: Temp, info: MaskInfo, cg: *CodeGen) void { |
| ... | ... | @@ -30248,7 +30757,20 @@ const Temp = struct { |
| 30248 | 30757 | try cg.asmOpOnly(.{ .@"rep _sb", .sto }); |
| 30249 | 30758 | } |
| 30250 | 30759 | |
| 30251 | | fn moveTo(temp: Temp, inst: Air.Inst.Index, cg: *CodeGen) !void { |
| 30760 | fn finish( |
| 30761 | temp: Temp, |
| 30762 | inst: Air.Inst.Index, |
| 30763 | op_refs: []const Air.Inst.Ref, |
| 30764 | op_temps: []const Temp, |
| 30765 | cg: *CodeGen, |
| 30766 | ) !void { |
| 30767 | const tomb_bits = cg.liveness.getTombBits(inst); |
| 30768 | for (0.., op_refs, op_temps) |op_index, op_ref, op_temp| { |
| 30769 | if (op_temp.index != temp.index) try op_temp.die(cg); |
| 30770 | if (tomb_bits & @as(Liveness.Bpi, 1) << @intCast(op_index) == 0) continue; |
| 30771 | if (cg.reused_operands.isSet(op_index)) continue; |
| 30772 | try cg.processDeath(op_ref.toIndexAllowNone() orelse continue); |
| 30773 | } |
| 30252 | 30774 | if (cg.liveness.isUnused(inst)) try temp.die(cg) else switch (temp.unwrap(cg)) { |
| 30253 | 30775 | .ref, .err_ret_trace => { |
| 30254 | 30776 | const result = try cg.allocRegOrMem(inst, true); |
| ... | ... | @@ -30529,35 +31051,94 @@ const Select = struct { |
| 30529 | 31051 | }; |
| 30530 | 31052 | switch (mir_tag[0]) { |
| 30531 | 31053 | .f_ => switch (mir_tag[1]) { |
| 30532 | | .abs, .st => {}, |
| 31054 | .@"2xm1", |
| 31055 | .abs, |
| 31056 | .add, |
| 31057 | .chs, |
| 31058 | .clex, |
| 31059 | .com, |
| 31060 | .comi, |
| 31061 | .cos, |
| 31062 | .div, |
| 31063 | .divr, |
| 31064 | .free, |
| 31065 | .mul, |
| 31066 | .nop, |
| 31067 | .prem, |
| 31068 | .rndint, |
| 31069 | .scale, |
| 31070 | .sin, |
| 31071 | .sqrt, |
| 31072 | .st, |
| 31073 | .sub, |
| 31074 | .subr, |
| 31075 | .tst, |
| 31076 | .ucom, |
| 31077 | .ucomi, |
| 31078 | .wait, |
| 31079 | .xam, |
| 31080 | .xch, |
| 31081 | => {}, |
| 31082 | .init, .save => s.top = 0, |
| 31083 | .ld, .ptan, .sincos, .xtract => s.top -%= 1, |
| 31084 | .patan, .yl2x => s.top +%= 1, |
| 31085 | .rstor => unreachable, |
| 31086 | else => unreachable, |
| 31087 | }, |
| 31088 | .f_1 => switch (mir_tag[1]) { |
| 30533 | 31089 | .ld => s.top -%= 1, |
| 30534 | | else => { |
| 30535 | | const fixes = @tagName(mir_tag[0]); |
| 30536 | | const fixes_blank = std.mem.indexOfScalar(u8, fixes, '_').?; |
| 30537 | | std.debug.panic("{s}: {s}{s}{s}\n", .{ |
| 30538 | | @src().fn_name, |
| 30539 | | fixes[0..fixes_blank], |
| 30540 | | @tagName(mir_tag[1]), |
| 30541 | | fixes[fixes_blank + 1 ..], |
| 30542 | | }); |
| 30543 | | }, |
| 31090 | .prem => {}, |
| 31091 | else => unreachable, |
| 30544 | 31092 | }, |
| 30545 | | .f_p => switch (mir_tag[1]) { |
| 31093 | .f_b, .f_be, .f_e, .f_nb, .f_nbe, .f_ne, .f_nu, .f_u => switch (mir_tag[1]) { |
| 31094 | .cmov => {}, |
| 31095 | else => unreachable, |
| 31096 | }, |
| 31097 | .f_cw, .f_env, .f_sw => switch (mir_tag[1]) { |
| 31098 | .ld, .st => {}, |
| 31099 | else => unreachable, |
| 31100 | }, |
| 31101 | .f_p1 => switch (mir_tag[1]) { |
| 31102 | .yl2x => s.top +%= 1, |
| 31103 | else => unreachable, |
| 31104 | }, |
| 31105 | .fb_ => switch (mir_tag[1]) { |
| 31106 | .ld => s.top -%= 1, |
| 31107 | else => unreachable, |
| 31108 | }, |
| 31109 | .fb_p => switch (mir_tag[1]) { |
| 30546 | 31110 | .st => s.top +%= 1, |
| 30547 | | else => { |
| 30548 | | const fixes = @tagName(mir_tag[0]); |
| 30549 | | const fixes_blank = std.mem.indexOfScalar(u8, fixes, '_').?; |
| 30550 | | std.debug.panic("{s}: {s}{s}{s}\n", .{ |
| 30551 | | @src().fn_name, |
| 30552 | | fixes[0..fixes_blank], |
| 30553 | | @tagName(mir_tag[1]), |
| 30554 | | fixes[fixes_blank + 1 ..], |
| 30555 | | }); |
| 30556 | | }, |
| 31111 | else => unreachable, |
| 30557 | 31112 | }, |
| 30558 | | .f_1, |
| 30559 | | => switch (mir_tag[1]) { |
| 31113 | .fi_ => switch (mir_tag[1]) { |
| 31114 | .add, .com, .div, .divr, .mul, .st, .stt, .sub, .subr => {}, |
| 30560 | 31115 | .ld => s.top -%= 1, |
| 31116 | else => unreachable, |
| 31117 | }, |
| 31118 | .fi_p => switch (mir_tag[1]) { |
| 31119 | .com, .st => s.top +%= 1, |
| 31120 | else => unreachable, |
| 31121 | }, |
| 31122 | .fn_ => switch (mir_tag[1]) { |
| 31123 | .clex => {}, |
| 31124 | .init, .save => s.top = 0, |
| 31125 | else => unreachable, |
| 31126 | }, |
| 31127 | .fn_cw, .fn_env, .fn_sw => switch (mir_tag[1]) { |
| 31128 | .st => {}, |
| 31129 | else => unreachable, |
| 31130 | }, |
| 31131 | .f_cstp => switch (mir_tag[1]) { |
| 31132 | .de => s.top -%= 1, |
| 31133 | .in => s.top +%= 1, |
| 31134 | else => unreachable, |
| 31135 | }, |
| 31136 | .f_l2e, .f_l2t, .f_lg2, .f_ln2, .f_pi, .f_z => switch (mir_tag[1]) { |
| 31137 | .ld => s.top -%= 1, |
| 31138 | else => unreachable, |
| 31139 | }, |
| 31140 | .f_p => switch (mir_tag[1]) { |
| 31141 | .add, .com, .comi, .div, .divr, .mul, .st, .sub, .subr, .ucom, .ucomi => s.top +%= 1, |
| 30561 | 31142 | else => { |
| 30562 | 31143 | const fixes = @tagName(mir_tag[0]); |
| 30563 | 31144 | const fixes_blank = std.mem.indexOfScalar(u8, fixes, '_').?; |
| ... | ... | @@ -30569,41 +31150,13 @@ const Select = struct { |
| 30569 | 31150 | }); |
| 30570 | 31151 | }, |
| 30571 | 31152 | }, |
| 30572 | | .f_l2e, |
| 30573 | | .f_l2t, |
| 30574 | | .f_lg2, |
| 30575 | | .f_ln2, |
| 30576 | | .f_pi, |
| 30577 | | .f_z, |
| 30578 | | => switch (mir_tag[1]) { |
| 30579 | | .ld => s.top -%= 1, |
| 31153 | .f_pp => switch (mir_tag[1]) { |
| 31154 | .com, .ucom => s.top +%= 2, |
| 30580 | 31155 | else => unreachable, |
| 30581 | 31156 | }, |
| 30582 | | .f_b, |
| 30583 | | .f_be, |
| 30584 | | .f_cw, |
| 30585 | | .f_e, |
| 30586 | | .f_env, |
| 30587 | | .f_nb, |
| 30588 | | .f_nbe, |
| 30589 | | .f_ne, |
| 30590 | | .f_nu, |
| 30591 | | .f_p1, |
| 30592 | | .f_pp, |
| 30593 | | .f_sw, |
| 30594 | | .f_u, |
| 30595 | | .fb_, |
| 30596 | | .fb_p, |
| 30597 | | .fi_, |
| 30598 | | .fi_p, |
| 30599 | | .fn_, |
| 30600 | | .fn_cw, |
| 30601 | | .fn_env, |
| 30602 | | .fn_sw, |
| 30603 | | => {}, |
| 30604 | | .f_cstp => switch (mir_tag[1]) { |
| 30605 | | .de => s.top -%= 1, |
| 30606 | | .in => s.top +%= 1, |
| 31157 | .fx_ => switch (mir_tag[1]) { |
| 31158 | .rstor => unreachable, |
| 31159 | .save => {}, |
| 30607 | 31160 | else => unreachable, |
| 30608 | 31161 | }, |
| 30609 | 31162 | else => {}, |
| ... | ... | @@ -30620,9 +31173,13 @@ const Select = struct { |
| 30620 | 31173 | dst_constraints: [@intFromEnum(Select.Operand.Ref.src0) - @intFromEnum(Select.Operand.Ref.dst0)]Constraint = @splat(.any), |
| 30621 | 31174 | src_constraints: [@intFromEnum(Select.Operand.Ref.none) - @intFromEnum(Select.Operand.Ref.src0)]Constraint = @splat(.any), |
| 30622 | 31175 | patterns: []const Select.Pattern, |
| 31176 | call_frame: packed struct(u16) { size: u10, alignment: InternPool.Alignment } = .{ .size = 0, .alignment = .none }, |
| 30623 | 31177 | extra_temps: [@intFromEnum(Select.Operand.Ref.dst0) - @intFromEnum(Select.Operand.Ref.tmp0)]TempSpec = @splat(.unused), |
| 30624 | 31178 | dst_temps: [@intFromEnum(Select.Operand.Ref.src0) - @intFromEnum(Select.Operand.Ref.dst0)]TempSpec.Kind = @splat(.unused), |
| 30625 | | clobbers: struct { eflags: bool = false, st: u3 = 0 } = .{}, |
| 31179 | clobbers: packed struct { |
| 31180 | eflags: bool = false, |
| 31181 | caller_preserved: enum(u2) { none, ccc, zigcc } = .none, |
| 31182 | } = .{}, |
| 30626 | 31183 | each: union(enum) { |
| 30627 | 31184 | once: []const Instruction, |
| 30628 | 31185 | }, |
| ... | ... | @@ -30649,6 +31206,7 @@ const Select = struct { |
| 30649 | 31206 | scalar_signed_int: Memory.Size, |
| 30650 | 31207 | scalar_unsigned_int: Memory.Size, |
| 30651 | 31208 | scalar_remainder_int: struct { of: Memory.Size, is: Memory.Size }, |
| 31209 | float: Memory.Size, |
| 30652 | 31210 | scalar_float: struct { of: Memory.Size, is: Memory.Size }, |
| 30653 | 31211 | scalar_exact_float: struct { of: Memory.Size, is: Memory.Size }, |
| 30654 | 31212 | multiple_scalar_int: struct { of: Memory.Size, is: Memory.Size }, |
| ... | ... | @@ -30716,6 +31274,7 @@ const Select = struct { |
| 30716 | 31274 | of_is.is.bitSize(cg.target) >= (int_info.bits - 1) % of_is.of.bitSize(cg.target) + 1 |
| 30717 | 31275 | else |
| 30718 | 31276 | false, |
| 31277 | .float => |size| if (cg.floatBits(ty)) |float_bits| size.bitSize(cg.target) == float_bits else false, |
| 30719 | 31278 | .scalar_float => |of_is| @divExact(of_is.of.bitSize(cg.target), 8) >= ty.abiSize(zcu) and |
| 30720 | 31279 | if (cg.floatBits(ty.scalarType(zcu))) |float_bits| of_is.is.bitSize(cg.target) >= float_bits else false, |
| 30721 | 31280 | .scalar_exact_float => |of_is| @divExact(of_is.of.bitSize(cg.target), 8) >= ty.abiSize(zcu) and |
| ... | ... | @@ -30787,13 +31346,14 @@ const Select = struct { |
| 30787 | 31346 | src: [2]Src, |
| 30788 | 31347 | commute: struct { u8, u8 } = .{ 0, 0 }, |
| 30789 | 31348 | |
| 30790 | | const Src = enum { |
| 31349 | const Src = union(enum) { |
| 30791 | 31350 | none, |
| 30792 | 31351 | any, |
| 30793 | 31352 | imm8, |
| 30794 | 31353 | imm16, |
| 30795 | 31354 | imm32, |
| 30796 | 31355 | simm32, |
| 31356 | to_reg: Register, |
| 30797 | 31357 | mem, |
| 30798 | 31358 | to_mem, |
| 30799 | 31359 | mut_mem, |
| ... | ... | @@ -30846,6 +31406,7 @@ const Select = struct { |
| 30846 | 31406 | .mem => temp.tracking(cg).short.isMemory(), |
| 30847 | 31407 | .to_mem, .to_mut_mem => true, |
| 30848 | 31408 | .mut_mem => temp.isMut(cg) and temp.tracking(cg).short.isMemory(), |
| 31409 | .to_reg => true, |
| 30849 | 31410 | .gpr => temp.typeOf(cg).abiSize(cg.pt.zcu) <= 8 and switch (temp.tracking(cg).short) { |
| 30850 | 31411 | .register => |reg| reg.class() == .general_purpose, |
| 30851 | 31412 | .register_offset => |reg_off| reg_off.reg.class() == .general_purpose and reg_off.off == 0, |
| ... | ... | @@ -30920,6 +31481,7 @@ const Select = struct { |
| 30920 | 31481 | .none => unreachable, |
| 30921 | 31482 | .any, .imm8, .imm16, .imm32, .simm32 => false, |
| 30922 | 31483 | .mem, .to_mem, .mut_mem, .to_mut_mem => try temp.toBase(cg), |
| 31484 | .to_reg => |reg| try temp.toReg(reg, cg), |
| 30923 | 31485 | .gpr, .to_gpr => try temp.toRegClass(false, .general_purpose, cg), |
| 30924 | 31486 | .mut_gpr, .to_mut_gpr => try temp.toRegClass(true, .general_purpose, cg), |
| 30925 | 31487 | .x87, .to_x87 => try temp.toRegClass(false, .x87, cg), |
| ... | ... | @@ -30954,6 +31516,7 @@ const Select = struct { |
| 30954 | 31516 | ref: Select.Operand.Ref, |
| 30955 | 31517 | ref_mask: struct { ref: Select.Operand.Ref, info: MaskInfo }, |
| 30956 | 31518 | mut_reg: struct { ref: Select.Operand.Ref, rc: Register.Class }, |
| 31519 | symbol: *const struct { lib: ?[]const u8 = null, name: []const u8 }, |
| 30957 | 31520 | |
| 30958 | 31521 | const ConstInfo = struct { ref: Select.Operand.Ref, vectorize: bool = false }; |
| 30959 | 31522 | |
| ... | ... | @@ -31048,6 +31611,11 @@ const Select = struct { |
| 31048 | 31611 | }; |
| 31049 | 31612 | return try cg.tempAllocReg(spec.type, regSetForRegClass(ref_rc.rc)); |
| 31050 | 31613 | }, |
| 31614 | .symbol => |symbol| if (cg.bin_file.cast(.elf)) |elf_file| try cg.tempInit(spec.type, .{ .lea_symbol = .{ |
| 31615 | .sym_index = try elf_file.getGlobalSymbol(symbol.name, symbol.lib), |
| 31616 | } }) else if (cg.bin_file.cast(.macho)) |macho_file| try cg.tempInit(spec.type, .{ .lea_symbol = .{ |
| 31617 | .sym_index = try macho_file.getGlobalSymbol(symbol.name, symbol.lib), |
| 31618 | } }) else cg.fail("external symbols unimplemented for {s}", .{@tagName(cg.bin_file.tag)}), |
| 31051 | 31619 | }; |
| 31052 | 31620 | } |
| 31053 | 31621 | }; |
| ... | ... | @@ -31566,6 +32134,7 @@ const Select = struct { |
| 31566 | 32134 | } }, |
| 31567 | 32135 | else => |mcv| .{ .mem = try mcv.mem(s.cg, .{ .size = op.base.size }) }, |
| 31568 | 32136 | .register => |reg| .{ .reg = s.lowerReg(registerAlias(reg, @intCast(@divExact(op.base.size.bitSize(s.cg.target), 8)))) }, |
| 32137 | .lea_symbol => |sym_off| .{ .imm = .rel(sym_off) }, |
| 31569 | 32138 | }, |
| 31570 | 32139 | .simm => .{ .imm = .s(op.adjustedImm(i32, s)) }, |
| 31571 | 32140 | .uimm => .{ .imm = .u(@bitCast(op.adjustedImm(i64, s))) }, |
| ... | ... | @@ -31601,6 +32170,7 @@ fn select( |
| 31601 | 32170 | src_temps: []Temp, |
| 31602 | 32171 | cases: []const Select.Case, |
| 31603 | 32172 | ) !void { |
| 32173 | @setEvalBranchQuota(33_600); |
| 31604 | 32174 | cases: for (cases) |case| { |
| 31605 | 32175 | for (case.required_features) |required_feature| if (required_feature) |feature| if (!cg.hasFeature(feature)) continue :cases; |
| 31606 | 32176 | for (case.dst_constraints[0..dst_temps.len], dst_tys) |dst_constraint, dst_ty| if (!dst_constraint.accepts(dst_ty, cg)) continue :cases; |
| ... | ... | @@ -31613,6 +32183,16 @@ fn select( |
| 31613 | 32183 | for (pattern.src[0..src_temps.len], src_temps) |src_pattern, src_temp| if (!src_pattern.matches(src_temp, cg)) continue :patterns; |
| 31614 | 32184 | if (std.debug.runtime_safety) for (pattern.src[src_temps.len..]) |src_pattern| assert(src_pattern == .none); |
| 31615 | 32185 | |
| 32186 | if (case.call_frame.alignment != .none) { |
| 32187 | const frame_allocs_slice = cg.frame_allocs.slice(); |
| 32188 | const stack_frame_size = |
| 32189 | &frame_allocs_slice.items(.abi_size)[@intFromEnum(FrameIndex.call_frame)]; |
| 32190 | stack_frame_size.* = @max(stack_frame_size.*, case.call_frame.size); |
| 32191 | const stack_frame_align = |
| 32192 | &frame_allocs_slice.items(.abi_align)[@intFromEnum(FrameIndex.call_frame)]; |
| 32193 | stack_frame_align.* = stack_frame_align.max(case.call_frame.alignment); |
| 32194 | } |
| 32195 | |
| 31616 | 32196 | var s: Select = .{ |
| 31617 | 32197 | .cg = cg, |
| 31618 | 32198 | .temps = undefined, |
| ... | ... | @@ -31623,10 +32203,19 @@ fn select( |
| 31623 | 32203 | const dst_slots = s.temps[@intFromEnum(Select.Operand.Ref.dst0)..@intFromEnum(Select.Operand.Ref.src0)]; |
| 31624 | 32204 | const src_slots = s.temps[@intFromEnum(Select.Operand.Ref.src0)..@intFromEnum(Select.Operand.Ref.none)]; |
| 31625 | 32205 | |
| 31626 | | for (0..case.clobbers.st -| 1) |i| { |
| 31627 | | const tracked_index: RegisterManager.TrackedIndex = @intCast(RegisterManager.indexOfKnownRegIntoTracked(.st6).? - i); |
| 31628 | | try cg.register_manager.getRegIndex(tracked_index, null); |
| 31629 | | _ = cg.register_manager.lockRegIndexAssumeUnused(tracked_index); |
| 32206 | caller_preserved: { |
| 32207 | switch (switch (case.clobbers.caller_preserved) { |
| 32208 | .none => break :caller_preserved, |
| 32209 | .ccc => cg.target.cCallingConvention().?, |
| 32210 | .zigcc => .auto, |
| 32211 | }) { |
| 32212 | else => unreachable, |
| 32213 | inline .x86_64_sysv, .x86_64_win, .auto => |_, tag| inline for (comptime abi.getCallerPreservedRegs(tag)) |reg| { |
| 32214 | const tracked_index = RegisterManager.indexOfKnownRegIntoTracked(reg) orelse continue; |
| 32215 | try cg.register_manager.getRegIndex(tracked_index, null); |
| 32216 | assert(cg.register_manager.lockRegIndexAssumeUnused(tracked_index).tracked_index == tracked_index); |
| 32217 | }, |
| 32218 | } |
| 31630 | 32219 | } |
| 31631 | 32220 | |
| 31632 | 32221 | @memcpy(src_slots[0..src_temps.len], src_temps); |
| ... | ... | @@ -31653,9 +32242,20 @@ fn select( |
| 31653 | 32242 | } |
| 31654 | 32243 | assert(s.top == 0); |
| 31655 | 32244 | |
| 31656 | | for (0..case.clobbers.st -| 1) |i| cg.register_manager.unlockReg(.{ |
| 31657 | | .tracked_index = @intCast(RegisterManager.indexOfKnownRegIntoTracked(.st6).? - i), |
| 31658 | | }); |
| 32245 | caller_preserved: { |
| 32246 | switch (switch (case.clobbers.caller_preserved) { |
| 32247 | .none => break :caller_preserved, |
| 32248 | .ccc => cg.target.cCallingConvention().?, |
| 32249 | .zigcc => .auto, |
| 32250 | }) { |
| 32251 | else => unreachable, |
| 32252 | inline .x86_64_sysv, .x86_64_win, .auto => |_, tag| inline for (comptime abi.getCallerPreservedRegs(tag)) |reg| { |
| 32253 | cg.register_manager.unlockReg(.{ |
| 32254 | .tracked_index = RegisterManager.indexOfKnownRegIntoTracked(reg) orelse continue, |
| 32255 | }); |
| 32256 | }, |
| 32257 | } |
| 32258 | } |
| 31659 | 32259 | for (dst_temps, case.dst_temps[0..dst_temps.len]) |dst_temp, dst_kind| dst_kind.finish(dst_temp, &s); |
| 31660 | 32260 | for (case.extra_temps, tmp_slots) |spec, temp| if (spec.kind != .unused) try temp.die(cg); |
| 31661 | 32261 | return; |
Jacob Young <jacobly@ziglang.org> 2025-01-21 08:46:12-05:00
src