| ... | @@ -2291,7 +2291,7 @@ fn genBodyBlock(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { | ... | @@ -2291,7 +2291,7 @@ fn genBodyBlock(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 2291 | } | 2291 | } |
| 2292 | | 2292 | |
| 2293 | fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { | 2293 | fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 2294 | @setEvalBranchQuota(29_400); | 2294 | @setEvalBranchQuota(29_500); |
| 2295 | const pt = cg.pt; | 2295 | const pt = cg.pt; |
| 2296 | const zcu = pt.zcu; | 2296 | const zcu = pt.zcu; |
| 2297 | const ip = &zcu.intern_pool; | 2297 | const ip = &zcu.intern_pool; |
| ... | @@ -86774,52 +86774,313 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { | ... | @@ -86774,52 +86774,313 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 86774 | const is_non_err = try cg.tempInit(.bool, .{ .eflags = .e }); | 86774 | const is_non_err = try cg.tempInit(.bool, .{ .eflags = .e }); |
| 86775 | try is_non_err.finish(inst, &.{un_op}, &ops, cg); | 86775 | try is_non_err.finish(inst, &.{un_op}, &ops, cg); |
| 86776 | }, | 86776 | }, |
| 86777 | .load => fallback: { | 86777 | .load => { |
| 86778 | const ty_op = air_datas[@intFromEnum(inst)].ty_op; | 86778 | const ty_op = air_datas[@intFromEnum(inst)].ty_op; |
| 86779 | const val_ty = ty_op.ty.toType(); | 86779 | const val_ty = ty_op.ty.toType(); |
| 86780 | const ptr_ty = cg.typeOf(ty_op.operand); | | |
| 86781 | const ptr_info = ptr_ty.ptrInfo(zcu); | | |
| 86782 | if (ptr_info.packed_offset.host_size > 0 and | | |
| 86783 | (ptr_info.flags.vector_index == .none or val_ty.toIntern() == .bool_type)) | | |
| 86784 | break :fallback try cg.airLoad(inst); | | |
| 86785 | var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); | 86780 | var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); |
| 86786 | const res = try ops[0].load(val_ty, .{ | 86781 | var res: [1]Temp = undefined; |
| 86787 | .disp = switch (ptr_info.flags.vector_index) { | 86782 | cg.select(&res, &.{val_ty}, &ops, comptime &.{ .{ |
| 86788 | .none => 0, | 86783 | .src_constraints = .{ .{ .ptr_bool_vec_elem = .byte }, .any, .any }, |
| 86789 | .runtime => unreachable, | 86784 | .patterns = &.{ |
| 86790 | else => |vector_index| @intCast(val_ty.abiSize(zcu) * @intFromEnum(vector_index)), | 86785 | .{ .src = .{ .to_gpr, .none, .none } }, |
| 86791 | }, | 86786 | }, |
| 86792 | }, cg); | 86787 | .extra_temps = .{ |
| 86793 | try res.finish(inst, &.{ty_op.operand}, &ops, cg); | 86788 | .{ .type = .u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } } }, |
| | 86789 | .unused, |
| | 86790 | .unused, |
| | 86791 | .unused, |
| | 86792 | .unused, |
| | 86793 | .unused, |
| | 86794 | .unused, |
| | 86795 | .unused, |
| | 86796 | .unused, |
| | 86797 | .unused, |
| | 86798 | .unused, |
| | 86799 | }, |
| | 86800 | .dst_temps = .{ .{ .cc = .c }, .unused }, |
| | 86801 | .clobbers = .{ .eflags = true }, |
| | 86802 | .each = .{ .once = &.{ |
| | 86803 | .{ ._, ._, .movzx, .tmp0d, .lea(.src0b), ._, ._ }, |
| | 86804 | .{ ._, ._, .bt, .tmp0d, .ua(.src0, .add_vector_index), ._, ._ }, |
| | 86805 | } }, |
| | 86806 | }, .{ |
| | 86807 | .src_constraints = .{ .{ .ptr_bool_vec_elem = .word }, .any, .any }, |
| | 86808 | .patterns = &.{ |
| | 86809 | .{ .src = .{ .to_gpr, .none, .none } }, |
| | 86810 | }, |
| | 86811 | .dst_temps = .{ .{ .cc = .c }, .unused }, |
| | 86812 | .clobbers = .{ .eflags = true }, |
| | 86813 | .each = .{ .once = &.{ |
| | 86814 | .{ ._, ._, .bt, .lea(.src0w), .ua(.src0, .add_vector_index), ._, ._ }, |
| | 86815 | } }, |
| | 86816 | }, .{ |
| | 86817 | .src_constraints = .{ .ptr_any_bool_vec_elem, .any, .any }, |
| | 86818 | .patterns = &.{ |
| | 86819 | .{ .src = .{ .to_gpr, .none, .none } }, |
| | 86820 | }, |
| | 86821 | .dst_temps = .{ .{ .cc = .c }, .unused }, |
| | 86822 | .clobbers = .{ .eflags = true }, |
| | 86823 | .each = .{ .once = &.{ |
| | 86824 | .{ ._, ._, .bt, .leaa(.src0d, .add_vector_index_div_8_down_4), .ua(.src0, .add_vector_index_rem_32), ._, ._ }, |
| | 86825 | } }, |
| | 86826 | } }) catch |err| switch (err) { |
| | 86827 | error.SelectFailed => res[0] = try ops[0].load(val_ty, .{ |
| | 86828 | .disp = switch (cg.typeOf(ty_op.operand).ptrInfo(zcu).flags.vector_index) { |
| | 86829 | .none => 0, |
| | 86830 | .runtime => unreachable, |
| | 86831 | else => |vector_index| @intCast(val_ty.abiSize(zcu) * @intFromEnum(vector_index)), |
| | 86832 | }, |
| | 86833 | }, cg), |
| | 86834 | else => |e| return e, |
| | 86835 | }; |
| | 86836 | try res[0].finish(inst, &.{ty_op.operand}, &ops, cg); |
| 86794 | }, | 86837 | }, |
| 86795 | .ret => try cg.airRet(inst, false), | 86838 | .ret => try cg.airRet(inst, false), |
| 86796 | .ret_safe => try cg.airRet(inst, true), | 86839 | .ret_safe => try cg.airRet(inst, true), |
| 86797 | .ret_load => try cg.airRetLoad(inst), | 86840 | .ret_load => try cg.airRetLoad(inst), |
| 86798 | .store, .store_safe => |air_tag| fallback: { | 86841 | .store, .store_safe => |air_tag| { |
| 86799 | const bin_op = air_datas[@intFromEnum(inst)].bin_op; | 86842 | const bin_op = air_datas[@intFromEnum(inst)].bin_op; |
| 86800 | const ptr_ty = cg.typeOf(bin_op.lhs); | | |
| 86801 | const ptr_info = ptr_ty.ptrInfo(zcu); | | |
| 86802 | const val_ty = cg.typeOf(bin_op.rhs); | | |
| 86803 | if (ptr_info.packed_offset.host_size > 0 and | | |
| 86804 | (ptr_info.flags.vector_index == .none or val_ty.toIntern() == .bool_type)) | | |
| 86805 | break :fallback try cg.airStore(inst, switch (air_tag) { | | |
| 86806 | else => unreachable, | | |
| 86807 | .store => false, | | |
| 86808 | .store_safe => true, | | |
| 86809 | }); | | |
| 86810 | var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs }); | 86843 | var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs }); |
| 86811 | try ops[0].store(&ops[1], .{ | 86844 | cg.select(&.{}, &.{}, &ops, comptime &.{ .{ |
| 86812 | .disp = switch (ptr_info.flags.vector_index) { | 86845 | .src_constraints = .{ .{ .ptr_bool_vec_elem = .byte }, .bool, .any }, |
| 86813 | .none => 0, | 86846 | .patterns = &.{ |
| 86814 | .runtime => unreachable, | 86847 | .{ .src = .{ .to_gpr, .{ .imm = 0 }, .none } }, |
| 86815 | else => |vector_index| @intCast(val_ty.abiSize(zcu) * @intFromEnum(vector_index)), | | |
| 86816 | }, | 86848 | }, |
| 86817 | .safe = switch (air_tag) { | 86849 | .extra_temps = .{ |
| 86818 | else => unreachable, | 86850 | .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, |
| 86819 | .store => false, | 86851 | .unused, |
| 86820 | .store_safe => true, | 86852 | .unused, |
| | 86853 | .unused, |
| | 86854 | .unused, |
| | 86855 | .unused, |
| | 86856 | .unused, |
| | 86857 | .unused, |
| | 86858 | .unused, |
| | 86859 | .unused, |
| | 86860 | .unused, |
| 86821 | }, | 86861 | }, |
| 86822 | }, cg); | 86862 | .clobbers = .{ .eflags = true }, |
| | 86863 | .each = .{ .once = &.{ |
| | 86864 | .{ ._, ._, .movzx, .tmp0d, .lea(.src0b), ._, ._ }, |
| | 86865 | .{ ._, ._r, .bt, .tmp0d, .ua(.src0, .add_vector_index), ._, ._ }, |
| | 86866 | .{ ._, ._, .mov, .lea(.src0b), .tmp0b, ._, ._ }, |
| | 86867 | } }, |
| | 86868 | }, .{ |
| | 86869 | .src_constraints = .{ .{ .ptr_bool_vec_elem = .byte }, .bool, .any }, |
| | 86870 | .patterns = &.{ |
| | 86871 | .{ .src = .{ .to_gpr, .{ .imm = 1 }, .none } }, |
| | 86872 | }, |
| | 86873 | .extra_temps = .{ |
| | 86874 | .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, |
| | 86875 | .unused, |
| | 86876 | .unused, |
| | 86877 | .unused, |
| | 86878 | .unused, |
| | 86879 | .unused, |
| | 86880 | .unused, |
| | 86881 | .unused, |
| | 86882 | .unused, |
| | 86883 | .unused, |
| | 86884 | .unused, |
| | 86885 | }, |
| | 86886 | .clobbers = .{ .eflags = true }, |
| | 86887 | .each = .{ .once = &.{ |
| | 86888 | .{ ._, ._, .movzx, .tmp0d, .lea(.src0b), ._, ._ }, |
| | 86889 | .{ ._, ._s, .bt, .tmp0d, .ua(.src0, .add_vector_index), ._, ._ }, |
| | 86890 | .{ ._, ._, .mov, .lea(.src0b), .tmp0b, ._, ._ }, |
| | 86891 | } }, |
| | 86892 | }, .{ |
| | 86893 | .required_features = .{ .cmov, null, null, null }, |
| | 86894 | .src_constraints = .{ .{ .ptr_bool_vec_elem = .byte }, .bool, .any }, |
| | 86895 | .patterns = &.{ |
| | 86896 | .{ .src = .{ .to_gpr, .to_gpr, .none } }, |
| | 86897 | }, |
| | 86898 | .extra_temps = .{ |
| | 86899 | .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, |
| | 86900 | .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, |
| | 86901 | .unused, |
| | 86902 | .unused, |
| | 86903 | .unused, |
| | 86904 | .unused, |
| | 86905 | .unused, |
| | 86906 | .unused, |
| | 86907 | .unused, |
| | 86908 | .unused, |
| | 86909 | .unused, |
| | 86910 | }, |
| | 86911 | .clobbers = .{ .eflags = true }, |
| | 86912 | .each = .{ .once = &.{ |
| | 86913 | .{ ._, ._, .movzx, .tmp0d, .lea(.src0b), ._, ._ }, |
| | 86914 | .{ ._, ._, .mov, .tmp1d, .tmp0d, ._, ._ }, |
| | 86915 | .{ ._, ._r, .bt, .tmp1d, .ua(.src0, .add_vector_index), ._, ._ }, |
| | 86916 | .{ ._, ._s, .bt, .tmp0d, .ua(.src0, .add_vector_index), ._, ._ }, |
| | 86917 | .{ ._, ._, .@"test", .src1b, .si(1), ._, ._ }, |
| | 86918 | .{ ._, ._z, .cmov, .tmp0d, .tmp1d, ._, ._ }, |
| | 86919 | .{ ._, ._, .mov, .lea(.src0b), .tmp0b, ._, ._ }, |
| | 86920 | } }, |
| | 86921 | }, .{ |
| | 86922 | .src_constraints = .{ .{ .ptr_bool_vec_elem = .byte }, .bool, .any }, |
| | 86923 | .patterns = &.{ |
| | 86924 | .{ .src = .{ .to_gpr, .to_gpr, .none } }, |
| | 86925 | }, |
| | 86926 | .extra_temps = .{ |
| | 86927 | .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, |
| | 86928 | .unused, |
| | 86929 | .unused, |
| | 86930 | .unused, |
| | 86931 | .unused, |
| | 86932 | .unused, |
| | 86933 | .unused, |
| | 86934 | .unused, |
| | 86935 | .unused, |
| | 86936 | .unused, |
| | 86937 | .unused, |
| | 86938 | }, |
| | 86939 | .clobbers = .{ .eflags = true }, |
| | 86940 | .each = .{ .once = &.{ |
| | 86941 | .{ ._, ._, .movzx, .tmp0d, .lea(.src0b), ._, ._ }, |
| | 86942 | .{ ._, ._, .@"test", .src1b, .si(1), ._, ._ }, |
| | 86943 | .{ ._, ._nz, .j, .@"0f", ._, ._, ._ }, |
| | 86944 | .{ ._, ._r, .bt, .tmp0d, .ua(.src0, .add_vector_index), ._, ._ }, |
| | 86945 | .{ ._, ._mp, .j, .@"1f", ._, ._, ._ }, |
| | 86946 | .{ .@"0:", ._s, .bt, .tmp0d, .ua(.src0, .add_vector_index), ._, ._ }, |
| | 86947 | .{ .@"1:", ._, .mov, .lea(.src0b), .tmp0b, ._, ._ }, |
| | 86948 | } }, |
| | 86949 | }, .{ |
| | 86950 | .src_constraints = .{ .{ .ptr_bool_vec_elem = .word }, .bool, .any }, |
| | 86951 | .patterns = &.{ |
| | 86952 | .{ .src = .{ .to_gpr, .{ .imm = 0 }, .none } }, |
| | 86953 | }, |
| | 86954 | .clobbers = .{ .eflags = true }, |
| | 86955 | .each = .{ .once = &.{ |
| | 86956 | .{ ._, ._r, .bt, .lea(.src0w), .ua(.src0, .add_vector_index), ._, ._ }, |
| | 86957 | } }, |
| | 86958 | }, .{ |
| | 86959 | .src_constraints = .{ .{ .ptr_bool_vec_elem = .word }, .bool, .any }, |
| | 86960 | .patterns = &.{ |
| | 86961 | .{ .src = .{ .to_gpr, .{ .imm = 1 }, .none } }, |
| | 86962 | }, |
| | 86963 | .clobbers = .{ .eflags = true }, |
| | 86964 | .each = .{ .once = &.{ |
| | 86965 | .{ ._, ._s, .bt, .lea(.src0w), .ua(.src0, .add_vector_index), ._, ._ }, |
| | 86966 | } }, |
| | 86967 | }, .{ |
| | 86968 | .required_features = .{ .cmov, null, null, null }, |
| | 86969 | .src_constraints = .{ .{ .ptr_bool_vec_elem = .word }, .bool, .any }, |
| | 86970 | .patterns = &.{ |
| | 86971 | .{ .src = .{ .to_gpr, .to_gpr, .none } }, |
| | 86972 | }, |
| | 86973 | .extra_temps = .{ |
| | 86974 | .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, |
| | 86975 | .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, |
| | 86976 | .unused, |
| | 86977 | .unused, |
| | 86978 | .unused, |
| | 86979 | .unused, |
| | 86980 | .unused, |
| | 86981 | .unused, |
| | 86982 | .unused, |
| | 86983 | .unused, |
| | 86984 | .unused, |
| | 86985 | }, |
| | 86986 | .clobbers = .{ .eflags = true }, |
| | 86987 | .each = .{ .once = &.{ |
| | 86988 | .{ ._, ._, .movzx, .tmp0d, .lea(.src0w), ._, ._ }, |
| | 86989 | .{ ._, ._, .mov, .tmp1d, .tmp0d, ._, ._ }, |
| | 86990 | .{ ._, ._r, .bt, .tmp1d, .ua(.src0, .add_vector_index), ._, ._ }, |
| | 86991 | .{ ._, ._s, .bt, .tmp0d, .ua(.src0, .add_vector_index), ._, ._ }, |
| | 86992 | .{ ._, ._, .@"test", .src1b, .si(1), ._, ._ }, |
| | 86993 | .{ ._, ._z, .cmov, .tmp0d, .tmp1d, ._, ._ }, |
| | 86994 | .{ ._, ._, .mov, .lea(.src0w), .tmp0w, ._, ._ }, |
| | 86995 | } }, |
| | 86996 | }, .{ |
| | 86997 | .src_constraints = .{ .{ .ptr_bool_vec_elem = .word }, .bool, .any }, |
| | 86998 | .patterns = &.{ |
| | 86999 | .{ .src = .{ .to_gpr, .to_gpr, .none } }, |
| | 87000 | }, |
| | 87001 | .clobbers = .{ .eflags = true }, |
| | 87002 | .each = .{ .once = &.{ |
| | 87003 | .{ ._, ._, .@"test", .src1b, .si(1), ._, ._ }, |
| | 87004 | .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, |
| | 87005 | .{ ._, ._r, .bt, .lea(.src0w), .ua(.src0, .add_vector_index), ._, ._ }, |
| | 87006 | .{ ._, ._mp, .j, .@"0f", ._, ._, ._ }, |
| | 87007 | .{ .@"1:", ._s, .bt, .lea(.src0w), .ua(.src0, .add_vector_index), ._, ._ }, |
| | 87008 | } }, |
| | 87009 | }, .{ |
| | 87010 | .src_constraints = .{ .ptr_any_bool_vec_elem, .bool, .any }, |
| | 87011 | .patterns = &.{ |
| | 87012 | .{ .src = .{ .to_gpr, .{ .imm = 0 }, .none } }, |
| | 87013 | }, |
| | 87014 | .clobbers = .{ .eflags = true }, |
| | 87015 | .each = .{ .once = &.{ |
| | 87016 | .{ ._, ._r, .bt, .leaa(.src0d, .add_vector_index_div_8_down_4), .ua(.src0, .add_vector_index_rem_32), ._, ._ }, |
| | 87017 | } }, |
| | 87018 | }, .{ |
| | 87019 | .src_constraints = .{ .ptr_any_bool_vec_elem, .bool, .any }, |
| | 87020 | .patterns = &.{ |
| | 87021 | .{ .src = .{ .to_gpr, .{ .imm = 1 }, .none } }, |
| | 87022 | }, |
| | 87023 | .clobbers = .{ .eflags = true }, |
| | 87024 | .each = .{ .once = &.{ |
| | 87025 | .{ ._, ._s, .bt, .leaa(.src0d, .add_vector_index_div_8_down_4), .ua(.src0, .add_vector_index_rem_32), ._, ._ }, |
| | 87026 | } }, |
| | 87027 | }, .{ |
| | 87028 | .required_features = .{ .cmov, null, null, null }, |
| | 87029 | .src_constraints = .{ .ptr_any_bool_vec_elem, .bool, .any }, |
| | 87030 | .patterns = &.{ |
| | 87031 | .{ .src = .{ .to_gpr, .to_gpr, .none } }, |
| | 87032 | }, |
| | 87033 | .extra_temps = .{ |
| | 87034 | .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, |
| | 87035 | .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, |
| | 87036 | .unused, |
| | 87037 | .unused, |
| | 87038 | .unused, |
| | 87039 | .unused, |
| | 87040 | .unused, |
| | 87041 | .unused, |
| | 87042 | .unused, |
| | 87043 | .unused, |
| | 87044 | .unused, |
| | 87045 | }, |
| | 87046 | .clobbers = .{ .eflags = true }, |
| | 87047 | .each = .{ .once = &.{ |
| | 87048 | .{ ._, ._, .mov, .tmp0d, .leaa(.src0d, .add_vector_index_div_8_down_4), ._, ._ }, |
| | 87049 | .{ ._, ._, .mov, .tmp1d, .tmp0d, ._, ._ }, |
| | 87050 | .{ ._, ._r, .bt, .tmp1d, .ua(.src0, .add_vector_index_rem_32), ._, ._ }, |
| | 87051 | .{ ._, ._s, .bt, .tmp0d, .ua(.src0, .add_vector_index_rem_32), ._, ._ }, |
| | 87052 | .{ ._, ._, .@"test", .src1b, .si(1), ._, ._ }, |
| | 87053 | .{ ._, ._z, .cmov, .tmp0d, .tmp1d, ._, ._ }, |
| | 87054 | .{ ._, ._, .mov, .leaa(.src0d, .add_vector_index_div_8_down_4), .tmp0d, ._, ._ }, |
| | 87055 | } }, |
| | 87056 | }, .{ |
| | 87057 | .src_constraints = .{ .ptr_any_bool_vec_elem, .bool, .any }, |
| | 87058 | .patterns = &.{ |
| | 87059 | .{ .src = .{ .to_gpr, .to_gpr, .none } }, |
| | 87060 | }, |
| | 87061 | .clobbers = .{ .eflags = true }, |
| | 87062 | .each = .{ .once = &.{ |
| | 87063 | .{ ._, ._, .@"test", .src1b, .si(1), ._, ._ }, |
| | 87064 | .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, |
| | 87065 | .{ ._, ._r, .bt, .leaa(.src0d, .add_vector_index_div_8_down_4), .ua(.src0, .add_vector_index_rem_32), ._, ._ }, |
| | 87066 | .{ ._, ._mp, .j, .@"0f", ._, ._, ._ }, |
| | 87067 | .{ .@"1:", ._s, .bt, .leaa(.src0d, .add_vector_index_div_8_down_4), .ua(.src0, .add_vector_index_rem_32), ._, ._ }, |
| | 87068 | } }, |
| | 87069 | } }) catch |err| switch (err) { |
| | 87070 | error.SelectFailed => try ops[0].store(&ops[1], .{ |
| | 87071 | .disp = switch (cg.typeOf(bin_op.lhs).ptrInfo(zcu).flags.vector_index) { |
| | 87072 | .none => 0, |
| | 87073 | .runtime => unreachable, |
| | 87074 | else => |vector_index| @intCast(cg.typeOf(bin_op.rhs).abiSize(zcu) * @intFromEnum(vector_index)), |
| | 87075 | }, |
| | 87076 | .safe = switch (air_tag) { |
| | 87077 | else => unreachable, |
| | 87078 | .store => false, |
| | 87079 | .store_safe => true, |
| | 87080 | }, |
| | 87081 | }, cg), |
| | 87082 | else => |e| return e, |
| | 87083 | }; |
| 86823 | for (ops) |op| try op.die(cg); | 87084 | for (ops) |op| try op.die(cg); |
| 86824 | }, | 87085 | }, |
| 86825 | .unreach => {}, | 87086 | .unreach => {}, |
| ... | @@ -100863,7 +101124,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { | ... | @@ -100863,7 +101124,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 100863 | .dst_temps = .{ .{ .cc = .c }, .unused }, | 101124 | .dst_temps = .{ .{ .cc = .c }, .unused }, |
| 100864 | .clobbers = .{ .eflags = true }, | 101125 | .clobbers = .{ .eflags = true }, |
| 100865 | .each = .{ .once = &.{ | 101126 | .each = .{ .once = &.{ |
| 100866 | .{ ._, ._, .bt, .src0d, .ua(.none, .add_src1_rem_32), ._, ._ }, | 101127 | .{ ._, ._, .bt, .src0d, .ua(.none, .add_src1), ._, ._ }, |
| 100867 | } }, | 101128 | } }, |
| 100868 | }, .{ | 101129 | }, .{ |
| 100869 | .src_constraints = .{ .{ .bool_vec = .dword }, .any, .any }, | 101130 | .src_constraints = .{ .{ .bool_vec = .dword }, .any, .any }, |
| ... | @@ -100884,7 +101145,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { | ... | @@ -100884,7 +101145,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 100884 | .dst_temps = .{ .{ .cc = .c }, .unused }, | 101145 | .dst_temps = .{ .{ .cc = .c }, .unused }, |
| 100885 | .clobbers = .{ .eflags = true }, | 101146 | .clobbers = .{ .eflags = true }, |
| 100886 | .each = .{ .once = &.{ | 101147 | .each = .{ .once = &.{ |
| 100887 | .{ ._, ._, .bt, .src0q, .ua(.none, .add_src1_rem_64), ._, ._ }, | 101148 | .{ ._, ._, .bt, .src0q, .ua(.none, .add_src1), ._, ._ }, |
| 100888 | } }, | 101149 | } }, |
| 100889 | }, .{ | 101150 | }, .{ |
| 100890 | .required_features = .{ .@"64bit", null, null, null }, | 101151 | .required_features = .{ .@"64bit", null, null, null }, |
| ... | @@ -174481,114 +174742,6 @@ fn reuseOperandAdvanced( | ... | @@ -174481,114 +174742,6 @@ fn reuseOperandAdvanced( |
| 174481 | return true; | 174742 | return true; |
| 174482 | } | 174743 | } |
| 174483 | | 174744 | |
| 174484 | fn packedLoad(self: *CodeGen, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) InnerError!void { | | |
| 174485 | const pt = self.pt; | | |
| 174486 | const zcu = pt.zcu; | | |
| 174487 | | | |
| 174488 | const ptr_info = ptr_ty.ptrInfo(zcu); | | |
| 174489 | const val_ty: Type = .fromInterned(ptr_info.child); | | |
| 174490 | if (!val_ty.hasRuntimeBitsIgnoreComptime(zcu)) return; | | |
| 174491 | const val_abi_size: u32 = @intCast(val_ty.abiSize(zcu)); | | |
| 174492 | | | |
| 174493 | const val_bit_size: u32 = @intCast(val_ty.bitSize(zcu)); | | |
| 174494 | const ptr_bit_off = ptr_info.packed_offset.bit_offset + switch (ptr_info.flags.vector_index) { | | |
| 174495 | .none => 0, | | |
| 174496 | .runtime => unreachable, | | |
| 174497 | else => |vector_index| @intFromEnum(vector_index) * val_bit_size, | | |
| 174498 | }; | | |
| 174499 | if (ptr_bit_off % 8 == 0) { | | |
| 174500 | { | | |
| 174501 | const mat_ptr_mcv: MCValue = switch (ptr_mcv) { | | |
| 174502 | .immediate, .register, .register_offset, .lea_frame => ptr_mcv, | | |
| 174503 | else => .{ .register = try self.copyToTmpRegister(ptr_ty, ptr_mcv) }, | | |
| 174504 | }; | | |
| 174505 | const mat_ptr_lock = switch (mat_ptr_mcv) { | | |
| 174506 | .register => |mat_ptr_reg| self.register_manager.lockReg(mat_ptr_reg), | | |
| 174507 | else => null, | | |
| 174508 | }; | | |
| 174509 | defer if (mat_ptr_lock) |lock| self.register_manager.unlockReg(lock); | | |
| 174510 | | | |
| 174511 | try self.load(dst_mcv, ptr_ty, mat_ptr_mcv.offset(@intCast(@divExact(ptr_bit_off, 8)))); | | |
| 174512 | } | | |
| 174513 | | | |
| 174514 | if (val_abi_size * 8 > val_bit_size) { | | |
| 174515 | if (dst_mcv.isRegister()) { | | |
| 174516 | try self.truncateRegister(val_ty, dst_mcv.getReg().?); | | |
| 174517 | } else { | | |
| 174518 | const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); | | |
| 174519 | const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); | | |
| 174520 | defer self.register_manager.unlockReg(tmp_lock); | | |
| 174521 | | | |
| 174522 | const hi_mcv = dst_mcv.address().offset(@intCast(val_bit_size / 64 * 8)).deref(); | | |
| 174523 | try self.genSetReg(tmp_reg, .usize, hi_mcv, .{}); | | |
| 174524 | try self.truncateRegister(val_ty, tmp_reg); | | |
| 174525 | try self.genCopy(.usize, hi_mcv, .{ .register = tmp_reg }, .{}); | | |
| 174526 | } | | |
| 174527 | } | | |
| 174528 | return; | | |
| 174529 | } | | |
| 174530 | | | |
| 174531 | if (val_abi_size > 8) return self.fail("TODO implement packed load of {f}", .{val_ty.fmt(pt)}); | | |
| 174532 | | | |
| 174533 | const limb_abi_size: u31 = @min(val_abi_size, 8); | | |
| 174534 | const limb_abi_bits = limb_abi_size * 8; | | |
| 174535 | const val_byte_off: i32 = @intCast(ptr_bit_off / limb_abi_bits * limb_abi_size); | | |
| 174536 | const val_bit_off = ptr_bit_off % limb_abi_bits; | | |
| 174537 | const val_extra_bits = self.regExtraBits(val_ty); | | |
| 174538 | | | |
| 174539 | const ptr_reg = try self.copyToTmpRegister(ptr_ty, ptr_mcv); | | |
| 174540 | const ptr_lock = self.register_manager.lockRegAssumeUnused(ptr_reg); | | |
| 174541 | defer self.register_manager.unlockReg(ptr_lock); | | |
| 174542 | | | |
| 174543 | const dst_reg = switch (dst_mcv) { | | |
| 174544 | .register => |reg| reg, | | |
| 174545 | else => try self.register_manager.allocReg(null, abi.RegisterClass.gp), | | |
| 174546 | }; | | |
| 174547 | const dst_lock = self.register_manager.lockReg(dst_reg); | | |
| 174548 | defer if (dst_lock) |lock| self.register_manager.unlockReg(lock); | | |
| 174549 | | | |
| 174550 | const load_abi_size = | | |
| 174551 | if (val_bit_off < val_extra_bits) val_abi_size else val_abi_size * 2; | | |
| 174552 | if (load_abi_size <= 8) { | | |
| 174553 | const load_reg = registerAlias(dst_reg, load_abi_size); | | |
| 174554 | try self.asmRegisterMemory(.{ ._, .mov }, load_reg, .{ | | |
| 174555 | .base = .{ .reg = ptr_reg }, | | |
| 174556 | .mod = .{ .rm = .{ | | |
| 174557 | .size = .fromSize(load_abi_size), | | |
| 174558 | .disp = val_byte_off, | | |
| 174559 | } }, | | |
| 174560 | }); | | |
| 174561 | try self.spillEflagsIfOccupied(); | | |
| 174562 | try self.asmRegisterImmediate(.{ ._r, .sh }, load_reg, .u(val_bit_off)); | | |
| 174563 | } else { | | |
| 174564 | const tmp_reg = | | |
| 174565 | registerAlias(try self.register_manager.allocReg(null, abi.RegisterClass.gp), val_abi_size); | | |
| 174566 | const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); | | |
| 174567 | defer self.register_manager.unlockReg(tmp_lock); | | |
| 174568 | | | |
| 174569 | const dst_alias = registerAlias(dst_reg, val_abi_size); | | |
| 174570 | try self.asmRegisterMemory(.{ ._, .mov }, dst_alias, .{ | | |
| 174571 | .base = .{ .reg = ptr_reg }, | | |
| 174572 | .mod = .{ .rm = .{ | | |
| 174573 | .size = .fromSize(val_abi_size), | | |
| 174574 | .disp = val_byte_off, | | |
| 174575 | } }, | | |
| 174576 | }); | | |
| 174577 | try self.asmRegisterMemory(.{ ._, .mov }, tmp_reg, .{ | | |
| 174578 | .base = .{ .reg = ptr_reg }, | | |
| 174579 | .mod = .{ .rm = .{ | | |
| 174580 | .size = .fromSize(val_abi_size), | | |
| 174581 | .disp = val_byte_off + limb_abi_size, | | |
| 174582 | } }, | | |
| 174583 | }); | | |
| 174584 | try self.spillEflagsIfOccupied(); | | |
| 174585 | try self.asmRegisterRegisterImmediate(.{ ._rd, .sh }, dst_alias, tmp_reg, .u(val_bit_off)); | | |
| 174586 | } | | |
| 174587 | | | |
| 174588 | if (val_extra_bits > 0) try self.truncateRegister(val_ty, dst_reg); | | |
| 174589 | try self.genCopy(val_ty, dst_mcv, .{ .register = dst_reg }, .{}); | | |
| 174590 | } | | |
| 174591 | | | |
| 174592 | fn load(self: *CodeGen, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) InnerError!void { | 174745 | fn load(self: *CodeGen, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) InnerError!void { |
| 174593 | const pt = self.pt; | 174746 | const pt = self.pt; |
| 174594 | const zcu = pt.zcu; | 174747 | const zcu = pt.zcu; |
| ... | @@ -174636,174 +174789,6 @@ fn load(self: *CodeGen, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) InnerE | ... | @@ -174636,174 +174789,6 @@ fn load(self: *CodeGen, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) InnerE |
| 174636 | } | 174789 | } |
| 174637 | } | 174790 | } |
| 174638 | | 174791 | |
| 174639 | fn airLoad(self: *CodeGen, inst: Air.Inst.Index) !void { | | |
| 174640 | const pt = self.pt; | | |
| 174641 | const zcu = pt.zcu; | | |
| 174642 | const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; | | |
| 174643 | const elem_ty = self.typeOfIndex(inst); | | |
| 174644 | const result: MCValue = result: { | | |
| 174645 | if (!elem_ty.hasRuntimeBitsIgnoreComptime(zcu)) break :result .none; | | |
| 174646 | | | |
| 174647 | try self.spillRegisters(&.{ .rdi, .rsi, .rcx }); | | |
| 174648 | const reg_locks = self.register_manager.lockRegsAssumeUnused(3, .{ .rdi, .rsi, .rcx }); | | |
| 174649 | defer for (reg_locks) |lock| self.register_manager.unlockReg(lock); | | |
| 174650 | | | |
| 174651 | const ptr_ty = self.typeOf(ty_op.operand); | | |
| 174652 | const elem_size = elem_ty.abiSize(zcu); | | |
| 174653 | | | |
| 174654 | const elem_rs = self.regSetForType(elem_ty); | | |
| 174655 | const ptr_rs = self.regSetForType(ptr_ty); | | |
| 174656 | | | |
| 174657 | const ptr_mcv = try self.resolveInst(ty_op.operand); | | |
| 174658 | const dst_mcv = if (elem_size <= 8 and std.math.isPowerOfTwo(elem_size) and | | |
| 174659 | elem_rs.supersetOf(ptr_rs) and self.reuseOperand(inst, ty_op.operand, 0, ptr_mcv)) | | |
| 174660 | // The MCValue that holds the pointer can be re-used as the value. | | |
| 174661 | ptr_mcv | | |
| 174662 | else | | |
| 174663 | try self.allocRegOrMem(inst, true); | | |
| 174664 | | | |
| 174665 | const ptr_info = ptr_ty.ptrInfo(zcu); | | |
| 174666 | if (ptr_info.flags.vector_index != .none or ptr_info.packed_offset.host_size > 0) { | | |
| 174667 | try self.packedLoad(dst_mcv, ptr_ty, ptr_mcv); | | |
| 174668 | } else { | | |
| 174669 | try self.load(dst_mcv, ptr_ty, ptr_mcv); | | |
| 174670 | } | | |
| 174671 | | | |
| 174672 | if (elem_ty.isAbiInt(zcu) and elem_size * 8 > elem_ty.bitSize(zcu)) { | | |
| 174673 | const high_mcv: MCValue = switch (dst_mcv) { | | |
| 174674 | .register => |dst_reg| .{ .register = dst_reg }, | | |
| 174675 | .register_pair => |dst_regs| .{ .register = dst_regs[1] }, | | |
| 174676 | else => dst_mcv.address().offset(@intCast((elem_size - 1) / 8 * 8)).deref(), | | |
| 174677 | }; | | |
| 174678 | const high_reg = if (high_mcv.isRegister()) | | |
| 174679 | high_mcv.getReg().? | | |
| 174680 | else | | |
| 174681 | try self.copyToTmpRegister(.usize, high_mcv); | | |
| 174682 | const high_lock = self.register_manager.lockReg(high_reg); | | |
| 174683 | defer if (high_lock) |lock| self.register_manager.unlockReg(lock); | | |
| 174684 | | | |
| 174685 | try self.truncateRegister(elem_ty, high_reg); | | |
| 174686 | if (!high_mcv.isRegister()) try self.genCopy( | | |
| 174687 | if (elem_size <= 8) elem_ty else .usize, | | |
| 174688 | high_mcv, | | |
| 174689 | .{ .register = high_reg }, | | |
| 174690 | .{}, | | |
| 174691 | ); | | |
| 174692 | } | | |
| 174693 | break :result dst_mcv; | | |
| 174694 | }; | | |
| 174695 | return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); | | |
| 174696 | } | | |
| 174697 | | | |
| 174698 | fn packedStore(self: *CodeGen, ptr_ty: Type, ptr_mcv: MCValue, src_mcv: MCValue) InnerError!void { | | |
| 174699 | const pt = self.pt; | | |
| 174700 | const zcu = pt.zcu; | | |
| 174701 | const ptr_info = ptr_ty.ptrInfo(zcu); | | |
| 174702 | const src_ty: Type = .fromInterned(ptr_info.child); | | |
| 174703 | if (!src_ty.hasRuntimeBitsIgnoreComptime(zcu)) return; | | |
| 174704 | | | |
| 174705 | const limb_abi_size: u16 = @min(ptr_info.packed_offset.host_size, 8); | | |
| 174706 | const limb_abi_bits = limb_abi_size * 8; | | |
| 174707 | const limb_ty = try pt.intType(.unsigned, limb_abi_bits); | | |
| 174708 | | | |
| 174709 | const src_bit_size = src_ty.bitSize(zcu); | | |
| 174710 | const ptr_bit_off = ptr_info.packed_offset.bit_offset + switch (ptr_info.flags.vector_index) { | | |
| 174711 | .none => 0, | | |
| 174712 | .runtime => unreachable, | | |
| 174713 | else => |vector_index| @intFromEnum(vector_index) * src_bit_size, | | |
| 174714 | }; | | |
| 174715 | const src_byte_off: i32 = @intCast(ptr_bit_off / limb_abi_bits * limb_abi_size); | | |
| 174716 | const src_bit_off = ptr_bit_off % limb_abi_bits; | | |
| 174717 | | | |
| 174718 | const ptr_reg = try self.copyToTmpRegister(ptr_ty, ptr_mcv); | | |
| 174719 | const ptr_lock = self.register_manager.lockRegAssumeUnused(ptr_reg); | | |
| 174720 | defer self.register_manager.unlockReg(ptr_lock); | | |
| 174721 | | | |
| 174722 | const mat_src_mcv: MCValue = mat_src_mcv: switch (src_mcv) { | | |
| 174723 | .register => if (src_bit_size > 64) { | | |
| 174724 | const frame_index = try self.allocFrameIndex(.initSpill(src_ty, self.pt.zcu)); | | |
| 174725 | try self.genSetMem(.{ .frame = frame_index }, 0, src_ty, src_mcv, .{}); | | |
| 174726 | break :mat_src_mcv .{ .load_frame = .{ .index = frame_index } }; | | |
| 174727 | } else src_mcv, | | |
| 174728 | else => src_mcv, | | |
| 174729 | }; | | |
| 174730 | | | |
| 174731 | var limb_i: u16 = 0; | | |
| 174732 | while (limb_i * limb_abi_bits < src_bit_off + src_bit_size) : (limb_i += 1) { | | |
| 174733 | const part_bit_off = if (limb_i == 0) src_bit_off else 0; | | |
| 174734 | const part_bit_size = | | |
| 174735 | @min(src_bit_off + src_bit_size - limb_i * limb_abi_bits, limb_abi_bits) - part_bit_off; | | |
| 174736 | const limb_mem: Memory = .{ | | |
| 174737 | .base = .{ .reg = ptr_reg }, | | |
| 174738 | .mod = .{ .rm = .{ | | |
| 174739 | .size = .fromSize(limb_abi_size), | | |
| 174740 | .disp = src_byte_off + limb_i * limb_abi_size, | | |
| 174741 | } }, | | |
| 174742 | }; | | |
| 174743 | | | |
| 174744 | const part_mask = (@as(u64, std.math.maxInt(u64)) >> @intCast(64 - part_bit_size)) << | | |
| 174745 | @intCast(part_bit_off); | | |
| 174746 | const part_mask_not = part_mask ^ (@as(u64, std.math.maxInt(u64)) >> @intCast(64 - limb_abi_bits)); | | |
| 174747 | if (limb_abi_size <= 4) { | | |
| 174748 | try self.asmMemoryImmediate(.{ ._, .@"and" }, limb_mem, .u(part_mask_not)); | | |
| 174749 | } else if (std.math.cast(i32, @as(i64, @bitCast(part_mask_not)))) |small| { | | |
| 174750 | try self.asmMemoryImmediate(.{ ._, .@"and" }, limb_mem, .s(small)); | | |
| 174751 | } else { | | |
| 174752 | const part_mask_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); | | |
| 174753 | try self.asmRegisterImmediate(.{ ._, .mov }, part_mask_reg, .u(part_mask_not)); | | |
| 174754 | try self.asmMemoryRegister(.{ ._, .@"and" }, limb_mem, part_mask_reg); | | |
| 174755 | } | | |
| 174756 | | | |
| 174757 | if (src_bit_size <= 64) { | | |
| 174758 | const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); | | |
| 174759 | const tmp_mcv = MCValue{ .register = tmp_reg }; | | |
| 174760 | const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); | | |
| 174761 | defer self.register_manager.unlockReg(tmp_lock); | | |
| 174762 | | | |
| 174763 | try self.genSetReg(tmp_reg, limb_ty, mat_src_mcv, .{}); | | |
| 174764 | switch (limb_i) { | | |
| 174765 | 0 => try self.genShiftBinOpMir( | | |
| 174766 | .{ ._l, .sh }, | | |
| 174767 | limb_ty, | | |
| 174768 | tmp_mcv, | | |
| 174769 | .u8, | | |
| 174770 | .{ .immediate = src_bit_off }, | | |
| 174771 | ), | | |
| 174772 | 1 => try self.genShiftBinOpMir( | | |
| 174773 | .{ ._r, .sh }, | | |
| 174774 | limb_ty, | | |
| 174775 | tmp_mcv, | | |
| 174776 | .u8, | | |
| 174777 | .{ .immediate = limb_abi_bits - src_bit_off }, | | |
| 174778 | ), | | |
| 174779 | else => unreachable, | | |
| 174780 | } | | |
| 174781 | try self.genBinOpMir(.{ ._, .@"and" }, limb_ty, tmp_mcv, .{ .immediate = part_mask }); | | |
| 174782 | try self.asmMemoryRegister( | | |
| 174783 | .{ ._, .@"or" }, | | |
| 174784 | limb_mem, | | |
| 174785 | registerAlias(tmp_reg, limb_abi_size), | | |
| 174786 | ); | | |
| 174787 | } else if (src_bit_size <= 128 and src_bit_off == 0) { | | |
| 174788 | const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); | | |
| 174789 | const tmp_mcv = MCValue{ .register = tmp_reg }; | | |
| 174790 | const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); | | |
| 174791 | defer self.register_manager.unlockReg(tmp_lock); | | |
| 174792 | | | |
| 174793 | try self.genSetReg(tmp_reg, limb_ty, switch (limb_i) { | | |
| 174794 | 0 => mat_src_mcv, | | |
| 174795 | else => mat_src_mcv.address().offset(limb_i * limb_abi_size).deref(), | | |
| 174796 | }, .{}); | | |
| 174797 | try self.genBinOpMir(.{ ._, .@"and" }, limb_ty, tmp_mcv, .{ .immediate = part_mask }); | | |
| 174798 | try self.asmMemoryRegister( | | |
| 174799 | .{ ._, .@"or" }, | | |
| 174800 | limb_mem, | | |
| 174801 | registerAlias(tmp_reg, limb_abi_size), | | |
| 174802 | ); | | |
| 174803 | } else return self.fail("TODO: implement packed store of {f}", .{src_ty.fmt(pt)}); | | |
| 174804 | } | | |
| 174805 | } | | |
| 174806 | | | |
| 174807 | fn store( | 174792 | fn store( |
| 174808 | self: *CodeGen, | 174793 | self: *CodeGen, |
| 174809 | ptr_ty: Type, | 174794 | ptr_ty: Type, |
| ... | @@ -174857,35 +174842,6 @@ fn store( | ... | @@ -174857,35 +174842,6 @@ fn store( |
| 174857 | } | 174842 | } |
| 174858 | } | 174843 | } |
| 174859 | | 174844 | |
| 174860 | fn airStore(self: *CodeGen, inst: Air.Inst.Index, safety: bool) !void { | | |
| 174861 | const pt = self.pt; | | |
| 174862 | const zcu = pt.zcu; | | |
| 174863 | const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; | | |
| 174864 | | | |
| 174865 | result: { | | |
| 174866 | if (!safety and (try self.resolveInst(bin_op.rhs)) == .undef) break :result; | | |
| 174867 | | | |
| 174868 | try self.spillRegisters(&.{ .rdi, .rsi, .rcx }); | | |
| 174869 | const reg_locks = self.register_manager.lockRegsAssumeUnused(3, .{ .rdi, .rsi, .rcx }); | | |
| 174870 | defer for (reg_locks) |lock| self.register_manager.unlockReg(lock); | | |
| 174871 | | | |
| 174872 | const ptr_ty = self.typeOf(bin_op.lhs); | | |
| 174873 | const ptr_info = ptr_ty.ptrInfo(zcu); | | |
| 174874 | const is_packed = ptr_info.flags.vector_index != .none or ptr_info.packed_offset.host_size > 0; | | |
| 174875 | if (is_packed) try self.spillEflagsIfOccupied(); | | |
| 174876 | | | |
| 174877 | const src_mcv = try self.resolveInst(bin_op.rhs); | | |
| 174878 | const ptr_mcv = try self.resolveInst(bin_op.lhs); | | |
| 174879 | | | |
| 174880 | if (is_packed) { | | |
| 174881 | try self.packedStore(ptr_ty, ptr_mcv, src_mcv); | | |
| 174882 | } else { | | |
| 174883 | try self.store(ptr_ty, ptr_mcv, src_mcv, .{ .safety = safety }); | | |
| 174884 | } | | |
| 174885 | } | | |
| 174886 | return self.finishAir(inst, .none, .{ bin_op.lhs, bin_op.rhs, .none }); | | |
| 174887 | } | | |
| 174888 | | | |
| 174889 | fn genUnOp(self: *CodeGen, maybe_inst: ?Air.Inst.Index, tag: Air.Inst.Tag, src_air: Air.Inst.Ref) !MCValue { | 174845 | fn genUnOp(self: *CodeGen, maybe_inst: ?Air.Inst.Index, tag: Air.Inst.Tag, src_air: Air.Inst.Ref) !MCValue { |
| 174890 | const pt = self.pt; | 174846 | const pt = self.pt; |
| 174891 | const zcu = pt.zcu; | 174847 | const zcu = pt.zcu; |
| ... | @@ -192171,6 +192127,8 @@ const Select = struct { | ... | @@ -192171,6 +192127,8 @@ const Select = struct { |
| 192171 | exact_bool_vec: u16, | 192127 | exact_bool_vec: u16, |
| 192172 | ptr_any_bool_vec, | 192128 | ptr_any_bool_vec, |
| 192173 | ptr_bool_vec: Memory.Size, | 192129 | ptr_bool_vec: Memory.Size, |
| | 192130 | ptr_any_bool_vec_elem, |
| | 192131 | ptr_bool_vec_elem: Memory.Size, |
| 192174 | remainder_bool_vec: OfIsSizes, | 192132 | remainder_bool_vec: OfIsSizes, |
| 192175 | exact_remainder_bool_vec: struct { of: Memory.Size, is: u16 }, | 192133 | exact_remainder_bool_vec: struct { of: Memory.Size, is: u16 }, |
| 192176 | signed_int_vec: Memory.Size, | 192134 | signed_int_vec: Memory.Size, |
| ... | @@ -192273,6 +192231,22 @@ const Select = struct { | ... | @@ -192273,6 +192231,22 @@ const Select = struct { |
| 192273 | .vector_type => |vector_type| vector_type.child == .bool_type and size.bitSize(cg.target) >= vector_type.len, | 192231 | .vector_type => |vector_type| vector_type.child == .bool_type and size.bitSize(cg.target) >= vector_type.len, |
| 192274 | else => false, | 192232 | else => false, |
| 192275 | }, | 192233 | }, |
| | 192234 | .ptr_any_bool_vec_elem => { |
| | 192235 | const ptr_info = ty.ptrInfo(zcu); |
| | 192236 | return switch (ptr_info.flags.vector_index) { |
| | 192237 | .none => false, |
| | 192238 | .runtime => unreachable, |
| | 192239 | else => ptr_info.child == .bool_type, |
| | 192240 | }; |
| | 192241 | }, |
| | 192242 | .ptr_bool_vec_elem => |size| { |
| | 192243 | const ptr_info = ty.ptrInfo(zcu); |
| | 192244 | return switch (ptr_info.flags.vector_index) { |
| | 192245 | .none => false, |
| | 192246 | .runtime => unreachable, |
| | 192247 | else => ptr_info.child == .bool_type and size.bitSize(cg.target) >= ptr_info.packed_offset.host_size, |
| | 192248 | }; |
| | 192249 | }, |
| 192276 | .remainder_bool_vec => |of_is| ty.isVector(zcu) and ty.scalarType(zcu).toIntern() == .bool_type and | 192250 | .remainder_bool_vec => |of_is| ty.isVector(zcu) and ty.scalarType(zcu).toIntern() == .bool_type and |
| 192277 | of_is.is.bitSize(cg.target) >= (ty.vectorLen(zcu) - 1) % of_is.of.bitSize(cg.target) + 1, | 192251 | of_is.is.bitSize(cg.target) >= (ty.vectorLen(zcu) - 1) % of_is.of.bitSize(cg.target) + 1, |
| 192278 | .exact_remainder_bool_vec => |of_is| ty.isVector(zcu) and ty.scalarType(zcu).toIntern() == .bool_type and | 192252 | .exact_remainder_bool_vec => |of_is| ty.isVector(zcu) and ty.scalarType(zcu).toIntern() == .bool_type and |
| ... | @@ -193266,7 +193240,7 @@ const Select = struct { | ... | @@ -193266,7 +193240,7 @@ const Select = struct { |
| 193266 | ref: Ref, | 193240 | ref: Ref, |
| 193267 | scale: Memory.Scale = .@"1", | 193241 | scale: Memory.Scale = .@"1", |
| 193268 | } = .{ .ref = .none }, | 193242 | } = .{ .ref = .none }, |
| 193269 | unused: u3 = 0, | 193243 | unused: u2 = 0, |
| 193270 | }, | 193244 | }, |
| 193271 | imm: i32 = 0, | 193245 | imm: i32 = 0, |
| 193272 | | 193246 | |
| ... | @@ -193279,9 +193253,9 @@ const Select = struct { | ... | @@ -193279,9 +193253,9 @@ const Select = struct { |
| 193279 | lea, | 193253 | lea, |
| 193280 | mem, | 193254 | mem, |
| 193281 | }; | 193255 | }; |
| 193282 | const Adjust = packed struct(u10) { | 193256 | const Adjust = packed struct(u11) { |
| 193283 | sign: enum(u1) { neg, pos }, | 193257 | sign: enum(u1) { neg, pos }, |
| 193284 | lhs: enum(u5) { | 193258 | lhs: enum(u6) { |
| 193285 | none, | 193259 | none, |
| 193286 | ptr_size, | 193260 | ptr_size, |
| 193287 | ptr_bit_size, | 193261 | ptr_bit_size, |
| ... | @@ -193303,6 +193277,7 @@ const Select = struct { | ... | @@ -193303,6 +193277,7 @@ const Select = struct { |
| 193303 | src0_elem_size, | 193277 | src0_elem_size, |
| 193304 | dst0_elem_size, | 193278 | dst0_elem_size, |
| 193305 | src0_elem_size_mul_src1, | 193279 | src0_elem_size_mul_src1, |
| | 193280 | vector_index, |
| 193306 | src1, | 193281 | src1, |
| 193307 | src1_sub_bit_size, | 193282 | src1_sub_bit_size, |
| 193308 | log2_src0_elem_size, | 193283 | log2_src0_elem_size, |
| ... | @@ -193373,9 +193348,13 @@ const Select = struct { | ... | @@ -193373,9 +193348,13 @@ const Select = struct { |
| 193373 | const sub_src0_elem_size: Adjust = .{ .sign = .neg, .lhs = .src0_elem_size, .op = .mul, .rhs = .@"1" }; | 193348 | const sub_src0_elem_size: Adjust = .{ .sign = .neg, .lhs = .src0_elem_size, .op = .mul, .rhs = .@"1" }; |
| 193374 | const add_src0_elem_size_mul_src1: Adjust = .{ .sign = .pos, .lhs = .src0_elem_size_mul_src1, .op = .mul, .rhs = .@"1" }; | 193349 | const add_src0_elem_size_mul_src1: Adjust = .{ .sign = .pos, .lhs = .src0_elem_size_mul_src1, .op = .mul, .rhs = .@"1" }; |
| 193375 | const sub_src0_elem_size_mul_src1: Adjust = .{ .sign = .neg, .lhs = .src0_elem_size_mul_src1, .op = .mul, .rhs = .@"1" }; | 193350 | const sub_src0_elem_size_mul_src1: Adjust = .{ .sign = .neg, .lhs = .src0_elem_size_mul_src1, .op = .mul, .rhs = .@"1" }; |
| | 193351 | const add_vector_index: Adjust = .{ .sign = .pos, .lhs = .vector_index, .op = .mul, .rhs = .@"1" }; |
| | 193352 | const add_vector_index_rem_32: Adjust = .{ .sign = .pos, .lhs = .vector_index, .op = .rem_8_mul, .rhs = .@"4" }; |
| | 193353 | const add_vector_index_div_8_down_4: Adjust = .{ .sign = .pos, .lhs = .vector_index, .op = .div_8_down, .rhs = .@"4" }; |
| 193376 | const add_dst0_elem_size: Adjust = .{ .sign = .pos, .lhs = .dst0_elem_size, .op = .mul, .rhs = .@"1" }; | 193354 | const add_dst0_elem_size: Adjust = .{ .sign = .pos, .lhs = .dst0_elem_size, .op = .mul, .rhs = .@"1" }; |
| 193377 | const sub_dst0_elem_size: Adjust = .{ .sign = .neg, .lhs = .dst0_elem_size, .op = .mul, .rhs = .@"1" }; | 193355 | const sub_dst0_elem_size: Adjust = .{ .sign = .neg, .lhs = .dst0_elem_size, .op = .mul, .rhs = .@"1" }; |
| 193378 | const add_src1_div_8_down_4: Adjust = .{ .sign = .pos, .lhs = .src1, .op = .div_8_down, .rhs = .@"4" }; | 193356 | const add_src1_div_8_down_4: Adjust = .{ .sign = .pos, .lhs = .src1, .op = .div_8_down, .rhs = .@"4" }; |
| | 193357 | const add_src1: Adjust = .{ .sign = .pos, .lhs = .src1, .op = .mul, .rhs = .@"1" }; |
| 193379 | const add_src1_rem_32: Adjust = .{ .sign = .pos, .lhs = .src1, .op = .rem_8_mul, .rhs = .@"4" }; | 193358 | const add_src1_rem_32: Adjust = .{ .sign = .pos, .lhs = .src1, .op = .rem_8_mul, .rhs = .@"4" }; |
| 193380 | const add_src1_rem_64: Adjust = .{ .sign = .pos, .lhs = .src1, .op = .rem_8_mul, .rhs = .@"8" }; | 193359 | const add_src1_rem_64: Adjust = .{ .sign = .pos, .lhs = .src1, .op = .rem_8_mul, .rhs = .@"8" }; |
| 193381 | const add_src1_sub_bit_size: Adjust = .{ .sign = .pos, .lhs = .src1_sub_bit_size, .op = .mul, .rhs = .@"1" }; | 193360 | const add_src1_sub_bit_size: Adjust = .{ .sign = .pos, .lhs = .src1_sub_bit_size, .op = .mul, .rhs = .@"1" }; |
| ... | @@ -194258,6 +194237,10 @@ const Select = struct { | ... | @@ -194258,6 +194237,10 @@ const Select = struct { |
| 194258 | .dst0_elem_size => @intCast(Select.Operand.Ref.dst0.typeOf(s).elemType2(s.cg.pt.zcu).abiSize(s.cg.pt.zcu)), | 194237 | .dst0_elem_size => @intCast(Select.Operand.Ref.dst0.typeOf(s).elemType2(s.cg.pt.zcu).abiSize(s.cg.pt.zcu)), |
| 194259 | .src0_elem_size_mul_src1 => @intCast(Select.Operand.Ref.src0.typeOf(s).elemType2(s.cg.pt.zcu).abiSize(s.cg.pt.zcu) * | 194238 | .src0_elem_size_mul_src1 => @intCast(Select.Operand.Ref.src0.typeOf(s).elemType2(s.cg.pt.zcu).abiSize(s.cg.pt.zcu) * |
| 194260 | Select.Operand.Ref.src1.valueOf(s).immediate), | 194239 | Select.Operand.Ref.src1.valueOf(s).immediate), |
| | 194240 | .vector_index => switch (op.flags.base.ref.typeOf(s).ptrInfo(s.cg.pt.zcu).flags.vector_index) { |
| | 194241 | .none, .runtime => unreachable, |
| | 194242 | else => |vector_index| @intFromEnum(vector_index), |
| | 194243 | }, |
| 194261 | .src1 => @intCast(Select.Operand.Ref.src1.valueOf(s).immediate), | 194244 | .src1 => @intCast(Select.Operand.Ref.src1.valueOf(s).immediate), |
| 194262 | .src1_sub_bit_size => @as(SignedImm, @intCast(Select.Operand.Ref.src1.valueOf(s).immediate)) - | 194245 | .src1_sub_bit_size => @as(SignedImm, @intCast(Select.Operand.Ref.src1.valueOf(s).immediate)) - |
| 194263 | @as(SignedImm, @intCast(s.cg.nonBoolScalarBitSize(op.flags.base.ref.typeOf(s)))), | 194246 | @as(SignedImm, @intCast(s.cg.nonBoolScalarBitSize(op.flags.base.ref.typeOf(s)))), |
Jacob Young <jacobly@ziglang.org> 2025-09-11 08:40:17-04:00
Alex Rønne Petersen <alex@alexrp.com> 2025-09-21 21:36:23+02:00
Signed by SSH key SHA256:7B/LJ7bpR1eX8aCXSr4mtd5M45VMPKcx9zY8e95b5QM
src