| ... | ... | @@ -6587,12 +6587,15 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void { |
| 6587 | 6587 | const src_reg_lock = self.register_manager.lockRegAssumeUnused(src_reg); |
| 6588 | 6588 | defer self.register_manager.unlockReg(src_reg_lock); |
| 6589 | 6589 | |
| 6590 | | const dst_reg = if (field_rc.supersetOf(container_rc) and |
| 6591 | | self.reuseOperand(inst, operand, 0, src_mcv)) |
| 6590 | const src_in_field_rc = |
| 6591 | field_rc.isSet(RegisterManager.indexOfRegIntoTracked(src_reg).?); |
| 6592 | const dst_reg = if (src_in_field_rc and self.reuseOperand(inst, operand, 0, src_mcv)) |
| 6592 | 6593 | src_reg |
| 6594 | else if (field_off == 0) |
| 6595 | (try self.copyToRegisterWithInstTracking(inst, field_ty, src_mcv)).register |
| 6593 | 6596 | else |
| 6594 | 6597 | try self.copyToTmpRegister(Type.usize, .{ .register = src_reg }); |
| 6595 | | const dst_mcv = MCValue{ .register = dst_reg }; |
| 6598 | const dst_mcv: MCValue = .{ .register = dst_reg }; |
| 6596 | 6599 | const dst_lock = self.register_manager.lockReg(dst_reg); |
| 6597 | 6600 | defer if (dst_lock) |lock| self.register_manager.unlockReg(lock); |
| 6598 | 6601 | |
| ... | ... | @@ -6602,9 +6605,11 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void { |
| 6602 | 6605 | dst_mcv, |
| 6603 | 6606 | .{ .immediate = field_off }, |
| 6604 | 6607 | ); |
| 6605 | | if (self.regExtraBits(field_ty) > 0) try self.truncateRegister(field_ty, dst_reg); |
| 6608 | if (abi.RegisterClass.gp.isSet(RegisterManager.indexOfRegIntoTracked(dst_reg).?) and |
| 6609 | container_ty.abiSize(mod) * 8 > field_ty.bitSize(mod)) |
| 6610 | try self.truncateRegister(field_ty, dst_reg); |
| 6606 | 6611 | |
| 6607 | | break :result if (field_rc.supersetOf(abi.RegisterClass.gp)) |
| 6612 | break :result if (field_off == 0 or field_rc.supersetOf(abi.RegisterClass.gp)) |
| 6608 | 6613 | dst_mcv |
| 6609 | 6614 | else |
| 6610 | 6615 | try self.copyToRegisterWithInstTracking(inst, field_ty, dst_mcv); |
| ... | ... | @@ -9674,17 +9679,25 @@ fn airArg(self: *Self, inst: Air.Inst.Index) !void { |
| 9674 | 9679 | self.arg_index = arg_index + 1; |
| 9675 | 9680 | |
| 9676 | 9681 | const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else result: { |
| 9677 | | const dst_mcv = self.args[arg_index]; |
| 9678 | | switch (dst_mcv) { |
| 9679 | | .register, .register_pair, .load_frame => for (dst_mcv.getRegs()) |reg| |
| 9680 | | self.register_manager.getRegAssumeFree(reg, inst), |
| 9681 | | else => return self.fail("TODO implement arg for {}", .{dst_mcv}), |
| 9682 | | } |
| 9682 | const arg_ty = self.typeOfIndex(inst); |
| 9683 | const src_mcv = self.args[arg_index]; |
| 9684 | const dst_mcv = switch (src_mcv) { |
| 9685 | .register, .register_pair, .load_frame => dst: { |
| 9686 | for (src_mcv.getRegs()) |reg| self.register_manager.getRegAssumeFree(reg, inst); |
| 9687 | break :dst src_mcv; |
| 9688 | }, |
| 9689 | .indirect => |reg_off| dst: { |
| 9690 | self.register_manager.getRegAssumeFree(reg_off.reg, inst); |
| 9691 | const dst_mcv = try self.allocRegOrMem(inst, false); |
| 9692 | try self.genCopy(arg_ty, dst_mcv, src_mcv); |
| 9693 | break :dst dst_mcv; |
| 9694 | }, |
| 9695 | else => return self.fail("TODO implement arg for {}", .{src_mcv}), |
| 9696 | }; |
| 9683 | 9697 | |
| 9684 | | const ty = self.typeOfIndex(inst); |
| 9685 | 9698 | const src_index = self.air.instructions.items(.data)[inst].arg.src_index; |
| 9686 | 9699 | const name = mod.getParamName(self.owner.func_index, src_index); |
| 9687 | | try self.genArgDbgInfo(ty, name, dst_mcv); |
| 9700 | try self.genArgDbgInfo(arg_ty, name, src_mcv); |
| 9688 | 9701 | |
| 9689 | 9702 | break :result dst_mcv; |
| 9690 | 9703 | }; |
| ... | ... | @@ -9863,7 +9876,8 @@ fn genCall(self: *Self, info: union(enum) { |
| 9863 | 9876 | |
| 9864 | 9877 | const ExpectedContents = extern struct { |
| 9865 | 9878 | var_args: [16][@sizeOf(Type)]u8 align(@alignOf(Type)), |
| 9866 | | arg_regs: [16][@sizeOf(?RegisterLock)]u8 align(@alignOf(?RegisterLock)), |
| 9879 | frame_indices: [16]FrameIndex, |
| 9880 | reg_locks: [16][@sizeOf(?RegisterLock)]u8 align(@alignOf(?RegisterLock)), |
| 9867 | 9881 | }; |
| 9868 | 9882 | var stack align(@max(@alignOf(ExpectedContents), @alignOf(std.heap.StackFallbackAllocator(0)))) = |
| 9869 | 9883 | std.heap.stackFallback(@sizeOf(ExpectedContents), self.gpa); |
| ... | ... | @@ -9873,10 +9887,13 @@ fn genCall(self: *Self, info: union(enum) { |
| 9873 | 9887 | defer allocator.free(var_args); |
| 9874 | 9888 | for (var_args, arg_types[fn_info.param_types.len..]) |*var_arg, arg_ty| var_arg.* = arg_ty; |
| 9875 | 9889 | |
| 9876 | | var arg_locks = std.ArrayList(?RegisterLock).init(allocator); |
| 9877 | | defer arg_locks.deinit(); |
| 9878 | | try arg_locks.ensureTotalCapacity(16); |
| 9879 | | defer for (arg_locks.items) |arg_lock| if (arg_lock) |lock| self.register_manager.unlockReg(lock); |
| 9890 | const frame_indices = try allocator.alloc(FrameIndex, args.len); |
| 9891 | defer allocator.free(frame_indices); |
| 9892 | |
| 9893 | var reg_locks = std.ArrayList(?RegisterLock).init(allocator); |
| 9894 | defer reg_locks.deinit(); |
| 9895 | try reg_locks.ensureTotalCapacity(16); |
| 9896 | defer for (reg_locks.items) |reg_lock| if (reg_lock) |lock| self.register_manager.unlockReg(lock); |
| 9880 | 9897 | |
| 9881 | 9898 | var call_info = try self.resolveCallingConventionValues(fn_info, var_args, .call_frame); |
| 9882 | 9899 | defer call_info.deinit(self); |
| ... | ... | @@ -9908,44 +9925,69 @@ fn genCall(self: *Self, info: union(enum) { |
| 9908 | 9925 | .indirect => |reg_off| try self.spillRegisters(&.{reg_off.reg}), |
| 9909 | 9926 | else => unreachable, |
| 9910 | 9927 | } |
| 9911 | | for (call_info.args, arg_types, args) |dst_arg, arg_ty, src_arg| switch (dst_arg) { |
| 9912 | | .none => {}, |
| 9913 | | .register => |reg| { |
| 9914 | | try self.spillRegisters(&.{reg}); |
| 9915 | | try arg_locks.append(self.register_manager.lockReg(reg)); |
| 9916 | | }, |
| 9917 | | .register_pair => |regs| { |
| 9918 | | try self.spillRegisters(&regs); |
| 9919 | | try arg_locks.appendSlice(&self.register_manager.lockRegs(2, regs)); |
| 9920 | | }, |
| 9921 | | .load_frame => { |
| 9922 | | try self.genCopy(arg_ty, dst_arg, src_arg); |
| 9923 | | try self.freeValue(src_arg); |
| 9924 | | }, |
| 9925 | | else => unreachable, |
| 9926 | | }; |
| 9928 | for (call_info.args, arg_types, args, frame_indices) |dst_arg, arg_ty, src_arg, *frame_index| |
| 9929 | switch (dst_arg) { |
| 9930 | .none => {}, |
| 9931 | .register => |reg| { |
| 9932 | try self.spillRegisters(&.{reg}); |
| 9933 | try reg_locks.append(self.register_manager.lockReg(reg)); |
| 9934 | }, |
| 9935 | .register_pair => |regs| { |
| 9936 | try self.spillRegisters(&regs); |
| 9937 | try reg_locks.appendSlice(&self.register_manager.lockRegs(2, regs)); |
| 9938 | }, |
| 9939 | .indirect => |reg_off| { |
| 9940 | frame_index.* = try self.allocFrameIndex(FrameAlloc.initType(arg_ty, mod)); |
| 9941 | try self.genSetMem(.{ .frame = frame_index.* }, 0, arg_ty, src_arg); |
| 9942 | try self.spillRegisters(&.{reg_off.reg}); |
| 9943 | try reg_locks.append(self.register_manager.lockReg(reg_off.reg)); |
| 9944 | }, |
| 9945 | .load_frame => { |
| 9946 | try self.genCopy(arg_ty, dst_arg, src_arg); |
| 9947 | try self.freeValue(src_arg); |
| 9948 | }, |
| 9949 | else => unreachable, |
| 9950 | }; |
| 9927 | 9951 | |
| 9928 | 9952 | // now we are free to set register arguments |
| 9929 | | const ret_lock = switch (call_info.return_value.long) { |
| 9930 | | .none, .unreach => null, |
| 9931 | | .indirect => |reg_off| lock: { |
| 9953 | switch (call_info.return_value.long) { |
| 9954 | .none, .unreach => {}, |
| 9955 | .indirect => |reg_off| { |
| 9932 | 9956 | const ret_ty = fn_info.return_type.toType(); |
| 9933 | 9957 | const frame_index = try self.allocFrameIndex(FrameAlloc.initType(ret_ty, mod)); |
| 9934 | 9958 | try self.genSetReg(reg_off.reg, Type.usize, .{ |
| 9935 | 9959 | .lea_frame = .{ .index = frame_index, .off = -reg_off.off }, |
| 9936 | 9960 | }); |
| 9937 | 9961 | call_info.return_value.short = .{ .load_frame = .{ .index = frame_index } }; |
| 9938 | | break :lock self.register_manager.lockRegAssumeUnused(reg_off.reg); |
| 9962 | try reg_locks.append(self.register_manager.lockReg(reg_off.reg)); |
| 9939 | 9963 | }, |
| 9940 | 9964 | else => unreachable, |
| 9941 | | }; |
| 9942 | | defer if (ret_lock) |lock| self.register_manager.unlockReg(lock); |
| 9965 | } |
| 9943 | 9966 | |
| 9944 | | for (call_info.args, arg_types, args) |dst_arg, arg_ty, src_arg| switch (dst_arg) { |
| 9945 | | .none, .load_frame => {}, |
| 9946 | | .register, .register_pair => try self.genCopy(arg_ty, dst_arg, src_arg), |
| 9947 | | else => unreachable, |
| 9948 | | }; |
| 9967 | for (call_info.args, arg_types, args, frame_indices) |dst_arg, arg_ty, src_arg, frame_index| |
| 9968 | switch (dst_arg) { |
| 9969 | .none, .load_frame => {}, |
| 9970 | .register => |dst_reg| switch (fn_info.cc) { |
| 9971 | else => try self.genSetReg( |
| 9972 | registerAlias(dst_reg, @intCast(arg_ty.abiSize(mod))), |
| 9973 | arg_ty, |
| 9974 | src_arg, |
| 9975 | ), |
| 9976 | .C, .SysV, .Win64 => { |
| 9977 | const promoted_ty = self.promoteInt(arg_ty); |
| 9978 | const promoted_abi_size: u32 = @intCast(promoted_ty.abiSize(mod)); |
| 9979 | const dst_alias = registerAlias(dst_reg, promoted_abi_size); |
| 9980 | try self.genSetReg(dst_alias, promoted_ty, src_arg); |
| 9981 | if (promoted_ty.toIntern() != arg_ty.toIntern()) |
| 9982 | try self.truncateRegister(arg_ty, dst_alias); |
| 9983 | }, |
| 9984 | }, |
| 9985 | .register_pair => try self.genCopy(arg_ty, dst_arg, src_arg), |
| 9986 | .indirect => |reg_off| try self.genSetReg(reg_off.reg, Type.usize, .{ |
| 9987 | .lea_frame = .{ .index = frame_index, .off = -reg_off.off }, |
| 9988 | }), |
| 9989 | else => unreachable, |
| 9990 | }; |
| 9949 | 9991 | |
| 9950 | 9992 | if (fn_info.is_var_args) |
| 9951 | 9993 | try self.asmRegisterImmediate(.{ ._, .mov }, .al, Immediate.u(call_info.fp_count)); |
| ... | ... | @@ -10054,7 +10096,7 @@ fn airRetLoad(self: *Self, inst: Air.Inst.Index) !void { |
| 10054 | 10096 | const ptr_ty = self.typeOf(un_op); |
| 10055 | 10097 | switch (self.ret_mcv.short) { |
| 10056 | 10098 | .none => {}, |
| 10057 | | .register => try self.load(self.ret_mcv.short, ptr_ty, ptr), |
| 10099 | .register, .register_pair => try self.load(self.ret_mcv.short, ptr_ty, ptr), |
| 10058 | 10100 | .indirect => |reg_off| try self.genSetReg(reg_off.reg, ptr_ty, ptr), |
| 10059 | 10101 | else => unreachable, |
| 10060 | 10102 | } |
| ... | ... | @@ -10115,20 +10157,28 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void { |
| 10115 | 10157 | try self.spillEflagsIfOccupied(); |
| 10116 | 10158 | |
| 10117 | 10159 | const lhs_mcv = try self.resolveInst(bin_op.lhs); |
| 10118 | | const lhs_lock = switch (lhs_mcv) { |
| 10119 | | .register => |reg| self.register_manager.lockRegAssumeUnused(reg), |
| 10120 | | .register_offset => |ro| self.register_manager.lockRegAssumeUnused(ro.reg), |
| 10121 | | else => null, |
| 10160 | const lhs_locks: [2]?RegisterLock = switch (lhs_mcv) { |
| 10161 | .register => |lhs_reg| .{ self.register_manager.lockRegAssumeUnused(lhs_reg), null }, |
| 10162 | .register_pair => |lhs_regs| locks: { |
| 10163 | const locks = self.register_manager.lockRegsAssumeUnused(2, lhs_regs); |
| 10164 | break :locks .{ locks[0], locks[1] }; |
| 10165 | }, |
| 10166 | .register_offset => |lhs_ro| .{ |
| 10167 | self.register_manager.lockRegAssumeUnused(lhs_ro.reg), |
| 10168 | null, |
| 10169 | }, |
| 10170 | else => .{null} ** 2, |
| 10122 | 10171 | }; |
| 10123 | | defer if (lhs_lock) |lock| self.register_manager.unlockReg(lock); |
| 10172 | defer for (lhs_locks) |lhs_lock| if (lhs_lock) |lock| self.register_manager.unlockReg(lock); |
| 10124 | 10173 | |
| 10125 | 10174 | const rhs_mcv = try self.resolveInst(bin_op.rhs); |
| 10126 | | const rhs_lock = switch (rhs_mcv) { |
| 10127 | | .register => |reg| self.register_manager.lockReg(reg), |
| 10128 | | .register_offset => |ro| self.register_manager.lockReg(ro.reg), |
| 10129 | | else => null, |
| 10175 | const rhs_locks: [2]?RegisterLock = switch (rhs_mcv) { |
| 10176 | .register => |rhs_reg| .{ self.register_manager.lockReg(rhs_reg), null }, |
| 10177 | .register_pair => |rhs_regs| self.register_manager.lockRegs(2, rhs_regs), |
| 10178 | .register_offset => |rhs_ro| .{ self.register_manager.lockReg(rhs_ro.reg), null }, |
| 10179 | else => .{null} ** 2, |
| 10130 | 10180 | }; |
| 10131 | | defer if (rhs_lock) |lock| self.register_manager.unlockReg(lock); |
| 10181 | defer for (rhs_locks) |rhs_lock| if (rhs_lock) |lock| self.register_manager.unlockReg(lock); |
| 10132 | 10182 | |
| 10133 | 10183 | switch (ty.zigTypeTag(mod)) { |
| 10134 | 10184 | else => { |
| ... | ... | @@ -10159,7 +10209,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void { |
| 10159 | 10209 | if (dst_mcv.getReg()) |reg| self.register_manager.lockReg(reg) else null; |
| 10160 | 10210 | defer if (dst_lock) |lock| self.register_manager.unlockReg(lock); |
| 10161 | 10211 | |
| 10162 | | const src_mcv = if (flipped) lhs_mcv else rhs_mcv; |
| 10212 | const src_mcv = try self.resolveInst(if (flipped) bin_op.lhs else bin_op.rhs); |
| 10163 | 10213 | const src_lock = |
| 10164 | 10214 | if (src_mcv.getReg()) |reg| self.register_manager.lockReg(reg) else null; |
| 10165 | 10215 | defer if (src_lock) |lock| self.register_manager.unlockReg(lock); |
| ... | ... | @@ -10350,13 +10400,10 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void { |
| 10350 | 10400 | }, |
| 10351 | 10401 | ); |
| 10352 | 10402 | |
| 10353 | | if (limb_i > 0) try self.asmRegisterRegister( |
| 10354 | | .{ ._, .@"or" }, |
| 10355 | | acc_reg, |
| 10356 | | tmp_reg, |
| 10357 | | ); |
| 10403 | if (limb_i > 0) |
| 10404 | try self.asmRegisterRegister(.{ ._, .@"or" }, acc_reg, tmp_reg); |
| 10358 | 10405 | } |
| 10359 | | try self.asmRegisterRegister(.{ ._, .@"test" }, acc_reg, acc_reg); |
| 10406 | assert(limbs_len >= 2); // use flags from or |
| 10360 | 10407 | break :result_op flipped_op; |
| 10361 | 10408 | }, |
| 10362 | 10409 | }; |
| ... | ... | @@ -11798,6 +11845,7 @@ const MoveStrategy = union(enum) { |
| 11798 | 11845 | .move => |tag| try self.asmRegisterMemory(tag, dst_reg, src_mem), |
| 11799 | 11846 | .x87_load_store => { |
| 11800 | 11847 | try self.asmMemory(.{ .f_, .ld }, src_mem); |
| 11848 | assert(dst_reg != .st7); |
| 11801 | 11849 | try self.asmRegister(.{ .f_p, .st }, @enumFromInt(@intFromEnum(dst_reg) + 1)); |
| 11802 | 11850 | }, |
| 11803 | 11851 | .insert_extract => |ie| try self.asmRegisterMemoryImmediate( |
| ... | ... | @@ -11831,188 +11879,284 @@ const MoveStrategy = union(enum) { |
| 11831 | 11879 | } |
| 11832 | 11880 | } |
| 11833 | 11881 | }; |
| 11834 | | fn moveStrategy(self: *Self, ty: Type, aligned: bool) !MoveStrategy { |
| 11882 | fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !MoveStrategy { |
| 11835 | 11883 | const mod = self.bin_file.options.module.?; |
| 11836 | | switch (ty.zigTypeTag(mod)) { |
| 11837 | | else => return .{ .move = .{ ._, .mov } }, |
| 11838 | | .Float => switch (ty.floatBits(self.target.*)) { |
| 11839 | | 16 => return if (self.hasFeature(.avx)) .{ .vex_insert_extract = .{ |
| 11840 | | .insert = .{ .vp_w, .insr }, |
| 11841 | | .extract = .{ .vp_w, .extr }, |
| 11842 | | } } else .{ .insert_extract = .{ |
| 11843 | | .insert = .{ .p_w, .insr }, |
| 11844 | | .extract = .{ .p_w, .extr }, |
| 11845 | | } }, |
| 11846 | | 32 => return .{ .move = if (self.hasFeature(.avx)) .{ .v_ss, .mov } else .{ ._ss, .mov } }, |
| 11847 | | 64 => return .{ .move = if (self.hasFeature(.avx)) .{ .v_sd, .mov } else .{ ._sd, .mov } }, |
| 11848 | | 80 => return .x87_load_store, |
| 11849 | | 128 => return .{ .move = if (self.hasFeature(.avx)) |
| 11850 | | if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu } |
| 11851 | | else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } }, |
| 11852 | | else => {}, |
| 11853 | | }, |
| 11854 | | .Vector => switch (ty.childType(mod).zigTypeTag(mod)) { |
| 11855 | | .Bool => return .{ .move = .{ ._, .mov } }, |
| 11856 | | .Int => switch (ty.childType(mod).intInfo(mod).bits) { |
| 11857 | | 8 => switch (ty.vectorLen(mod)) { |
| 11858 | | 1 => if (self.hasFeature(.avx)) return .{ .vex_insert_extract = .{ |
| 11859 | | .insert = .{ .vp_b, .insr }, |
| 11860 | | .extract = .{ .vp_b, .extr }, |
| 11861 | | } } else if (self.hasFeature(.sse4_2)) return .{ .insert_extract = .{ |
| 11862 | | .insert = .{ .p_b, .insr }, |
| 11863 | | .extract = .{ .p_b, .extr }, |
| 11864 | | } }, |
| 11865 | | 2 => return if (self.hasFeature(.avx)) .{ .vex_insert_extract = .{ |
| 11866 | | .insert = .{ .vp_w, .insr }, |
| 11867 | | .extract = .{ .vp_w, .extr }, |
| 11868 | | } } else .{ .insert_extract = .{ |
| 11869 | | .insert = .{ .p_w, .insr }, |
| 11870 | | .extract = .{ .p_w, .extr }, |
| 11871 | | } }, |
| 11872 | | 3...4 => return .{ .move = if (self.hasFeature(.avx)) |
| 11873 | | .{ .v_d, .mov } |
| 11874 | | else |
| 11875 | | .{ ._d, .mov } }, |
| 11876 | | 5...8 => return .{ .move = if (self.hasFeature(.avx)) |
| 11877 | | .{ .v_q, .mov } |
| 11878 | | else |
| 11879 | | .{ ._q, .mov } }, |
| 11880 | | 9...16 => return .{ .move = if (self.hasFeature(.avx)) |
| 11881 | | if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu } |
| 11882 | | else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } }, |
| 11883 | | 17...32 => if (self.hasFeature(.avx)) |
| 11884 | | return .{ .move = if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu } }, |
| 11885 | | else => {}, |
| 11886 | | }, |
| 11887 | | 16 => switch (ty.vectorLen(mod)) { |
| 11888 | | 1 => return if (self.hasFeature(.avx)) .{ .vex_insert_extract = .{ |
| 11889 | | .insert = .{ .vp_w, .insr }, |
| 11890 | | .extract = .{ .vp_w, .extr }, |
| 11891 | | } } else .{ .insert_extract = .{ |
| 11892 | | .insert = .{ .p_w, .insr }, |
| 11893 | | .extract = .{ .p_w, .extr }, |
| 11894 | | } }, |
| 11895 | | 2 => return .{ .move = if (self.hasFeature(.avx)) |
| 11896 | | .{ .v_d, .mov } |
| 11897 | | else |
| 11898 | | .{ ._d, .mov } }, |
| 11899 | | 3...4 => return .{ .move = if (self.hasFeature(.avx)) |
| 11900 | | .{ .v_q, .mov } |
| 11901 | | else |
| 11902 | | .{ ._q, .mov } }, |
| 11903 | | 5...8 => return .{ .move = if (self.hasFeature(.avx)) |
| 11904 | | if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu } |
| 11905 | | else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } }, |
| 11906 | | 9...16 => if (self.hasFeature(.avx)) |
| 11907 | | return .{ .move = if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu } }, |
| 11908 | | else => {}, |
| 11909 | | }, |
| 11910 | | 32 => switch (ty.vectorLen(mod)) { |
| 11911 | | 1 => return .{ .move = if (self.hasFeature(.avx)) |
| 11912 | | .{ .v_d, .mov } |
| 11913 | | else |
| 11914 | | .{ ._d, .mov } }, |
| 11915 | | 2 => return .{ .move = if (self.hasFeature(.avx)) |
| 11916 | | .{ .v_q, .mov } |
| 11917 | | else |
| 11918 | | .{ ._q, .mov } }, |
| 11919 | | 3...4 => return .{ .move = if (self.hasFeature(.avx)) |
| 11920 | | if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu } |
| 11921 | | else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } }, |
| 11922 | | 5...8 => if (self.hasFeature(.avx)) |
| 11923 | | return .{ .move = if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu } }, |
| 11924 | | else => {}, |
| 11925 | | }, |
| 11926 | | 64 => switch (ty.vectorLen(mod)) { |
| 11927 | | 1 => return .{ .move = if (self.hasFeature(.avx)) |
| 11928 | | .{ .v_q, .mov } |
| 11929 | | else |
| 11930 | | .{ ._q, .mov } }, |
| 11931 | | 2 => return .{ .move = if (self.hasFeature(.avx)) |
| 11932 | | if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu } |
| 11933 | | else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } }, |
| 11934 | | 3...4 => if (self.hasFeature(.avx)) |
| 11935 | | return .{ .move = if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu } }, |
| 11936 | | else => {}, |
| 11937 | | }, |
| 11938 | | 128 => switch (ty.vectorLen(mod)) { |
| 11939 | | 1 => return .{ .move = if (self.hasFeature(.avx)) |
| 11940 | | if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu } |
| 11941 | | else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } }, |
| 11942 | | 2 => if (self.hasFeature(.avx)) |
| 11943 | | return .{ .move = if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu } }, |
| 11944 | | else => {}, |
| 11945 | | }, |
| 11946 | | 256 => switch (ty.vectorLen(mod)) { |
| 11947 | | 1 => if (self.hasFeature(.avx)) |
| 11948 | | return .{ .move = if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu } }, |
| 11949 | | else => {}, |
| 11884 | switch (class) { |
| 11885 | .general_purpose, .segment => return .{ .move = .{ ._, .mov } }, |
| 11886 | .x87 => return .x87_load_store, |
| 11887 | .mmx => {}, |
| 11888 | .sse => { |
| 11889 | switch (ty.zigTypeTag(mod)) { |
| 11890 | else => { |
| 11891 | const classes = mem.sliceTo(&abi.classifySystemV(ty, mod, .other), .none); |
| 11892 | assert(std.mem.indexOfNone(abi.Class, classes, &.{ |
| 11893 | .integer, .sse, .float, .float_combine, |
| 11894 | }) == null); |
| 11895 | const abi_size = ty.abiSize(mod); |
| 11896 | if (abi_size < 4 or |
| 11897 | std.mem.indexOfScalar(abi.Class, classes, .integer) != null) switch (abi_size) { |
| 11898 | 1 => if (self.hasFeature(.avx)) return .{ .vex_insert_extract = .{ |
| 11899 | .insert = .{ .vp_b, .insr }, |
| 11900 | .extract = .{ .vp_b, .extr }, |
| 11901 | } } else if (self.hasFeature(.sse4_2)) return .{ .insert_extract = .{ |
| 11902 | .insert = .{ .p_b, .insr }, |
| 11903 | .extract = .{ .p_b, .extr }, |
| 11904 | } }, |
| 11905 | 2 => return if (self.hasFeature(.avx)) .{ .vex_insert_extract = .{ |
| 11906 | .insert = .{ .vp_w, .insr }, |
| 11907 | .extract = .{ .vp_w, .extr }, |
| 11908 | } } else .{ .insert_extract = .{ |
| 11909 | .insert = .{ .p_w, .insr }, |
| 11910 | .extract = .{ .p_w, .extr }, |
| 11911 | } }, |
| 11912 | 3...4 => return .{ .move = if (self.hasFeature(.avx)) |
| 11913 | .{ .v_d, .mov } |
| 11914 | else |
| 11915 | .{ ._d, .mov } }, |
| 11916 | 5...8 => return .{ .move = if (self.hasFeature(.avx)) |
| 11917 | .{ .v_q, .mov } |
| 11918 | else |
| 11919 | .{ ._q, .mov } }, |
| 11920 | 9...16 => return .{ .move = if (self.hasFeature(.avx)) |
| 11921 | if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu } |
| 11922 | else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } }, |
| 11923 | 17...32 => if (self.hasFeature(.avx)) |
| 11924 | return .{ .move = if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu } }, |
| 11925 | else => {}, |
| 11926 | } else switch (abi_size) { |
| 11927 | 4 => return .{ .move = if (self.hasFeature(.avx)) |
| 11928 | .{ .v_ss, .mov } |
| 11929 | else |
| 11930 | .{ ._ss, .mov } }, |
| 11931 | 5...8 => return .{ .move = if (self.hasFeature(.avx)) |
| 11932 | .{ .v_sd, .mov } |
| 11933 | else |
| 11934 | .{ ._sd, .mov } }, |
| 11935 | 9...16 => return .{ .move = if (self.hasFeature(.avx)) |
| 11936 | if (aligned) .{ .v_pd, .mova } else .{ .v_pd, .movu } |
| 11937 | else if (aligned) .{ ._pd, .mova } else .{ ._pd, .movu } }, |
| 11938 | 17...32 => if (self.hasFeature(.avx)) return .{ .move = if (aligned) |
| 11939 | .{ .v_pd, .mova } |
| 11940 | else |
| 11941 | .{ .v_pd, .movu } }, |
| 11942 | else => {}, |
| 11943 | } |
| 11950 | 11944 | }, |
| 11951 | | else => {}, |
| 11952 | | }, |
| 11953 | | .Float => switch (ty.childType(mod).floatBits(self.target.*)) { |
| 11954 | | 16 => switch (ty.vectorLen(mod)) { |
| 11955 | | 1 => return if (self.hasFeature(.avx)) .{ .vex_insert_extract = .{ |
| 11945 | .Float => switch (ty.floatBits(self.target.*)) { |
| 11946 | 16 => return if (self.hasFeature(.avx)) .{ .vex_insert_extract = .{ |
| 11956 | 11947 | .insert = .{ .vp_w, .insr }, |
| 11957 | 11948 | .extract = .{ .vp_w, .extr }, |
| 11958 | 11949 | } } else .{ .insert_extract = .{ |
| 11959 | 11950 | .insert = .{ .p_w, .insr }, |
| 11960 | 11951 | .extract = .{ .p_w, .extr }, |
| 11961 | 11952 | } }, |
| 11962 | | 2 => return .{ .move = if (self.hasFeature(.avx)) |
| 11963 | | .{ .v_d, .mov } |
| 11964 | | else |
| 11965 | | .{ ._d, .mov } }, |
| 11966 | | 3...4 => return .{ .move = if (self.hasFeature(.avx)) |
| 11967 | | .{ .v_q, .mov } |
| 11968 | | else |
| 11969 | | .{ ._q, .mov } }, |
| 11970 | | 5...8 => return .{ .move = if (self.hasFeature(.avx)) |
| 11971 | | if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu } |
| 11972 | | else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } }, |
| 11973 | | 9...16 => if (self.hasFeature(.avx)) |
| 11974 | | return .{ .move = if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu } }, |
| 11975 | | else => {}, |
| 11976 | | }, |
| 11977 | | 32 => switch (ty.vectorLen(mod)) { |
| 11978 | | 1 => return .{ .move = if (self.hasFeature(.avx)) |
| 11953 | 32 => return .{ .move = if (self.hasFeature(.avx)) |
| 11979 | 11954 | .{ .v_ss, .mov } |
| 11980 | 11955 | else |
| 11981 | 11956 | .{ ._ss, .mov } }, |
| 11982 | | 2 => return .{ .move = if (self.hasFeature(.avx)) |
| 11957 | 64 => return .{ .move = if (self.hasFeature(.avx)) |
| 11983 | 11958 | .{ .v_sd, .mov } |
| 11984 | 11959 | else |
| 11985 | 11960 | .{ ._sd, .mov } }, |
| 11986 | | 3...4 => return .{ .move = if (self.hasFeature(.avx)) |
| 11987 | | if (aligned) .{ .v_ps, .mova } else .{ .v_ps, .movu } |
| 11988 | | else if (aligned) .{ ._ps, .mova } else .{ ._ps, .movu } }, |
| 11989 | | 5...8 => if (self.hasFeature(.avx)) |
| 11990 | | return .{ .move = if (aligned) .{ .v_ps, .mova } else .{ .v_ps, .movu } }, |
| 11991 | | else => {}, |
| 11992 | | }, |
| 11993 | | 64 => switch (ty.vectorLen(mod)) { |
| 11994 | | 1 => return .{ .move = if (self.hasFeature(.avx)) |
| 11995 | | .{ .v_sd, .mov } |
| 11996 | | else |
| 11997 | | .{ ._sd, .mov } }, |
| 11998 | | 2 => return .{ .move = if (self.hasFeature(.avx)) |
| 11999 | | if (aligned) .{ .v_pd, .mova } else .{ .v_pd, .movu } |
| 12000 | | else if (aligned) .{ ._pd, .mova } else .{ ._pd, .movu } }, |
| 12001 | | 3...4 => if (self.hasFeature(.avx)) |
| 12002 | | return .{ .move = if (aligned) .{ .v_pd, .mova } else .{ .v_pd, .movu } }, |
| 12003 | | else => {}, |
| 12004 | | }, |
| 12005 | | 128 => switch (ty.vectorLen(mod)) { |
| 12006 | | 1 => return .{ .move = if (self.hasFeature(.avx)) |
| 11961 | 128 => return .{ .move = if (self.hasFeature(.avx)) |
| 12007 | 11962 | if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu } |
| 12008 | 11963 | else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } }, |
| 12009 | | 2 => if (self.hasFeature(.avx)) |
| 12010 | | return .{ .move = if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu } }, |
| 12011 | 11964 | else => {}, |
| 12012 | 11965 | }, |
| 12013 | | else => {}, |
| 12014 | | }, |
| 12015 | | else => {}, |
| 11966 | .Vector => switch (ty.childType(mod).zigTypeTag(mod)) { |
| 11967 | .Bool => return .{ .move = .{ ._, .mov } }, |
| 11968 | .Int => switch (ty.childType(mod).intInfo(mod).bits) { |
| 11969 | 8 => switch (ty.vectorLen(mod)) { |
| 11970 | 1 => if (self.hasFeature(.avx)) return .{ .vex_insert_extract = .{ |
| 11971 | .insert = .{ .vp_b, .insr }, |
| 11972 | .extract = .{ .vp_b, .extr }, |
| 11973 | } } else if (self.hasFeature(.sse4_2)) return .{ .insert_extract = .{ |
| 11974 | .insert = .{ .p_b, .insr }, |
| 11975 | .extract = .{ .p_b, .extr }, |
| 11976 | } }, |
| 11977 | 2 => return if (self.hasFeature(.avx)) .{ .vex_insert_extract = .{ |
| 11978 | .insert = .{ .vp_w, .insr }, |
| 11979 | .extract = .{ .vp_w, .extr }, |
| 11980 | } } else .{ .insert_extract = .{ |
| 11981 | .insert = .{ .p_w, .insr }, |
| 11982 | .extract = .{ .p_w, .extr }, |
| 11983 | } }, |
| 11984 | 3...4 => return .{ .move = if (self.hasFeature(.avx)) |
| 11985 | .{ .v_d, .mov } |
| 11986 | else |
| 11987 | .{ ._d, .mov } }, |
| 11988 | 5...8 => return .{ .move = if (self.hasFeature(.avx)) |
| 11989 | .{ .v_q, .mov } |
| 11990 | else |
| 11991 | .{ ._q, .mov } }, |
| 11992 | 9...16 => return .{ .move = if (self.hasFeature(.avx)) |
| 11993 | if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu } |
| 11994 | else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } }, |
| 11995 | 17...32 => if (self.hasFeature(.avx)) |
| 11996 | return .{ .move = if (aligned) |
| 11997 | .{ .v_, .movdqa } |
| 11998 | else |
| 11999 | .{ .v_, .movdqu } }, |
| 12000 | else => {}, |
| 12001 | }, |
| 12002 | 16 => switch (ty.vectorLen(mod)) { |
| 12003 | 1 => return if (self.hasFeature(.avx)) .{ .vex_insert_extract = .{ |
| 12004 | .insert = .{ .vp_w, .insr }, |
| 12005 | .extract = .{ .vp_w, .extr }, |
| 12006 | } } else .{ .insert_extract = .{ |
| 12007 | .insert = .{ .p_w, .insr }, |
| 12008 | .extract = .{ .p_w, .extr }, |
| 12009 | } }, |
| 12010 | 2 => return .{ .move = if (self.hasFeature(.avx)) |
| 12011 | .{ .v_d, .mov } |
| 12012 | else |
| 12013 | .{ ._d, .mov } }, |
| 12014 | 3...4 => return .{ .move = if (self.hasFeature(.avx)) |
| 12015 | .{ .v_q, .mov } |
| 12016 | else |
| 12017 | .{ ._q, .mov } }, |
| 12018 | 5...8 => return .{ .move = if (self.hasFeature(.avx)) |
| 12019 | if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu } |
| 12020 | else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } }, |
| 12021 | 9...16 => if (self.hasFeature(.avx)) |
| 12022 | return .{ .move = if (aligned) |
| 12023 | .{ .v_, .movdqa } |
| 12024 | else |
| 12025 | .{ .v_, .movdqu } }, |
| 12026 | else => {}, |
| 12027 | }, |
| 12028 | 32 => switch (ty.vectorLen(mod)) { |
| 12029 | 1 => return .{ .move = if (self.hasFeature(.avx)) |
| 12030 | .{ .v_d, .mov } |
| 12031 | else |
| 12032 | .{ ._d, .mov } }, |
| 12033 | 2 => return .{ .move = if (self.hasFeature(.avx)) |
| 12034 | .{ .v_q, .mov } |
| 12035 | else |
| 12036 | .{ ._q, .mov } }, |
| 12037 | 3...4 => return .{ .move = if (self.hasFeature(.avx)) |
| 12038 | if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu } |
| 12039 | else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } }, |
| 12040 | 5...8 => if (self.hasFeature(.avx)) |
| 12041 | return .{ .move = if (aligned) |
| 12042 | .{ .v_, .movdqa } |
| 12043 | else |
| 12044 | .{ .v_, .movdqu } }, |
| 12045 | else => {}, |
| 12046 | }, |
| 12047 | 64 => switch (ty.vectorLen(mod)) { |
| 12048 | 1 => return .{ .move = if (self.hasFeature(.avx)) |
| 12049 | .{ .v_q, .mov } |
| 12050 | else |
| 12051 | .{ ._q, .mov } }, |
| 12052 | 2 => return .{ .move = if (self.hasFeature(.avx)) |
| 12053 | if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu } |
| 12054 | else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } }, |
| 12055 | 3...4 => if (self.hasFeature(.avx)) |
| 12056 | return .{ .move = if (aligned) |
| 12057 | .{ .v_, .movdqa } |
| 12058 | else |
| 12059 | .{ .v_, .movdqu } }, |
| 12060 | else => {}, |
| 12061 | }, |
| 12062 | 128 => switch (ty.vectorLen(mod)) { |
| 12063 | 1 => return .{ .move = if (self.hasFeature(.avx)) |
| 12064 | if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu } |
| 12065 | else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } }, |
| 12066 | 2 => if (self.hasFeature(.avx)) |
| 12067 | return .{ .move = if (aligned) |
| 12068 | .{ .v_, .movdqa } |
| 12069 | else |
| 12070 | .{ .v_, .movdqu } }, |
| 12071 | else => {}, |
| 12072 | }, |
| 12073 | 256 => switch (ty.vectorLen(mod)) { |
| 12074 | 1 => if (self.hasFeature(.avx)) |
| 12075 | return .{ .move = if (aligned) |
| 12076 | .{ .v_, .movdqa } |
| 12077 | else |
| 12078 | .{ .v_, .movdqu } }, |
| 12079 | else => {}, |
| 12080 | }, |
| 12081 | else => {}, |
| 12082 | }, |
| 12083 | .Float => switch (ty.childType(mod).floatBits(self.target.*)) { |
| 12084 | 16 => switch (ty.vectorLen(mod)) { |
| 12085 | 1 => return if (self.hasFeature(.avx)) .{ .vex_insert_extract = .{ |
| 12086 | .insert = .{ .vp_w, .insr }, |
| 12087 | .extract = .{ .vp_w, .extr }, |
| 12088 | } } else .{ .insert_extract = .{ |
| 12089 | .insert = .{ .p_w, .insr }, |
| 12090 | .extract = .{ .p_w, .extr }, |
| 12091 | } }, |
| 12092 | 2 => return .{ .move = if (self.hasFeature(.avx)) |
| 12093 | .{ .v_d, .mov } |
| 12094 | else |
| 12095 | .{ ._d, .mov } }, |
| 12096 | 3...4 => return .{ .move = if (self.hasFeature(.avx)) |
| 12097 | .{ .v_q, .mov } |
| 12098 | else |
| 12099 | .{ ._q, .mov } }, |
| 12100 | 5...8 => return .{ .move = if (self.hasFeature(.avx)) |
| 12101 | if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu } |
| 12102 | else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } }, |
| 12103 | 9...16 => if (self.hasFeature(.avx)) |
| 12104 | return .{ .move = if (aligned) |
| 12105 | .{ .v_, .movdqa } |
| 12106 | else |
| 12107 | .{ .v_, .movdqu } }, |
| 12108 | else => {}, |
| 12109 | }, |
| 12110 | 32 => switch (ty.vectorLen(mod)) { |
| 12111 | 1 => return .{ .move = if (self.hasFeature(.avx)) |
| 12112 | .{ .v_ss, .mov } |
| 12113 | else |
| 12114 | .{ ._ss, .mov } }, |
| 12115 | 2 => return .{ .move = if (self.hasFeature(.avx)) |
| 12116 | .{ .v_sd, .mov } |
| 12117 | else |
| 12118 | .{ ._sd, .mov } }, |
| 12119 | 3...4 => return .{ .move = if (self.hasFeature(.avx)) |
| 12120 | if (aligned) .{ .v_ps, .mova } else .{ .v_ps, .movu } |
| 12121 | else if (aligned) .{ ._ps, .mova } else .{ ._ps, .movu } }, |
| 12122 | 5...8 => if (self.hasFeature(.avx)) |
| 12123 | return .{ .move = if (aligned) |
| 12124 | .{ .v_ps, .mova } |
| 12125 | else |
| 12126 | .{ .v_ps, .movu } }, |
| 12127 | else => {}, |
| 12128 | }, |
| 12129 | 64 => switch (ty.vectorLen(mod)) { |
| 12130 | 1 => return .{ .move = if (self.hasFeature(.avx)) |
| 12131 | .{ .v_sd, .mov } |
| 12132 | else |
| 12133 | .{ ._sd, .mov } }, |
| 12134 | 2 => return .{ .move = if (self.hasFeature(.avx)) |
| 12135 | if (aligned) .{ .v_pd, .mova } else .{ .v_pd, .movu } |
| 12136 | else if (aligned) .{ ._pd, .mova } else .{ ._pd, .movu } }, |
| 12137 | 3...4 => if (self.hasFeature(.avx)) |
| 12138 | return .{ .move = if (aligned) |
| 12139 | .{ .v_pd, .mova } |
| 12140 | else |
| 12141 | .{ .v_pd, .movu } }, |
| 12142 | else => {}, |
| 12143 | }, |
| 12144 | 128 => switch (ty.vectorLen(mod)) { |
| 12145 | 1 => return .{ .move = if (self.hasFeature(.avx)) |
| 12146 | if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu } |
| 12147 | else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } }, |
| 12148 | 2 => if (self.hasFeature(.avx)) |
| 12149 | return .{ .move = if (aligned) |
| 12150 | .{ .v_, .movdqa } |
| 12151 | else |
| 12152 | .{ .v_, .movdqu } }, |
| 12153 | else => {}, |
| 12154 | }, |
| 12155 | else => {}, |
| 12156 | }, |
| 12157 | else => {}, |
| 12158 | }, |
| 12159 | } |
| 12016 | 12160 | }, |
| 12017 | 12161 | } |
| 12018 | 12162 | return self.fail("TODO moveStrategy for {}", .{ty.fmt(mod)}); |
| ... | ... | @@ -12060,28 +12204,49 @@ fn genCopy(self: *Self, ty: Type, dst_mcv: MCValue, src_mcv: MCValue) InnerError |
| 12060 | 12204 | } }, |
| 12061 | 12205 | }), |
| 12062 | 12206 | .register_pair => |dst_regs| { |
| 12063 | | switch (src_mcv) { |
| 12207 | const src_info: ?struct { addr_reg: Register, addr_lock: RegisterLock } = switch (src_mcv) { |
| 12208 | .register_pair, .memory, .indirect, .load_frame => null, |
| 12209 | .load_direct, .load_got, .load_extern_got, .load_tlv => src: { |
| 12210 | const src_addr_reg = |
| 12211 | (try self.register_manager.allocReg(null, abi.RegisterClass.gp)).to64(); |
| 12212 | const src_addr_lock = self.register_manager.lockRegAssumeUnused(src_addr_reg); |
| 12213 | errdefer self.register_manager.unlockReg(src_addr_lock); |
| 12214 | |
| 12215 | try self.genSetReg(src_addr_reg, Type.usize, src_mcv.address()); |
| 12216 | break :src .{ .addr_reg = src_addr_reg, .addr_lock = src_addr_lock }; |
| 12217 | }, |
| 12064 | 12218 | .air_ref => |src_ref| return self.genCopy(ty, dst_mcv, try self.resolveInst(src_ref)), |
| 12065 | | else => {}, |
| 12066 | | } |
| 12219 | else => return self.fail("TODO implement genCopy for {s} of {}", .{ |
| 12220 | @tagName(src_mcv), ty.fmt(mod), |
| 12221 | }), |
| 12222 | }; |
| 12223 | defer if (src_info) |info| self.register_manager.unlockReg(info.addr_lock); |
| 12224 | |
| 12067 | 12225 | const classes = mem.sliceTo(&abi.classifySystemV(ty, mod, .other), .none); |
| 12068 | 12226 | for (dst_regs, classes, 0..) |dst_reg, class, dst_reg_i| { |
| 12069 | 12227 | const class_ty = switch (class) { |
| 12070 | 12228 | .integer => Type.usize, |
| 12071 | | .sse => Type.f64, |
| 12229 | .sse, .float, .float_combine => Type.f64, |
| 12072 | 12230 | else => unreachable, |
| 12073 | 12231 | }; |
| 12232 | const off: i32 = @intCast(dst_reg_i * 8); |
| 12074 | 12233 | switch (src_mcv) { |
| 12075 | 12234 | .register_pair => |src_regs| try self.genSetReg( |
| 12076 | 12235 | dst_reg, |
| 12077 | 12236 | class_ty, |
| 12078 | 12237 | .{ .register = src_regs[dst_reg_i] }, |
| 12079 | 12238 | ), |
| 12080 | | else => try self.genSetReg( |
| 12239 | .memory, .indirect, .load_frame => try self.genSetReg( |
| 12240 | dst_reg, |
| 12241 | class_ty, |
| 12242 | src_mcv.address().offset(off).deref(), |
| 12243 | ), |
| 12244 | .load_direct, .load_got, .load_extern_got, .load_tlv => try self.genSetReg( |
| 12081 | 12245 | dst_reg, |
| 12082 | 12246 | class_ty, |
| 12083 | | src_mcv.address().offset(@intCast(dst_reg_i * 8)).deref(), |
| 12247 | .{ .indirect = .{ .reg = src_info.?.addr_reg, .off = off } }, |
| 12084 | 12248 | ), |
| 12249 | else => unreachable, |
| 12085 | 12250 | } |
| 12086 | 12251 | } |
| 12087 | 12252 | }, |
| ... | ... | @@ -12157,6 +12322,7 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr |
| 12157 | 12322 | registerAlias(dst_reg, abi_size), |
| 12158 | 12323 | src_reg, |
| 12159 | 12324 | ), |
| 12325 | .x87, .mmx => unreachable, |
| 12160 | 12326 | .sse => try self.asmRegisterRegister( |
| 12161 | 12327 | switch (abi_size) { |
| 12162 | 12328 | 1...4 => if (self.hasFeature(.avx)) .{ .v_d, .mov } else .{ ._d, .mov }, |
| ... | ... | @@ -12166,17 +12332,30 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr |
| 12166 | 12332 | registerAlias(dst_reg, @max(abi_size, 4)), |
| 12167 | 12333 | src_reg.to128(), |
| 12168 | 12334 | ), |
| 12169 | | .x87, .mmx => unreachable, |
| 12170 | 12335 | }, |
| 12171 | 12336 | .segment => try self.asmRegisterRegister( |
| 12172 | 12337 | .{ ._, .mov }, |
| 12173 | 12338 | dst_reg, |
| 12174 | 12339 | switch (src_reg.class()) { |
| 12175 | 12340 | .general_purpose, .segment => registerAlias(src_reg, abi_size), |
| 12176 | | .sse => try self.copyToTmpRegister(ty, src_mcv), |
| 12177 | 12341 | .x87, .mmx => unreachable, |
| 12342 | .sse => try self.copyToTmpRegister(ty, src_mcv), |
| 12178 | 12343 | }, |
| 12179 | 12344 | ), |
| 12345 | .x87 => switch (src_reg.class()) { |
| 12346 | .general_purpose, .segment => unreachable, |
| 12347 | .x87 => switch (src_reg) { |
| 12348 | .st0 => try self.asmRegister(.{ .f_, .st }, dst_reg), |
| 12349 | .st1, .st2, .st3, .st4, .st5, .st6 => { |
| 12350 | try self.asmRegister(.{ .f_, .ld }, src_reg); |
| 12351 | assert(dst_reg != .st7); |
| 12352 | try self.asmRegister(.{ .f_p, .st }, @enumFromInt(@intFromEnum(dst_reg) + 1)); |
| 12353 | }, |
| 12354 | else => unreachable, |
| 12355 | }, |
| 12356 | .mmx, .sse => unreachable, |
| 12357 | }, |
| 12358 | .mmx => unreachable, |
| 12180 | 12359 | .sse => switch (src_reg.class()) { |
| 12181 | 12360 | .general_purpose => try self.asmRegisterRegister( |
| 12182 | 12361 | switch (abi_size) { |
| ... | ... | @@ -12192,6 +12371,7 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr |
| 12192 | 12371 | ty, |
| 12193 | 12372 | .{ .register = try self.copyToTmpRegister(ty, src_mcv) }, |
| 12194 | 12373 | ), |
| 12374 | .x87, .mmx => unreachable, |
| 12195 | 12375 | .sse => try self.asmRegisterRegister( |
| 12196 | 12376 | @as(?Mir.Inst.FixedTag, switch (ty.scalarType(mod).zigTypeTag(mod)) { |
| 12197 | 12377 | else => switch (abi_size) { |
| ... | ... | @@ -12217,9 +12397,7 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr |
| 12217 | 12397 | registerAlias(dst_reg, abi_size), |
| 12218 | 12398 | registerAlias(src_reg, abi_size), |
| 12219 | 12399 | ), |
| 12220 | | .x87, .mmx => unreachable, |
| 12221 | 12400 | }, |
| 12222 | | .x87, .mmx => unreachable, |
| 12223 | 12401 | }, |
| 12224 | 12402 | .register_pair => |src_regs| try self.genSetReg(dst_reg, ty, .{ .register = src_regs[0] }), |
| 12225 | 12403 | .register_offset, |
| ... | ... | @@ -12231,9 +12409,10 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr |
| 12231 | 12409 | 0 => return self.genSetReg(dst_reg, ty, .{ .register = reg_off.reg }), |
| 12232 | 12410 | else => .{ .move = .{ ._, .lea } }, |
| 12233 | 12411 | }, |
| 12234 | | .indirect => try self.moveStrategy(ty, false), |
| 12412 | .indirect => try self.moveStrategy(ty, dst_reg.class(), false), |
| 12235 | 12413 | .load_frame => |frame_addr| try self.moveStrategy( |
| 12236 | 12414 | ty, |
| 12415 | dst_reg.class(), |
| 12237 | 12416 | self.getFrameAddrAlignment(frame_addr).compare(.gte, ty.abiAlignment(mod)), |
| 12238 | 12417 | ), |
| 12239 | 12418 | .lea_frame => .{ .move = .{ ._, .lea } }, |
| ... | ... | @@ -12257,13 +12436,14 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr |
| 12257 | 12436 | .memory => |addr| if (math.cast(i32, @as(i64, @bitCast(addr)))) |small_addr| |
| 12258 | 12437 | return (try self.moveStrategy( |
| 12259 | 12438 | ty, |
| 12439 | dst_reg.class(), |
| 12260 | 12440 | ty.abiAlignment(mod).check(@as(u32, @bitCast(small_addr))), |
| 12261 | 12441 | )).read(self, registerAlias(dst_reg, abi_size), Memory.sib( |
| 12262 | 12442 | self.memPtrSize(ty), |
| 12263 | 12443 | .{ .base = .{ .reg = .ds }, .disp = small_addr }, |
| 12264 | 12444 | )), |
| 12265 | | .load_direct => |sym_index| switch (ty.zigTypeTag(mod)) { |
| 12266 | | else => { |
| 12445 | .load_direct => |sym_index| switch (dst_reg.class()) { |
| 12446 | .general_purpose => { |
| 12267 | 12447 | const atom_index = try self.owner.getSymbolIndex(self); |
| 12268 | 12448 | _ = try self.addInst(.{ |
| 12269 | 12449 | .tag = .mov, |
| ... | ... | @@ -12278,7 +12458,8 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr |
| 12278 | 12458 | }); |
| 12279 | 12459 | return; |
| 12280 | 12460 | }, |
| 12281 | | .Float, .Vector => {}, |
| 12461 | .segment, .mmx => unreachable, |
| 12462 | .x87, .sse => {}, |
| 12282 | 12463 | }, |
| 12283 | 12464 | .load_got, .load_extern_got, .load_tlv => {}, |
| 12284 | 12465 | else => unreachable, |
| ... | ... | @@ -12288,7 +12469,7 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr |
| 12288 | 12469 | const addr_lock = self.register_manager.lockRegAssumeUnused(addr_reg); |
| 12289 | 12470 | defer self.register_manager.unlockReg(addr_lock); |
| 12290 | 12471 | |
| 12291 | | try (try self.moveStrategy(ty, false)).read( |
| 12472 | try (try self.moveStrategy(ty, dst_reg.class(), false)).read( |
| 12292 | 12473 | self, |
| 12293 | 12474 | registerAlias(dst_reg, abi_size), |
| 12294 | 12475 | Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = .{ .reg = addr_reg } }), |
| ... | ... | @@ -12393,7 +12574,7 @@ fn genSetMem(self: *Self, base: Memory.Base, disp: i32, ty: Type, src_mcv: MCVal |
| 12393 | 12574 | }, |
| 12394 | 12575 | }, |
| 12395 | 12576 | .eflags => |cc| try self.asmSetccMemory(cc, Memory.sib(.byte, .{ .base = base, .disp = disp })), |
| 12396 | | .register => |src_reg| try (try self.moveStrategy(ty, switch (base) { |
| 12577 | .register => |src_reg| try (try self.moveStrategy(ty, src_reg.class(), switch (base) { |
| 12397 | 12578 | .none => ty.abiAlignment(mod).check(@as(u32, @bitCast(disp))), |
| 12398 | 12579 | .reg => |reg| switch (reg) { |
| 12399 | 12580 | .es, .cs, .ss, .ds => ty.abiAlignment(mod).check(@as(u32, @bitCast(disp))), |
| ... | ... | @@ -12408,17 +12589,21 @@ fn genSetMem(self: *Self, base: Memory.Base, disp: i32, ty: Type, src_mcv: MCVal |
| 12408 | 12589 | registerAlias(src_reg, abi_size), |
| 12409 | 12590 | ), |
| 12410 | 12591 | .register_pair => |src_regs| for (src_regs, 0..) |src_reg, src_reg_i| { |
| 12411 | | const part_size = @min(abi_size - src_reg_i * 8, 8); |
| 12412 | | try (try self.moveStrategy(ty, switch (base) { |
| 12413 | | .none => ty.abiAlignment(mod).check(@as(u32, @bitCast(disp))), |
| 12414 | | .reg => |reg| switch (reg) { |
| 12415 | | .es, .cs, .ss, .ds => ty.abiAlignment(mod).check(@as(u32, @bitCast(disp))), |
| 12416 | | else => false, |
| 12592 | const part_size: u16 = @min(abi_size - src_reg_i * 8, 8); |
| 12593 | try (try self.moveStrategy( |
| 12594 | try mod.intType(.unsigned, part_size * 8), |
| 12595 | src_reg.class(), |
| 12596 | switch (base) { |
| 12597 | .none => ty.abiAlignment(mod).check(@as(u32, @bitCast(disp))), |
| 12598 | .reg => |reg| switch (reg) { |
| 12599 | .es, .cs, .ss, .ds => ty.abiAlignment(mod).check(@as(u32, @bitCast(disp))), |
| 12600 | else => false, |
| 12601 | }, |
| 12602 | .frame => |frame_index| self.getFrameAddrAlignment( |
| 12603 | .{ .index = frame_index, .off = disp }, |
| 12604 | ).compare(.gte, ty.abiAlignment(mod)), |
| 12417 | 12605 | }, |
| 12418 | | .frame => |frame_index| self.getFrameAddrAlignment( |
| 12419 | | .{ .index = frame_index, .off = disp }, |
| 12420 | | ).compare(.gte, ty.abiAlignment(mod)), |
| 12421 | | })).write(self, Memory.sib( |
| 12606 | )).write(self, Memory.sib( |
| 12422 | 12607 | Memory.PtrSize.fromSize(part_size), |
| 12423 | 12608 | .{ .base = base, .disp = disp + @as(i32, @intCast(src_reg_i * 8)) }, |
| 12424 | 12609 | ), registerAlias(src_reg, part_size)); |
| ... | ... | @@ -12482,9 +12667,9 @@ fn genSetMem(self: *Self, base: Memory.Base, disp: i32, ty: Type, src_mcv: MCVal |
| 12482 | 12667 | } |
| 12483 | 12668 | |
| 12484 | 12669 | fn genInlineMemcpy(self: *Self, dst_ptr: MCValue, src_ptr: MCValue, len: MCValue) InnerError!void { |
| 12485 | | try self.spillRegisters(&.{ .rdi, .rsi, .rcx }); |
| 12486 | | try self.genSetReg(.rdi, Type.usize, dst_ptr); |
| 12670 | try self.spillRegisters(&.{ .rsi, .rdi, .rcx }); |
| 12487 | 12671 | try self.genSetReg(.rsi, Type.usize, src_ptr); |
| 12672 | try self.genSetReg(.rdi, Type.usize, dst_ptr); |
| 12488 | 12673 | try self.genSetReg(.rcx, Type.usize, len); |
| 12489 | 12674 | try self.asmOpOnly(.{ .@"rep _sb", .mov }); |
| 12490 | 12675 | } |
| ... | ... | @@ -14730,16 +14915,22 @@ fn resolveCallingConventionValues( |
| 14730 | 14915 | arg_mcv_i += 1; |
| 14731 | 14916 | }, |
| 14732 | 14917 | .sseup => assert(arg_mcv[arg_mcv_i - 1].register.class() == .sse), |
| 14733 | | .x87, .x87up, .complex_x87, .memory => break, |
| 14734 | | .none => unreachable, |
| 14735 | | .win_i128 => { |
| 14736 | | const param_int_reg = |
| 14737 | | abi.getCAbiIntParamRegs(resolved_cc)[param_int_reg_i].to64(); |
| 14738 | | param_int_reg_i += 1; |
| 14739 | | |
| 14740 | | arg_mcv[arg_mcv_i] = .{ .indirect = .{ .reg = param_int_reg } }; |
| 14741 | | arg_mcv_i += 1; |
| 14918 | .x87, .x87up, .complex_x87, .memory, .win_i128 => switch (resolved_cc) { |
| 14919 | .SysV => switch (class) { |
| 14920 | .x87, .x87up, .complex_x87, .memory => break, |
| 14921 | else => unreachable, |
| 14922 | }, |
| 14923 | .Win64 => if (ty.abiSize(mod) > 8) { |
| 14924 | const param_int_reg = |
| 14925 | abi.getCAbiIntParamRegs(resolved_cc)[param_int_reg_i].to64(); |
| 14926 | param_int_reg_i += 1; |
| 14927 | |
| 14928 | arg_mcv[arg_mcv_i] = .{ .indirect = .{ .reg = param_int_reg } }; |
| 14929 | arg_mcv_i += 1; |
| 14930 | } else break, |
| 14931 | else => unreachable, |
| 14742 | 14932 | }, |
| 14933 | .none => unreachable, |
| 14743 | 14934 | } else { |
| 14744 | 14935 | arg.* = switch (arg_mcv_i) { |
| 14745 | 14936 | else => unreachable, |
| ... | ... | @@ -15017,34 +15208,32 @@ fn floatLibcAbiSuffix(ty: Type) []const u8 { |
| 15017 | 15208 | }; |
| 15018 | 15209 | } |
| 15019 | 15210 | |
| 15020 | | fn promoteVarArg(self: *Self, ty: Type) Type { |
| 15211 | fn promoteInt(self: *Self, ty: Type) Type { |
| 15021 | 15212 | const mod = self.bin_file.options.module.?; |
| 15022 | | switch (ty.zigTypeTag(mod)) { |
| 15023 | | .Bool => return Type.c_int, |
| 15024 | | else => { |
| 15025 | | const int_info = ty.intInfo(mod); |
| 15026 | | for ([_]Type{ |
| 15027 | | Type.c_int, Type.c_uint, |
| 15028 | | Type.c_long, Type.c_ulong, |
| 15029 | | Type.c_longlong, Type.c_ulonglong, |
| 15030 | | }) |promote_ty| { |
| 15031 | | const promote_info = promote_ty.intInfo(mod); |
| 15032 | | if (int_info.signedness == .signed and promote_info.signedness == .unsigned) continue; |
| 15033 | | if (int_info.bits + @intFromBool(int_info.signedness == .unsigned and |
| 15034 | | promote_info.signedness == .signed) <= promote_info.bits) return promote_ty; |
| 15035 | | } |
| 15036 | | unreachable; |
| 15037 | | }, |
| 15038 | | .Float => switch (ty.floatBits(self.target.*)) { |
| 15039 | | 32, 64 => return Type.f64, |
| 15040 | | else => |float_bits| { |
| 15041 | | assert(float_bits == self.target.c_type_bit_size(.longdouble)); |
| 15042 | | return Type.c_longdouble; |
| 15043 | | }, |
| 15044 | | }, |
| 15045 | | .Pointer => { |
| 15046 | | assert(!ty.isSlice(mod)); |
| 15047 | | return ty; |
| 15213 | const int_info: InternPool.Key.IntType = switch (ty.toIntern()) { |
| 15214 | .bool_type => .{ .signedness = .unsigned, .bits = 1 }, |
| 15215 | else => if (ty.isAbiInt(mod)) ty.intInfo(mod) else return ty, |
| 15216 | }; |
| 15217 | for ([_]Type{ |
| 15218 | Type.c_int, Type.c_uint, |
| 15219 | Type.c_long, Type.c_ulong, |
| 15220 | Type.c_longlong, Type.c_ulonglong, |
| 15221 | }) |promote_ty| { |
| 15222 | const promote_info = promote_ty.intInfo(mod); |
| 15223 | if (int_info.signedness == .signed and promote_info.signedness == .unsigned) continue; |
| 15224 | if (int_info.bits + @intFromBool(int_info.signedness == .unsigned and |
| 15225 | promote_info.signedness == .signed) <= promote_info.bits) return promote_ty; |
| 15226 | } |
| 15227 | return ty; |
| 15228 | } |
| 15229 | |
| 15230 | fn promoteVarArg(self: *Self, ty: Type) Type { |
| 15231 | if (!ty.isRuntimeFloat()) return self.promoteInt(ty); |
| 15232 | switch (ty.floatBits(self.target.*)) { |
| 15233 | 32, 64 => return Type.f64, |
| 15234 | else => |float_bits| { |
| 15235 | assert(float_bits == self.target.c_type_bit_size(.longdouble)); |
| 15236 | return Type.c_longdouble; |
| 15048 | 15237 | }, |
| 15049 | 15238 | } |
| 15050 | 15239 | } |