| ... | ... | @@ -2224,6 +2224,10 @@ fn getFrameAddrAlignment(self: *Self, frame_addr: FrameAddr) u32 { |
| 2224 | 2224 | return @min(alloc_align, @bitCast(u32, frame_addr.off) & (alloc_align - 1)); |
| 2225 | 2225 | } |
| 2226 | 2226 | |
| 2227 | fn getFrameAddrSize(self: *Self, frame_addr: FrameAddr) u32 { |
| 2228 | return self.frame_allocs.get(@enumToInt(frame_addr.index)).abi_size - @intCast(u31, frame_addr.off); |
| 2229 | } |
| 2230 | |
| 2227 | 2231 | fn allocFrameIndex(self: *Self, alloc: FrameAlloc) !FrameIndex { |
| 2228 | 2232 | const frame_allocs_slice = self.frame_allocs.slice(); |
| 2229 | 2233 | const frame_size = frame_allocs_slice.items(.abi_size); |
| ... | ... | @@ -2615,87 +2619,90 @@ fn airFpext(self: *Self, inst: Air.Inst.Index) !void { |
| 2615 | 2619 | |
| 2616 | 2620 | fn airIntCast(self: *Self, inst: Air.Inst.Index) !void { |
| 2617 | 2621 | const ty_op = self.air.instructions.items(.data)[inst].ty_op; |
| 2622 | const result: MCValue = result: { |
| 2623 | const src_ty = self.air.typeOf(ty_op.operand); |
| 2624 | const src_int_info = src_ty.intInfo(self.target.*); |
| 2618 | 2625 | |
| 2619 | | const src_ty = self.air.typeOf(ty_op.operand); |
| 2620 | | const src_int_info = src_ty.intInfo(self.target.*); |
| 2621 | | const src_abi_size = @intCast(u32, src_ty.abiSize(self.target.*)); |
| 2622 | | const src_mcv = try self.resolveInst(ty_op.operand); |
| 2623 | | const src_lock = switch (src_mcv) { |
| 2624 | | .register => |reg| self.register_manager.lockRegAssumeUnused(reg), |
| 2625 | | else => null, |
| 2626 | | }; |
| 2627 | | defer if (src_lock) |lock| self.register_manager.unlockReg(lock); |
| 2626 | const dst_ty = self.air.typeOfIndex(inst); |
| 2627 | const dst_int_info = dst_ty.intInfo(self.target.*); |
| 2628 | const abi_size = @intCast(u32, dst_ty.abiSize(self.target.*)); |
| 2628 | 2629 | |
| 2629 | | const dst_ty = self.air.typeOfIndex(inst); |
| 2630 | | const dst_int_info = dst_ty.intInfo(self.target.*); |
| 2631 | | const dst_abi_size = @intCast(u32, dst_ty.abiSize(self.target.*)); |
| 2632 | | const dst_mcv = if (dst_abi_size <= src_abi_size and |
| 2633 | | self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) |
| 2634 | | src_mcv |
| 2635 | | else |
| 2636 | | try self.allocRegOrMem(inst, true); |
| 2630 | const min_ty = if (dst_int_info.bits < src_int_info.bits) dst_ty else src_ty; |
| 2631 | const extend = switch (src_int_info.signedness) { |
| 2632 | .signed => dst_int_info, |
| 2633 | .unsigned => src_int_info, |
| 2634 | }.signedness; |
| 2637 | 2635 | |
| 2638 | | const min_ty = if (dst_int_info.bits < src_int_info.bits) dst_ty else src_ty; |
| 2639 | | const signedness: std.builtin.Signedness = if (dst_int_info.signedness == .signed and |
| 2640 | | src_int_info.signedness == .signed) .signed else .unsigned; |
| 2641 | | switch (dst_mcv) { |
| 2642 | | .register => |dst_reg| { |
| 2643 | | const min_abi_size = @min(dst_abi_size, src_abi_size); |
| 2644 | | const tag: Mir.Inst.FixedTag = switch (signedness) { |
| 2645 | | .signed => if (min_abi_size >= 4) .{ ._d, .movsx } else .{ ._, .movsx }, |
| 2646 | | .unsigned => if (min_abi_size >= 4) .{ ._, .mov } else .{ ._, .movzx }, |
| 2647 | | }; |
| 2648 | | const dst_alias = switch (tag[1]) { |
| 2649 | | .movsx => dst_reg.to64(), |
| 2650 | | .mov, .movzx => if (min_abi_size > 4) dst_reg.to64() else dst_reg.to32(), |
| 2651 | | else => unreachable, |
| 2636 | const src_mcv = try self.resolveInst(ty_op.operand); |
| 2637 | const src_storage_bits = switch (src_mcv) { |
| 2638 | .register, .register_offset => 64, |
| 2639 | .load_frame => |frame_addr| self.getFrameAddrSize(frame_addr) * 8, |
| 2640 | else => src_int_info.bits, |
| 2641 | }; |
| 2642 | |
| 2643 | const dst_mcv = if (dst_int_info.bits <= src_storage_bits and |
| 2644 | self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) src_mcv else dst: { |
| 2645 | const dst_mcv = try self.allocRegOrMem(inst, true); |
| 2646 | try self.genCopy(min_ty, dst_mcv, src_mcv); |
| 2647 | break :dst dst_mcv; |
| 2648 | }; |
| 2649 | |
| 2650 | if (dst_int_info.bits <= src_int_info.bits) break :result if (dst_mcv.isRegister()) |
| 2651 | .{ .register = registerAlias(dst_mcv.getReg().?, abi_size) } |
| 2652 | else |
| 2653 | dst_mcv; |
| 2654 | |
| 2655 | if (dst_mcv.isRegister()) { |
| 2656 | try self.truncateRegister(src_ty, dst_mcv.getReg().?); |
| 2657 | break :result .{ .register = registerAlias(dst_mcv.getReg().?, abi_size) }; |
| 2658 | } |
| 2659 | |
| 2660 | const src_limbs_len = std.math.divCeil(u16, src_int_info.bits, 64) catch unreachable; |
| 2661 | const dst_limbs_len = std.math.divCeil(u16, dst_int_info.bits, 64) catch unreachable; |
| 2662 | |
| 2663 | const high_mcv = dst_mcv.address().offset((src_limbs_len - 1) * 8).deref(); |
| 2664 | const high_reg = try self.copyToTmpRegister(switch (src_int_info.signedness) { |
| 2665 | .signed => Type.isize, |
| 2666 | .unsigned => Type.usize, |
| 2667 | }, high_mcv); |
| 2668 | const high_lock = self.register_manager.lockRegAssumeUnused(high_reg); |
| 2669 | defer self.register_manager.unlockReg(high_lock); |
| 2670 | |
| 2671 | const high_bits = src_int_info.bits % 64; |
| 2672 | if (high_bits > 0) { |
| 2673 | var high_pl = Type.Payload.Bits{ |
| 2674 | .base = .{ .tag = switch (extend) { |
| 2675 | .signed => .int_signed, |
| 2676 | .unsigned => .int_unsigned, |
| 2677 | } }, |
| 2678 | .data = high_bits, |
| 2652 | 2679 | }; |
| 2653 | | switch (src_mcv) { |
| 2654 | | .register => |src_reg| { |
| 2655 | | try self.asmRegisterRegister( |
| 2656 | | tag, |
| 2657 | | dst_alias, |
| 2658 | | registerAlias(src_reg, min_abi_size), |
| 2680 | const high_ty = Type.initPayload(&high_pl.base); |
| 2681 | try self.truncateRegister(high_ty, high_reg); |
| 2682 | try self.genCopy(Type.usize, high_mcv, .{ .register = high_reg }); |
| 2683 | } |
| 2684 | |
| 2685 | if (dst_limbs_len > src_limbs_len) try self.genInlineMemset( |
| 2686 | dst_mcv.address().offset(src_limbs_len * 8), |
| 2687 | switch (extend) { |
| 2688 | .signed => extend: { |
| 2689 | const extend_mcv = MCValue{ .register = high_reg }; |
| 2690 | try self.genShiftBinOpMir( |
| 2691 | .{ ._r, .sa }, |
| 2692 | Type.isize, |
| 2693 | extend_mcv, |
| 2694 | .{ .immediate = 63 }, |
| 2659 | 2695 | ); |
| 2696 | break :extend extend_mcv; |
| 2660 | 2697 | }, |
| 2661 | | .memory, .indirect, .load_frame => try self.asmRegisterMemory( |
| 2662 | | tag, |
| 2663 | | dst_alias, |
| 2664 | | src_mcv.mem(Memory.PtrSize.fromSize(min_abi_size)), |
| 2665 | | ), |
| 2666 | | else => return self.fail("TODO airIntCast from {s} to {s}", .{ |
| 2667 | | @tagName(src_mcv), |
| 2668 | | @tagName(dst_mcv), |
| 2669 | | }), |
| 2670 | | } |
| 2671 | | if (self.regExtraBits(min_ty) > 0) try self.truncateRegister(min_ty, dst_reg); |
| 2672 | | }, |
| 2673 | | else => { |
| 2674 | | try self.genCopy(min_ty, dst_mcv, src_mcv); |
| 2675 | | const extra = dst_abi_size * 8 - dst_int_info.bits; |
| 2676 | | if (extra > 0) { |
| 2677 | | try self.genShiftBinOpMir( |
| 2678 | | switch (signedness) { |
| 2679 | | .signed => .{ ._l, .sa }, |
| 2680 | | .unsigned => .{ ._l, .sh }, |
| 2681 | | }, |
| 2682 | | dst_ty, |
| 2683 | | dst_mcv, |
| 2684 | | .{ .immediate = extra }, |
| 2685 | | ); |
| 2686 | | try self.genShiftBinOpMir( |
| 2687 | | switch (signedness) { |
| 2688 | | .signed => .{ ._r, .sa }, |
| 2689 | | .unsigned => .{ ._r, .sh }, |
| 2690 | | }, |
| 2691 | | dst_ty, |
| 2692 | | dst_mcv, |
| 2693 | | .{ .immediate = extra }, |
| 2694 | | ); |
| 2695 | | } |
| 2696 | | }, |
| 2697 | | } |
| 2698 | | return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none }); |
| 2698 | .unsigned => .{ .immediate = 0 }, |
| 2699 | }, |
| 2700 | .{ .immediate = (dst_limbs_len - src_limbs_len) * 8 }, |
| 2701 | ); |
| 2702 | |
| 2703 | break :result dst_mcv; |
| 2704 | }; |
| 2705 | return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); |
| 2699 | 2706 | } |
| 2700 | 2707 | |
| 2701 | 2708 | fn airTrunc(self: *Self, inst: Air.Inst.Index) !void { |
| ... | ... | @@ -9879,63 +9886,6 @@ fn genSetMem(self: *Self, base: Memory.Base, disp: i32, ty: Type, src_mcv: MCVal |
| 9879 | 9886 | } |
| 9880 | 9887 | } |
| 9881 | 9888 | |
| 9882 | | /// Like `genInlineMemcpy` but copies value from a register to an address via dereferencing |
| 9883 | | /// of destination register. |
| 9884 | | /// Boils down to MOV r/m64, r64. |
| 9885 | | fn genInlineMemcpyRegisterRegister( |
| 9886 | | self: *Self, |
| 9887 | | ty: Type, |
| 9888 | | dst_reg: Register, |
| 9889 | | src_reg: Register, |
| 9890 | | offset: i32, |
| 9891 | | ) InnerError!void { |
| 9892 | | assert(dst_reg.bitSize() == 64); |
| 9893 | | |
| 9894 | | const dst_reg_lock = self.register_manager.lockReg(dst_reg); |
| 9895 | | defer if (dst_reg_lock) |lock| self.register_manager.unlockReg(lock); |
| 9896 | | |
| 9897 | | const src_reg_lock = self.register_manager.lockReg(src_reg); |
| 9898 | | defer if (src_reg_lock) |lock| self.register_manager.unlockReg(lock); |
| 9899 | | |
| 9900 | | const abi_size = @intCast(u32, ty.abiSize(self.target.*)); |
| 9901 | | |
| 9902 | | if (!math.isPowerOfTwo(abi_size)) { |
| 9903 | | const tmp_reg = try self.copyToTmpRegister(ty, .{ .register = src_reg }); |
| 9904 | | |
| 9905 | | var next_offset = offset; |
| 9906 | | var remainder = abi_size; |
| 9907 | | while (remainder > 0) { |
| 9908 | | const nearest_power_of_two = @as(u6, 1) << math.log2_int(u3, @intCast(u3, remainder)); |
| 9909 | | try self.asmMemoryRegister( |
| 9910 | | .{ ._, .mov }, |
| 9911 | | Memory.sib(Memory.PtrSize.fromSize(nearest_power_of_two), .{ |
| 9912 | | .base = dst_reg, |
| 9913 | | .disp = -next_offset, |
| 9914 | | }), |
| 9915 | | registerAlias(tmp_reg, nearest_power_of_two), |
| 9916 | | ); |
| 9917 | | |
| 9918 | | if (nearest_power_of_two > 1) { |
| 9919 | | try self.genShiftBinOpMir(.{ ._r, .sh }, ty, .{ .register = tmp_reg }, .{ |
| 9920 | | .immediate = nearest_power_of_two * 8, |
| 9921 | | }); |
| 9922 | | } |
| 9923 | | |
| 9924 | | remainder -= nearest_power_of_two; |
| 9925 | | next_offset -= nearest_power_of_two; |
| 9926 | | } |
| 9927 | | } else { |
| 9928 | | try self.asmMemoryRegister( |
| 9929 | | switch (src_reg.class()) { |
| 9930 | | .general_purpose, .segment => .{ ._, .mov }, |
| 9931 | | .sse => .{ ._ss, .mov }, |
| 9932 | | }, |
| 9933 | | Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = dst_reg, .disp = -offset }), |
| 9934 | | registerAlias(src_reg, abi_size), |
| 9935 | | ); |
| 9936 | | } |
| 9937 | | } |
| 9938 | | |
| 9939 | 9889 | fn genInlineMemcpy(self: *Self, dst_ptr: MCValue, src_ptr: MCValue, len: MCValue) InnerError!void { |
| 9940 | 9890 | try self.spillRegisters(&.{ .rdi, .rsi, .rcx }); |
| 9941 | 9891 | try self.genSetReg(.rdi, Type.usize, dst_ptr); |
| ... | ... | @@ -10036,20 +9986,56 @@ fn airBitCast(self: *Self, inst: Air.Inst.Index) !void { |
| 10036 | 9986 | const result = result: { |
| 10037 | 9987 | const dst_rc = regClassForType(dst_ty); |
| 10038 | 9988 | const src_rc = regClassForType(src_ty); |
| 10039 | | const operand = try self.resolveInst(ty_op.operand); |
| 10040 | | if (dst_rc.supersetOf(src_rc) and self.reuseOperand(inst, ty_op.operand, 0, operand)) |
| 10041 | | break :result operand; |
| 9989 | const src_mcv = try self.resolveInst(ty_op.operand); |
| 9990 | if (dst_rc.supersetOf(src_rc) and self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) |
| 9991 | break :result src_mcv; |
| 10042 | 9992 | |
| 10043 | | const operand_lock = switch (operand) { |
| 10044 | | .register => |reg| self.register_manager.lockReg(reg), |
| 10045 | | .register_overflow => |ro| self.register_manager.lockReg(ro.reg), |
| 10046 | | else => null, |
| 10047 | | }; |
| 10048 | | defer if (operand_lock) |lock| self.register_manager.unlockReg(lock); |
| 9993 | const src_lock = if (src_mcv.getReg()) |reg| self.register_manager.lockReg(reg) else null; |
| 9994 | defer if (src_lock) |lock| self.register_manager.unlockReg(lock); |
| 10049 | 9995 | |
| 10050 | | const dest = try self.allocRegOrMem(inst, true); |
| 10051 | | try self.genCopy(if (!dest.isMemory() or operand.isMemory()) dst_ty else src_ty, dest, operand); |
| 10052 | | break :result dest; |
| 9996 | const dst_mcv = try self.allocRegOrMem(inst, true); |
| 9997 | try self.genCopy( |
| 9998 | if (!dst_mcv.isMemory() or src_mcv.isMemory()) dst_ty else src_ty, |
| 9999 | dst_mcv, |
| 10000 | src_mcv, |
| 10001 | ); |
| 10002 | |
| 10003 | const dst_signedness = |
| 10004 | if (dst_ty.isAbiInt()) dst_ty.intInfo(self.target.*).signedness else .unsigned; |
| 10005 | const src_signedness = |
| 10006 | if (src_ty.isAbiInt()) src_ty.intInfo(self.target.*).signedness else .unsigned; |
| 10007 | const abi_size = @intCast(u16, dst_ty.abiSize(self.target.*)); |
| 10008 | const bit_size = @intCast(u16, dst_ty.bitSize(self.target.*)); |
| 10009 | const dst_limbs_len = std.math.divCeil(u16, bit_size, 64) catch unreachable; |
| 10010 | if (dst_signedness != src_signedness and abi_size * 8 > bit_size) { |
| 10011 | const high_reg = if (dst_mcv.isRegister()) |
| 10012 | dst_mcv.getReg().? |
| 10013 | else |
| 10014 | try self.copyToTmpRegister( |
| 10015 | Type.usize, |
| 10016 | dst_mcv.address().offset((dst_limbs_len - 1) * 8).deref(), |
| 10017 | ); |
| 10018 | const high_lock = self.register_manager.lockReg(high_reg); |
| 10019 | defer if (high_lock) |lock| self.register_manager.unlockReg(lock); |
| 10020 | |
| 10021 | var high_pl = Type.Payload.Bits{ |
| 10022 | .base = .{ .tag = switch (dst_signedness) { |
| 10023 | .signed => .int_signed, |
| 10024 | .unsigned => .int_unsigned, |
| 10025 | } }, |
| 10026 | .data = bit_size % 64, |
| 10027 | }; |
| 10028 | const high_ty = Type.initPayload(&high_pl.base); |
| 10029 | |
| 10030 | try self.truncateRegister(high_ty, high_reg); |
| 10031 | if (!dst_mcv.isRegister()) try self.genCopy( |
| 10032 | Type.usize, |
| 10033 | dst_mcv.address().offset((dst_limbs_len - 1) * 8).deref(), |
| 10034 | .{ .register = high_reg }, |
| 10035 | ); |
| 10036 | } |
| 10037 | |
| 10038 | break :result dst_mcv; |
| 10053 | 10039 | }; |
| 10054 | 10040 | return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); |
| 10055 | 10041 | } |
Jacob Young <jacobly@ziglang.org> 2023-05-15 23:34:57-04:00
src