| ... | @@ -2224,6 +2224,10 @@ fn getFrameAddrAlignment(self: *Self, frame_addr: FrameAddr) u32 { | ... | @@ -2224,6 +2224,10 @@ fn getFrameAddrAlignment(self: *Self, frame_addr: FrameAddr) u32 { |
| 2224 | return @min(alloc_align, @bitCast(u32, frame_addr.off) & (alloc_align - 1)); | 2224 | return @min(alloc_align, @bitCast(u32, frame_addr.off) & (alloc_align - 1)); |
| 2225 | } | 2225 | } |
| 2226 | | 2226 | |
| | 2227 | fn getFrameAddrSize(self: *Self, frame_addr: FrameAddr) u32 { |
| | 2228 | return self.frame_allocs.get(@enumToInt(frame_addr.index)).abi_size - @intCast(u31, frame_addr.off); |
| | 2229 | } |
| | 2230 | |
| 2227 | fn allocFrameIndex(self: *Self, alloc: FrameAlloc) !FrameIndex { | 2231 | fn allocFrameIndex(self: *Self, alloc: FrameAlloc) !FrameIndex { |
| 2228 | const frame_allocs_slice = self.frame_allocs.slice(); | 2232 | const frame_allocs_slice = self.frame_allocs.slice(); |
| 2229 | const frame_size = frame_allocs_slice.items(.abi_size); | 2233 | const frame_size = frame_allocs_slice.items(.abi_size); |
| ... | @@ -2615,87 +2619,90 @@ fn airFpext(self: *Self, inst: Air.Inst.Index) !void { | ... | @@ -2615,87 +2619,90 @@ fn airFpext(self: *Self, inst: Air.Inst.Index) !void { |
| 2615 | | 2619 | |
| 2616 | fn airIntCast(self: *Self, inst: Air.Inst.Index) !void { | 2620 | fn airIntCast(self: *Self, inst: Air.Inst.Index) !void { |
| 2617 | const ty_op = self.air.instructions.items(.data)[inst].ty_op; | 2621 | const ty_op = self.air.instructions.items(.data)[inst].ty_op; |
| | 2622 | const result: MCValue = result: { |
| | 2623 | const src_ty = self.air.typeOf(ty_op.operand); |
| | 2624 | const src_int_info = src_ty.intInfo(self.target.*); |
| 2618 | | 2625 | |
| 2619 | const src_ty = self.air.typeOf(ty_op.operand); | 2626 | const dst_ty = self.air.typeOfIndex(inst); |
| 2620 | const src_int_info = src_ty.intInfo(self.target.*); | 2627 | const dst_int_info = dst_ty.intInfo(self.target.*); |
| 2621 | const src_abi_size = @intCast(u32, src_ty.abiSize(self.target.*)); | 2628 | const abi_size = @intCast(u32, dst_ty.abiSize(self.target.*)); |
| 2622 | const src_mcv = try self.resolveInst(ty_op.operand); | | |
| 2623 | const src_lock = switch (src_mcv) { | | |
| 2624 | .register => |reg| self.register_manager.lockRegAssumeUnused(reg), | | |
| 2625 | else => null, | | |
| 2626 | }; | | |
| 2627 | defer if (src_lock) |lock| self.register_manager.unlockReg(lock); | | |
| 2628 | | 2629 | |
| 2629 | const dst_ty = self.air.typeOfIndex(inst); | 2630 | const min_ty = if (dst_int_info.bits < src_int_info.bits) dst_ty else src_ty; |
| 2630 | const dst_int_info = dst_ty.intInfo(self.target.*); | 2631 | const extend = switch (src_int_info.signedness) { |
| 2631 | const dst_abi_size = @intCast(u32, dst_ty.abiSize(self.target.*)); | 2632 | .signed => dst_int_info, |
| 2632 | const dst_mcv = if (dst_abi_size <= src_abi_size and | 2633 | .unsigned => src_int_info, |
| 2633 | self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) | 2634 | }.signedness; |
| 2634 | src_mcv | | |
| 2635 | else | | |
| 2636 | try self.allocRegOrMem(inst, true); | | |
| 2637 | | 2635 | |
| 2638 | const min_ty = if (dst_int_info.bits < src_int_info.bits) dst_ty else src_ty; | 2636 | const src_mcv = try self.resolveInst(ty_op.operand); |
| 2639 | const signedness: std.builtin.Signedness = if (dst_int_info.signedness == .signed and | 2637 | const src_storage_bits = switch (src_mcv) { |
| 2640 | src_int_info.signedness == .signed) .signed else .unsigned; | 2638 | .register, .register_offset => 64, |
| 2641 | switch (dst_mcv) { | 2639 | .load_frame => |frame_addr| self.getFrameAddrSize(frame_addr) * 8, |
| 2642 | .register => |dst_reg| { | 2640 | else => src_int_info.bits, |
| 2643 | const min_abi_size = @min(dst_abi_size, src_abi_size); | 2641 | }; |
| 2644 | const tag: Mir.Inst.FixedTag = switch (signedness) { | 2642 | |
| 2645 | .signed => if (min_abi_size >= 4) .{ ._d, .movsx } else .{ ._, .movsx }, | 2643 | const dst_mcv = if (dst_int_info.bits <= src_storage_bits and |
| 2646 | .unsigned => if (min_abi_size >= 4) .{ ._, .mov } else .{ ._, .movzx }, | 2644 | self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) src_mcv else dst: { |
| 2647 | }; | 2645 | const dst_mcv = try self.allocRegOrMem(inst, true); |
| 2648 | const dst_alias = switch (tag[1]) { | 2646 | try self.genCopy(min_ty, dst_mcv, src_mcv); |
| 2649 | .movsx => dst_reg.to64(), | 2647 | break :dst dst_mcv; |
| 2650 | .mov, .movzx => if (min_abi_size > 4) dst_reg.to64() else dst_reg.to32(), | 2648 | }; |
| 2651 | else => unreachable, | 2649 | |
| | 2650 | if (dst_int_info.bits <= src_int_info.bits) break :result if (dst_mcv.isRegister()) |
| | 2651 | .{ .register = registerAlias(dst_mcv.getReg().?, abi_size) } |
| | 2652 | else |
| | 2653 | dst_mcv; |
| | 2654 | |
| | 2655 | if (dst_mcv.isRegister()) { |
| | 2656 | try self.truncateRegister(src_ty, dst_mcv.getReg().?); |
| | 2657 | break :result .{ .register = registerAlias(dst_mcv.getReg().?, abi_size) }; |
| | 2658 | } |
| | 2659 | |
| | 2660 | const src_limbs_len = std.math.divCeil(u16, src_int_info.bits, 64) catch unreachable; |
| | 2661 | const dst_limbs_len = std.math.divCeil(u16, dst_int_info.bits, 64) catch unreachable; |
| | 2662 | |
| | 2663 | const high_mcv = dst_mcv.address().offset((src_limbs_len - 1) * 8).deref(); |
| | 2664 | const high_reg = try self.copyToTmpRegister(switch (src_int_info.signedness) { |
| | 2665 | .signed => Type.isize, |
| | 2666 | .unsigned => Type.usize, |
| | 2667 | }, high_mcv); |
| | 2668 | const high_lock = self.register_manager.lockRegAssumeUnused(high_reg); |
| | 2669 | defer self.register_manager.unlockReg(high_lock); |
| | 2670 | |
| | 2671 | const high_bits = src_int_info.bits % 64; |
| | 2672 | if (high_bits > 0) { |
| | 2673 | var high_pl = Type.Payload.Bits{ |
| | 2674 | .base = .{ .tag = switch (extend) { |
| | 2675 | .signed => .int_signed, |
| | 2676 | .unsigned => .int_unsigned, |
| | 2677 | } }, |
| | 2678 | .data = high_bits, |
| 2652 | }; | 2679 | }; |
| 2653 | switch (src_mcv) { | 2680 | const high_ty = Type.initPayload(&high_pl.base); |
| 2654 | .register => |src_reg| { | 2681 | try self.truncateRegister(high_ty, high_reg); |
| 2655 | try self.asmRegisterRegister( | 2682 | try self.genCopy(Type.usize, high_mcv, .{ .register = high_reg }); |
| 2656 | tag, | 2683 | } |
| 2657 | dst_alias, | 2684 | |
| 2658 | registerAlias(src_reg, min_abi_size), | 2685 | if (dst_limbs_len > src_limbs_len) try self.genInlineMemset( |
| | 2686 | dst_mcv.address().offset(src_limbs_len * 8), |
| | 2687 | switch (extend) { |
| | 2688 | .signed => extend: { |
| | 2689 | const extend_mcv = MCValue{ .register = high_reg }; |
| | 2690 | try self.genShiftBinOpMir( |
| | 2691 | .{ ._r, .sa }, |
| | 2692 | Type.isize, |
| | 2693 | extend_mcv, |
| | 2694 | .{ .immediate = 63 }, |
| 2659 | ); | 2695 | ); |
| | 2696 | break :extend extend_mcv; |
| 2660 | }, | 2697 | }, |
| 2661 | .memory, .indirect, .load_frame => try self.asmRegisterMemory( | 2698 | .unsigned => .{ .immediate = 0 }, |
| 2662 | tag, | 2699 | }, |
| 2663 | dst_alias, | 2700 | .{ .immediate = (dst_limbs_len - src_limbs_len) * 8 }, |
| 2664 | src_mcv.mem(Memory.PtrSize.fromSize(min_abi_size)), | 2701 | ); |
| 2665 | ), | 2702 | |
| 2666 | else => return self.fail("TODO airIntCast from {s} to {s}", .{ | 2703 | break :result dst_mcv; |
| 2667 | @tagName(src_mcv), | 2704 | }; |
| 2668 | @tagName(dst_mcv), | 2705 | return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); |
| 2669 | }), | | |
| 2670 | } | | |
| 2671 | if (self.regExtraBits(min_ty) > 0) try self.truncateRegister(min_ty, dst_reg); | | |
| 2672 | }, | | |
| 2673 | else => { | | |
| 2674 | try self.genCopy(min_ty, dst_mcv, src_mcv); | | |
| 2675 | const extra = dst_abi_size * 8 - dst_int_info.bits; | | |
| 2676 | if (extra > 0) { | | |
| 2677 | try self.genShiftBinOpMir( | | |
| 2678 | switch (signedness) { | | |
| 2679 | .signed => .{ ._l, .sa }, | | |
| 2680 | .unsigned => .{ ._l, .sh }, | | |
| 2681 | }, | | |
| 2682 | dst_ty, | | |
| 2683 | dst_mcv, | | |
| 2684 | .{ .immediate = extra }, | | |
| 2685 | ); | | |
| 2686 | try self.genShiftBinOpMir( | | |
| 2687 | switch (signedness) { | | |
| 2688 | .signed => .{ ._r, .sa }, | | |
| 2689 | .unsigned => .{ ._r, .sh }, | | |
| 2690 | }, | | |
| 2691 | dst_ty, | | |
| 2692 | dst_mcv, | | |
| 2693 | .{ .immediate = extra }, | | |
| 2694 | ); | | |
| 2695 | } | | |
| 2696 | }, | | |
| 2697 | } | | |
| 2698 | return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none }); | | |
| 2699 | } | 2706 | } |
| 2700 | | 2707 | |
| 2701 | fn airTrunc(self: *Self, inst: Air.Inst.Index) !void { | 2708 | fn airTrunc(self: *Self, inst: Air.Inst.Index) !void { |
| ... | @@ -9879,63 +9886,6 @@ fn genSetMem(self: *Self, base: Memory.Base, disp: i32, ty: Type, src_mcv: MCVal | ... | @@ -9879,63 +9886,6 @@ fn genSetMem(self: *Self, base: Memory.Base, disp: i32, ty: Type, src_mcv: MCVal |
| 9879 | } | 9886 | } |
| 9880 | } | 9887 | } |
| 9881 | | 9888 | |
| 9882 | /// Like `genInlineMemcpy` but copies value from a register to an address via dereferencing | | |
| 9883 | /// of destination register. | | |
| 9884 | /// Boils down to MOV r/m64, r64. | | |
| 9885 | fn genInlineMemcpyRegisterRegister( | | |
| 9886 | self: *Self, | | |
| 9887 | ty: Type, | | |
| 9888 | dst_reg: Register, | | |
| 9889 | src_reg: Register, | | |
| 9890 | offset: i32, | | |
| 9891 | ) InnerError!void { | | |
| 9892 | assert(dst_reg.bitSize() == 64); | | |
| 9893 | | | |
| 9894 | const dst_reg_lock = self.register_manager.lockReg(dst_reg); | | |
| 9895 | defer if (dst_reg_lock) |lock| self.register_manager.unlockReg(lock); | | |
| 9896 | | | |
| 9897 | const src_reg_lock = self.register_manager.lockReg(src_reg); | | |
| 9898 | defer if (src_reg_lock) |lock| self.register_manager.unlockReg(lock); | | |
| 9899 | | | |
| 9900 | const abi_size = @intCast(u32, ty.abiSize(self.target.*)); | | |
| 9901 | | | |
| 9902 | if (!math.isPowerOfTwo(abi_size)) { | | |
| 9903 | const tmp_reg = try self.copyToTmpRegister(ty, .{ .register = src_reg }); | | |
| 9904 | | | |
| 9905 | var next_offset = offset; | | |
| 9906 | var remainder = abi_size; | | |
| 9907 | while (remainder > 0) { | | |
| 9908 | const nearest_power_of_two = @as(u6, 1) << math.log2_int(u3, @intCast(u3, remainder)); | | |
| 9909 | try self.asmMemoryRegister( | | |
| 9910 | .{ ._, .mov }, | | |
| 9911 | Memory.sib(Memory.PtrSize.fromSize(nearest_power_of_two), .{ | | |
| 9912 | .base = dst_reg, | | |
| 9913 | .disp = -next_offset, | | |
| 9914 | }), | | |
| 9915 | registerAlias(tmp_reg, nearest_power_of_two), | | |
| 9916 | ); | | |
| 9917 | | | |
| 9918 | if (nearest_power_of_two > 1) { | | |
| 9919 | try self.genShiftBinOpMir(.{ ._r, .sh }, ty, .{ .register = tmp_reg }, .{ | | |
| 9920 | .immediate = nearest_power_of_two * 8, | | |
| 9921 | }); | | |
| 9922 | } | | |
| 9923 | | | |
| 9924 | remainder -= nearest_power_of_two; | | |
| 9925 | next_offset -= nearest_power_of_two; | | |
| 9926 | } | | |
| 9927 | } else { | | |
| 9928 | try self.asmMemoryRegister( | | |
| 9929 | switch (src_reg.class()) { | | |
| 9930 | .general_purpose, .segment => .{ ._, .mov }, | | |
| 9931 | .sse => .{ ._ss, .mov }, | | |
| 9932 | }, | | |
| 9933 | Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = dst_reg, .disp = -offset }), | | |
| 9934 | registerAlias(src_reg, abi_size), | | |
| 9935 | ); | | |
| 9936 | } | | |
| 9937 | } | | |
| 9938 | | | |
| 9939 | fn genInlineMemcpy(self: *Self, dst_ptr: MCValue, src_ptr: MCValue, len: MCValue) InnerError!void { | 9889 | fn genInlineMemcpy(self: *Self, dst_ptr: MCValue, src_ptr: MCValue, len: MCValue) InnerError!void { |
| 9940 | try self.spillRegisters(&.{ .rdi, .rsi, .rcx }); | 9890 | try self.spillRegisters(&.{ .rdi, .rsi, .rcx }); |
| 9941 | try self.genSetReg(.rdi, Type.usize, dst_ptr); | 9891 | try self.genSetReg(.rdi, Type.usize, dst_ptr); |
| ... | @@ -10036,20 +9986,56 @@ fn airBitCast(self: *Self, inst: Air.Inst.Index) !void { | ... | @@ -10036,20 +9986,56 @@ fn airBitCast(self: *Self, inst: Air.Inst.Index) !void { |
| 10036 | const result = result: { | 9986 | const result = result: { |
| 10037 | const dst_rc = regClassForType(dst_ty); | 9987 | const dst_rc = regClassForType(dst_ty); |
| 10038 | const src_rc = regClassForType(src_ty); | 9988 | const src_rc = regClassForType(src_ty); |
| 10039 | const operand = try self.resolveInst(ty_op.operand); | 9989 | const src_mcv = try self.resolveInst(ty_op.operand); |
| 10040 | if (dst_rc.supersetOf(src_rc) and self.reuseOperand(inst, ty_op.operand, 0, operand)) | 9990 | if (dst_rc.supersetOf(src_rc) and self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) |
| 10041 | break :result operand; | 9991 | break :result src_mcv; |
| 10042 | | 9992 | |
| 10043 | const operand_lock = switch (operand) { | 9993 | const src_lock = if (src_mcv.getReg()) |reg| self.register_manager.lockReg(reg) else null; |
| 10044 | .register => |reg| self.register_manager.lockReg(reg), | 9994 | defer if (src_lock) |lock| self.register_manager.unlockReg(lock); |
| 10045 | .register_overflow => |ro| self.register_manager.lockReg(ro.reg), | | |
| 10046 | else => null, | | |
| 10047 | }; | | |
| 10048 | defer if (operand_lock) |lock| self.register_manager.unlockReg(lock); | | |
| 10049 | | 9995 | |
| 10050 | const dest = try self.allocRegOrMem(inst, true); | 9996 | const dst_mcv = try self.allocRegOrMem(inst, true); |
| 10051 | try self.genCopy(if (!dest.isMemory() or operand.isMemory()) dst_ty else src_ty, dest, operand); | 9997 | try self.genCopy( |
| 10052 | break :result dest; | 9998 | if (!dst_mcv.isMemory() or src_mcv.isMemory()) dst_ty else src_ty, |
| | 9999 | dst_mcv, |
| | 10000 | src_mcv, |
| | 10001 | ); |
| | 10002 | |
| | 10003 | const dst_signedness = |
| | 10004 | if (dst_ty.isAbiInt()) dst_ty.intInfo(self.target.*).signedness else .unsigned; |
| | 10005 | const src_signedness = |
| | 10006 | if (src_ty.isAbiInt()) src_ty.intInfo(self.target.*).signedness else .unsigned; |
| | 10007 | const abi_size = @intCast(u16, dst_ty.abiSize(self.target.*)); |
| | 10008 | const bit_size = @intCast(u16, dst_ty.bitSize(self.target.*)); |
| | 10009 | const dst_limbs_len = std.math.divCeil(u16, bit_size, 64) catch unreachable; |
| | 10010 | if (dst_signedness != src_signedness and abi_size * 8 > bit_size) { |
| | 10011 | const high_reg = if (dst_mcv.isRegister()) |
| | 10012 | dst_mcv.getReg().? |
| | 10013 | else |
| | 10014 | try self.copyToTmpRegister( |
| | 10015 | Type.usize, |
| | 10016 | dst_mcv.address().offset((dst_limbs_len - 1) * 8).deref(), |
| | 10017 | ); |
| | 10018 | const high_lock = self.register_manager.lockReg(high_reg); |
| | 10019 | defer if (high_lock) |lock| self.register_manager.unlockReg(lock); |
| | 10020 | |
| | 10021 | var high_pl = Type.Payload.Bits{ |
| | 10022 | .base = .{ .tag = switch (dst_signedness) { |
| | 10023 | .signed => .int_signed, |
| | 10024 | .unsigned => .int_unsigned, |
| | 10025 | } }, |
| | 10026 | .data = bit_size % 64, |
| | 10027 | }; |
| | 10028 | const high_ty = Type.initPayload(&high_pl.base); |
| | 10029 | |
| | 10030 | try self.truncateRegister(high_ty, high_reg); |
| | 10031 | if (!dst_mcv.isRegister()) try self.genCopy( |
| | 10032 | Type.usize, |
| | 10033 | dst_mcv.address().offset((dst_limbs_len - 1) * 8).deref(), |
| | 10034 | .{ .register = high_reg }, |
| | 10035 | ); |
| | 10036 | } |
| | 10037 | |
| | 10038 | break :result dst_mcv; |
| 10053 | }; | 10039 | }; |
| 10054 | return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); | 10040 | return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); |
| 10055 | } | 10041 | } |
Jacob Young <jacobly@ziglang.org> 2023-05-15 23:34:57-04:00
src