authorgravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2022-02-16 21:40:08+01:00
committergravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2022-02-16 21:40:08+01:00
log94474ec7c783b94face643f448426bcb91adfc93
treefb70f73bc21f6130e55edd742b4e7df57bd96cea
parent09d468b237e58dd70cfa0a4311efab5ddabc9f95

x64: refactor use of inline memcpy

1 files changed, 84 insertions(+), 213 deletions(-)

src/arch/x86_64/CodeGen.zig+84-213
...@@ -1782,36 +1782,7 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo...@@ -1782,36 +1782,7 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo
1782 return self.genSetStack(elem_ty, off, MCValue{ .register = tmp_reg });1782 return self.genSetStack(elem_ty, off, MCValue{ .register = tmp_reg });
1783 }1783 }
17841784
1785 self.register_manager.freezeRegs(&.{ .rax, .rcx });1785 try self.genInlineMemcpy(off, .rbp, elem_ty, ptr);
1786 defer self.register_manager.unfreezeRegs(&.{ .rax, .rcx });
1787
1788 const regs = try self.register_manager.allocRegs(3, .{ null, null, null });
1789 const addr_reg = regs[0];
1790 const count_reg = regs[1];
1791 const tmp_reg = regs[2];
1792
1793 _ = try self.addInst(.{
1794 .tag = .mov,
1795 .ops = (Mir.Ops{
1796 .reg1 = registerAlias(addr_reg, @divExact(reg.size(), 8)),
1797 .reg2 = reg,
1798 }).encode(),
1799 .data = undefined,
1800 });
1801
1802 try self.register_manager.getReg(.rax, null);
1803 try self.register_manager.getReg(.rcx, null);
1804
1805 // TODO allow for abi size to be u64
1806 try self.genSetReg(Type.u32, count_reg, .{ .immediate = @intCast(u32, abi_size) });
1807
1808 return self.genInlineMemcpy(
1809 -off,
1810 .rbp,
1811 registerAlias(addr_reg, @divExact(reg.size(), 8)),
1812 count_reg.to64(),
1813 tmp_reg.to8(),
1814 );
1815 },1786 },
1816 else => return self.fail("TODO implement loading from register into {}", .{dst_mcv}),1787 else => return self.fail("TODO implement loading from register into {}", .{dst_mcv}),
1817 }1788 }
...@@ -3626,62 +3597,7 @@ fn genSetStackArg(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue) InnerE...@@ -3626,62 +3597,7 @@ fn genSetStackArg(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue) InnerE
3626 return self.genSetStackArg(ty, stack_offset, MCValue{ .register = reg });3597 return self.genSetStackArg(ty, stack_offset, MCValue{ .register = reg });
3627 }3598 }
36283599
3629 self.register_manager.freezeRegs(&.{ .rax, .rcx });3600 try self.genInlineMemcpy(stack_offset, .rsp, ty, mcv);
3630 defer self.register_manager.unfreezeRegs(&.{ .rax, .rcx });
3631
3632 const addr_reg: Register = blk: {
3633 switch (mcv) {
3634 .got_load,
3635 .direct_load,
3636 => |sym_index| {
3637 const flags: u2 = switch (mcv) {
3638 .got_load => 0b00,
3639 .direct_load => 0b01,
3640 else => unreachable,
3641 };
3642 const addr_reg = try self.register_manager.allocReg(null);
3643 _ = try self.addInst(.{
3644 .tag = .lea_pie,
3645 .ops = (Mir.Ops{
3646 .reg1 = addr_reg.to64(),
3647 .flags = flags,
3648 }).encode(),
3649 .data = .{
3650 .load_reloc = .{
3651 .atom_index = self.mod_fn.owner_decl.link.macho.local_sym_index,
3652 .sym_index = sym_index,
3653 },
3654 },
3655 });
3656 break :blk addr_reg;
3657 },
3658 .memory => |addr| {
3659 const addr_reg = try self.copyToTmpRegister(Type.usize, .{ .immediate = addr });
3660 break :blk addr_reg;
3661 },
3662 else => unreachable,
3663 }
3664 };
3665
3666 self.register_manager.freezeRegs(&.{addr_reg});
3667 defer self.register_manager.unfreezeRegs(&.{addr_reg});
3668
3669 const regs = try self.register_manager.allocRegs(2, .{ null, null });
3670 const count_reg = regs[0];
3671 const tmp_reg = regs[1];
3672
3673 try self.register_manager.getReg(.rax, null);
3674 try self.register_manager.getReg(.rcx, null);
3675
3676 // TODO allow for abi_size to be u64
3677 try self.genSetReg(Type.u32, count_reg, .{ .immediate = @intCast(u32, abi_size) });
3678 try self.genInlineMemcpy(
3679 -stack_offset,
3680 .rsp,
3681 addr_reg.to64(),
3682 count_reg.to64(),
3683 tmp_reg.to8(),
3684 );
3685 },3601 },
3686 .register => |reg| {3602 .register => |reg| {
3687 _ = try self.addInst(.{3603 _ = try self.addInst(.{
...@@ -3698,41 +3614,13 @@ fn genSetStackArg(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue) InnerE...@@ -3698,41 +3614,13 @@ fn genSetStackArg(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue) InnerE
3698 const reg = try self.copyToTmpRegister(ty, mcv);3614 const reg = try self.copyToTmpRegister(ty, mcv);
3699 return self.genSetStackArg(ty, stack_offset, MCValue{ .register = reg });3615 return self.genSetStackArg(ty, stack_offset, MCValue{ .register = reg });
3700 },3616 },
3701 .stack_offset => |off| {3617 .stack_offset => {
3702 if (abi_size <= 8) {3618 if (abi_size <= 8) {
3703 const reg = try self.copyToTmpRegister(ty, mcv);3619 const reg = try self.copyToTmpRegister(ty, mcv);
3704 return self.genSetStackArg(ty, stack_offset, MCValue{ .register = reg });3620 return self.genSetStackArg(ty, stack_offset, MCValue{ .register = reg });
3705 }3621 }
37063622
3707 self.register_manager.freezeRegs(&.{ .rax, .rcx });3623 try self.genInlineMemcpy(stack_offset, .rsp, ty, mcv);
3708 defer self.register_manager.unfreezeRegs(&.{ .rax, .rcx });
3709
3710 const regs = try self.register_manager.allocRegs(3, .{ null, null, null });
3711 const addr_reg = regs[0];
3712 const count_reg = regs[1];
3713 const tmp_reg = regs[2];
3714
3715 try self.register_manager.getReg(.rax, null);
3716 try self.register_manager.getReg(.rcx, null);
3717
3718 _ = try self.addInst(.{
3719 .tag = .lea,
3720 .ops = (Mir.Ops{
3721 .reg1 = addr_reg.to64(),
3722 .reg2 = .rbp,
3723 }).encode(),
3724 .data = .{ .imm = @bitCast(u32, -off) },
3725 });
3726
3727 // TODO allow for abi_size to be u64
3728 try self.genSetReg(Type.u32, count_reg, .{ .immediate = @intCast(u32, abi_size) });
3729 try self.genInlineMemcpy(
3730 -stack_offset,
3731 .rsp,
3732 addr_reg.to64(),
3733 count_reg.to64(),
3734 tmp_reg.to8(),
3735 );
3736 },3624 },
3737 else => return self.fail("TODO implement args on stack for {}", .{mcv}),3625 else => return self.fail("TODO implement args on stack for {}", .{mcv}),
3738 }3626 }
...@@ -3847,65 +3735,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue) InnerErro...@@ -3847,65 +3735,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue) InnerErro
3847 return self.genSetStack(ty, stack_offset, MCValue{ .register = reg });3735 return self.genSetStack(ty, stack_offset, MCValue{ .register = reg });
3848 }3736 }
38493737
3850 try self.register_manager.getReg(.rax, null);3738 try self.genInlineMemcpy(stack_offset, .rbp, ty, mcv);
3851 try self.register_manager.getReg(.rcx, null);
3852
3853 self.register_manager.freezeRegs(&.{ .rax, .rcx, .rbp });
3854 defer self.register_manager.unfreezeRegs(&.{ .rax, .rcx, .rbp });
3855
3856 const addr_reg: Register = blk: {
3857 switch (mcv) {
3858 .memory => |addr| {
3859 const reg = try self.copyToTmpRegister(Type.usize, .{ .immediate = addr });
3860 break :blk reg;
3861 },
3862 .direct_load,
3863 .got_load,
3864 => |sym_index| {
3865 const flags: u2 = switch (mcv) {
3866 .got_load => 0b00,
3867 .direct_load => 0b01,
3868 else => unreachable,
3869 };
3870 const addr_reg = try self.register_manager.allocReg(null);
3871 _ = try self.addInst(.{
3872 .tag = .lea_pie,
3873 .ops = (Mir.Ops{
3874 .reg1 = addr_reg.to64(),
3875 .flags = flags,
3876 }).encode(),
3877 .data = .{
3878 .load_reloc = .{
3879 .atom_index = self.mod_fn.owner_decl.link.macho.local_sym_index,
3880 .sym_index = sym_index,
3881 },
3882 },
3883 });
3884 break :blk addr_reg;
3885 },
3886 else => {
3887 return self.fail("TODO implement memcpy for setting stack from {}", .{mcv});
3888 },
3889 }
3890 };
3891
3892 self.register_manager.freezeRegs(&.{addr_reg});
3893 defer self.register_manager.unfreezeRegs(&.{addr_reg});
3894
3895 const regs = try self.register_manager.allocRegs(2, .{ null, null });
3896 const count_reg = regs[0];
3897 const tmp_reg = regs[1];
3898
3899 // TODO allow for abi_size to be u64
3900 try self.genSetReg(Type.u32, count_reg, .{ .immediate = @intCast(u32, abi_size) });
3901
3902 return self.genInlineMemcpy(
3903 -stack_offset,
3904 .rbp,
3905 addr_reg.to64(),
3906 count_reg.to64(),
3907 tmp_reg.to8(),
3908 );
3909 },3739 },
3910 .ptr_stack_offset => {3740 .ptr_stack_offset => {
3911 const reg = try self.copyToTmpRegister(ty, mcv);3741 const reg = try self.copyToTmpRegister(ty, mcv);
...@@ -3922,48 +3752,89 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue) InnerErro...@@ -3922,48 +3752,89 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue) InnerErro
3922 return self.genSetStack(ty, stack_offset, MCValue{ .register = reg });3752 return self.genSetStack(ty, stack_offset, MCValue{ .register = reg });
3923 }3753 }
39243754
3925 self.register_manager.freezeRegs(&.{ .rax, .rcx, .rbp });3755 try self.genInlineMemcpy(stack_offset, .rbp, ty, mcv);
3926 defer self.register_manager.unfreezeRegs(&.{ .rax, .rcx, .rbp });3756 },
3757 }
3758}
39273759
3928 const regs = try self.register_manager.allocRegs(3, .{ null, null, null });3760fn genInlineMemcpy(self: *Self, stack_offset: i32, stack_reg: Register, ty: Type, val: MCValue) InnerError!void {
3929 const addr_reg = regs[0];3761 const abi_size = ty.abiSize(self.target.*);
3930 const count_reg = regs[1];
3931 const tmp_reg = regs[2];
39323762
3933 try self.register_manager.getReg(.rax, null);3763 try self.register_manager.getReg(.rax, null);
3934 try self.register_manager.getReg(.rcx, null);3764 try self.register_manager.getReg(.rcx, null);
39353765
3936 _ = try self.addInst(.{3766 self.register_manager.freezeRegs(&.{ .rax, .rcx, .rbp });
3937 .tag = .lea,3767 defer self.register_manager.unfreezeRegs(&.{ .rax, .rcx, .rbp });
3938 .ops = (Mir.Ops{3768
3939 .reg1 = addr_reg.to64(),3769 const addr_reg: Register = blk: {
3940 .reg2 = .rbp,3770 switch (val) {
3941 }).encode(),3771 .memory => |addr| {
3942 .data = .{ .imm = @bitCast(u32, -off) },3772 const reg = try self.copyToTmpRegister(Type.usize, .{ .immediate = addr });
3943 });3773 break :blk reg;
3774 },
3775 .direct_load,
3776 .got_load,
3777 => |sym_index| {
3778 const flags: u2 = switch (val) {
3779 .got_load => 0b00,
3780 .direct_load => 0b01,
3781 else => unreachable,
3782 };
3783 const addr_reg = (try self.register_manager.allocReg(null)).to64();
3784 _ = try self.addInst(.{
3785 .tag = .lea_pie,
3786 .ops = (Mir.Ops{
3787 .reg1 = addr_reg,
3788 .flags = flags,
3789 }).encode(),
3790 .data = .{
3791 .load_reloc = .{
3792 .atom_index = self.mod_fn.owner_decl.link.macho.local_sym_index,
3793 .sym_index = sym_index,
3794 },
3795 },
3796 });
3797 break :blk addr_reg;
3798 },
3799 .stack_offset => |off| {
3800 const addr_reg = (try self.register_manager.allocReg(null)).to64();
3801 _ = try self.addInst(.{
3802 .tag = .lea,
3803 .ops = (Mir.Ops{
3804 .reg1 = addr_reg,
3805 .reg2 = .rbp,
3806 }).encode(),
3807 .data = .{ .imm = @bitCast(u32, -off) },
3808 });
3809 break :blk addr_reg;
3810 },
3811 .register => |reg| {
3812 const addr_reg = try self.register_manager.allocReg(null);
3813 _ = try self.addInst(.{
3814 .tag = .mov,
3815 .ops = (Mir.Ops{
3816 .reg1 = registerAlias(addr_reg, @divExact(reg.size(), 8)),
3817 .reg2 = reg,
3818 }).encode(),
3819 .data = undefined,
3820 });
3821 break :blk addr_reg.to64();
3822 },
3823 else => {
3824 return self.fail("TODO implement memcpy for setting stack from {}", .{val});
3825 },
3826 }
3827 };
39443828
3945 // TODO allow for abi_size to be u643829 self.register_manager.freezeRegs(&.{addr_reg});
3946 try self.genSetReg(Type.u32, count_reg, .{ .immediate = @intCast(u32, abi_size) });3830 defer self.register_manager.unfreezeRegs(&.{addr_reg});
39473831
3948 return self.genInlineMemcpy(3832 const regs = try self.register_manager.allocRegs(2, .{ null, null });
3949 -stack_offset,3833 const count_reg = regs[0].to64();
3950 .rbp,3834 const tmp_reg = regs[1].to8();
3951 addr_reg.to64(),3835
3952 count_reg.to64(),3836 try self.genSetReg(Type.u32, count_reg, .{ .immediate = @intCast(u32, abi_size) });
3953 tmp_reg.to8(),
3954 );
3955 },
3956 }
3957}
39583837
3959fn genInlineMemcpy(
3960 self: *Self,
3961 stack_offset: i32,
3962 stack_reg: Register,
3963 addr_reg: Register,
3964 count_reg: Register,
3965 tmp_reg: Register,
3966) InnerError!void {
3967 // mov rcx, 03838 // mov rcx, 0
3968 _ = try self.addInst(.{3839 _ = try self.addInst(.{
3969 .tag = .mov,3840 .tag = .mov,
...@@ -4016,7 +3887,7 @@ fn genInlineMemcpy(...@@ -4016,7 +3887,7 @@ fn genInlineMemcpy(
4016 .reg1 = stack_reg,3887 .reg1 = stack_reg,
4017 .reg2 = tmp_reg.to8(),3888 .reg2 = tmp_reg.to8(),
4018 }).encode(),3889 }).encode(),
4019 .data = .{ .imm = @bitCast(u32, stack_offset) },3890 .data = .{ .imm = @bitCast(u32, -stack_offset) },
4020 });3891 });
40213892
4022 // add rcx, 13893 // add rcx, 1
...@@ -4959,8 +4830,8 @@ fn parseRegName(name: []const u8) ?Register {...@@ -4959,8 +4830,8 @@ fn parseRegName(name: []const u8) ?Register {
4959 return std.meta.stringToEnum(Register, name);4830 return std.meta.stringToEnum(Register, name);
4960}4831}
49614832
4833/// Returns register wide enough to hold at least `size_bytes`.
4962fn registerAlias(reg: Register, size_bytes: u32) Register {4834fn registerAlias(reg: Register, size_bytes: u32) Register {
4963 // For x86_64 we have to pick a smaller register alias depending on abi size.
4964 if (size_bytes == 0) {4835 if (size_bytes == 0) {
4965 unreachable; // should be comptime known4836 unreachable; // should be comptime known
4966 } else if (size_bytes <= 1) {4837 } else if (size_bytes <= 1) {