| ... | ... | @@ -1625,24 +1625,68 @@ fn airPtrArithmetic(self: *Self, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void |
| 1625 | 1625 | return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); |
| 1626 | 1626 | } |
| 1627 | 1627 | |
| 1628 | fn activeIntBits(self: *Self, dst_air: Air.Inst.Ref) u16 { |
| 1629 | const air_tag = self.air.instructions.items(.tag); |
| 1630 | const air_data = self.air.instructions.items(.data); |
| 1631 | |
| 1632 | const dst_ty = self.air.typeOf(dst_air); |
| 1633 | const dst_info = dst_ty.intInfo(self.target.*); |
| 1634 | if (Air.refToIndex(dst_air)) |inst| { |
| 1635 | switch (air_tag[inst]) { |
| 1636 | .constant => { |
| 1637 | const src_val = self.air.values[air_data[inst].ty_pl.payload]; |
| 1638 | var space: Value.BigIntSpace = undefined; |
| 1639 | const src_int = src_val.toBigInt(&space, self.target.*); |
| 1640 | return @intCast(u16, src_int.bitCountTwosComp()) + |
| 1641 | @boolToInt(src_int.positive and dst_info.signedness == .signed); |
| 1642 | }, |
| 1643 | .intcast => { |
| 1644 | const src_ty = self.air.typeOf(air_data[inst].ty_op.operand); |
| 1645 | const src_info = src_ty.intInfo(self.target.*); |
| 1646 | return @min(switch (src_info.signedness) { |
| 1647 | .signed => switch (dst_info.signedness) { |
| 1648 | .signed => src_info.bits, |
| 1649 | .unsigned => src_info.bits - 1, |
| 1650 | }, |
| 1651 | .unsigned => switch (dst_info.signedness) { |
| 1652 | .signed => src_info.bits + 1, |
| 1653 | .unsigned => src_info.bits, |
| 1654 | }, |
| 1655 | }, dst_info.bits); |
| 1656 | }, |
| 1657 | else => {}, |
| 1658 | } |
| 1659 | } |
| 1660 | return dst_info.bits; |
| 1661 | } |
| 1662 | |
| 1628 | 1663 | fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void { |
| 1629 | 1664 | const bin_op = self.air.instructions.items(.data)[inst].bin_op; |
| 1630 | | const result = result: { |
| 1631 | | if (self.liveness.isUnused(inst)) break :result .dead; |
| 1632 | | |
| 1665 | const result = if (self.liveness.isUnused(inst)) .dead else result: { |
| 1633 | 1666 | const tag = self.air.instructions.items(.tag)[inst]; |
| 1634 | | const ty = self.air.typeOfIndex(inst); |
| 1635 | | |
| 1636 | | if (ty.zigTypeTag() == .Float) { |
| 1667 | const dst_ty = self.air.typeOfIndex(inst); |
| 1668 | if (dst_ty.zigTypeTag() == .Float) |
| 1637 | 1669 | break :result try self.genBinOp(inst, tag, bin_op.lhs, bin_op.rhs); |
| 1638 | | } |
| 1639 | 1670 | |
| 1640 | | try self.spillRegisters(&.{ .rax, .rdx }); |
| 1671 | const dst_info = dst_ty.intInfo(self.target.*); |
| 1672 | var src_pl = Type.Payload.Bits{ .base = .{ .tag = switch (dst_info.signedness) { |
| 1673 | .signed => .int_signed, |
| 1674 | .unsigned => .int_unsigned, |
| 1675 | } }, .data = switch (tag) { |
| 1676 | else => unreachable, |
| 1677 | .mul, .mulwrap => std.math.max3( |
| 1678 | self.activeIntBits(bin_op.lhs), |
| 1679 | self.activeIntBits(bin_op.rhs), |
| 1680 | dst_info.bits / 2, |
| 1681 | ), |
| 1682 | .div_trunc, .div_floor, .div_exact, .rem, .mod => dst_info.bits, |
| 1683 | } }; |
| 1684 | const src_ty = Type.initPayload(&src_pl.base); |
| 1641 | 1685 | |
| 1686 | try self.spillRegisters(&.{ .rax, .rdx }); |
| 1642 | 1687 | const lhs = try self.resolveInst(bin_op.lhs); |
| 1643 | 1688 | const rhs = try self.resolveInst(bin_op.rhs); |
| 1644 | | |
| 1645 | | break :result try self.genMulDivBinOp(tag, inst, ty, lhs, rhs); |
| 1689 | break :result try self.genMulDivBinOp(tag, inst, dst_ty, src_ty, lhs, rhs); |
| 1646 | 1690 | }; |
| 1647 | 1691 | return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); |
| 1648 | 1692 | } |
| ... | ... | @@ -1795,7 +1839,7 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void { |
| 1795 | 1839 | break :cc .c; |
| 1796 | 1840 | }; |
| 1797 | 1841 | |
| 1798 | | const dst_mcv = try self.genMulDivBinOp(.mul, inst, ty, lhs_mcv, rhs_mcv); |
| 1842 | const dst_mcv = try self.genMulDivBinOp(.mul, inst, ty, ty, lhs_mcv, rhs_mcv); |
| 1799 | 1843 | const abi_size = @intCast(u32, @max(ty.abiSize(self.target.*), 2)); |
| 1800 | 1844 | try self.asmCmovccRegisterRegister( |
| 1801 | 1845 | registerAlias(dst_mcv.register, abi_size), |
| ... | ... | @@ -1809,9 +1853,9 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void { |
| 1809 | 1853 | |
| 1810 | 1854 | fn airAddSubShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void { |
| 1811 | 1855 | const ty_pl = self.air.instructions.items(.data)[inst].ty_pl; |
| 1812 | | const tag = self.air.instructions.items(.tag)[inst]; |
| 1813 | 1856 | const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data; |
| 1814 | | const result = if (self.liveness.isUnused(inst)) .dead else result: { |
| 1857 | const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: { |
| 1858 | const tag = self.air.instructions.items(.tag)[inst]; |
| 1815 | 1859 | const ty = self.air.typeOf(bin_op.lhs); |
| 1816 | 1860 | const abi_size = ty.abiSize(self.target.*); |
| 1817 | 1861 | switch (ty.zigTypeTag()) { |
| ... | ... | @@ -1842,21 +1886,17 @@ fn airAddSubShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void { |
| 1842 | 1886 | |
| 1843 | 1887 | const int_info = ty.intInfo(self.target.*); |
| 1844 | 1888 | |
| 1845 | | if (math.isPowerOfTwo(int_info.bits) and int_info.bits >= 8) { |
| 1889 | if (int_info.bits >= 8 and math.isPowerOfTwo(int_info.bits)) { |
| 1846 | 1890 | self.eflags_inst = inst; |
| 1847 | | |
| 1848 | | const cc: Condition = switch (int_info.signedness) { |
| 1849 | | .unsigned => .c, |
| 1850 | | .signed => .o, |
| 1851 | | }; |
| 1852 | | break :result MCValue{ .register_overflow = .{ |
| 1891 | break :result .{ .register_overflow = .{ |
| 1853 | 1892 | .reg = partial.register, |
| 1854 | | .eflags = cc, |
| 1893 | .eflags = switch (int_info.signedness) { |
| 1894 | .unsigned => .c, |
| 1895 | .signed => .o, |
| 1896 | }, |
| 1855 | 1897 | } }; |
| 1856 | 1898 | } |
| 1857 | 1899 | |
| 1858 | | self.eflags_inst = null; |
| 1859 | | |
| 1860 | 1900 | const tuple_ty = self.air.typeOfIndex(inst); |
| 1861 | 1901 | const tuple_size = @intCast(u32, tuple_ty.abiSize(self.target.*)); |
| 1862 | 1902 | const tuple_align = tuple_ty.abiAlignment(self.target.*); |
| ... | ... | @@ -1865,12 +1905,11 @@ fn airAddSubShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void { |
| 1865 | 1905 | |
| 1866 | 1906 | try self.genSetStackTruncatedOverflowCompare(ty, stack_offset, overflow_bit_offset, partial.register); |
| 1867 | 1907 | |
| 1868 | | break :result MCValue{ .stack_offset = stack_offset }; |
| 1908 | break :result .{ .stack_offset = stack_offset }; |
| 1869 | 1909 | }, |
| 1870 | 1910 | else => unreachable, |
| 1871 | 1911 | } |
| 1872 | 1912 | }; |
| 1873 | | |
| 1874 | 1913 | return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); |
| 1875 | 1914 | } |
| 1876 | 1915 | |
| ... | ... | @@ -1931,48 +1970,56 @@ fn genSetStackTruncatedOverflowCompare( |
| 1931 | 1970 | fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void { |
| 1932 | 1971 | const ty_pl = self.air.instructions.items(.data)[inst].ty_pl; |
| 1933 | 1972 | const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data; |
| 1934 | | |
| 1935 | | if (self.liveness.isUnused(inst)) { |
| 1936 | | return self.finishAir(inst, .dead, .{ bin_op.lhs, bin_op.rhs, .none }); |
| 1937 | | } |
| 1938 | | |
| 1939 | | const ty = self.air.typeOf(bin_op.lhs); |
| 1940 | | const abi_size = ty.abiSize(self.target.*); |
| 1941 | | const result: MCValue = result: { |
| 1942 | | switch (ty.zigTypeTag()) { |
| 1973 | const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: { |
| 1974 | const dst_ty = self.air.typeOf(bin_op.lhs); |
| 1975 | switch (dst_ty.zigTypeTag()) { |
| 1943 | 1976 | .Vector => return self.fail("TODO implement mul_with_overflow for Vector type", .{}), |
| 1944 | 1977 | .Int => { |
| 1945 | | if (abi_size > 8) { |
| 1946 | | return self.fail("TODO implement mul_with_overflow for Ints larger than 64bits", .{}); |
| 1947 | | } |
| 1948 | | |
| 1949 | | const int_info = ty.intInfo(self.target.*); |
| 1978 | try self.spillEflagsIfOccupied(); |
| 1950 | 1979 | |
| 1951 | | if (math.isPowerOfTwo(int_info.bits) and int_info.bits >= 8) { |
| 1952 | | try self.spillEflagsIfOccupied(); |
| 1953 | | self.eflags_inst = inst; |
| 1980 | const dst_info = dst_ty.intInfo(self.target.*); |
| 1981 | if (dst_info.bits >= 8 and math.isPowerOfTwo(dst_info.bits)) { |
| 1982 | var src_pl = Type.Payload.Bits{ .base = .{ .tag = switch (dst_info.signedness) { |
| 1983 | .signed => .int_signed, |
| 1984 | .unsigned => .int_unsigned, |
| 1985 | } }, .data = std.math.max3( |
| 1986 | self.activeIntBits(bin_op.lhs), |
| 1987 | self.activeIntBits(bin_op.rhs), |
| 1988 | dst_info.bits / 2, |
| 1989 | ) }; |
| 1990 | const src_ty = Type.initPayload(&src_pl.base); |
| 1954 | 1991 | |
| 1955 | 1992 | try self.spillRegisters(&.{ .rax, .rdx }); |
| 1956 | | |
| 1957 | 1993 | const lhs = try self.resolveInst(bin_op.lhs); |
| 1958 | 1994 | const rhs = try self.resolveInst(bin_op.rhs); |
| 1959 | 1995 | |
| 1960 | | const partial = try self.genMulDivBinOp(.mul, null, ty, lhs, rhs); |
| 1961 | | const cc: Condition = switch (int_info.signedness) { |
| 1996 | const partial = try self.genMulDivBinOp(.mul, null, dst_ty, src_ty, lhs, rhs); |
| 1997 | const cc: Condition = switch (dst_info.signedness) { |
| 1962 | 1998 | .unsigned => .c, |
| 1963 | 1999 | .signed => .o, |
| 1964 | 2000 | }; |
| 1965 | | break :result MCValue{ .register_overflow = .{ |
| 1966 | | .reg = partial.register, |
| 1967 | | .eflags = cc, |
| 1968 | | } }; |
| 1969 | | } |
| 2001 | switch (partial) { |
| 2002 | .register => |reg| { |
| 2003 | self.eflags_inst = inst; |
| 2004 | break :result .{ .register_overflow = .{ .reg = reg, .eflags = cc } }; |
| 2005 | }, |
| 2006 | else => {}, |
| 2007 | } |
| 1970 | 2008 | |
| 1971 | | try self.spillEflagsIfOccupied(); |
| 1972 | | self.eflags_inst = null; |
| 2009 | const dst_abi_size = @intCast(i32, dst_ty.abiSize(self.target.*)); |
| 2010 | const dst_mcv = try self.allocRegOrMem(inst, false); |
| 2011 | try self.genSetStack( |
| 2012 | Type.u1, |
| 2013 | dst_mcv.stack_offset - dst_abi_size, |
| 2014 | .{ .eflags = cc }, |
| 2015 | .{}, |
| 2016 | ); |
| 2017 | try self.genSetStack(dst_ty, dst_mcv.stack_offset, partial, .{}); |
| 2018 | break :result dst_mcv; |
| 2019 | } |
| 1973 | 2020 | |
| 1974 | 2021 | const dst_reg: Register = dst_reg: { |
| 1975 | | switch (int_info.signedness) { |
| 2022 | switch (dst_info.signedness) { |
| 1976 | 2023 | .signed => { |
| 1977 | 2024 | const lhs = try self.resolveInst(bin_op.lhs); |
| 1978 | 2025 | const rhs = try self.resolveInst(bin_op.rhs); |
| ... | ... | @@ -1985,14 +2032,14 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void { |
| 1985 | 2032 | |
| 1986 | 2033 | const dst_reg: Register = blk: { |
| 1987 | 2034 | if (lhs.isRegister()) break :blk lhs.register; |
| 1988 | | break :blk try self.copyToTmpRegister(ty, lhs); |
| 2035 | break :blk try self.copyToTmpRegister(dst_ty, lhs); |
| 1989 | 2036 | }; |
| 1990 | 2037 | const dst_reg_lock = self.register_manager.lockRegAssumeUnused(dst_reg); |
| 1991 | 2038 | defer self.register_manager.unlockReg(dst_reg_lock); |
| 1992 | 2039 | |
| 1993 | 2040 | const rhs_mcv: MCValue = blk: { |
| 1994 | 2041 | if (rhs.isRegister() or rhs.isMemory()) break :blk rhs; |
| 1995 | | break :blk MCValue{ .register = try self.copyToTmpRegister(ty, rhs) }; |
| 2042 | break :blk MCValue{ .register = try self.copyToTmpRegister(dst_ty, rhs) }; |
| 1996 | 2043 | }; |
| 1997 | 2044 | const rhs_mcv_lock: ?RegisterLock = switch (rhs_mcv) { |
| 1998 | 2045 | .register => |reg| self.register_manager.lockReg(reg), |
| ... | ... | @@ -2010,7 +2057,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void { |
| 2010 | 2057 | const lhs = try self.resolveInst(bin_op.lhs); |
| 2011 | 2058 | const rhs = try self.resolveInst(bin_op.rhs); |
| 2012 | 2059 | |
| 2013 | | const dst_mcv = try self.genMulDivBinOp(.mul, null, ty, lhs, rhs); |
| 2060 | const dst_mcv = try self.genMulDivBinOp(.mul, null, dst_ty, dst_ty, lhs, rhs); |
| 2014 | 2061 | break :dst_reg dst_mcv.register; |
| 2015 | 2062 | }, |
| 2016 | 2063 | } |
| ... | ... | @@ -2022,14 +2069,13 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void { |
| 2022 | 2069 | const overflow_bit_offset = @intCast(i32, tuple_ty.structFieldOffset(1, self.target.*)); |
| 2023 | 2070 | const stack_offset = @intCast(i32, try self.allocMem(inst, tuple_size, tuple_align)); |
| 2024 | 2071 | |
| 2025 | | try self.genSetStackTruncatedOverflowCompare(ty, stack_offset, overflow_bit_offset, dst_reg); |
| 2072 | try self.genSetStackTruncatedOverflowCompare(dst_ty, stack_offset, overflow_bit_offset, dst_reg); |
| 2026 | 2073 | |
| 2027 | | break :result MCValue{ .stack_offset = stack_offset }; |
| 2074 | break :result .{ .stack_offset = stack_offset }; |
| 2028 | 2075 | }, |
| 2029 | 2076 | else => unreachable, |
| 2030 | 2077 | } |
| 2031 | 2078 | }; |
| 2032 | | |
| 2033 | 2079 | return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); |
| 2034 | 2080 | } |
| 2035 | 2081 | |
| ... | ... | @@ -2040,7 +2086,6 @@ fn genIntMulDivOpMir( |
| 2040 | 2086 | self: *Self, |
| 2041 | 2087 | tag: Mir.Inst.Tag, |
| 2042 | 2088 | ty: Type, |
| 2043 | | signedness: std.builtin.Signedness, |
| 2044 | 2089 | lhs: MCValue, |
| 2045 | 2090 | rhs: MCValue, |
| 2046 | 2091 | ) !void { |
| ... | ... | @@ -2057,26 +2102,23 @@ fn genIntMulDivOpMir( |
| 2057 | 2102 | try self.genSetReg(ty, .rax, lhs); |
| 2058 | 2103 | } |
| 2059 | 2104 | |
| 2060 | | switch (signedness) { |
| 2061 | | .signed => try self.asmOpOnly(.cqo), |
| 2062 | | .unsigned => try self.asmRegisterRegister(.xor, .rdx, .rdx), |
| 2105 | switch (tag) { |
| 2106 | else => unreachable, |
| 2107 | .mul, .imul => {}, |
| 2108 | .div => try self.asmRegisterRegister(.xor, .edx, .edx), |
| 2109 | .idiv => try self.asmOpOnly(.cqo), |
| 2063 | 2110 | } |
| 2064 | 2111 | |
| 2065 | | const factor = switch (rhs) { |
| 2066 | | .register => rhs, |
| 2067 | | .stack_offset => rhs, |
| 2068 | | else => blk: { |
| 2069 | | const reg = try self.copyToTmpRegister(ty, rhs); |
| 2070 | | break :blk MCValue{ .register = reg }; |
| 2071 | | }, |
| 2112 | const factor: MCValue = switch (rhs) { |
| 2113 | .register, .stack_offset => rhs, |
| 2114 | else => .{ .register = try self.copyToTmpRegister(ty, rhs) }, |
| 2072 | 2115 | }; |
| 2073 | | |
| 2074 | 2116 | switch (factor) { |
| 2075 | 2117 | .register => |reg| try self.asmRegister(tag, reg), |
| 2076 | | .stack_offset => |off| try self.asmMemory(tag, Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ |
| 2077 | | .base = .rbp, |
| 2078 | | .disp = -off, |
| 2079 | | })), |
| 2118 | .stack_offset => |off| try self.asmMemory(tag, Memory.sib( |
| 2119 | Memory.PtrSize.fromSize(abi_size), |
| 2120 | .{ .base = .rbp, .disp = -off }, |
| 2121 | )), |
| 2080 | 2122 | else => unreachable, |
| 2081 | 2123 | } |
| 2082 | 2124 | } |
| ... | ... | @@ -2102,7 +2144,7 @@ fn genInlineIntDivFloor(self: *Self, ty: Type, lhs: MCValue, rhs: MCValue) !MCVa |
| 2102 | 2144 | try self.genIntMulDivOpMir(switch (signedness) { |
| 2103 | 2145 | .signed => .idiv, |
| 2104 | 2146 | .unsigned => .div, |
| 2105 | | }, Type.isize, signedness, .{ .register = dividend }, .{ .register = divisor }); |
| 2147 | }, Type.isize, .{ .register = dividend }, .{ .register = divisor }); |
| 2106 | 2148 | |
| 2107 | 2149 | try self.asmRegisterRegister(.xor, divisor.to64(), dividend.to64()); |
| 2108 | 2150 | try self.asmRegisterImmediate(.sar, divisor.to64(), Immediate.u(63)); |
| ... | ... | @@ -3938,14 +3980,14 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void { |
| 3938 | 3980 | .register_overflow => |ro| { |
| 3939 | 3981 | switch (index) { |
| 3940 | 3982 | // Get wrapped value for overflow operation. |
| 3941 | | 0 => if (self.liveness.operandDies(inst, 0)) { |
| 3942 | | self.eflags_inst = null; |
| 3943 | | break :result .{ .register = ro.reg }; |
| 3944 | | } else break :result try self.copyToRegisterWithInstTracking( |
| 3945 | | inst, |
| 3946 | | Type.usize, |
| 3947 | | .{ .register = ro.reg }, |
| 3948 | | ), |
| 3983 | 0 => break :result if (self.liveness.operandDies(inst, 0)) |
| 3984 | .{ .register = ro.reg } |
| 3985 | else |
| 3986 | try self.copyToRegisterWithInstTracking( |
| 3987 | inst, |
| 3988 | Type.usize, |
| 3989 | .{ .register = ro.reg }, |
| 3990 | ), |
| 3949 | 3991 | // Get overflow bit. |
| 3950 | 3992 | 1 => if (self.liveness.operandDies(inst, 0)) { |
| 3951 | 3993 | self.eflags_inst = inst; |
| ... | ... | @@ -4318,20 +4360,26 @@ fn genMulDivBinOp( |
| 4318 | 4360 | self: *Self, |
| 4319 | 4361 | tag: Air.Inst.Tag, |
| 4320 | 4362 | maybe_inst: ?Air.Inst.Index, |
| 4321 | | ty: Type, |
| 4363 | dst_ty: Type, |
| 4364 | src_ty: Type, |
| 4322 | 4365 | lhs: MCValue, |
| 4323 | 4366 | rhs: MCValue, |
| 4324 | 4367 | ) !MCValue { |
| 4325 | | if (ty.zigTypeTag() == .Vector or ty.zigTypeTag() == .Float) { |
| 4326 | | return self.fail("TODO implement genMulDivBinOp for {}", .{ty.fmtDebug()}); |
| 4327 | | } |
| 4328 | | const abi_size = @intCast(u32, ty.abiSize(self.target.*)); |
| 4329 | | if (abi_size > 8) { |
| 4330 | | return self.fail("TODO implement genMulDivBinOp for {}", .{ty.fmtDebug()}); |
| 4331 | | } |
| 4332 | | if (tag == .div_float) { |
| 4333 | | return self.fail("TODO implement genMulDivBinOp for div_float", .{}); |
| 4368 | if (dst_ty.zigTypeTag() == .Vector or dst_ty.zigTypeTag() == .Float) { |
| 4369 | return self.fail("TODO implement genMulDivBinOp for {}", .{dst_ty.fmtDebug()}); |
| 4334 | 4370 | } |
| 4371 | const dst_abi_size = @intCast(u32, dst_ty.abiSize(self.target.*)); |
| 4372 | const src_abi_size = @intCast(u32, src_ty.abiSize(self.target.*)); |
| 4373 | if (switch (tag) { |
| 4374 | else => unreachable, |
| 4375 | .mul, .mulwrap => dst_abi_size != src_abi_size and dst_abi_size != src_abi_size * 2, |
| 4376 | .div_trunc, .div_floor, .div_exact, .rem, .mod => dst_abi_size != src_abi_size, |
| 4377 | } or src_abi_size > 8) return self.fail("TODO implement genMulDivBinOp from {} to {}", .{ |
| 4378 | src_ty.fmt(self.bin_file.options.module.?), |
| 4379 | dst_ty.fmt(self.bin_file.options.module.?), |
| 4380 | }); |
| 4381 | const ty = if (dst_abi_size <= 8) dst_ty else src_ty; |
| 4382 | const abi_size = if (dst_abi_size <= 8) dst_abi_size else src_abi_size; |
| 4335 | 4383 | |
| 4336 | 4384 | assert(self.register_manager.isRegFree(.rax)); |
| 4337 | 4385 | assert(self.register_manager.isRegFree(.rdx)); |
| ... | ... | @@ -4339,9 +4387,7 @@ fn genMulDivBinOp( |
| 4339 | 4387 | const reg_locks = self.register_manager.lockRegs(2, .{ .rax, .rdx }); |
| 4340 | 4388 | defer for (reg_locks) |reg_lock| if (reg_lock) |lock| self.register_manager.unlockReg(lock); |
| 4341 | 4389 | |
| 4342 | | const int_info = ty.intInfo(self.target.*); |
| 4343 | | const signedness = int_info.signedness; |
| 4344 | | |
| 4390 | const signedness = ty.intInfo(self.target.*).signedness; |
| 4345 | 4391 | switch (tag) { |
| 4346 | 4392 | .mul, |
| 4347 | 4393 | .mulwrap, |
| ... | ... | @@ -4349,11 +4395,12 @@ fn genMulDivBinOp( |
| 4349 | 4395 | .div_trunc, |
| 4350 | 4396 | .div_exact, |
| 4351 | 4397 | => { |
| 4352 | | const track_inst_rax: ?Air.Inst.Index = switch (tag) { |
| 4353 | | .mul, .mulwrap, .div_exact, .div_trunc => maybe_inst, |
| 4398 | const track_inst_rax = switch (tag) { |
| 4399 | .mul, .mulwrap => if (dst_abi_size <= 8) maybe_inst else null, |
| 4400 | .div_exact, .div_trunc => maybe_inst, |
| 4354 | 4401 | else => null, |
| 4355 | 4402 | }; |
| 4356 | | const track_inst_rdx: ?Air.Inst.Index = switch (tag) { |
| 4403 | const track_inst_rdx = switch (tag) { |
| 4357 | 4404 | .rem => maybe_inst, |
| 4358 | 4405 | else => null, |
| 4359 | 4406 | }; |
| ... | ... | @@ -4373,13 +4420,24 @@ fn genMulDivBinOp( |
| 4373 | 4420 | }, |
| 4374 | 4421 | }; |
| 4375 | 4422 | |
| 4376 | | try self.genIntMulDivOpMir(mir_tag, ty, .signed, lhs, rhs); |
| 4423 | try self.genIntMulDivOpMir(mir_tag, ty, lhs, rhs); |
| 4377 | 4424 | |
| 4378 | | return .{ .register = registerAlias(switch (tag) { |
| 4425 | if (dst_abi_size <= 8) return .{ .register = registerAlias(switch (tag) { |
| 4379 | 4426 | .mul, .mulwrap, .div_trunc, .div_exact => .rax, |
| 4380 | 4427 | .rem => .rdx, |
| 4381 | 4428 | else => unreachable, |
| 4382 | | }, abi_size) }; |
| 4429 | }, dst_abi_size) }; |
| 4430 | |
| 4431 | const dst_mcv = try self.allocRegOrMemAdvanced(dst_ty, maybe_inst, false); |
| 4432 | try self.asmMemoryRegister(.mov, Memory.sib(.qword, .{ |
| 4433 | .base = .rbp, |
| 4434 | .disp = 0 - dst_mcv.stack_offset, |
| 4435 | }), .rax); |
| 4436 | try self.asmMemoryRegister(.mov, Memory.sib(.qword, .{ |
| 4437 | .base = .rbp, |
| 4438 | .disp = 8 - dst_mcv.stack_offset, |
| 4439 | }), .rdx); |
| 4440 | return dst_mcv; |
| 4383 | 4441 | }, |
| 4384 | 4442 | |
| 4385 | 4443 | .mod => { |
| ... | ... | @@ -4402,7 +4460,7 @@ fn genMulDivBinOp( |
| 4402 | 4460 | return result; |
| 4403 | 4461 | }, |
| 4404 | 4462 | .unsigned => { |
| 4405 | | try self.genIntMulDivOpMir(.div, ty, .unsigned, lhs, rhs); |
| 4463 | try self.genIntMulDivOpMir(.div, ty, lhs, rhs); |
| 4406 | 4464 | return .{ .register = registerAlias(.rdx, abi_size) }; |
| 4407 | 4465 | }, |
| 4408 | 4466 | } |
| ... | ... | @@ -4441,18 +4499,13 @@ fn genMulDivBinOp( |
| 4441 | 4499 | }; |
| 4442 | 4500 | defer if (rhs_lock) |lock| self.register_manager.unlockReg(lock); |
| 4443 | 4501 | |
| 4444 | | const result: MCValue = result: { |
| 4445 | | switch (signedness) { |
| 4446 | | .signed => break :result try self.genInlineIntDivFloor(ty, lhs, actual_rhs), |
| 4447 | | .unsigned => { |
| 4448 | | try self.genIntMulDivOpMir(.div, ty, .unsigned, lhs, actual_rhs); |
| 4449 | | break :result MCValue{ |
| 4450 | | .register = registerAlias(.rax, @intCast(u32, ty.abiSize(self.target.*))), |
| 4451 | | }; |
| 4452 | | }, |
| 4453 | | } |
| 4454 | | }; |
| 4455 | | return result; |
| 4502 | switch (signedness) { |
| 4503 | .signed => return try self.genInlineIntDivFloor(ty, lhs, actual_rhs), |
| 4504 | .unsigned => { |
| 4505 | try self.genIntMulDivOpMir(.div, ty, lhs, actual_rhs); |
| 4506 | return .{ .register = registerAlias(.rax, abi_size) }; |
| 4507 | }, |
| 4508 | } |
| 4456 | 4509 | }, |
| 4457 | 4510 | |
| 4458 | 4511 | else => unreachable, |
| ... | ... | @@ -6718,6 +6771,7 @@ fn genSetStackArg(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue) InnerE |
| 6718 | 6771 | } |
| 6719 | 6772 | |
| 6720 | 6773 | fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: InlineMemcpyOpts) InnerError!void { |
| 6774 | const base_reg = opts.dest_stack_base orelse .rbp; |
| 6721 | 6775 | const abi_size = @intCast(u32, ty.abiSize(self.target.*)); |
| 6722 | 6776 | switch (mcv) { |
| 6723 | 6777 | .dead => unreachable, |
| ... | ... | @@ -6733,12 +6787,17 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl |
| 6733 | 6787 | 4 => 0xaaaaaaaa, |
| 6734 | 6788 | else => unreachable, |
| 6735 | 6789 | }; |
| 6736 | | return self.asmMemoryImmediate(.mov, Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ |
| 6737 | | .base = opts.dest_stack_base orelse .rbp, |
| 6738 | | .disp = -stack_offset, |
| 6739 | | }), Immediate.u(value)); |
| 6790 | return self.asmMemoryImmediate(.mov, Memory.sib( |
| 6791 | Memory.PtrSize.fromSize(abi_size), |
| 6792 | .{ .base = base_reg, .disp = -stack_offset }, |
| 6793 | ), Immediate.u(value)); |
| 6740 | 6794 | }, |
| 6741 | | 8 => return self.genSetStack(ty, stack_offset, .{ .immediate = 0xaaaaaaaaaaaaaaaa }, opts), |
| 6795 | 8 => return self.genSetStack( |
| 6796 | ty, |
| 6797 | stack_offset, |
| 6798 | .{ .immediate = 0xaaaaaaaaaaaaaaaa }, |
| 6799 | opts, |
| 6800 | ), |
| 6742 | 6801 | else => |x| return self.genInlineMemset( |
| 6743 | 6802 | .{ .stack_offset = stack_offset }, |
| 6744 | 6803 | .{ .immediate = 0xaa }, |
| ... | ... | @@ -6766,12 +6825,11 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl |
| 6766 | 6825 | .{}, |
| 6767 | 6826 | ); |
| 6768 | 6827 | }, |
| 6769 | | .eflags => { |
| 6770 | | const reg = try self.copyToTmpRegister(ty, mcv); |
| 6771 | | return self.genSetStack(ty, stack_offset, .{ .register = reg }, opts); |
| 6772 | | }, |
| 6773 | | .immediate => |x_big| { |
| 6774 | | const base_reg = opts.dest_stack_base orelse .rbp; |
| 6828 | .eflags => |cc| try self.asmSetccMemory( |
| 6829 | Memory.sib(.byte, .{ .base = base_reg, .disp = -stack_offset }), |
| 6830 | cc, |
| 6831 | ), |
| 6832 | .immediate => |imm| { |
| 6775 | 6833 | // TODO |
| 6776 | 6834 | switch (abi_size) { |
| 6777 | 6835 | 0 => { |
| ... | ... | @@ -6779,13 +6837,13 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl |
| 6779 | 6837 | try self.asmMemoryImmediate(.mov, Memory.sib(.byte, .{ |
| 6780 | 6838 | .base = base_reg, |
| 6781 | 6839 | .disp = -stack_offset, |
| 6782 | | }), Immediate.u(@truncate(u8, x_big))); |
| 6840 | }), Immediate.u(@truncate(u8, imm))); |
| 6783 | 6841 | }, |
| 6784 | 6842 | 1, 2, 4 => { |
| 6785 | 6843 | const immediate = if (ty.isSignedInt()) |
| 6786 | | Immediate.s(@truncate(i32, @bitCast(i64, x_big))) |
| 6844 | Immediate.s(@truncate(i32, @bitCast(i64, imm))) |
| 6787 | 6845 | else |
| 6788 | | Immediate.u(@intCast(u32, x_big)); |
| 6846 | Immediate.u(@intCast(u32, imm)); |
| 6789 | 6847 | try self.asmMemoryImmediate(.mov, Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ |
| 6790 | 6848 | .base = base_reg, |
| 6791 | 6849 | .disp = -stack_offset, |
| ... | ... | @@ -6795,27 +6853,32 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl |
| 6795 | 6853 | else => { |
| 6796 | 6854 | // 64 bit write to memory would take two mov's anyways so we |
| 6797 | 6855 | // insted just use two 32 bit writes to avoid register allocation |
| 6798 | | var offset: i32 = 0; |
| 6799 | | while (offset < abi_size) : (offset += 4) try self.asmMemoryImmediate( |
| 6800 | | .mov, |
| 6801 | | Memory.sib(.dword, .{ .base = base_reg, .disp = offset - stack_offset }), |
| 6802 | | if (ty.isSignedInt()) |
| 6803 | | Immediate.s(@truncate( |
| 6804 | | i32, |
| 6805 | | @bitCast(i64, x_big) >> (math.cast(u6, offset * 8) orelse 63), |
| 6806 | | )) |
| 6807 | | else |
| 6808 | | Immediate.u(@truncate( |
| 6809 | | u32, |
| 6810 | | if (math.cast(u6, offset * 8)) |shift| x_big >> shift else 0, |
| 6811 | | )), |
| 6812 | | ); |
| 6856 | if (std.math.cast(i32, @bitCast(i64, imm))) |small| { |
| 6857 | try self.asmMemoryImmediate(.mov, Memory.sib( |
| 6858 | Memory.PtrSize.fromSize(abi_size), |
| 6859 | .{ .base = base_reg, .disp = -stack_offset }, |
| 6860 | ), Immediate.s(small)); |
| 6861 | } else { |
| 6862 | var offset: i32 = 0; |
| 6863 | while (offset < abi_size) : (offset += 4) try self.asmMemoryImmediate( |
| 6864 | .mov, |
| 6865 | Memory.sib(.dword, .{ .base = base_reg, .disp = offset - stack_offset }), |
| 6866 | if (ty.isSignedInt()) |
| 6867 | Immediate.s(@truncate( |
| 6868 | i32, |
| 6869 | @bitCast(i64, imm) >> (math.cast(u6, offset * 8) orelse 63), |
| 6870 | )) |
| 6871 | else |
| 6872 | Immediate.u(@truncate( |
| 6873 | u32, |
| 6874 | if (math.cast(u6, offset * 8)) |shift| imm >> shift else 0, |
| 6875 | )), |
| 6876 | ); |
| 6877 | } |
| 6813 | 6878 | }, |
| 6814 | 6879 | } |
| 6815 | 6880 | }, |
| 6816 | 6881 | .register => |reg| { |
| 6817 | | const base_reg = opts.dest_stack_base orelse .rbp; |
| 6818 | | |
| 6819 | 6882 | switch (ty.zigTypeTag()) { |
| 6820 | 6883 | .Float => { |
| 6821 | 6884 | if (intrinsicsAllowed(self.target.*, ty)) { |
Jacob Young <jacobly@ziglang.org> 2023-04-02 04:34:14-04:00
src