authorgravatar for 81774659+gracefuu@users.noreply.github.comgracefu <81774659+gracefuu@users.noreply.github.com> 2021-04-09 13:43:42+08:00
committergravatar for 81774659+gracefuu@users.noreply.github.comgracefu <81774659+gracefuu@users.noreply.github.com> 2021-04-16 15:21:17+08:00
log36df1526da0e703a9f3d5bd6c8775d3f0e0f0a33
tree04d8a15225424eaf01e885220c5c9d34043de16c
parente1959ccd4e74612d792f098dfbe7c0ae31813653
signaturelock-open Commit is signed but in an unrecognized format.

stage2 x86_64: refactor codegen to use inst encoder

There are parts of it that I didn't modify because the byte representation was important (e.g. we need to know what the exact byte position where we store the address into the offset table is)

1 files changed, 332 insertions(+), 158 deletions(-)

src/codegen.zig+332-158
...@@ -1034,7 +1034,7 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {...@@ -1034,7 +1034,7 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
1034 },1034 },
1035 .val = Value.initTag(.bool_true),1035 .val = Value.initTag(.bool_true),
1036 };1036 };
1037 return try self.genX8664BinMath(&inst.base, inst.operand, &imm.base, 6, 0x30);1037 return try self.genX8664BinMath(&inst.base, inst.operand, &imm.base);
1038 },1038 },
1039 .arm, .armeb => {1039 .arm, .armeb => {
1040 var imm = ir.Inst.Constant{1040 var imm = ir.Inst.Constant{
...@@ -1058,7 +1058,7 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {...@@ -1058,7 +1058,7 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
1058 return MCValue.dead;1058 return MCValue.dead;
1059 switch (arch) {1059 switch (arch) {
1060 .x86_64 => {1060 .x86_64 => {
1061 return try self.genX8664BinMath(&inst.base, inst.lhs, inst.rhs, 0, 0x00);1061 return try self.genX8664BinMath(&inst.base, inst.lhs, inst.rhs);
1062 },1062 },
1063 .arm, .armeb => return try self.genArmBinOp(&inst.base, inst.lhs, inst.rhs, .add),1063 .arm, .armeb => return try self.genArmBinOp(&inst.base, inst.lhs, inst.rhs, .add),
1064 else => return self.fail(inst.base.src, "TODO implement add for {}", .{self.target.cpu.arch}),1064 else => return self.fail(inst.base.src, "TODO implement add for {}", .{self.target.cpu.arch}),
...@@ -1352,7 +1352,7 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {...@@ -1352,7 +1352,7 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
1352 return MCValue.dead;1352 return MCValue.dead;
1353 switch (arch) {1353 switch (arch) {
1354 .x86_64 => {1354 .x86_64 => {
1355 return try self.genX8664BinMath(&inst.base, inst.lhs, inst.rhs, 5, 0x28);1355 return try self.genX8664BinMath(&inst.base, inst.lhs, inst.rhs);
1356 },1356 },
1357 .arm, .armeb => return try self.genArmBinOp(&inst.base, inst.lhs, inst.rhs, .sub),1357 .arm, .armeb => return try self.genArmBinOp(&inst.base, inst.lhs, inst.rhs, .sub),
1358 else => return self.fail(inst.base.src, "TODO implement sub for {}", .{self.target.cpu.arch}),1358 else => return self.fail(inst.base.src, "TODO implement sub for {}", .{self.target.cpu.arch}),
...@@ -1497,8 +1497,14 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {...@@ -1497,8 +1497,14 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
1497 return dst_mcv;1497 return dst_mcv;
1498 }1498 }
14991499
1500 /// Perform "binary" operators, excluding comparisons.
1501 /// Currently, the following ops are supported:
1500 /// ADD, SUB, XOR, OR, AND1502 /// ADD, SUB, XOR, OR, AND
1501 fn genX8664BinMath(self: *Self, inst: *ir.Inst, op_lhs: *ir.Inst, op_rhs: *ir.Inst, opx: u8, mr: u8) !MCValue {1503 fn genX8664BinMath(self: *Self, inst: *ir.Inst, op_lhs: *ir.Inst, op_rhs: *ir.Inst) !MCValue {
1504 // We'll handle these ops in two steps.
1505 // 1) Prepare an output register, and put one of the arguments in it
1506 // 2) Perform the op with the other argument
1507
1502 try self.code.ensureCapacity(self.code.items.len + 8);1508 try self.code.ensureCapacity(self.code.items.len + 8);
15031509
1504 const lhs = try self.resolveInst(op_lhs);1510 const lhs = try self.resolveInst(op_lhs);
...@@ -1559,18 +1565,108 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {...@@ -1559,18 +1565,108 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
1559 else => {},1565 else => {},
1560 }1566 }
15611567
1562 try self.genX8664BinMathCode(inst.src, inst.ty, dst_mcv, src_mcv, opx, mr);1568 // Now for step 2, we perform the actual op
1569 switch (inst.tag) {
1570 // TODO: Generate wrapping and non-wrapping versions separately
1571 .add, .addwrap => try self.genX8664BinMathCode(inst.src, inst.ty, dst_mcv, src_mcv, 0, 0x00),
1572 .bool_or, .bit_or => try self.genX8664BinMathCode(inst.src, inst.ty, dst_mcv, src_mcv, 1, 0x08),
1573 .bool_and, .bit_and => try self.genX8664BinMathCode(inst.src, inst.ty, dst_mcv, src_mcv, 4, 0x20),
1574 .sub, .subwrap => try self.genX8664BinMathCode(inst.src, inst.ty, dst_mcv, src_mcv, 5, 0x28),
1575 .xor, .not => try self.genX8664BinMathCode(inst.src, inst.ty, dst_mcv, src_mcv, 6, 0x30),
1576
1577 else => unreachable,
1578 }
15631579
1564 return dst_mcv;1580 return dst_mcv;
1565 }1581 }
15661582
1583 /// Wrap over Instruction.encodeInto to translate errors
1584 fn encodeX8664Instruction(
1585 self: *Self,
1586 src: LazySrcLoc,
1587 inst: Instruction,
1588 ) !void {
1589 inst.encodeInto(self.code) catch |err| {
1590 if (err == error.OutOfMemory)
1591 return error.OutOfMemory
1592 else
1593 return self.fail(src, "Instruction.encodeInto failed because {s}", .{@errorName(err)});
1594 };
1595 }
1596
1597 /// This function encodes a binary operation for x86_64
1598 /// intended for use with the following opcode ranges
1599 /// because they share the same structure.
1600 ///
1601 /// Thus not all binary operations can be used here
1602 /// -- multiplication needs to be done with imul,
1603 /// which doesn't have as convenient an interface.
1604 ///
1605 /// "opx"-style instructions use the opcode extension field to indicate which instruction to execute:
1606 ///
1607 /// opx = /0: add
1608 /// opx = /1: or
1609 /// opx = /2: adc
1610 /// opx = /3: sbb
1611 /// opx = /4: and
1612 /// opx = /5: sub
1613 /// opx = /6: xor
1614 /// opx = /7: cmp
1615 ///
1616 /// opcode | operand shape
1617 /// --------+----------------------
1618 /// 80 /opx | r/m8, imm8
1619 /// 81 /opx | r/m16/32/64, imm16/32
1620 /// 83 /opx | r/m16/32/64, imm8
1621 ///
1622 /// "mr"-style instructions use the low bits of opcode to indicate shape of instruction:
1623 ///
1624 /// mr = 00: add
1625 /// mr = 08: or
1626 /// mr = 10: adc
1627 /// mr = 18: sbb
1628 /// mr = 20: and
1629 /// mr = 28: sub
1630 /// mr = 30: xor
1631 /// mr = 38: cmp
1632 ///
1633 /// opcode | operand shape
1634 /// -------+-------------------------
1635 /// mr + 0 | r/m8, r8
1636 /// mr + 1 | r/m16/32/64, r16/32/64
1637 /// mr + 2 | r8, r/m8
1638 /// mr + 3 | r16/32/64, r/m16/32/64
1639 /// mr + 4 | AL, imm8
1640 /// mr + 5 | rAX, imm16/32
1641 ///
1642 /// TODO: rotates and shifts share the same structure, so we can potentially implement them
1643 /// at a later date with very similar code.
1644 /// They have "opx"-style instructions, but no "mr"-style instructions.
1645 ///
1646 /// opx = /0: rol,
1647 /// opx = /1: ror,
1648 /// opx = /2: rcl,
1649 /// opx = /3: rcr,
1650 /// opx = /4: shl sal,
1651 /// opx = /5: shr,
1652 /// opx = /6: sal shl,
1653 /// opx = /7: sar,
1654 ///
1655 /// opcode | operand shape
1656 /// --------+------------------
1657 /// c0 /opx | r/m8, imm8
1658 /// c1 /opx | r/m16/32/64, imm8
1659 /// d0 /opx | r/m8, 1
1660 /// d1 /opx | r/m16/32/64, 1
1661 /// d2 /opx | r/m8, CL (for context, CL is register 1)
1662 /// d3 /opx | r/m16/32/64, CL (for context, CL is register 1)
1567 fn genX8664BinMathCode(1663 fn genX8664BinMathCode(
1568 self: *Self,1664 self: *Self,
1569 src: LazySrcLoc,1665 src: LazySrcLoc,
1570 dst_ty: Type,1666 dst_ty: Type,
1571 dst_mcv: MCValue,1667 dst_mcv: MCValue,
1572 src_mcv: MCValue,1668 src_mcv: MCValue,
1573 opx: u8,1669 opx: u3,
1574 mr: u8,1670 mr: u8,
1575 ) !void {1671 ) !void {
1576 switch (dst_mcv) {1672 switch (dst_mcv) {
...@@ -1589,31 +1685,78 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {...@@ -1589,31 +1685,78 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
1589 .ptr_stack_offset => unreachable,1685 .ptr_stack_offset => unreachable,
1590 .ptr_embedded_in_code => unreachable,1686 .ptr_embedded_in_code => unreachable,
1591 .register => |src_reg| {1687 .register => |src_reg| {
1592 self.rex(.{ .b = dst_reg.isExtended(), .r = src_reg.isExtended(), .w = dst_reg.size() == 64 });1688 // register, register use mr + 1 addressing mode: r/m16/32/64, r16/32/64
1593 self.code.appendSliceAssumeCapacity(&[_]u8{ mr + 0x1, 0xC0 | (@as(u8, src_reg.id() & 0b111) << 3) | @as(u8, dst_reg.id() & 0b111) });1689 try self.encodeX8664Instruction(src, Instruction{
1690 .operand_size_64 = dst_reg.size() == 64,
1691 .primary_opcode_1b = mr + 1,
1692 // TODO: Explicit optional wrap due to stage 1 miscompilation :(
1693 // https://github.com/ziglang/zig/issues/6515
1694 .modrm = @as(
1695 ?Instruction.ModrmEffectiveAddress,
1696 Instruction.ModrmEffectiveAddress{ .reg = dst_reg },
1697 ),
1698 .reg = src_reg,
1699 });
1594 },1700 },
1595 .immediate => |imm| {1701 .immediate => |imm| {
1702 // register, immediate use opx = 81 or 83 addressing modes:
1703 // opx = 81: r/m16/32/64, imm16/32
1704 // opx = 83: r/m16/32/64, imm8
1596 const imm32 = @intCast(u31, imm); // This case must be handled before calling genX8664BinMathCode.1705 const imm32 = @intCast(u31, imm); // This case must be handled before calling genX8664BinMathCode.
1597 // 81 /opx id
1598 if (imm32 <= math.maxInt(u7)) {1706 if (imm32 <= math.maxInt(u7)) {
1599 self.rex(.{ .b = dst_reg.isExtended(), .w = dst_reg.size() == 64 });1707 try self.encodeX8664Instruction(src, Instruction{
1600 self.code.appendSliceAssumeCapacity(&[_]u8{1708 .operand_size_64 = dst_reg.size() == 64,
1601 0x83,1709 .primary_opcode_1b = 0x83,
1602 0xC0 | (opx << 3) | @truncate(u3, dst_reg.id()),1710 .opcode_extension = opx,
1603 @intCast(u8, imm32),1711 // TODO: Explicit optional wrap due to stage 1 miscompilation :(
1712 // https://github.com/ziglang/zig/issues/6515
1713 .modrm = @as(
1714 ?Instruction.ModrmEffectiveAddress,
1715 Instruction.ModrmEffectiveAddress{ .reg = dst_reg },
1716 ),
1717 .immediate_bytes = 1,
1718 .immediate = imm32,
1604 });1719 });
1605 } else {1720 } else {
1606 self.rex(.{ .r = dst_reg.isExtended(), .w = dst_reg.size() == 64 });1721 try self.encodeX8664Instruction(src, Instruction{
1607 self.code.appendSliceAssumeCapacity(&[_]u8{1722 .operand_size_64 = dst_reg.size() == 64,
1608 0x81,1723 .primary_opcode_1b = 0x81,
1609 0xC0 | (opx << 3) | @truncate(u3, dst_reg.id()),1724 .opcode_extension = opx,
1725 // TODO: Explicit optional wrap due to stage 1 miscompilation :(
1726 // https://github.com/ziglang/zig/issues/6515
1727 .modrm = @as(
1728 ?Instruction.ModrmEffectiveAddress,
1729 Instruction.ModrmEffectiveAddress{ .reg = dst_reg },
1730 ),
1731 .immediate_bytes = 4,
1732 .immediate = imm32,
1610 });1733 });
1611 std.mem.writeIntLittle(u32, self.code.addManyAsArrayAssumeCapacity(4), imm32);
1612 }1734 }
1613 },1735 },
1614 .embedded_in_code, .memory, .stack_offset => {1736 .embedded_in_code, .memory => {
1615 return self.fail(src, "TODO implement x86 ADD/SUB/CMP source memory", .{});1737 return self.fail(src, "TODO implement x86 ADD/SUB/CMP source memory", .{});
1616 },1738 },
1739 .stack_offset => |off| {
1740 const abi_size = dst_ty.abiSize(self.target.*);
1741 const adj_off = off + abi_size;
1742 if (off > math.maxInt(i32)) {
1743 return self.fail(src, "stack offset too large", .{});
1744 }
1745 try self.encodeX8664Instruction(src, Instruction{
1746 .operand_size_64 = dst_reg.size() == 64,
1747 .primary_opcode_1b = mr + 0x3,
1748 .reg = dst_reg,
1749 // TODO: Explicit optional wrap due to stage 1 miscompilation :(
1750 // https://github.com/ziglang/zig/issues/6515
1751 .modrm = @as(
1752 ?Instruction.ModrmEffectiveAddress,
1753 Instruction.ModrmEffectiveAddress{ .mem_disp = .{
1754 .reg = Register.ebp,
1755 .disp = -@intCast(i32, adj_off),
1756 } },
1757 ),
1758 });
1759 },
1617 .compare_flags_unsigned => {1760 .compare_flags_unsigned => {
1618 return self.fail(src, "TODO implement x86 ADD/SUB/CMP source compare flag (unsigned)", .{});1761 return self.fail(src, "TODO implement x86 ADD/SUB/CMP source compare flag (unsigned)", .{});
1619 },1762 },
...@@ -1655,25 +1798,23 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {...@@ -1655,25 +1798,23 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
1655 fn genX8664ModRMRegToStack(self: *Self, src: LazySrcLoc, ty: Type, off: u32, reg: Register, opcode: u8) !void {1798 fn genX8664ModRMRegToStack(self: *Self, src: LazySrcLoc, ty: Type, off: u32, reg: Register, opcode: u8) !void {
1656 const abi_size = ty.abiSize(self.target.*);1799 const abi_size = ty.abiSize(self.target.*);
1657 const adj_off = off + abi_size;1800 const adj_off = off + abi_size;
1658 try self.code.ensureCapacity(self.code.items.len + 7);1801 if (off > math.maxInt(i32)) {
1659 self.rex(.{ .w = reg.size() == 64, .r = reg.isExtended() });
1660 const reg_id: u8 = @truncate(u3, reg.id());
1661 if (adj_off <= 128) {
1662 // example: 48 89 55 7f mov QWORD PTR [rbp+0x7f],rdx
1663 const RM = @as(u8, 0b01_000_101) | (reg_id << 3);
1664 const negative_offset = @intCast(i8, -@intCast(i32, adj_off));
1665 const twos_comp = @bitCast(u8, negative_offset);
1666 self.code.appendSliceAssumeCapacity(&[_]u8{ opcode, RM, twos_comp });
1667 } else if (adj_off <= 2147483648) {
1668 // example: 48 89 95 80 00 00 00 mov QWORD PTR [rbp+0x80],rdx
1669 const RM = @as(u8, 0b10_000_101) | (reg_id << 3);
1670 const negative_offset = @intCast(i32, -@intCast(i33, adj_off));
1671 const twos_comp = @bitCast(u32, negative_offset);
1672 self.code.appendSliceAssumeCapacity(&[_]u8{ opcode, RM });
1673 mem.writeIntLittle(u32, self.code.addManyAsArrayAssumeCapacity(4), twos_comp);
1674 } else {
1675 return self.fail(src, "stack offset too large", .{});1802 return self.fail(src, "stack offset too large", .{});
1676 }1803 }
1804 try self.encodeX8664Instruction(src, Instruction{
1805 .operand_size_64 = reg.size() == 64,
1806 .primary_opcode_1b = opcode,
1807 .reg = reg,
1808 // TODO: Explicit optional wrap due to stage 1 miscompilation :(
1809 // https://github.com/ziglang/zig/issues/6515
1810 .modrm = @as(
1811 ?Instruction.ModrmEffectiveAddress,
1812 Instruction.ModrmEffectiveAddress{ .mem_disp = .{
1813 .reg = Register.ebp,
1814 .disp = -@intCast(i32, adj_off),
1815 } },
1816 ),
1817 });
1677 }1818 }
16781819
1679 fn genArgDbgInfo(self: *Self, inst: *ir.Inst.Arg, mcv: MCValue) !void {1820 fn genArgDbgInfo(self: *Self, inst: *ir.Inst.Arg, mcv: MCValue) !void {
...@@ -2340,15 +2481,24 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {...@@ -2340,15 +2481,24 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
2340 },2481 },
2341 .register => |reg| blk: {2482 .register => |reg| blk: {
2342 // test reg, 12483 // test reg, 1
2343 // TODO detect al, ax, eax2484 try self.encodeX8664Instruction(inst.base.src, Instruction{
2344 try self.code.ensureCapacity(self.code.items.len + 4);2485 // TODO audit this codegen: we force w = true here to make
2345 // TODO audit this codegen: we force w = true here to make2486 // the value affect the big register
2346 // the value affect the big register2487 .operand_size_64 = true,
2347 self.rex(.{ .b = reg.isExtended(), .w = true });2488
2348 self.code.appendSliceAssumeCapacity(&[_]u8{2489 .primary_opcode_1b = 0xf6, // f6/0 is TEST r/m8, imm8
2349 0xf6,2490 .opcode_extension = 0,
2350 @as(u8, 0xC0) | (0 << 3) | @truncate(u3, reg.id()),2491
2351 0x01,2492 // TODO: Explicit optional wrap due to stage 1 miscompilation :(
2493 // https://github.com/ziglang/zig/issues/6515
2494 // TODO detect al, ax, eax, there's another opcode 0xa8 for that
2495 .modrm = @as(
2496 ?Instruction.ModrmEffectiveAddress,
2497 Instruction.ModrmEffectiveAddress{ .reg = reg },
2498 ),
2499
2500 .immediate_bytes = 1,
2501 .immediate = 1,
2352 });2502 });
2353 break :blk 0x84;2503 break :blk 0x84;
2354 },2504 },
...@@ -2662,9 +2812,9 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {...@@ -2662,9 +2812,9 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
2662 switch (arch) {2812 switch (arch) {
2663 .x86_64 => switch (inst.base.tag) {2813 .x86_64 => switch (inst.base.tag) {
2664 // lhs AND rhs2814 // lhs AND rhs
2665 .bool_and => return try self.genX8664BinMath(&inst.base, inst.lhs, inst.rhs, 4, 0x20),2815 .bool_and => return try self.genX8664BinMath(&inst.base, inst.lhs, inst.rhs),
2666 // lhs OR rhs2816 // lhs OR rhs
2667 .bool_or => return try self.genX8664BinMath(&inst.base, inst.lhs, inst.rhs, 1, 0x08),2817 .bool_or => return try self.genX8664BinMath(&inst.base, inst.lhs, inst.rhs),
2668 else => unreachable, // Not a boolean operation2818 else => unreachable, // Not a boolean operation
2669 },2819 },
2670 .arm, .armeb => switch (inst.base.tag) {2820 .arm, .armeb => switch (inst.base.tag) {
...@@ -3451,20 +3601,27 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {...@@ -3451,20 +3601,27 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
3451 }3601 }
3452 },3602 },
3453 .compare_flags_unsigned => |op| {3603 .compare_flags_unsigned => |op| {
3454 try self.code.ensureCapacity(self.code.items.len + 3);3604 try self.encodeX8664Instruction(src, Instruction{
3455 // TODO audit this codegen: we force w = true here to make3605 // TODO audit this codegen: we force w = true here to make
3456 // the value affect the big register3606 // the value affect the big register
3457 self.rex(.{ .b = reg.isExtended(), .w = true });3607 .operand_size_64 = true,
3458 const opcode: u8 = switch (op) {3608
3459 .gte => 0x93,3609 .primary_opcode_2b = switch (op) {
3460 .gt => 0x97,3610 .gte => 0x93,
3461 .neq => 0x95,3611 .gt => 0x97,
3462 .lt => 0x92,3612 .neq => 0x95,
3463 .lte => 0x96,3613 .lt => 0x92,
3464 .eq => 0x94,3614 .lte => 0x96,
3465 };3615 .eq => 0x94,
3466 const id = @as(u8, reg.id() & 0b111);3616 },
3467 self.code.appendSliceAssumeCapacity(&[_]u8{ 0x0f, opcode, 0xC0 | id });3617
3618 // TODO: Explicit optional wrap due to stage 1 miscompilation :(
3619 // https://github.com/ziglang/zig/issues/6515
3620 .modrm = @as(
3621 ?Instruction.ModrmEffectiveAddress,
3622 Instruction.ModrmEffectiveAddress{ .reg = reg },
3623 ),
3624 });
3468 },3625 },
3469 .compare_flags_signed => |op| {3626 .compare_flags_signed => |op| {
3470 return self.fail(src, "TODO set register with compare flags value (signed)", .{});3627 return self.fail(src, "TODO set register with compare flags value (signed)", .{});
...@@ -3476,38 +3633,32 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {...@@ -3476,38 +3633,32 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
3476 // The encoding for `xor r32, r32` is `0x31 /r`.3633 // The encoding for `xor r32, r32` is `0x31 /r`.
3477 // Section 3.1.1.1 of the Intel x64 Manual states that "/r indicates that the3634 // Section 3.1.1.1 of the Intel x64 Manual states that "/r indicates that the
3478 // ModR/M byte of the instruction contains a register operand and an r/m operand."3635 // ModR/M byte of the instruction contains a register operand and an r/m operand."
3479 //3636 try self.encodeX8664Instruction(src, Instruction{
3480 // R/M bytes are composed of two bits for the mode, then three bits for the register,3637 .primary_opcode_1b = 0x31,
3481 // then three bits for the operand. Since we're zeroing a register, the two three-bit3638
3482 // values will be identical, and the mode is three (the raw register value).3639 // TODO: Explicit optional wrap due to stage 1 miscompilation :(
3483 //3640 // https://github.com/ziglang/zig/issues/6515
3484 // If we're accessing e.g. r8d, we need to use a REX prefix before the actual operation. Since3641 .reg = @as(?Register, reg),
3485 // this is a 32-bit operation, the W flag is set to zero. X is also zero, as we're not using a SIB.3642 .modrm = @as(
3486 // Both R and B are set, as we're extending, in effect, the register bits *and* the operand.3643 ?Instruction.ModrmEffectiveAddress,
3487 try self.code.ensureCapacity(self.code.items.len + 3);3644 Instruction.ModrmEffectiveAddress{ .reg = reg },
3488 self.rex(.{ .r = reg.isExtended(), .b = reg.isExtended() });3645 ),
3489 const id = @as(u8, reg.id() & 0b111);3646 });
3490 self.code.appendSliceAssumeCapacity(&[_]u8{ 0x31, 0xC0 | id << 3 | id });
3491 return;3647 return;
3492 }3648 }
3493 if (x <= math.maxInt(u32)) {3649 if (x <= math.maxInt(u32)) {
3494 // Next best case: if we set the lower four bytes, the upper four will be zeroed.3650 // Next best case: if we set the lower four bytes, the upper four will be zeroed.
3495 //3651 //
3496 // The encoding for `mov IMM32 -> REG` is (0xB8 + R) IMM.3652 // The encoding for `mov IMM32 -> REG` is (0xB8 + R) IMM.
3497 if (reg.isExtended()) {3653 try self.encodeX8664Instruction(src, Instruction{
3498 // Just as with XORing, we need a REX prefix. This time though, we only3654 // B8 + R
3499 // need the B bit set, as we're extending the opcode's register field,3655 .primary_opcode_1b = 0xB8,
3500 // and there is no Mod R/M byte.3656 .opcode_reg = @as(?Register, reg),
3501 //3657
3502 // Thus, we need b01000001, or 0x41.3658 // IMM32
3503 try self.code.resize(self.code.items.len + 6);3659 .immediate_bytes = 4,
3504 self.code.items[self.code.items.len - 6] = 0x41;3660 .immediate = x,
3505 } else {3661 });
3506 try self.code.resize(self.code.items.len + 5);
3507 }
3508 self.code.items[self.code.items.len - 5] = 0xB8 | @as(u8, reg.id() & 0b111);
3509 const imm_ptr = self.code.items[self.code.items.len - 4 ..][0..4];
3510 mem.writeIntLittle(u32, imm_ptr, @intCast(u32, x));
3511 return;3662 return;
3512 }3663 }
3513 // Worst case: we need to load the 64-bit register with the IMM. GNU's assemblers calls3664 // Worst case: we need to load the 64-bit register with the IMM. GNU's assemblers calls
...@@ -3517,50 +3668,58 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {...@@ -3517,50 +3668,58 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
3517 // This encoding is, in fact, the *same* as the one used for 32-bit loads. The only3668 // This encoding is, in fact, the *same* as the one used for 32-bit loads. The only
3518 // difference is that we set REX.W before the instruction, which extends the load to3669 // difference is that we set REX.W before the instruction, which extends the load to
3519 // 64-bit and uses the full bit-width of the register.3670 // 64-bit and uses the full bit-width of the register.
3520 //3671 try self.encodeX8664Instruction(src, Instruction{
3521 // Since we always need a REX here, let's just check if we also need to set REX.B.3672 .operand_size_64 = true,
3522 //3673 // B8 + R
3523 // In this case, the encoding of the REX byte is 0b0100100B3674 .primary_opcode_1b = 0xB8,
3524 try self.code.ensureCapacity(self.code.items.len + 10);3675 .opcode_reg = @as(?Register, reg),
3525 self.rex(.{ .w = reg.size() == 64, .b = reg.isExtended() });3676
3526 self.code.items.len += 9;3677 // IMM64
3527 self.code.items[self.code.items.len - 9] = 0xB8 | @as(u8, reg.id() & 0b111);3678 .immediate_bytes = 8,
3528 const imm_ptr = self.code.items[self.code.items.len - 8 ..][0..8];3679 .immediate = x,
3529 mem.writeIntLittle(u64, imm_ptr, x);3680 });
3530 },3681 },
3531 .embedded_in_code => |code_offset| {3682 .embedded_in_code => |code_offset| {
3532 // We need the offset from RIP in a signed i32 twos complement.3683 // 64-bit LEA is encoded as REX.W 8D /r.
3533 // The instruction is 7 bytes long and RIP points to the next instruction.
3534 try self.code.ensureCapacity(self.code.items.len + 7);
3535 // 64-bit LEA is encoded as REX.W 8D /r. If the register is extended, the REX byte is modified,
3536 // but the operation size is unchanged. Since we're using a disp32, we want mode 0 and lower three
3537 // bits as five.
3538 // REX 0x8D 0b00RRR101, where RRR is the lower three bits of the id.
3539 self.rex(.{ .w = reg.size() == 64, .b = reg.isExtended() });
3540 self.code.items.len += 6;
3541 const rip = self.code.items.len;3684 const rip = self.code.items.len;
3542 const big_offset = @intCast(i64, code_offset) - @intCast(i64, rip);3685 const big_offset = @intCast(i64, code_offset) - @intCast(i64, rip);
3543 const offset = @intCast(i32, big_offset);3686 const offset = @intCast(i32, big_offset);
3544 self.code.items[self.code.items.len - 6] = 0x8D;3687 try self.encodeX8664Instruction(src, Instruction{
3545 self.code.items[self.code.items.len - 5] = 0b101 | (@as(u8, reg.id() & 0b111) << 3);3688 .operand_size_64 = true,
3546 const imm_ptr = self.code.items[self.code.items.len - 4 ..][0..4];3689
3547 mem.writeIntLittle(i32, imm_ptr, offset);3690 // LEA
3691 .primary_opcode_1b = 0x8D,
3692
3693 .reg = reg,
3694
3695 // TODO: Explicit optional wrap due to stage 1 miscompilation :(
3696 // https://github.com/ziglang/zig/issues/6515
3697 .modrm = @as(
3698 ?Instruction.ModrmEffectiveAddress,
3699 Instruction.ModrmEffectiveAddress{ .disp32 = @bitCast(i32, offset) },
3700 ),
3701 });
3548 },3702 },
3549 .register => |src_reg| {3703 .register => |src_reg| {
3550 // If the registers are the same, nothing to do.3704 // If the registers are the same, nothing to do.
3551 if (src_reg.id() == reg.id())3705 if (src_reg.id() == reg.id())
3552 return;3706 return;
35533707
3554 // This is a variant of 8B /r. Since we're using 64-bit moves, we require a REX.3708 // This is a variant of 8B /r.
3555 // This is thus three bytes: REX 0x8B R/M.3709 try self.encodeX8664Instruction(src, Instruction{
3556 // If the destination is extended, the R field must be 1.3710 .operand_size_64 = reg.size() == 64,
3557 // If the *source* is extended, the B field must be 1.3711
3558 // Since the register is being accessed directly, the R/M mode is three. The reg field (the middle3712 .primary_opcode_1b = 0x8B,
3559 // three bits) contain the destination, and the R/M field (the lower three bits) contain the source.3713
3560 try self.code.ensureCapacity(self.code.items.len + 3);3714 .reg = reg,
3561 self.rex(.{ .w = reg.size() == 64, .r = reg.isExtended(), .b = src_reg.isExtended() });3715
3562 const R = 0xC0 | (@as(u8, reg.id() & 0b111) << 3) | @as(u8, src_reg.id() & 0b111);3716 // TODO: Explicit optional wrap due to stage 1 miscompilation :(
3563 self.code.appendSliceAssumeCapacity(&[_]u8{ 0x8B, R });3717 // https://github.com/ziglang/zig/issues/6515
3718 .modrm = @as(
3719 ?Instruction.ModrmEffectiveAddress,
3720 Instruction.ModrmEffectiveAddress{ .reg = src_reg },
3721 ),
3722 });
3564 },3723 },
3565 .memory => |x| {3724 .memory => |x| {
3566 if (self.bin_file.options.pie) {3725 if (self.bin_file.options.pie) {
...@@ -3577,6 +3736,9 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {...@@ -3577,6 +3736,9 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
3577 } else {3736 } else {
3578 return self.fail(src, "TODO implement genSetReg for PIE GOT indirection on this platform", .{});3737 return self.fail(src, "TODO implement genSetReg for PIE GOT indirection on this platform", .{});
3579 }3738 }
3739
3740 // LEA reg, [<offset>]
3741 // manually do this instruction to make sure the offset into the disp32 field won't change.
3580 try self.code.ensureCapacity(self.code.items.len + 7);3742 try self.code.ensureCapacity(self.code.items.len + 7);
3581 self.rex(.{ .w = reg.size() == 64, .r = reg.isExtended() });3743 self.rex(.{ .w = reg.size() == 64, .r = reg.isExtended() });
3582 self.code.appendSliceAssumeCapacity(&[_]u8{3744 self.code.appendSliceAssumeCapacity(&[_]u8{
...@@ -3585,10 +3747,21 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {...@@ -3585,10 +3747,21 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
3585 });3747 });
3586 mem.writeIntLittle(u32, self.code.addManyAsArrayAssumeCapacity(4), 0);3748 mem.writeIntLittle(u32, self.code.addManyAsArrayAssumeCapacity(4), 0);
35873749
3588 try self.code.ensureCapacity(self.code.items.len + 3);3750 // MOV reg, [reg]
3589 self.rex(.{ .w = reg.size() == 64, .b = reg.isExtended(), .r = reg.isExtended() });3751 try self.encodeX8664Instruction(src, Instruction{
3590 const RM = (@as(u8, reg.id() & 0b111) << 3) | @truncate(u3, reg.id());3752 .operand_size_64 = reg.size() == 64,
3591 self.code.appendSliceAssumeCapacity(&[_]u8{ 0x8B, RM });3753
3754 .primary_opcode_1b = 0x8B,
3755
3756 .reg = reg,
3757
3758 // TODO: Explicit optional wrap due to stage 1 miscompilation :(
3759 // https://github.com/ziglang/zig/issues/6515
3760 .modrm = @as(
3761 ?Instruction.ModrmEffectiveAddress,
3762 Instruction.ModrmEffectiveAddress{ .mem = reg },
3763 ),
3764 });
3592 } else if (x <= math.maxInt(u32)) {3765 } else if (x <= math.maxInt(u32)) {
3593 // Moving from memory to a register is a variant of `8B /r`.3766 // Moving from memory to a register is a variant of `8B /r`.
3594 // Since we're using 64-bit moves, we require a REX.3767 // Since we're using 64-bit moves, we require a REX.
...@@ -3612,12 +3785,13 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {...@@ -3612,12 +3785,13 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
3612 // REX.W 0xA1 moffs64*3785 // REX.W 0xA1 moffs64*
3613 // moffs64* is a 64-bit offset "relative to segment base", which really just means the3786 // moffs64* is a 64-bit offset "relative to segment base", which really just means the
3614 // absolute address for all practical purposes.3787 // absolute address for all practical purposes.
3615 try self.code.resize(self.code.items.len + 10);3788
3616 // REX.W == 0x483789 try self.encodeX8664Instruction(src, Instruction{
3617 self.code.items[self.code.items.len - 10] = 0x48;3790 .operand_size_64 = true,
3618 self.code.items[self.code.items.len - 9] = 0xA1;3791 .primary_opcode_1b = 0xa1,
3619 const imm_ptr = self.code.items[self.code.items.len - 8 ..][0..8];3792 .immediate_bytes = 8,
3620 mem.writeIntLittle(u64, imm_ptr, x);3793 .immediate = x,
3794 });
3621 } else {3795 } else {
3622 // This requires two instructions; a move imm as used above, followed by an indirect load using the register3796 // This requires two instructions; a move imm as used above, followed by an indirect load using the register
3623 // as the address and the register as the destination.3797 // as the address and the register as the destination.
...@@ -3634,41 +3808,41 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {...@@ -3634,41 +3808,41 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
3634 // Now, the register contains the address of the value to load into it3808 // Now, the register contains the address of the value to load into it
3635 // Currently, we're only allowing 64-bit registers, so we need the `REX.W 8B /r` variant.3809 // Currently, we're only allowing 64-bit registers, so we need the `REX.W 8B /r` variant.
3636 // TODO: determine whether to allow other sized registers, and if so, handle them properly.3810 // TODO: determine whether to allow other sized registers, and if so, handle them properly.
3637 // This operation requires three bytes: REX 0x8B R/M3811 try self.encodeX8664Instruction(src, Instruction{
3638 try self.code.ensureCapacity(self.code.items.len + 3);3812 .operand_size_64 = reg.size() == 64,
3639 // For this operation, we want R/M mode *zero* (use register indirectly), and the two register3813 .primary_opcode_1b = 0x8B,
3640 // values must match. Thus, it's 00ABCABC where ABC is the lower three bits of the register ID.3814 .reg = reg,
3641 //3815 // TODO: Explicit optional wrap due to stage 1 miscompilation :(
3642 // Furthermore, if this is an extended register, both B and R must be set in the REX byte, as *both*3816 // https://github.com/ziglang/zig/issues/6515
3643 // register operands need to be marked as extended.3817 .modrm = @as(
3644 self.rex(.{ .w = reg.size() == 64, .b = reg.isExtended(), .r = reg.isExtended() });3818 ?Instruction.ModrmEffectiveAddress,
3645 const RM = (@as(u8, reg.id() & 0b111) << 3) | @truncate(u3, reg.id());3819 Instruction.ModrmEffectiveAddress{ .mem = reg },
3646 self.code.appendSliceAssumeCapacity(&[_]u8{ 0x8B, RM });3820 ),
3821 });
3647 }3822 }
3648 }3823 }
3649 },3824 },
3650 .stack_offset => |unadjusted_off| {3825 .stack_offset => |unadjusted_off| {
3651 try self.code.ensureCapacity(self.code.items.len + 7);
3652 const size_bytes = @divExact(reg.size(), 8);3826 const size_bytes = @divExact(reg.size(), 8);
3653 const off = unadjusted_off + size_bytes;3827 const off = unadjusted_off + size_bytes;
3654 self.rex(.{ .w = reg.size() == 64, .r = reg.isExtended() });3828 if (off < std.math.minInt(i32) or off > std.math.maxInt(i32)) {
3655 const reg_id: u8 = @truncate(u3, reg.id());
3656 if (off <= 128) {
3657 // Example: 48 8b 4d 7f mov rcx,QWORD PTR [rbp+0x7f]
3658 const RM = @as(u8, 0b01_000_101) | (reg_id << 3);
3659 const negative_offset = @intCast(i8, -@intCast(i32, off));
3660 const twos_comp = @bitCast(u8, negative_offset);
3661 self.code.appendSliceAssumeCapacity(&[_]u8{ 0x8b, RM, twos_comp });
3662 } else if (off <= 2147483648) {
3663 // Example: 48 8b 8d 80 00 00 00 mov rcx,QWORD PTR [rbp+0x80]
3664 const RM = @as(u8, 0b10_000_101) | (reg_id << 3);
3665 const negative_offset = @intCast(i32, -@intCast(i33, off));
3666 const twos_comp = @bitCast(u32, negative_offset);
3667 self.code.appendSliceAssumeCapacity(&[_]u8{ 0x8b, RM });
3668 mem.writeIntLittle(u32, self.code.addManyAsArrayAssumeCapacity(4), twos_comp);
3669 } else {
3670 return self.fail(src, "stack offset too large", .{});3829 return self.fail(src, "stack offset too large", .{});
3671 }3830 }
3831 const ioff = -@intCast(i32, off);
3832 try self.encodeX8664Instruction(src, Instruction{
3833 .operand_size_64 = reg.size() == 64,
3834 .primary_opcode_1b = 0x8B,
3835 .reg = reg,
3836 // TODO: Explicit optional wrap due to stage 1 miscompilation :(
3837 // https://github.com/ziglang/zig/issues/6515
3838 .modrm = @as(
3839 ?Instruction.ModrmEffectiveAddress,
3840 Instruction.ModrmEffectiveAddress{ .mem_disp = .{
3841 .reg = Register.ebp,
3842 .disp = ioff,
3843 } },
3844 ),
3845 });
3672 },3846 },
3673 },3847 },
3674 else => return self.fail(src, "TODO implement getSetReg for {}", .{self.target.cpu.arch}),3848 else => return self.fail(src, "TODO implement getSetReg for {}", .{self.target.cpu.arch}),