| author |  Jacob Young <jacobly@ziglang.org> 2025-03-27 12:03:02-04:00 |
| committer | Jacob Young <jacobly@ziglang.org> 2025-04-10 02:04:52-04:00 |
| log | bbf8abf5d3eb6870b2235f5c18e23239872cda69 |
| tree | c25c3784a8c6f59451a0980fd587a79e2b022fb6 |
| parent | f4a31bed98c5ee65c0e8494e0e0e15e418697cb8 |
x86_64: support rip-relative addressing to labels in inline asm
6 files changed, 61 insertions(+), 16 deletions(-)
src/arch/x86_64/CodeGen.zig+40-7
| ... | @@ -103178,6 +103178,19 @@ fn performReloc(self: *CodeGen, reloc: Mir.Inst.Index) void { | ... | @@ -103178,6 +103178,19 @@ fn performReloc(self: *CodeGen, reloc: Mir.Inst.Index) void { |
| 103178 | .pseudo_j_z_and_np_inst, .pseudo_j_nz_or_p_inst => {}, | 103178 | .pseudo_j_z_and_np_inst, .pseudo_j_nz_or_p_inst => {}, |
| 103179 | else => unreachable, | 103179 | else => unreachable, |
| 103180 | }, | 103180 | }, |
| 103181 | .lea => switch (self.mir_instructions.items(.ops)[reloc]) { | ||
| 103182 | .rm => { | ||
| 103183 | const rx = self.mir_instructions.items(.data)[reloc].rx; | ||
| 103184 | assert(rx.fixes == ._); | ||
| 103185 | const mem_info: Mir.Memory.Info = @bitCast( | ||
| 103186 | self.mir_extra.items[rx.payload + std.meta.fieldIndex(Mir.Memory, "info").?], | ||
| 103187 | ); | ||
| 103188 | assert(mem_info.base == .rip_inst); | ||
| 103189 | self.mir_extra.items[rx.payload + std.meta.fieldIndex(Mir.Memory, "base").?] = next_inst; | ||
| 103190 | return; | ||
| 103191 | }, | ||
| 103192 | else => unreachable, | ||
| 103193 | }, | ||
| 103181 | else => unreachable, | 103194 | else => unreachable, |
| 103182 | } | 103195 | } |
| 103183 | self.mir_instructions.items(.data)[reloc].inst.inst = next_inst; | 103196 | self.mir_instructions.items(.data)[reloc].inst.inst = next_inst; |
| ... | @@ -103461,7 +103474,10 @@ fn airAsm(self: *CodeGen, inst: Air.Inst.Index) !void { | ... | @@ -103461,7 +103474,10 @@ fn airAsm(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 103461 | // for the string, we still use the next u32 for the null terminator. | 103474 | // for the string, we still use the next u32 for the null terminator. |
| 103462 | extra_i += clobber.len / 4 + 1; | 103475 | extra_i += clobber.len / 4 + 1; |
| 103463 | 103476 | ||
| 103464 | if (std.mem.eql(u8, clobber, "") or std.mem.eql(u8, clobber, "memory")) { | 103477 | if (std.mem.eql(u8, clobber, "") or std.mem.eql(u8, clobber, "memory") or |
| 103478 | std.mem.eql(u8, clobber, "fpsr") or std.mem.eql(u8, clobber, "fpcr") or | ||
| 103479 | std.mem.eql(u8, clobber, "mxcsr") or std.mem.eql(u8, clobber, "dirflag")) | ||
| 103480 | { | ||
| 103465 | // ok, sure | 103481 | // ok, sure |
| 103466 | } else if (std.mem.eql(u8, clobber, "cc") or | 103482 | } else if (std.mem.eql(u8, clobber, "cc") or |
| 103467 | std.mem.eql(u8, clobber, "flags") or | 103483 | std.mem.eql(u8, clobber, "flags") or |
| ... | @@ -103726,7 +103742,24 @@ fn airAsm(self: *CodeGen, inst: Air.Inst.Index) !void { | ... | @@ -103726,7 +103742,24 @@ fn airAsm(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 103726 | else | 103742 | else |
| 103727 | .@"1"; | 103743 | .@"1"; |
| 103728 | if (sib_it.next()) |_| return self.fail("invalid memory operand: '{s}'", .{op_str}); | 103744 | if (sib_it.next()) |_| return self.fail("invalid memory operand: '{s}'", .{op_str}); |
| 103729 | op.* = if (std.mem.eql(u8, base_str, "%%dx") and index_str.len == 0) .{ .reg = .dx } else .{ .mem = .{ | 103745 | op.* = if (std.mem.eql(u8, base_str, "%%dx") and index_str.len == 0) |
| 103746 | .{ .reg = .dx } | ||
| 103747 | else if (std.mem.eql(u8, base_str, "%%rip") and index_str.len == 0 and | ||
| 103748 | Label.isValid(.reference, op_str[0..open])) | ||
| 103749 | op: { | ||
| 103750 | const anon = std.ascii.isDigit(op_str[0]); | ||
| 103751 | const label_gop = try labels.getOrPut(self.gpa, op_str[0..if (anon) 1 else open]); | ||
| 103752 | if (anon and (op_str[1] == 'b' or op_str[1] == 'B') and !label_gop.found_existing) | ||
| 103753 | return self.fail("undefined label: '{s}'", .{op_str}); | ||
| 103754 | if (!label_gop.found_existing) label_gop.value_ptr.* = .{}; | ||
| 103755 | const pending_relocs = &label_gop.value_ptr.pending_relocs; | ||
| 103756 | if (if (anon) | ||
| 103757 | op_str[1] == 'f' or op_str[1] == 'F' | ||
| 103758 | else | ||
| 103759 | !label_gop.found_existing or pending_relocs.items.len > 0) | ||
| 103760 | try pending_relocs.append(self.gpa, @intCast(self.mir_instructions.len)); | ||
| 103761 | break :op .{ .mem = .{ .base = .{ .rip_inst = label_gop.value_ptr.target } } }; | ||
| 103762 | } else .{ .mem = .{ | ||
| 103730 | .base = if (base_str.len > 0) | 103763 | .base = if (base_str.len > 0) |
| 103731 | .{ .reg = parseRegName(base_str["%%".len..]) orelse | 103764 | .{ .reg = parseRegName(base_str["%%".len..]) orelse |
| 103732 | return self.fail("invalid base register: '{s}'", .{base_str}) } | 103765 | return self.fail("invalid base register: '{s}'", .{base_str}) } |
| ... | @@ -103770,9 +103803,9 @@ fn airAsm(self: *CodeGen, inst: Air.Inst.Index) !void { | ... | @@ -103770,9 +103803,9 @@ fn airAsm(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 103770 | } else if (Label.isValid(.reference, op_str)) { | 103803 | } else if (Label.isValid(.reference, op_str)) { |
| 103771 | const anon = std.ascii.isDigit(op_str[0]); | 103804 | const anon = std.ascii.isDigit(op_str[0]); |
| 103772 | const label_gop = try labels.getOrPut(self.gpa, op_str[0..if (anon) 1 else op_str.len]); | 103805 | const label_gop = try labels.getOrPut(self.gpa, op_str[0..if (anon) 1 else op_str.len]); |
| 103773 | if (!label_gop.found_existing) label_gop.value_ptr.* = .{}; | ||
| 103774 | if (anon and (op_str[1] == 'b' or op_str[1] == 'B') and !label_gop.found_existing) | 103806 | if (anon and (op_str[1] == 'b' or op_str[1] == 'B') and !label_gop.found_existing) |
| 103775 | return self.fail("undefined label: '{s}'", .{op_str}); | 103807 | return self.fail("undefined label: '{s}'", .{op_str}); |
| 103808 | if (!label_gop.found_existing) label_gop.value_ptr.* = .{}; | ||
| 103776 | const pending_relocs = &label_gop.value_ptr.pending_relocs; | 103809 | const pending_relocs = &label_gop.value_ptr.pending_relocs; |
| 103777 | if (if (anon) | 103810 | if (if (anon) |
| 103778 | op_str[1] == 'f' or op_str[1] == 'F' | 103811 | op_str[1] == 'f' or op_str[1] == 'F' |
| ... | @@ -105008,7 +105041,7 @@ fn genSetMem( | ... | @@ -105008,7 +105041,7 @@ fn genSetMem( |
| 105008 | .none => .{ .immediate = @bitCast(@as(i64, disp)) }, | 105041 | .none => .{ .immediate = @bitCast(@as(i64, disp)) }, |
| 105009 | .reg => |base_reg| .{ .register_offset = .{ .reg = base_reg, .off = disp } }, | 105042 | .reg => |base_reg| .{ .register_offset = .{ .reg = base_reg, .off = disp } }, |
| 105010 | .frame => |base_frame_index| .{ .lea_frame = .{ .index = base_frame_index, .off = disp } }, | 105043 | .frame => |base_frame_index| .{ .lea_frame = .{ .index = base_frame_index, .off = disp } }, |
| 105011 | .table => unreachable, | 105044 | .table, .rip_inst => unreachable, |
| 105012 | .reloc => |sym_index| .{ .lea_symbol = .{ .sym_index = sym_index, .off = disp } }, | 105045 | .reloc => |sym_index| .{ .lea_symbol = .{ .sym_index = sym_index, .off = disp } }, |
| 105013 | }; | 105046 | }; |
| 105014 | switch (src_mcv) { | 105047 | switch (src_mcv) { |
| ... | @@ -105118,7 +105151,7 @@ fn genSetMem( | ... | @@ -105118,7 +105151,7 @@ fn genSetMem( |
| 105118 | .index = frame_index, | 105151 | .index = frame_index, |
| 105119 | .off = disp, | 105152 | .off = disp, |
| 105120 | }).compare(.gte, src_align), | 105153 | }).compare(.gte, src_align), |
| 105121 | .table => unreachable, | 105154 | .table, .rip_inst => unreachable, |
| 105122 | .reloc => false, | 105155 | .reloc => false, |
| 105123 | })).write( | 105156 | })).write( |
| 105124 | self, | 105157 | self, |
| ... | @@ -105767,7 +105800,7 @@ fn airCmpxchg(self: *CodeGen, inst: Air.Inst.Index) !void { | ... | @@ -105767,7 +105800,7 @@ fn airCmpxchg(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 105767 | const ptr_lock = switch (ptr_mem.base) { | 105800 | const ptr_lock = switch (ptr_mem.base) { |
| 105768 | .none, .frame, .reloc => null, | 105801 | .none, .frame, .reloc => null, |
| 105769 | .reg => |reg| self.register_manager.lockReg(reg), | 105802 | .reg => |reg| self.register_manager.lockReg(reg), |
| 105770 | .table => unreachable, | 105803 | .table, .rip_inst => unreachable, |
| 105771 | }; | 105804 | }; |
| 105772 | defer if (ptr_lock) |lock| self.register_manager.unlockReg(lock); | 105805 | defer if (ptr_lock) |lock| self.register_manager.unlockReg(lock); |
| 105773 | 105806 | ||
| ... | @@ -105850,7 +105883,7 @@ fn atomicOp( | ... | @@ -105850,7 +105883,7 @@ fn atomicOp( |
| 105850 | const mem_lock = switch (ptr_mem.base) { | 105883 | const mem_lock = switch (ptr_mem.base) { |
| 105851 | .none, .frame, .reloc => null, | 105884 | .none, .frame, .reloc => null, |
| 105852 | .reg => |reg| self.register_manager.lockReg(reg), | 105885 | .reg => |reg| self.register_manager.lockReg(reg), |
| 105853 | .table => unreachable, | 105886 | .table, .rip_inst => unreachable, |
| 105854 | }; | 105887 | }; |
| 105855 | defer if (mem_lock) |lock| self.register_manager.unlockReg(lock); | 105888 | defer if (mem_lock) |lock| self.register_manager.unlockReg(lock); |
| 105856 | 105889 |
src/arch/x86_64/Emit.zig+8-6
| ... | @@ -97,12 +97,14 @@ pub fn emitMir(emit: *Emit) Error!void { | ... | @@ -97,12 +97,14 @@ pub fn emitMir(emit: *Emit) Error!void { |
| 97 | op_index -= 1; | 97 | op_index -= 1; |
| 98 | const op = lowered_inst.encoding.data.ops[op_index]; | 98 | const op = lowered_inst.encoding.data.ops[op_index]; |
| 99 | if (op == .none) continue; | 99 | if (op == .none) continue; |
| 100 | const enc_length: u4 = @intCast( | 100 | const is_mem = op.isMemory(); |
| 101 | std.math.divCeil(u7, @intCast(op.immBitSize()), 8) catch unreachable, | 101 | const enc_length: u4 = if (is_mem) switch (lowered_inst.ops[op_index].mem.sib.base) { |
| 102 | ); | 102 | .rip_inst => 4, |
| 103 | else => unreachable, | ||
| 104 | } else @intCast(std.math.divCeil(u7, @intCast(op.immBitSize()), 8) catch unreachable); | ||
| 103 | reloc_offset -= enc_length; | 105 | reloc_offset -= enc_length; |
| 104 | if (op_index == lowered_relocs[0].op_index) | 106 | if (op_index == lowered_relocs[0].op_index) break :reloc_offset_length .{ reloc_offset, enc_length }; |
| 105 | break :reloc_offset_length .{ reloc_offset, enc_length }; | 107 | std.debug.assert(!is_mem); |
| 106 | } | 108 | } |
| 107 | }; | 109 | }; |
| 108 | try relocs.append(emit.lower.allocator, .{ | 110 | try relocs.append(emit.lower.allocator, .{ |
| ... | @@ -434,7 +436,7 @@ pub fn emitMir(emit: *Emit) Error!void { | ... | @@ -434,7 +436,7 @@ pub fn emitMir(emit: *Emit) Error!void { |
| 434 | loc_buf[0] = switch (mem.base()) { | 436 | loc_buf[0] = switch (mem.base()) { |
| 435 | .none => .{ .constu = 0 }, | 437 | .none => .{ .constu = 0 }, |
| 436 | .reg => |reg| .{ .breg = reg.dwarfNum() }, | 438 | .reg => |reg| .{ .breg = reg.dwarfNum() }, |
| 437 | .frame, .table => unreachable, | 439 | .frame, .table, .rip_inst => unreachable, |
| 438 | .reloc => |sym_index| .{ .addr = .{ .sym = sym_index } }, | 440 | .reloc => |sym_index| .{ .addr = .{ .sym = sym_index } }, |
| 439 | }; | 441 | }; |
| 440 | break :base &loc_buf[0]; | 442 | break :base &loc_buf[0]; |
src/arch/x86_64/Lower.zig+1
| ... | @@ -395,6 +395,7 @@ pub fn mem(lower: *Lower, op_index: InstOpIndex, payload: u32) Memory { | ... | @@ -395,6 +395,7 @@ pub fn mem(lower: *Lower, op_index: InstOpIndex, payload: u32) Memory { |
| 395 | .sib => |*sib| switch (sib.base) { | 395 | .sib => |*sib| switch (sib.base) { |
| 396 | else => {}, | 396 | else => {}, |
| 397 | .table => sib.disp = lower.reloc(op_index, .table, sib.disp).signed, | 397 | .table => sib.disp = lower.reloc(op_index, .table, sib.disp).signed, |
| 398 | .rip_inst => |inst_index| sib.disp = lower.reloc(op_index, .{ .inst = inst_index }, sib.disp).signed, | ||
| 398 | }, | 399 | }, |
| 399 | else => {}, | 400 | else => {}, |
| 400 | } | 401 | } |
src/arch/x86_64/Mir.zig+3-1
| ... | @@ -1742,6 +1742,7 @@ pub const Memory = struct { | ... | @@ -1742,6 +1742,7 @@ pub const Memory = struct { |
| 1742 | .reg => |reg| @intFromEnum(reg), | 1742 | .reg => |reg| @intFromEnum(reg), |
| 1743 | .frame => |frame_index| @intFromEnum(frame_index), | 1743 | .frame => |frame_index| @intFromEnum(frame_index), |
| 1744 | .reloc => |sym_index| sym_index, | 1744 | .reloc => |sym_index| sym_index, |
| 1745 | .rip_inst => |inst_index| inst_index, | ||
| 1745 | }, | 1746 | }, |
| 1746 | .off = switch (mem.mod) { | 1747 | .off = switch (mem.mod) { |
| 1747 | .rm => |rm| @bitCast(rm.disp), | 1748 | .rm => |rm| @bitCast(rm.disp), |
| ... | @@ -1769,6 +1770,7 @@ pub const Memory = struct { | ... | @@ -1769,6 +1770,7 @@ pub const Memory = struct { |
| 1769 | .frame => .{ .frame = @enumFromInt(mem.base) }, | 1770 | .frame => .{ .frame = @enumFromInt(mem.base) }, |
| 1770 | .table => .table, | 1771 | .table => .table, |
| 1771 | .reloc => .{ .reloc = mem.base }, | 1772 | .reloc => .{ .reloc = mem.base }, |
| 1773 | .rip_inst => .{ .rip_inst = mem.base }, | ||
| 1772 | }, | 1774 | }, |
| 1773 | .scale_index = switch (mem.info.index) { | 1775 | .scale_index = switch (mem.info.index) { |
| 1774 | .none => null, | 1776 | .none => null, |
| ... | @@ -1832,7 +1834,7 @@ pub fn resolveFrameAddr(mir: Mir, frame_addr: bits.FrameAddr) bits.RegisterOffse | ... | @@ -1832,7 +1834,7 @@ pub fn resolveFrameAddr(mir: Mir, frame_addr: bits.FrameAddr) bits.RegisterOffse |
| 1832 | 1834 | ||
| 1833 | pub fn resolveFrameLoc(mir: Mir, mem: Memory) Memory { | 1835 | pub fn resolveFrameLoc(mir: Mir, mem: Memory) Memory { |
| 1834 | return switch (mem.info.base) { | 1836 | return switch (mem.info.base) { |
| 1835 | .none, .reg, .table, .reloc => mem, | 1837 | .none, .reg, .table, .reloc, .rip_inst => mem, |
| 1836 | .frame => if (mir.frame_locs.len > 0) .{ | 1838 | .frame => if (mir.frame_locs.len > 0) .{ |
| 1837 | .info = .{ | 1839 | .info = .{ |
| 1838 | .base = .reg, | 1840 | .base = .reg, |
src/arch/x86_64/bits.zig+3-1
| ... | @@ -4,6 +4,7 @@ const expect = std.testing.expect; | ... | @@ -4,6 +4,7 @@ const expect = std.testing.expect; |
| 4 | 4 | ||
| 5 | const Allocator = std.mem.Allocator; | 5 | const Allocator = std.mem.Allocator; |
| 6 | const ArrayList = std.ArrayList; | 6 | const ArrayList = std.ArrayList; |
| 7 | const Mir = @import("Mir.zig"); | ||
| 7 | 8 | ||
| 8 | /// EFLAGS condition codes | 9 | /// EFLAGS condition codes |
| 9 | pub const Condition = enum(u5) { | 10 | pub const Condition = enum(u5) { |
| ... | @@ -678,12 +679,13 @@ pub const Memory = struct { | ... | @@ -678,12 +679,13 @@ pub const Memory = struct { |
| 678 | frame: FrameIndex, | 679 | frame: FrameIndex, |
| 679 | table, | 680 | table, |
| 680 | reloc: u32, | 681 | reloc: u32, |
| 682 | rip_inst: Mir.Inst.Index, | ||
| 681 | 683 | ||
| 682 | pub const Tag = @typeInfo(Base).@"union".tag_type.?; | 684 | pub const Tag = @typeInfo(Base).@"union".tag_type.?; |
| 683 | 685 | ||
| 684 | pub fn isExtended(self: Base) bool { | 686 | pub fn isExtended(self: Base) bool { |
| 685 | return switch (self) { | 687 | return switch (self) { |
| 686 | .none, .frame, .table, .reloc => false, // rsp, rbp, and rip are not extended | 688 | .none, .frame, .table, .reloc, .rip_inst => false, // rsp, rbp, and rip are not extended |
| 687 | .reg => |reg| reg.isExtended(), | 689 | .reg => |reg| reg.isExtended(), |
| 688 | }; | 690 | }; |
| 689 | } | 691 | } |
src/arch/x86_64/encoder.zig+6-1
| ... | @@ -138,7 +138,7 @@ pub const Instruction = struct { | ... | @@ -138,7 +138,7 @@ pub const Instruction = struct { |
| 138 | .moffs => true, | 138 | .moffs => true, |
| 139 | .rip => false, | 139 | .rip => false, |
| 140 | .sib => |s| switch (s.base) { | 140 | .sib => |s| switch (s.base) { |
| 141 | .none, .frame, .table, .reloc => false, | 141 | .none, .frame, .table, .reloc, .rip_inst => false, |
| 142 | .reg => |reg| reg.class() == .segment, | 142 | .reg => |reg| reg.class() == .segment, |
| 143 | }, | 143 | }, |
| 144 | }; | 144 | }; |
| ... | @@ -279,6 +279,7 @@ pub const Instruction = struct { | ... | @@ -279,6 +279,7 @@ pub const Instruction = struct { |
| 279 | .frame => |frame_index| try writer.print("{}", .{frame_index}), | 279 | .frame => |frame_index| try writer.print("{}", .{frame_index}), |
| 280 | .table => try writer.print("Table", .{}), | 280 | .table => try writer.print("Table", .{}), |
| 281 | .reloc => |sym_index| try writer.print("Symbol({d})", .{sym_index}), | 281 | .reloc => |sym_index| try writer.print("Symbol({d})", .{sym_index}), |
| 282 | .rip_inst => |inst_index| try writer.print("RipInst({d})", .{inst_index}), | ||
| 282 | } | 283 | } |
| 283 | if (mem.scaleIndex()) |si| { | 284 | if (mem.scaleIndex()) |si| { |
| 284 | if (any) try writer.writeAll(" + "); | 285 | if (any) try writer.writeAll(" + "); |
| ... | @@ -705,6 +706,10 @@ pub const Instruction = struct { | ... | @@ -705,6 +706,10 @@ pub const Instruction = struct { |
| 705 | try encoder.modRm_indirectDisp32(operand_enc, 0); | 706 | try encoder.modRm_indirectDisp32(operand_enc, 0); |
| 706 | try encoder.disp32(undefined); | 707 | try encoder.disp32(undefined); |
| 707 | } else return error.CannotEncode, | 708 | } else return error.CannotEncode, |
| 709 | .rip_inst => { | ||
| 710 | try encoder.modRm_RIPDisp32(operand_enc); | ||
| 711 | try encoder.disp32(sib.disp); | ||
| 712 | }, | ||
| 708 | }, | 713 | }, |
| 709 | .rip => |rip| { | 714 | .rip => |rip| { |
| 710 | try encoder.modRm_RIPDisp32(operand_enc); | 715 | try encoder.modRm_RIPDisp32(operand_enc); |