authorgravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2022-01-15 18:36:13+01:00
committergravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2022-01-15 18:36:13+01:00
log6315bcb32aa48858c43373542d9c50a366913f7c
tree6045349f24a9a82c8c8cc0365dc667399eb84b93
parent4d4bbd76240e155151d0bf964debe09ea05fc7ef

stage2: rename Isel to Emit for x86_64

Clean up generated errors in Emit.

4 files changed, 2096 insertions(+), 2099 deletions(-)

CMakeLists.txt+1-1
......@@ -578,7 +578,7 @@ set(ZIG_STAGE2_SOURCES
578578 "${CMAKE_SOURCE_DIR}/src/arch/wasm/Emit.zig"
579579 "${CMAKE_SOURCE_DIR}/src/arch/wasm/Mir.zig"
580580 "${CMAKE_SOURCE_DIR}/src/arch/x86_64/CodeGen.zig"
581 "${CMAKE_SOURCE_DIR}/src/arch/x86_64/Isel.zig"
581 "${CMAKE_SOURCE_DIR}/src/arch/x86_64/Emit.zig"
582582 "${CMAKE_SOURCE_DIR}/src/arch/x86_64/Mir.zig"
583583 "${CMAKE_SOURCE_DIR}/src/arch/x86_64/bits.zig"
584584 "${CMAKE_SOURCE_DIR}/src/clang.zig"
src/arch/x86_64/CodeGen.zig+5-5
......@@ -17,7 +17,7 @@ const DW = std.dwarf;
1717const ErrorMsg = Module.ErrorMsg;
1818const FnResult = @import("../../codegen.zig").FnResult;
1919const GenerateSymbolError = @import("../../codegen.zig").GenerateSymbolError;
20const Isel = @import("Isel.zig");
20const Emit = @import("Emit.zig");
2121const Liveness = @import("../../Liveness.zig");
2222const Mir = @import("Mir.zig");
2323const Module = @import("../../Module.zig");
......@@ -309,7 +309,7 @@ pub fn generate(
309309 };
310310 defer mir.deinit(bin_file.allocator);
311311
312 var isel = Isel{
312 var emit = Emit{
313313 .mir = mir,
314314 .bin_file = bin_file,
315315 .debug_output = debug_output,
......@@ -320,9 +320,9 @@ pub fn generate(
320320 .prev_di_line = module_fn.lbrace_line,
321321 .prev_di_column = module_fn.lbrace_column,
322322 };
323 defer isel.deinit();
324 isel.lowerMir() catch |err| switch (err) {
325 error.IselFail => return FnResult{ .fail = isel.err_msg.? },
323 defer emit.deinit();
324 emit.lowerMir() catch |err| switch (err) {
325 error.EmitFail => return FnResult{ .fail = emit.err_msg.? },
326326 else => |e| return e,
327327 };
328328
src/arch/x86_64/Emit.zig created+2090
......@@ -0,0 +1,2090 @@
1//! This file contains the functionality for lowering x86_64 MIR into
2//! machine code
3
4const Emit = @This();
5
6const std = @import("std");
7const assert = std.debug.assert;
8const bits = @import("bits.zig");
9const leb128 = std.leb;
10const link = @import("../../link.zig");
11const log = std.log.scoped(.codegen);
12const math = std.math;
13const mem = std.mem;
14const testing = std.testing;
15
16const Air = @import("../../Air.zig");
17const Allocator = mem.Allocator;
18const DebugInfoOutput = @import("../../codegen.zig").DebugInfoOutput;
19const DW = std.dwarf;
20const Encoder = bits.Encoder;
21const ErrorMsg = Module.ErrorMsg;
22const MCValue = @import("CodeGen.zig").MCValue;
23const Mir = @import("Mir.zig");
24const Module = @import("../../Module.zig");
25const Instruction = bits.Instruction;
26const Register = bits.Register;
27const Type = @import("../../type.zig").Type;
28
29mir: Mir,
30bin_file: *link.File,
31debug_output: DebugInfoOutput,
32target: *const std.Target,
33err_msg: ?*ErrorMsg = null,
34src_loc: Module.SrcLoc,
35code: *std.ArrayList(u8),
36
37prev_di_line: u32,
38prev_di_column: u32,
39/// Relative to the beginning of `code`.
40prev_di_pc: usize,
41
42code_offset_mapping: std.AutoHashMapUnmanaged(Mir.Inst.Index, usize) = .{},
43relocs: std.ArrayListUnmanaged(Reloc) = .{},
44
45const InnerError = error{
46 OutOfMemory,
47 EmitFail,
48};
49
50const Reloc = struct {
51 /// Offset of the instruction.
52 source: usize,
53 /// Target of the relocation.
54 target: Mir.Inst.Index,
55 /// Offset of the relocation within the instruction.
56 offset: usize,
57 /// Length of the instruction.
58 length: u5,
59};
60
61pub fn lowerMir(emit: *Emit) InnerError!void {
62 const mir_tags = emit.mir.instructions.items(.tag);
63
64 for (mir_tags) |tag, index| {
65 const inst = @intCast(u32, index);
66 try emit.code_offset_mapping.putNoClobber(emit.bin_file.allocator, inst, emit.code.items.len);
67 switch (tag) {
68 .adc => try emit.mirArith(.adc, inst),
69 .add => try emit.mirArith(.add, inst),
70 .sub => try emit.mirArith(.sub, inst),
71 .xor => try emit.mirArith(.xor, inst),
72 .@"and" => try emit.mirArith(.@"and", inst),
73 .@"or" => try emit.mirArith(.@"or", inst),
74 .sbb => try emit.mirArith(.sbb, inst),
75 .cmp => try emit.mirArith(.cmp, inst),
76 .mov => try emit.mirArith(.mov, inst),
77
78 .adc_mem_imm => try emit.mirArithMemImm(.adc, inst),
79 .add_mem_imm => try emit.mirArithMemImm(.add, inst),
80 .sub_mem_imm => try emit.mirArithMemImm(.sub, inst),
81 .xor_mem_imm => try emit.mirArithMemImm(.xor, inst),
82 .and_mem_imm => try emit.mirArithMemImm(.@"and", inst),
83 .or_mem_imm => try emit.mirArithMemImm(.@"or", inst),
84 .sbb_mem_imm => try emit.mirArithMemImm(.sbb, inst),
85 .cmp_mem_imm => try emit.mirArithMemImm(.cmp, inst),
86 .mov_mem_imm => try emit.mirArithMemImm(.mov, inst),
87
88 .adc_scale_src => try emit.mirArithScaleSrc(.adc, inst),
89 .add_scale_src => try emit.mirArithScaleSrc(.add, inst),
90 .sub_scale_src => try emit.mirArithScaleSrc(.sub, inst),
91 .xor_scale_src => try emit.mirArithScaleSrc(.xor, inst),
92 .and_scale_src => try emit.mirArithScaleSrc(.@"and", inst),
93 .or_scale_src => try emit.mirArithScaleSrc(.@"or", inst),
94 .sbb_scale_src => try emit.mirArithScaleSrc(.sbb, inst),
95 .cmp_scale_src => try emit.mirArithScaleSrc(.cmp, inst),
96 .mov_scale_src => try emit.mirArithScaleSrc(.mov, inst),
97
98 .adc_scale_dst => try emit.mirArithScaleDst(.adc, inst),
99 .add_scale_dst => try emit.mirArithScaleDst(.add, inst),
100 .sub_scale_dst => try emit.mirArithScaleDst(.sub, inst),
101 .xor_scale_dst => try emit.mirArithScaleDst(.xor, inst),
102 .and_scale_dst => try emit.mirArithScaleDst(.@"and", inst),
103 .or_scale_dst => try emit.mirArithScaleDst(.@"or", inst),
104 .sbb_scale_dst => try emit.mirArithScaleDst(.sbb, inst),
105 .cmp_scale_dst => try emit.mirArithScaleDst(.cmp, inst),
106 .mov_scale_dst => try emit.mirArithScaleDst(.mov, inst),
107
108 .adc_scale_imm => try emit.mirArithScaleImm(.adc, inst),
109 .add_scale_imm => try emit.mirArithScaleImm(.add, inst),
110 .sub_scale_imm => try emit.mirArithScaleImm(.sub, inst),
111 .xor_scale_imm => try emit.mirArithScaleImm(.xor, inst),
112 .and_scale_imm => try emit.mirArithScaleImm(.@"and", inst),
113 .or_scale_imm => try emit.mirArithScaleImm(.@"or", inst),
114 .sbb_scale_imm => try emit.mirArithScaleImm(.sbb, inst),
115 .cmp_scale_imm => try emit.mirArithScaleImm(.cmp, inst),
116 .mov_scale_imm => try emit.mirArithScaleImm(.mov, inst),
117
118 .movabs => try emit.mirMovabs(inst),
119
120 .lea => try emit.mirLea(inst),
121
122 .imul_complex => try emit.mirIMulComplex(inst),
123
124 .push => try emit.mirPushPop(.push, inst),
125 .pop => try emit.mirPushPop(.pop, inst),
126
127 .jmp => try emit.mirJmpCall(.jmp_near, inst),
128 .call => try emit.mirJmpCall(.call_near, inst),
129
130 .cond_jmp_greater_less,
131 .cond_jmp_above_below,
132 .cond_jmp_eq_ne,
133 => try emit.mirCondJmp(tag, inst),
134
135 .cond_set_byte_greater_less,
136 .cond_set_byte_above_below,
137 .cond_set_byte_eq_ne,
138 => try emit.mirCondSetByte(tag, inst),
139
140 .ret => try emit.mirRet(inst),
141
142 .syscall => try emit.mirSyscall(),
143
144 .@"test" => try emit.mirTest(inst),
145
146 .brk => try emit.mirBrk(),
147 .nop => try emit.mirNop(),
148
149 .call_extern => try emit.mirCallExtern(inst),
150
151 .dbg_line => try emit.mirDbgLine(inst),
152 .dbg_prologue_end => try emit.mirDbgPrologueEnd(inst),
153 .dbg_epilogue_begin => try emit.mirDbgEpilogueBegin(inst),
154 .arg_dbg_info => try emit.mirArgDbgInfo(inst),
155
156 .push_regs_from_callee_preserved_regs => try emit.mirPushPopRegsFromCalleePreservedRegs(.push, inst),
157 .pop_regs_from_callee_preserved_regs => try emit.mirPushPopRegsFromCalleePreservedRegs(.pop, inst),
158
159 else => {
160 return emit.fail("Implement MIR->Emit lowering for x86_64 for pseudo-inst: {s}", .{tag});
161 },
162 }
163 }
164
165 try emit.fixupRelocs();
166}
167
168pub fn deinit(emit: *Emit) void {
169 emit.relocs.deinit(emit.bin_file.allocator);
170 emit.code_offset_mapping.deinit(emit.bin_file.allocator);
171 emit.* = undefined;
172}
173
174fn fail(emit: *Emit, comptime format: []const u8, args: anytype) InnerError {
175 @setCold(true);
176 assert(emit.err_msg == null);
177 emit.err_msg = try ErrorMsg.create(emit.bin_file.allocator, emit.src_loc, format, args);
178 return error.EmitFail;
179}
180
181fn failWithLoweringError(emit: *Emit, err: LoweringError) InnerError {
182 return switch (err) {
183 error.RaxOperandExpected => emit.fail("Register.rax expected as destination operand", .{}),
184 error.OperandSizeMismatch => emit.fail("operand size mismatch", .{}),
185 else => |e| e,
186 };
187}
188
189fn fixupRelocs(emit: *Emit) InnerError!void {
190 // TODO this function currently assumes all relocs via JMP/CALL instructions are 32bit in size.
191 // This should be reversed like it is done in aarch64 MIR emit code: start with the smallest
192 // possible resolution, i.e., 8bit, and iteratively converge on the minimum required resolution
193 // until the entire decl is correctly emitted with all JMP/CALL instructions within range.
194 for (emit.relocs.items) |reloc| {
195 const target = emit.code_offset_mapping.get(reloc.target) orelse
196 return emit.fail("JMP/CALL relocation target not found!", .{});
197 const disp = @intCast(i32, @intCast(i64, target) - @intCast(i64, reloc.source + reloc.length));
198 mem.writeIntLittle(i32, emit.code.items[reloc.offset..][0..4], disp);
199 }
200}
201
202fn mirBrk(emit: *Emit) InnerError!void {
203 return lowerToZoEnc(.brk, emit.code) catch |err| emit.failWithLoweringError(err);
204}
205
206fn mirNop(emit: *Emit) InnerError!void {
207 return lowerToZoEnc(.nop, emit.code) catch |err| emit.failWithLoweringError(err);
208}
209
210fn mirSyscall(emit: *Emit) InnerError!void {
211 return lowerToZoEnc(.syscall, emit.code) catch |err| emit.failWithLoweringError(err);
212}
213
214fn mirPushPop(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
215 const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
216 switch (ops.flags) {
217 0b00 => {
218 // PUSH/POP reg
219 return lowerToOEnc(tag, ops.reg1, emit.code) catch |err| emit.failWithLoweringError(err);
220 },
221 0b01 => {
222 // PUSH/POP r/m64
223 const imm = emit.mir.instructions.items(.data)[inst].imm;
224 const ptr_size: Memory.PtrSize = switch (immOpSize(imm)) {
225 16 => .word_ptr,
226 else => .qword_ptr,
227 };
228 return lowerToMEnc(tag, RegisterOrMemory.mem(ptr_size, .{
229 .disp = imm,
230 .base = ops.reg1,
231 }), emit.code) catch |err| emit.failWithLoweringError(err);
232 },
233 0b10 => {
234 // PUSH imm32
235 assert(tag == .push);
236 const imm = emit.mir.instructions.items(.data)[inst].imm;
237 return lowerToIEnc(.push, imm, emit.code) catch |err|
238 emit.failWithLoweringError(err);
239 },
240 0b11 => unreachable,
241 }
242}
243fn mirPushPopRegsFromCalleePreservedRegs(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
244 const callee_preserved_regs = bits.callee_preserved_regs;
245 const regs = emit.mir.instructions.items(.data)[inst].regs_to_push_or_pop;
246 if (tag == .push) {
247 for (callee_preserved_regs) |reg, i| {
248 if ((regs >> @intCast(u5, i)) & 1 == 0) continue;
249 lowerToOEnc(.push, reg, emit.code) catch |err|
250 return emit.failWithLoweringError(err);
251 }
252 } else {
253 // pop in the reverse direction
254 var i = callee_preserved_regs.len;
255 while (i > 0) : (i -= 1) {
256 const reg = callee_preserved_regs[i - 1];
257 if ((regs >> @intCast(u5, i - 1)) & 1 == 0) continue;
258 lowerToOEnc(.pop, reg, emit.code) catch |err|
259 return emit.failWithLoweringError(err);
260 }
261 }
262}
263
264fn mirJmpCall(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
265 const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
266 switch (ops.flags) {
267 0b00 => {
268 const target = emit.mir.instructions.items(.data)[inst].inst;
269 const source = emit.code.items.len;
270 lowerToDEnc(tag, 0, emit.code) catch |err|
271 return emit.failWithLoweringError(err);
272 try emit.relocs.append(emit.bin_file.allocator, .{
273 .source = source,
274 .target = target,
275 .offset = emit.code.items.len - 4,
276 .length = 5,
277 });
278 },
279 0b01 => {
280 if (ops.reg1 == .none) {
281 // JMP/CALL [imm]
282 const imm = emit.mir.instructions.items(.data)[inst].imm;
283 const ptr_size: Memory.PtrSize = switch (immOpSize(imm)) {
284 16 => .word_ptr,
285 else => .qword_ptr,
286 };
287 return lowerToMEnc(tag, RegisterOrMemory.mem(ptr_size, .{ .disp = imm }), emit.code) catch |err|
288 emit.failWithLoweringError(err);
289 }
290 // JMP/CALL reg
291 return lowerToMEnc(tag, RegisterOrMemory.reg(ops.reg1), emit.code) catch |err| emit.failWithLoweringError(err);
292 },
293 0b10 => {
294 // JMP/CALL r/m64
295 const imm = emit.mir.instructions.items(.data)[inst].imm;
296 return lowerToMEnc(tag, RegisterOrMemory.mem(Memory.PtrSize.fromBits(ops.reg1.size()), .{
297 .disp = imm,
298 .base = ops.reg1,
299 }), emit.code) catch |err| emit.failWithLoweringError(err);
300 },
301 0b11 => return emit.fail("TODO unused JMP/CALL variant 0b11", .{}),
302 }
303}
304
305fn mirCondJmp(emit: *Emit, mir_tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerError!void {
306 const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
307 const target = emit.mir.instructions.items(.data)[inst].inst;
308 const tag = switch (mir_tag) {
309 .cond_jmp_greater_less => switch (ops.flags) {
310 0b00 => Tag.jge,
311 0b01 => Tag.jg,
312 0b10 => Tag.jl,
313 0b11 => Tag.jle,
314 },
315 .cond_jmp_above_below => switch (ops.flags) {
316 0b00 => Tag.jae,
317 0b01 => Tag.ja,
318 0b10 => Tag.jb,
319 0b11 => Tag.jbe,
320 },
321 .cond_jmp_eq_ne => switch (@truncate(u1, ops.flags)) {
322 0b0 => Tag.jne,
323 0b1 => Tag.je,
324 },
325 else => unreachable,
326 };
327 const source = emit.code.items.len;
328 lowerToDEnc(tag, 0, emit.code) catch |err|
329 return emit.failWithLoweringError(err);
330 try emit.relocs.append(emit.bin_file.allocator, .{
331 .source = source,
332 .target = target,
333 .offset = emit.code.items.len - 4,
334 .length = 6,
335 });
336}
337
338fn mirCondSetByte(emit: *Emit, mir_tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerError!void {
339 const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
340 const tag = switch (mir_tag) {
341 .cond_set_byte_greater_less => switch (ops.flags) {
342 0b00 => Tag.setge,
343 0b01 => Tag.setg,
344 0b10 => Tag.setl,
345 0b11 => Tag.setle,
346 },
347 .cond_set_byte_above_below => switch (ops.flags) {
348 0b00 => Tag.setae,
349 0b01 => Tag.seta,
350 0b10 => Tag.setb,
351 0b11 => Tag.setbe,
352 },
353 .cond_set_byte_eq_ne => switch (@truncate(u1, ops.flags)) {
354 0b0 => Tag.setne,
355 0b1 => Tag.sete,
356 },
357 else => unreachable,
358 };
359 return lowerToMEnc(tag, RegisterOrMemory.reg(ops.reg1.to8()), emit.code) catch |err|
360 emit.failWithLoweringError(err);
361}
362
363fn mirTest(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
364 const tag = emit.mir.instructions.items(.tag)[inst];
365 assert(tag == .@"test");
366 const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
367 switch (ops.flags) {
368 0b00 => {
369 if (ops.reg2 == .none) {
370 // TEST r/m64, imm32
371 // MI
372 const imm = emit.mir.instructions.items(.data)[inst].imm;
373 if (ops.reg1.to64() == .rax) {
374 // TEST rax, imm32
375 // I
376 return lowerToIEnc(.@"test", imm, emit.code) catch |err|
377 emit.failWithLoweringError(err);
378 }
379 return lowerToMiEnc(.@"test", RegisterOrMemory.reg(ops.reg1), imm, emit.code) catch |err|
380 emit.failWithLoweringError(err);
381 }
382 // TEST r/m64, r64
383 return emit.fail("TODO TEST r/m64, r64", .{});
384 },
385 else => return emit.fail("TODO more TEST alternatives", .{}),
386 }
387}
388
389fn mirRet(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
390 const tag = emit.mir.instructions.items(.tag)[inst];
391 assert(tag == .ret);
392 const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
393 switch (ops.flags) {
394 0b00 => {
395 // RETF imm16
396 // I
397 const imm = emit.mir.instructions.items(.data)[inst].imm;
398 return lowerToIEnc(.ret_far, imm, emit.code) catch |err| emit.failWithLoweringError(err);
399 },
400 0b01 => {
401 return lowerToZoEnc(.ret_far, emit.code) catch |err| emit.failWithLoweringError(err);
402 },
403 0b10 => {
404 // RET imm16
405 // I
406 const imm = emit.mir.instructions.items(.data)[inst].imm;
407 return lowerToIEnc(.ret_near, imm, emit.code) catch |err| emit.failWithLoweringError(err);
408 },
409 0b11 => {
410 return lowerToZoEnc(.ret_near, emit.code) catch |err| emit.failWithLoweringError(err);
411 },
412 }
413}
414
415fn mirArith(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
416 const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
417 switch (ops.flags) {
418 0b00 => {
419 if (ops.reg2 == .none) {
420 // mov reg1, imm32
421 // MI
422 const imm = emit.mir.instructions.items(.data)[inst].imm;
423 return lowerToMiEnc(tag, RegisterOrMemory.reg(ops.reg1), imm, emit.code) catch |err|
424 emit.failWithLoweringError(err);
425 }
426 // mov reg1, reg2
427 // RM
428 return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.reg(ops.reg2), emit.code) catch |err|
429 emit.failWithLoweringError(err);
430 },
431 0b01 => {
432 // mov reg1, [reg2 + imm32]
433 // RM
434 const imm = emit.mir.instructions.items(.data)[inst].imm;
435 const src_reg: ?Register = if (ops.reg2 == .none) null else ops.reg2;
436 return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.mem(Memory.PtrSize.fromBits(ops.reg1.size()), .{
437 .disp = imm,
438 .base = src_reg,
439 }), emit.code) catch |err| emit.failWithLoweringError(err);
440 },
441 0b10 => {
442 if (ops.reg2 == .none) {
443 return emit.fail("TODO unused variant: mov reg1, none, 0b10", .{});
444 }
445 // mov [reg1 + imm32], reg2
446 // MR
447 const imm = emit.mir.instructions.items(.data)[inst].imm;
448 return lowerToMrEnc(tag, RegisterOrMemory.mem(Memory.PtrSize.fromBits(ops.reg2.size()), .{
449 .disp = imm,
450 .base = ops.reg1,
451 }), ops.reg2, emit.code) catch |err| emit.failWithLoweringError(err);
452 },
453 0b11 => {
454 return emit.fail("TODO unused variant: mov reg1, reg2, 0b11", .{});
455 },
456 }
457}
458
459fn mirArithMemImm(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
460 const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
461 assert(ops.reg2 == .none);
462 const payload = emit.mir.instructions.items(.data)[inst].payload;
463 const imm_pair = emit.mir.extraData(Mir.ImmPair, payload).data;
464 const ptr_size: Memory.PtrSize = switch (ops.flags) {
465 0b00 => .byte_ptr,
466 0b01 => .word_ptr,
467 0b10 => .dword_ptr,
468 0b11 => .qword_ptr,
469 };
470 return lowerToMiEnc(tag, RegisterOrMemory.mem(ptr_size, .{
471 .disp = imm_pair.dest_off,
472 .base = ops.reg1,
473 }), imm_pair.operand, emit.code) catch |err| emit.failWithLoweringError(err);
474}
475
476inline fn setRexWRegister(reg: Register) bool {
477 if (reg.size() == 64) return true;
478 return switch (reg) {
479 .ah, .bh, .ch, .dh => true,
480 else => false,
481 };
482}
483
484inline fn immOpSize(u_imm: u32) u8 {
485 const imm = @bitCast(i32, u_imm);
486 if (math.minInt(i8) <= imm and imm <= math.maxInt(i8)) {
487 return 8;
488 }
489 if (math.minInt(i16) <= imm and imm <= math.maxInt(i16)) {
490 return 16;
491 }
492 return 32;
493}
494
495inline fn imm64OpSize(u_imm: u64) u8 {
496 const imm = @bitCast(i64, u_imm);
497 if (math.minInt(i8) <= imm and imm <= math.maxInt(i8)) {
498 return 8;
499 }
500 if (math.minInt(i16) <= imm and imm <= math.maxInt(i16)) {
501 return 16;
502 }
503 if (math.minInt(i32) <= imm and imm <= math.maxInt(i32)) {
504 return 32;
505 }
506 return 64;
507}
508
509fn mirArithScaleSrc(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
510 const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
511 const scale = ops.flags;
512 const imm = emit.mir.instructions.items(.data)[inst].imm;
513 // OP reg1, [reg2 + scale*rcx + imm32]
514 const scale_index = ScaleIndex{
515 .scale = scale,
516 .index = .rcx,
517 };
518 return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.mem(Memory.PtrSize.fromBits(ops.reg1.size()), .{
519 .disp = imm,
520 .base = ops.reg2,
521 .scale_index = scale_index,
522 }), emit.code) catch |err| emit.failWithLoweringError(err);
523}
524
525fn mirArithScaleDst(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
526 const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
527 const scale = ops.flags;
528 const imm = emit.mir.instructions.items(.data)[inst].imm;
529 const scale_index = ScaleIndex{
530 .scale = scale,
531 .index = .rax,
532 };
533 if (ops.reg2 == .none) {
534 // OP qword ptr [reg1 + scale*rax + 0], imm32
535 return lowerToMiEnc(tag, RegisterOrMemory.mem(.qword_ptr, .{
536 .disp = 0,
537 .base = ops.reg1,
538 .scale_index = scale_index,
539 }), imm, emit.code) catch |err| emit.failWithLoweringError(err);
540 }
541 // OP [reg1 + scale*rax + imm32], reg2
542 return lowerToMrEnc(tag, RegisterOrMemory.mem(Memory.PtrSize.fromBits(ops.reg2.size()), .{
543 .disp = imm,
544 .base = ops.reg1,
545 .scale_index = scale_index,
546 }), ops.reg2, emit.code) catch |err| emit.failWithLoweringError(err);
547}
548
549fn mirArithScaleImm(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
550 const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
551 const scale = ops.flags;
552 const payload = emit.mir.instructions.items(.data)[inst].payload;
553 const imm_pair = emit.mir.extraData(Mir.ImmPair, payload).data;
554 const scale_index = ScaleIndex{
555 .scale = scale,
556 .index = .rax,
557 };
558 // OP qword ptr [reg1 + scale*rax + imm32], imm32
559 return lowerToMiEnc(tag, RegisterOrMemory.mem(.qword_ptr, .{
560 .disp = imm_pair.dest_off,
561 .base = ops.reg1,
562 .scale_index = scale_index,
563 }), imm_pair.operand, emit.code) catch |err| emit.failWithLoweringError(err);
564}
565
566fn mirMovabs(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
567 const tag = emit.mir.instructions.items(.tag)[inst];
568 assert(tag == .movabs);
569 const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
570 const imm: u64 = if (ops.reg1.size() == 64) blk: {
571 const payload = emit.mir.instructions.items(.data)[inst].payload;
572 const imm = emit.mir.extraData(Mir.Imm64, payload).data;
573 break :blk imm.decode();
574 } else emit.mir.instructions.items(.data)[inst].imm;
575 if (ops.flags == 0b00) {
576 // movabs reg, imm64
577 // OI
578 return lowerToOiEnc(.mov, ops.reg1, imm, emit.code) catch |err| emit.failWithLoweringError(err);
579 }
580 if (ops.reg1 == .none) {
581 // movabs moffs64, rax
582 // TD
583 return lowerToTdEnc(.mov, imm, ops.reg2, emit.code) catch |err| emit.failWithLoweringError(err);
584 }
585 // movabs rax, moffs64
586 // FD
587 return lowerToFdEnc(.mov, ops.reg1, imm, emit.code) catch |err| emit.failWithLoweringError(err);
588}
589
590fn mirIMulComplex(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
591 const tag = emit.mir.instructions.items(.tag)[inst];
592 assert(tag == .imul_complex);
593 const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
594 switch (ops.flags) {
595 0b00 => {
596 return lowerToRmEnc(.imul, ops.reg1, RegisterOrMemory.reg(ops.reg2), emit.code) catch |err|
597 emit.failWithLoweringError(err);
598 },
599 0b10 => {
600 const imm = emit.mir.instructions.items(.data)[inst].imm;
601 return lowerToRmiEnc(.imul, ops.reg1, RegisterOrMemory.reg(ops.reg2), imm, emit.code) catch |err|
602 emit.failWithLoweringError(err);
603 },
604 else => return emit.fail("TODO implement imul", .{}),
605 }
606}
607
608fn mirLea(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
609 const tag = emit.mir.instructions.items(.tag)[inst];
610 assert(tag == .lea);
611 const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
612 switch (ops.flags) {
613 0b00 => {
614 // lea reg1, [reg2 + imm32]
615 // RM
616 const imm = emit.mir.instructions.items(.data)[inst].imm;
617 const src_reg: ?Register = if (ops.reg2 == .none) null else ops.reg2;
618 return lowerToRmEnc(
619 .lea,
620 ops.reg1,
621 RegisterOrMemory.mem(Memory.PtrSize.fromBits(ops.reg1.size()), .{
622 .disp = imm,
623 .base = src_reg,
624 }),
625 emit.code,
626 ) catch |err| emit.failWithLoweringError(err);
627 },
628 0b01 => {
629 // lea reg1, [rip + imm32]
630 // RM
631 const start_offset = emit.code.items.len;
632 lowerToRmEnc(
633 .lea,
634 ops.reg1,
635 RegisterOrMemory.rip(Memory.PtrSize.fromBits(ops.reg1.size()), 0),
636 emit.code,
637 ) catch |err| return emit.failWithLoweringError(err);
638 const end_offset = emit.code.items.len;
639 // Backpatch the displacement
640 const payload = emit.mir.instructions.items(.data)[inst].payload;
641 const imm = emit.mir.extraData(Mir.Imm64, payload).data.decode();
642 const disp = @intCast(i32, @intCast(i64, imm) - @intCast(i64, end_offset - start_offset));
643 mem.writeIntLittle(i32, emit.code.items[end_offset - 4 ..][0..4], disp);
644 },
645 0b10 => {
646 // lea reg1, [rip + reloc]
647 // RM
648 lowerToRmEnc(
649 .lea,
650 ops.reg1,
651 RegisterOrMemory.rip(Memory.PtrSize.fromBits(ops.reg1.size()), 0),
652 emit.code,
653 ) catch |err| return emit.failWithLoweringError(err);
654 const end_offset = emit.code.items.len;
655 const got_entry = emit.mir.instructions.items(.data)[inst].got_entry;
656 if (emit.bin_file.cast(link.File.MachO)) |macho_file| {
657 // TODO I think the reloc might be in the wrong place.
658 const decl = macho_file.active_decl.?;
659 try decl.link.macho.relocs.append(emit.bin_file.allocator, .{
660 .offset = @intCast(u32, end_offset - 4),
661 .target = .{ .local = got_entry },
662 .addend = 0,
663 .subtractor = null,
664 .pcrel = true,
665 .length = 2,
666 .@"type" = @enumToInt(std.macho.reloc_type_x86_64.X86_64_RELOC_GOT),
667 });
668 } else {
669 return emit.fail(
670 "TODO implement lea reg, [rip + reloc] for linking backends different than MachO",
671 .{},
672 );
673 }
674 },
675 0b11 => {
676 // lea reg, [rbp + rcx + imm32]
677 const imm = emit.mir.instructions.items(.data)[inst].imm;
678 const src_reg: ?Register = if (ops.reg2 == .none) null else ops.reg2;
679 const scale_index = ScaleIndex{
680 .scale = 0,
681 .index = .rcx,
682 };
683 return lowerToRmEnc(
684 .lea,
685 ops.reg1,
686 RegisterOrMemory.mem(Memory.PtrSize.fromBits(ops.reg1.size()), .{
687 .disp = imm,
688 .base = src_reg,
689 .scale_index = scale_index,
690 }),
691 emit.code,
692 ) catch |err| emit.failWithLoweringError(err);
693 },
694 }
695}
696
697fn mirCallExtern(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
698 const tag = emit.mir.instructions.items(.tag)[inst];
699 assert(tag == .call_extern);
700 const n_strx = emit.mir.instructions.items(.data)[inst].extern_fn;
701 const offset = blk: {
702 // callq
703 lowerToDEnc(.call_near, 0, emit.code) catch |err|
704 return emit.failWithLoweringError(err);
705 break :blk @intCast(u32, emit.code.items.len) - 4;
706 };
707 if (emit.bin_file.cast(link.File.MachO)) |macho_file| {
708 // Add relocation to the decl.
709 try macho_file.active_decl.?.link.macho.relocs.append(emit.bin_file.allocator, .{
710 .offset = offset,
711 .target = .{ .global = n_strx },
712 .addend = 0,
713 .subtractor = null,
714 .pcrel = true,
715 .length = 2,
716 .@"type" = @enumToInt(std.macho.reloc_type_x86_64.X86_64_RELOC_BRANCH),
717 });
718 } else {
719 return emit.fail("TODO implement call_extern for linking backends different than MachO", .{});
720 }
721}
722
723fn mirDbgLine(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
724 const tag = emit.mir.instructions.items(.tag)[inst];
725 assert(tag == .dbg_line);
726 const payload = emit.mir.instructions.items(.data)[inst].payload;
727 const dbg_line_column = emit.mir.extraData(Mir.DbgLineColumn, payload).data;
728 try emit.dbgAdvancePCAndLine(dbg_line_column.line, dbg_line_column.column);
729}
730
731fn dbgAdvancePCAndLine(emit: *Emit, line: u32, column: u32) InnerError!void {
732 const delta_line = @intCast(i32, line) - @intCast(i32, emit.prev_di_line);
733 const delta_pc: usize = emit.code.items.len - emit.prev_di_pc;
734 switch (emit.debug_output) {
735 .dwarf => |dbg_out| {
736 // TODO Look into using the DWARF special opcodes to compress this data.
737 // It lets you emit single-byte opcodes that add different numbers to
738 // both the PC and the line number at the same time.
739 try dbg_out.dbg_line.ensureUnusedCapacity(11);
740 dbg_out.dbg_line.appendAssumeCapacity(DW.LNS.advance_pc);
741 leb128.writeULEB128(dbg_out.dbg_line.writer(), delta_pc) catch unreachable;
742 if (delta_line != 0) {
743 dbg_out.dbg_line.appendAssumeCapacity(DW.LNS.advance_line);
744 leb128.writeILEB128(dbg_out.dbg_line.writer(), delta_line) catch unreachable;
745 }
746 dbg_out.dbg_line.appendAssumeCapacity(DW.LNS.copy);
747 emit.prev_di_pc = emit.code.items.len;
748 emit.prev_di_line = line;
749 emit.prev_di_column = column;
750 emit.prev_di_pc = emit.code.items.len;
751 },
752 .plan9 => |dbg_out| {
753 if (delta_pc <= 0) return; // only do this when the pc changes
754 // we have already checked the target in the linker to make sure it is compatable
755 const quant = @import("../../link/Plan9/aout.zig").getPCQuant(emit.target.cpu.arch) catch unreachable;
756
757 // increasing the line number
758 try @import("../../link/Plan9.zig").changeLine(dbg_out.dbg_line, delta_line);
759 // increasing the pc
760 const d_pc_p9 = @intCast(i64, delta_pc) - quant;
761 if (d_pc_p9 > 0) {
762 // minus one because if its the last one, we want to leave space to change the line which is one quanta
763 var diff = @divExact(d_pc_p9, quant) - quant;
764 while (diff > 0) {
765 if (diff < 64) {
766 try dbg_out.dbg_line.append(@intCast(u8, diff + 128));
767 diff = 0;
768 } else {
769 try dbg_out.dbg_line.append(@intCast(u8, 64 + 128));
770 diff -= 64;
771 }
772 }
773 if (dbg_out.pcop_change_index.*) |pci|
774 dbg_out.dbg_line.items[pci] += 1;
775 dbg_out.pcop_change_index.* = @intCast(u32, dbg_out.dbg_line.items.len - 1);
776 } else if (d_pc_p9 == 0) {
777 // we don't need to do anything, because adding the quant does it for us
778 } else unreachable;
779 if (dbg_out.start_line.* == null)
780 dbg_out.start_line.* = emit.prev_di_line;
781 dbg_out.end_line.* = line;
782 // only do this if the pc changed
783 emit.prev_di_line = line;
784 emit.prev_di_column = column;
785 emit.prev_di_pc = emit.code.items.len;
786 },
787 .none => {},
788 }
789}
790
791fn mirDbgPrologueEnd(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
792 const tag = emit.mir.instructions.items(.tag)[inst];
793 assert(tag == .dbg_prologue_end);
794 switch (emit.debug_output) {
795 .dwarf => |dbg_out| {
796 try dbg_out.dbg_line.append(DW.LNS.set_prologue_end);
797 try emit.dbgAdvancePCAndLine(emit.prev_di_line, emit.prev_di_column);
798 },
799 .plan9 => {},
800 .none => {},
801 }
802}
803
804fn mirDbgEpilogueBegin(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
805 const tag = emit.mir.instructions.items(.tag)[inst];
806 assert(tag == .dbg_epilogue_begin);
807 switch (emit.debug_output) {
808 .dwarf => |dbg_out| {
809 try dbg_out.dbg_line.append(DW.LNS.set_epilogue_begin);
810 try emit.dbgAdvancePCAndLine(emit.prev_di_line, emit.prev_di_column);
811 },
812 .plan9 => {},
813 .none => {},
814 }
815}
816
817fn mirArgDbgInfo(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
818 const tag = emit.mir.instructions.items(.tag)[inst];
819 assert(tag == .arg_dbg_info);
820 const payload = emit.mir.instructions.items(.data)[inst].payload;
821 const arg_dbg_info = emit.mir.extraData(Mir.ArgDbgInfo, payload).data;
822 const mcv = emit.mir.function.args[arg_dbg_info.arg_index];
823 try emit.genArgDbgInfo(arg_dbg_info.air_inst, mcv);
824}
825
826fn genArgDbgInfo(emit: *Emit, inst: Air.Inst.Index, mcv: MCValue) !void {
827 const ty_str = emit.mir.function.air.instructions.items(.data)[inst].ty_str;
828 const zir = &emit.mir.function.mod_fn.owner_decl.getFileScope().zir;
829 const name = zir.nullTerminatedString(ty_str.str);
830 const name_with_null = name.ptr[0 .. name.len + 1];
831 const ty = emit.mir.function.air.getRefType(ty_str.ty);
832
833 switch (mcv) {
834 .register => |reg| {
835 switch (emit.debug_output) {
836 .dwarf => |dbg_out| {
837 try dbg_out.dbg_info.ensureUnusedCapacity(3);
838 dbg_out.dbg_info.appendAssumeCapacity(link.File.Elf.abbrev_parameter);
839 dbg_out.dbg_info.appendSliceAssumeCapacity(&[2]u8{ // DW.AT.location, DW.FORM.exprloc
840 1, // ULEB128 dwarf expression length
841 reg.dwarfLocOp(),
842 });
843 try dbg_out.dbg_info.ensureUnusedCapacity(5 + name_with_null.len);
844 try emit.addDbgInfoTypeReloc(ty); // DW.AT.type, DW.FORM.ref4
845 dbg_out.dbg_info.appendSliceAssumeCapacity(name_with_null); // DW.AT.name, DW.FORM.string
846 },
847 .plan9 => {},
848 .none => {},
849 }
850 },
851 .stack_offset => {
852 switch (emit.debug_output) {
853 .dwarf => {},
854 .plan9 => {},
855 .none => {},
856 }
857 },
858 else => {},
859 }
860}
861
862/// Adds a Type to the .debug_info at the current position. The bytes will be populated later,
863/// after codegen for this symbol is done.
864fn addDbgInfoTypeReloc(emit: *Emit, ty: Type) !void {
865 switch (emit.debug_output) {
866 .dwarf => |dbg_out| {
867 assert(ty.hasCodeGenBits());
868 const index = dbg_out.dbg_info.items.len;
869 try dbg_out.dbg_info.resize(index + 4); // DW.AT.type, DW.FORM.ref4
870
871 const gop = try dbg_out.dbg_info_type_relocs.getOrPut(emit.bin_file.allocator, ty);
872 if (!gop.found_existing) {
873 gop.value_ptr.* = .{
874 .off = undefined,
875 .relocs = .{},
876 };
877 }
878 try gop.value_ptr.relocs.append(emit.bin_file.allocator, @intCast(u32, index));
879 },
880 .plan9 => {},
881 .none => {},
882 }
883}
884
885const Tag = enum {
886 adc,
887 add,
888 sub,
889 xor,
890 @"and",
891 @"or",
892 sbb,
893 cmp,
894 mov,
895 lea,
896 jmp_near,
897 call_near,
898 push,
899 pop,
900 @"test",
901 brk,
902 nop,
903 imul,
904 syscall,
905 ret_near,
906 ret_far,
907 jo,
908 jno,
909 jb,
910 jbe,
911 jc,
912 jnae,
913 jnc,
914 jae,
915 je,
916 jz,
917 jne,
918 jnz,
919 jna,
920 jnb,
921 jnbe,
922 ja,
923 js,
924 jns,
925 jpe,
926 jp,
927 jpo,
928 jnp,
929 jnge,
930 jl,
931 jge,
932 jnl,
933 jle,
934 jng,
935 jg,
936 jnle,
937 seto,
938 setno,
939 setb,
940 setc,
941 setnae,
942 setnb,
943 setnc,
944 setae,
945 sete,
946 setz,
947 setne,
948 setnz,
949 setbe,
950 setna,
951 seta,
952 setnbe,
953 sets,
954 setns,
955 setp,
956 setpe,
957 setnp,
958 setop,
959 setl,
960 setnge,
961 setnl,
962 setge,
963 setle,
964 setng,
965 setnle,
966 setg,
967
968 fn isSetCC(tag: Tag) bool {
969 return switch (tag) {
970 .seto,
971 .setno,
972 .setb,
973 .setc,
974 .setnae,
975 .setnb,
976 .setnc,
977 .setae,
978 .sete,
979 .setz,
980 .setne,
981 .setnz,
982 .setbe,
983 .setna,
984 .seta,
985 .setnbe,
986 .sets,
987 .setns,
988 .setp,
989 .setpe,
990 .setnp,
991 .setop,
992 .setl,
993 .setnge,
994 .setnl,
995 .setge,
996 .setle,
997 .setng,
998 .setnle,
999 .setg,
1000 => true,
1001 else => false,
1002 };
1003 }
1004};
1005
1006const Encoding = enum {
1007 /// OP
1008 zo,
1009
1010 /// OP rel32
1011 d,
1012
1013 /// OP r/m64
1014 m,
1015
1016 /// OP r64
1017 o,
1018
1019 /// OP imm32
1020 i,
1021
1022 /// OP r/m64, imm32
1023 mi,
1024
1025 /// OP r/m64, r64
1026 mr,
1027
1028 /// OP r64, r/m64
1029 rm,
1030
1031 /// OP r64, imm64
1032 oi,
1033
1034 /// OP al/ax/eax/rax, moffs
1035 fd,
1036
1037 /// OP moffs, al/ax/eax/rax
1038 td,
1039
1040 /// OP r64, r/m64, imm32
1041 rmi,
1042};
1043
1044const OpCode = union(enum) {
1045 one_byte: u8,
1046 two_byte: struct { _1: u8, _2: u8 },
1047
1048 fn oneByte(opc: u8) OpCode {
1049 return .{ .one_byte = opc };
1050 }
1051
1052 fn twoByte(opc1: u8, opc2: u8) OpCode {
1053 return .{ .two_byte = .{ ._1 = opc1, ._2 = opc2 } };
1054 }
1055
1056 fn encode(opc: OpCode, encoder: Encoder) void {
1057 switch (opc) {
1058 .one_byte => |v| encoder.opcode_1byte(v),
1059 .two_byte => |v| encoder.opcode_2byte(v._1, v._2),
1060 }
1061 }
1062
1063 fn encodeWithReg(opc: OpCode, encoder: Encoder, reg: Register) void {
1064 assert(opc == .one_byte);
1065 encoder.opcode_withReg(opc.one_byte, reg.lowId());
1066 }
1067};
1068
1069inline fn getOpCode(tag: Tag, enc: Encoding, is_one_byte: bool) ?OpCode {
1070 switch (enc) {
1071 .zo => return switch (tag) {
1072 .ret_near => OpCode.oneByte(0xc3),
1073 .ret_far => OpCode.oneByte(0xcb),
1074 .brk => OpCode.oneByte(0xcc),
1075 .nop => OpCode.oneByte(0x90),
1076 .syscall => OpCode.twoByte(0x0f, 0x05),
1077 else => null,
1078 },
1079 .d => return switch (tag) {
1080 .jmp_near => OpCode.oneByte(0xe9),
1081 .call_near => OpCode.oneByte(0xe8),
1082 .jo => if (is_one_byte) OpCode.oneByte(0x70) else OpCode.twoByte(0x0f, 0x80),
1083 .jno => if (is_one_byte) OpCode.oneByte(0x71) else OpCode.twoByte(0x0f, 0x81),
1084 .jb, .jc, .jnae => if (is_one_byte) OpCode.oneByte(0x72) else OpCode.twoByte(0x0f, 0x82),
1085 .jnb, .jnc, .jae => if (is_one_byte) OpCode.oneByte(0x73) else OpCode.twoByte(0x0f, 0x83),
1086 .je, .jz => if (is_one_byte) OpCode.oneByte(0x74) else OpCode.twoByte(0x0f, 0x84),
1087 .jne, .jnz => if (is_one_byte) OpCode.oneByte(0x75) else OpCode.twoByte(0x0f, 0x85),
1088 .jna, .jbe => if (is_one_byte) OpCode.oneByte(0x76) else OpCode.twoByte(0x0f, 0x86),
1089 .jnbe, .ja => if (is_one_byte) OpCode.oneByte(0x77) else OpCode.twoByte(0x0f, 0x87),
1090 .js => if (is_one_byte) OpCode.oneByte(0x78) else OpCode.twoByte(0x0f, 0x88),
1091 .jns => if (is_one_byte) OpCode.oneByte(0x79) else OpCode.twoByte(0x0f, 0x89),
1092 .jpe, .jp => if (is_one_byte) OpCode.oneByte(0x7a) else OpCode.twoByte(0x0f, 0x8a),
1093 .jpo, .jnp => if (is_one_byte) OpCode.oneByte(0x7b) else OpCode.twoByte(0x0f, 0x8b),
1094 .jnge, .jl => if (is_one_byte) OpCode.oneByte(0x7c) else OpCode.twoByte(0x0f, 0x8c),
1095 .jge, .jnl => if (is_one_byte) OpCode.oneByte(0x7d) else OpCode.twoByte(0x0f, 0x8d),
1096 .jle, .jng => if (is_one_byte) OpCode.oneByte(0x7e) else OpCode.twoByte(0x0f, 0x8e),
1097 .jg, .jnle => if (is_one_byte) OpCode.oneByte(0x7f) else OpCode.twoByte(0x0f, 0x8f),
1098 else => null,
1099 },
1100 .m => return switch (tag) {
1101 .jmp_near, .call_near, .push => OpCode.oneByte(0xff),
1102 .pop => OpCode.oneByte(0x8f),
1103 .seto => OpCode.twoByte(0x0f, 0x90),
1104 .setno => OpCode.twoByte(0x0f, 0x91),
1105 .setb, .setc, .setnae => OpCode.twoByte(0x0f, 0x92),
1106 .setnb, .setnc, .setae => OpCode.twoByte(0x0f, 0x93),
1107 .sete, .setz => OpCode.twoByte(0x0f, 0x94),
1108 .setne, .setnz => OpCode.twoByte(0x0f, 0x95),
1109 .setbe, .setna => OpCode.twoByte(0x0f, 0x96),
1110 .seta, .setnbe => OpCode.twoByte(0x0f, 0x97),
1111 .sets => OpCode.twoByte(0x0f, 0x98),
1112 .setns => OpCode.twoByte(0x0f, 0x99),
1113 .setp, .setpe => OpCode.twoByte(0x0f, 0x9a),
1114 .setnp, .setop => OpCode.twoByte(0x0f, 0x9b),
1115 .setl, .setnge => OpCode.twoByte(0x0f, 0x9c),
1116 .setnl, .setge => OpCode.twoByte(0x0f, 0x9d),
1117 .setle, .setng => OpCode.twoByte(0x0f, 0x9e),
1118 .setnle, .setg => OpCode.twoByte(0x0f, 0x9f),
1119 else => null,
1120 },
1121 .o => return switch (tag) {
1122 .push => OpCode.oneByte(0x50),
1123 .pop => OpCode.oneByte(0x58),
1124 else => null,
1125 },
1126 .i => return switch (tag) {
1127 .push => OpCode.oneByte(if (is_one_byte) 0x6a else 0x68),
1128 .@"test" => OpCode.oneByte(if (is_one_byte) 0xa8 else 0xa9),
1129 .ret_near => OpCode.oneByte(0xc2),
1130 .ret_far => OpCode.oneByte(0xca),
1131 else => null,
1132 },
1133 .mi => return switch (tag) {
1134 .adc, .add, .sub, .xor, .@"and", .@"or", .sbb, .cmp => OpCode.oneByte(if (is_one_byte) 0x80 else 0x81),
1135 .mov => OpCode.oneByte(if (is_one_byte) 0xc6 else 0xc7),
1136 .@"test" => OpCode.oneByte(if (is_one_byte) 0xf6 else 0xf7),
1137 else => null,
1138 },
1139 .mr => return switch (tag) {
1140 .adc => OpCode.oneByte(if (is_one_byte) 0x10 else 0x11),
1141 .add => OpCode.oneByte(if (is_one_byte) 0x00 else 0x01),
1142 .sub => OpCode.oneByte(if (is_one_byte) 0x28 else 0x29),
1143 .xor => OpCode.oneByte(if (is_one_byte) 0x30 else 0x31),
1144 .@"and" => OpCode.oneByte(if (is_one_byte) 0x20 else 0x21),
1145 .@"or" => OpCode.oneByte(if (is_one_byte) 0x08 else 0x09),
1146 .sbb => OpCode.oneByte(if (is_one_byte) 0x18 else 0x19),
1147 .cmp => OpCode.oneByte(if (is_one_byte) 0x38 else 0x39),
1148 .mov => OpCode.oneByte(if (is_one_byte) 0x88 else 0x89),
1149 else => null,
1150 },
1151 .rm => return switch (tag) {
1152 .adc => OpCode.oneByte(if (is_one_byte) 0x12 else 0x13),
1153 .add => OpCode.oneByte(if (is_one_byte) 0x02 else 0x03),
1154 .sub => OpCode.oneByte(if (is_one_byte) 0x2a else 0x2b),
1155 .xor => OpCode.oneByte(if (is_one_byte) 0x32 else 0x33),
1156 .@"and" => OpCode.oneByte(if (is_one_byte) 0x22 else 0x23),
1157 .@"or" => OpCode.oneByte(if (is_one_byte) 0x0b else 0x0b),
1158 .sbb => OpCode.oneByte(if (is_one_byte) 0x1a else 0x1b),
1159 .cmp => OpCode.oneByte(if (is_one_byte) 0x3a else 0x3b),
1160 .mov => OpCode.oneByte(if (is_one_byte) 0x8a else 0x8b),
1161 .lea => OpCode.oneByte(if (is_one_byte) 0x8c else 0x8d),
1162 .imul => OpCode.twoByte(0x0f, 0xaf),
1163 else => null,
1164 },
1165 .oi => return switch (tag) {
1166 .mov => OpCode.oneByte(if (is_one_byte) 0xb0 else 0xb8),
1167 else => null,
1168 },
1169 .fd => return switch (tag) {
1170 .mov => OpCode.oneByte(if (is_one_byte) 0xa0 else 0xa1),
1171 else => null,
1172 },
1173 .td => return switch (tag) {
1174 .mov => OpCode.oneByte(if (is_one_byte) 0xa2 else 0xa3),
1175 else => null,
1176 },
1177 .rmi => return switch (tag) {
1178 .imul => OpCode.oneByte(if (is_one_byte) 0x6b else 0x69),
1179 else => null,
1180 },
1181 }
1182}
1183
1184inline fn getModRmExt(tag: Tag) ?u3 {
1185 return switch (tag) {
1186 .adc => 0x2,
1187 .add => 0x0,
1188 .sub => 0x5,
1189 .xor => 0x6,
1190 .@"and" => 0x4,
1191 .@"or" => 0x1,
1192 .sbb => 0x3,
1193 .cmp => 0x7,
1194 .mov => 0x0,
1195 .jmp_near => 0x4,
1196 .call_near => 0x2,
1197 .push => 0x6,
1198 .pop => 0x0,
1199 .@"test" => 0x0,
1200 .seto,
1201 .setno,
1202 .setb,
1203 .setc,
1204 .setnae,
1205 .setnb,
1206 .setnc,
1207 .setae,
1208 .sete,
1209 .setz,
1210 .setne,
1211 .setnz,
1212 .setbe,
1213 .setna,
1214 .seta,
1215 .setnbe,
1216 .sets,
1217 .setns,
1218 .setp,
1219 .setpe,
1220 .setnp,
1221 .setop,
1222 .setl,
1223 .setnge,
1224 .setnl,
1225 .setge,
1226 .setle,
1227 .setng,
1228 .setnle,
1229 .setg,
1230 => 0x0,
1231 else => null,
1232 };
1233}
1234
1235const ScaleIndex = struct {
1236 scale: u2,
1237 index: Register,
1238};
1239
1240const Memory = struct {
1241 base: ?Register,
1242 rip: bool = false,
1243 disp: u32,
1244 ptr_size: PtrSize,
1245 scale_index: ?ScaleIndex = null,
1246
1247 const PtrSize = enum {
1248 byte_ptr,
1249 word_ptr,
1250 dword_ptr,
1251 qword_ptr,
1252
1253 fn fromBits(in_bits: u64) PtrSize {
1254 return switch (in_bits) {
1255 8 => .byte_ptr,
1256 16 => .word_ptr,
1257 32 => .dword_ptr,
1258 64 => .qword_ptr,
1259 else => unreachable,
1260 };
1261 }
1262
1263 /// Returns size in bits.
1264 fn size(ptr_size: PtrSize) u64 {
1265 return switch (ptr_size) {
1266 .byte_ptr => 8,
1267 .word_ptr => 16,
1268 .dword_ptr => 32,
1269 .qword_ptr => 64,
1270 };
1271 }
1272 };
1273
1274 fn encode(mem_op: Memory, encoder: Encoder, operand: u3) void {
1275 if (mem_op.base) |base| {
1276 const dst = base.lowId();
1277 const src = operand;
1278 if (dst == 4 or mem_op.scale_index != null) {
1279 if (mem_op.disp == 0 and dst != 5) {
1280 encoder.modRm_SIBDisp0(src);
1281 if (mem_op.scale_index) |si| {
1282 encoder.sib_scaleIndexBase(si.scale, si.index.lowId(), dst);
1283 } else {
1284 encoder.sib_base(dst);
1285 }
1286 } else if (immOpSize(mem_op.disp) == 8) {
1287 encoder.modRm_SIBDisp8(src);
1288 if (mem_op.scale_index) |si| {
1289 encoder.sib_scaleIndexBaseDisp8(si.scale, si.index.lowId(), dst);
1290 } else {
1291 encoder.sib_baseDisp8(dst);
1292 }
1293 encoder.disp8(@bitCast(i8, @truncate(u8, mem_op.disp)));
1294 } else {
1295 encoder.modRm_SIBDisp32(src);
1296 if (mem_op.scale_index) |si| {
1297 encoder.sib_scaleIndexBaseDisp32(si.scale, si.index.lowId(), dst);
1298 } else {
1299 encoder.sib_baseDisp32(dst);
1300 }
1301 encoder.disp32(@bitCast(i32, mem_op.disp));
1302 }
1303 } else {
1304 if (mem_op.disp == 0) {
1305 encoder.modRm_indirectDisp0(src, dst);
1306 } else if (immOpSize(mem_op.disp) == 8) {
1307 encoder.modRm_indirectDisp8(src, dst);
1308 encoder.disp8(@bitCast(i8, @truncate(u8, mem_op.disp)));
1309 } else {
1310 encoder.modRm_indirectDisp32(src, dst);
1311 encoder.disp32(@bitCast(i32, mem_op.disp));
1312 }
1313 }
1314 } else {
1315 if (mem_op.rip) {
1316 encoder.modRm_RIPDisp32(operand);
1317 } else {
1318 encoder.modRm_SIBDisp0(operand);
1319 if (mem_op.scale_index) |si| {
1320 encoder.sib_scaleIndexDisp32(si.scale, si.index.lowId());
1321 } else {
1322 encoder.sib_disp32();
1323 }
1324 }
1325 encoder.disp32(@bitCast(i32, mem_op.disp));
1326 }
1327 }
1328};
1329
1330fn encodeImm(encoder: Encoder, imm: u32, size: u64) void {
1331 switch (size) {
1332 8 => encoder.imm8(@bitCast(i8, @truncate(u8, imm))),
1333 16 => encoder.imm16(@bitCast(i16, @truncate(u16, imm))),
1334 32, 64 => encoder.imm32(@bitCast(i32, imm)),
1335 else => unreachable,
1336 }
1337}
1338
1339const RegisterOrMemory = union(enum) {
1340 register: Register,
1341 memory: Memory,
1342
1343 fn reg(register: Register) RegisterOrMemory {
1344 return .{ .register = register };
1345 }
1346
1347 fn mem(ptr_size: Memory.PtrSize, args: struct {
1348 disp: u32,
1349 base: ?Register = null,
1350 scale_index: ?ScaleIndex = null,
1351 }) RegisterOrMemory {
1352 return .{
1353 .memory = .{
1354 .base = args.base,
1355 .disp = args.disp,
1356 .ptr_size = ptr_size,
1357 .scale_index = args.scale_index,
1358 },
1359 };
1360 }
1361
1362 fn rip(ptr_size: Memory.PtrSize, disp: u32) RegisterOrMemory {
1363 return .{
1364 .memory = .{
1365 .base = null,
1366 .rip = true,
1367 .disp = disp,
1368 .ptr_size = ptr_size,
1369 },
1370 };
1371 }
1372};
1373
1374const LoweringError = error{
1375 OutOfMemory,
1376 OperandSizeMismatch,
1377 RaxOperandExpected,
1378};
1379
1380fn lowerToZoEnc(tag: Tag, code: *std.ArrayList(u8)) LoweringError!void {
1381 const opc = getOpCode(tag, .zo, false).?;
1382 const encoder = try Encoder.init(code, 1);
1383 opc.encode(encoder);
1384}
1385
1386fn lowerToIEnc(tag: Tag, imm: u32, code: *std.ArrayList(u8)) LoweringError!void {
1387 if (tag == .ret_far or tag == .ret_near) {
1388 const encoder = try Encoder.init(code, 3);
1389 const opc = getOpCode(tag, .i, false).?;
1390 opc.encode(encoder);
1391 encoder.imm16(@bitCast(i16, @truncate(u16, imm)));
1392 return;
1393 }
1394 const opc = getOpCode(tag, .i, immOpSize(imm) == 8).?;
1395 const encoder = try Encoder.init(code, 5);
1396 if (immOpSize(imm) == 16) {
1397 encoder.prefix16BitMode();
1398 }
1399 opc.encode(encoder);
1400 encodeImm(encoder, imm, immOpSize(imm));
1401}
1402
1403fn lowerToOEnc(tag: Tag, reg: Register, code: *std.ArrayList(u8)) LoweringError!void {
1404 if (reg.size() != 16 and reg.size() != 64) {
1405 return error.OperandSizeMismatch; // TODO correct for push/pop, but is it universal?
1406 }
1407 const opc = getOpCode(tag, .o, false).?;
1408 const encoder = try Encoder.init(code, 3);
1409 if (reg.size() == 16) {
1410 encoder.prefix16BitMode();
1411 }
1412 encoder.rex(.{
1413 .w = false,
1414 .b = reg.isExtended(),
1415 });
1416 opc.encodeWithReg(encoder, reg);
1417}
1418
1419fn lowerToDEnc(tag: Tag, imm: u32, code: *std.ArrayList(u8)) LoweringError!void {
1420 const opc = getOpCode(tag, .d, false).?;
1421 const encoder = try Encoder.init(code, 6);
1422 opc.encode(encoder);
1423 encoder.imm32(@bitCast(i32, imm));
1424}
1425
1426fn lowerToMEnc(tag: Tag, reg_or_mem: RegisterOrMemory, code: *std.ArrayList(u8)) LoweringError!void {
1427 const opc = getOpCode(tag, .m, false).?;
1428 const modrm_ext = getModRmExt(tag).?;
1429 switch (reg_or_mem) {
1430 .register => |reg| {
1431 const op_size_mismatch = blk: {
1432 if (tag.isSetCC() and reg.size() == 8)
1433 break :blk false;
1434 break :blk reg.size() != 64 and reg.size() != 16;
1435 };
1436 if (op_size_mismatch) {
1437 return error.OperandSizeMismatch;
1438 }
1439 const encoder = try Encoder.init(code, 4);
1440 if (reg.size() == 16) {
1441 encoder.prefix16BitMode();
1442 }
1443 encoder.rex(.{
1444 .w = switch (reg) {
1445 .ah, .bh, .ch, .dh => true,
1446 else => false,
1447 },
1448 .b = reg.isExtended(),
1449 });
1450 opc.encode(encoder);
1451 encoder.modRm_direct(modrm_ext, reg.lowId());
1452 },
1453 .memory => |mem_op| {
1454 if (mem_op.ptr_size != .qword_ptr and mem_op.ptr_size != .word_ptr) {
1455 return error.OperandSizeMismatch;
1456 }
1457 const encoder = try Encoder.init(code, 8);
1458 if (mem_op.ptr_size == .word_ptr) {
1459 encoder.prefix16BitMode();
1460 }
1461 if (mem_op.base) |base| {
1462 if (base.size() != 64) {
1463 return error.OperandSizeMismatch;
1464 }
1465 encoder.rex(.{
1466 .w = false,
1467 .b = base.isExtended(),
1468 });
1469 }
1470 opc.encode(encoder);
1471 mem_op.encode(encoder, modrm_ext);
1472 },
1473 }
1474}
1475
1476fn lowerToTdEnc(tag: Tag, moffs: u64, reg: Register, code: *std.ArrayList(u8)) LoweringError!void {
1477 return lowerToTdFdEnc(tag, reg, moffs, code, true);
1478}
1479
1480fn lowerToFdEnc(tag: Tag, reg: Register, moffs: u64, code: *std.ArrayList(u8)) LoweringError!void {
1481 return lowerToTdFdEnc(tag, reg, moffs, code, false);
1482}
1483
1484fn lowerToTdFdEnc(tag: Tag, reg: Register, moffs: u64, code: *std.ArrayList(u8), td: bool) LoweringError!void {
1485 if (reg.lowId() != Register.rax.lowId()) {
1486 return error.RaxOperandExpected;
1487 }
1488 if (reg.size() != imm64OpSize(moffs)) {
1489 return error.OperandSizeMismatch;
1490 }
1491 const opc = if (td)
1492 getOpCode(tag, .td, reg.size() == 8).?
1493 else
1494 getOpCode(tag, .fd, reg.size() == 8).?;
1495 const encoder = try Encoder.init(code, 10);
1496 if (reg.size() == 16) {
1497 encoder.prefix16BitMode();
1498 }
1499 encoder.rex(.{
1500 .w = setRexWRegister(reg),
1501 });
1502 opc.encode(encoder);
1503 switch (reg.size()) {
1504 8 => encoder.imm8(@bitCast(i8, @truncate(u8, moffs))),
1505 16 => encoder.imm16(@bitCast(i16, @truncate(u16, moffs))),
1506 32 => encoder.imm32(@bitCast(i32, @truncate(u32, moffs))),
1507 64 => encoder.imm64(moffs),
1508 else => unreachable,
1509 }
1510}
1511
1512fn lowerToOiEnc(tag: Tag, reg: Register, imm: u64, code: *std.ArrayList(u8)) LoweringError!void {
1513 if (reg.size() != imm64OpSize(imm)) {
1514 return error.OperandSizeMismatch;
1515 }
1516 const opc = getOpCode(tag, .oi, reg.size() == 8).?;
1517 const encoder = try Encoder.init(code, 10);
1518 if (reg.size() == 16) {
1519 encoder.prefix16BitMode();
1520 }
1521 encoder.rex(.{
1522 .w = setRexWRegister(reg),
1523 .b = reg.isExtended(),
1524 });
1525 opc.encodeWithReg(encoder, reg);
1526 switch (reg.size()) {
1527 8 => encoder.imm8(@bitCast(i8, @truncate(u8, imm))),
1528 16 => encoder.imm16(@bitCast(i16, @truncate(u16, imm))),
1529 32 => encoder.imm32(@bitCast(i32, @truncate(u32, imm))),
1530 64 => encoder.imm64(imm),
1531 else => unreachable,
1532 }
1533}
1534
1535fn lowerToMiEnc(tag: Tag, reg_or_mem: RegisterOrMemory, imm: u32, code: *std.ArrayList(u8)) LoweringError!void {
1536 const modrm_ext = getModRmExt(tag).?;
1537 switch (reg_or_mem) {
1538 .register => |dst_reg| {
1539 const opc = getOpCode(tag, .mi, dst_reg.size() == 8).?;
1540 const encoder = try Encoder.init(code, 7);
1541 if (dst_reg.size() == 16) {
1542 // 0x66 prefix switches to the non-default size; here we assume a switch from
1543 // the default 32bits to 16bits operand-size.
1544 // More info: https://www.cs.uni-potsdam.de/desn/lehre/ss15/64-ia-32-architectures-software-developer-instruction-set-reference-manual-325383.pdf#page=32&zoom=auto,-159,773
1545 encoder.prefix16BitMode();
1546 }
1547 encoder.rex(.{
1548 .w = setRexWRegister(dst_reg),
1549 .b = dst_reg.isExtended(),
1550 });
1551 opc.encode(encoder);
1552 encoder.modRm_direct(modrm_ext, dst_reg.lowId());
1553 encodeImm(encoder, imm, dst_reg.size());
1554 },
1555 .memory => |dst_mem| {
1556 const opc = getOpCode(tag, .mi, dst_mem.ptr_size == .byte_ptr).?;
1557 const encoder = try Encoder.init(code, 12);
1558 if (dst_mem.ptr_size == .word_ptr) {
1559 encoder.prefix16BitMode();
1560 }
1561 if (dst_mem.base) |base| {
1562 if (base.size() != 64) {
1563 return error.OperandSizeMismatch;
1564 }
1565 encoder.rex(.{
1566 .w = dst_mem.ptr_size == .qword_ptr,
1567 .b = base.isExtended(),
1568 });
1569 } else {
1570 encoder.rex(.{
1571 .w = dst_mem.ptr_size == .qword_ptr,
1572 });
1573 }
1574 opc.encode(encoder);
1575 dst_mem.encode(encoder, modrm_ext);
1576 encodeImm(encoder, imm, dst_mem.ptr_size.size());
1577 },
1578 }
1579}
1580
1581fn lowerToRmEnc(
1582 tag: Tag,
1583 reg: Register,
1584 reg_or_mem: RegisterOrMemory,
1585 code: *std.ArrayList(u8),
1586) LoweringError!void {
1587 const opc = getOpCode(tag, .rm, reg.size() == 8).?;
1588 switch (reg_or_mem) {
1589 .register => |src_reg| {
1590 if (reg.size() != src_reg.size()) {
1591 return error.OperandSizeMismatch;
1592 }
1593 const encoder = try Encoder.init(code, 3);
1594 encoder.rex(.{
1595 .w = setRexWRegister(reg) or setRexWRegister(src_reg),
1596 .r = reg.isExtended(),
1597 .b = src_reg.isExtended(),
1598 });
1599 opc.encode(encoder);
1600 encoder.modRm_direct(reg.lowId(), src_reg.lowId());
1601 },
1602 .memory => |src_mem| {
1603 if (reg.size() != src_mem.ptr_size.size()) {
1604 return error.OperandSizeMismatch;
1605 }
1606 const encoder = try Encoder.init(code, 9);
1607 if (reg.size() == 16) {
1608 encoder.prefix16BitMode();
1609 }
1610 if (src_mem.base) |base| {
1611 // TODO handle 32-bit base register - requires prefix 0x67
1612 // Intel Manual, Vol 1, chapter 3.6 and 3.6.1
1613 if (base.size() != 64) {
1614 return error.OperandSizeMismatch;
1615 }
1616 encoder.rex(.{
1617 .w = setRexWRegister(reg),
1618 .r = reg.isExtended(),
1619 .b = base.isExtended(),
1620 });
1621 } else {
1622 encoder.rex(.{
1623 .w = setRexWRegister(reg),
1624 .r = reg.isExtended(),
1625 });
1626 }
1627 opc.encode(encoder);
1628 src_mem.encode(encoder, reg.lowId());
1629 },
1630 }
1631}
1632
1633fn lowerToMrEnc(
1634 tag: Tag,
1635 reg_or_mem: RegisterOrMemory,
1636 reg: Register,
1637 code: *std.ArrayList(u8),
1638) LoweringError!void {
1639 const opc = getOpCode(tag, .mr, reg.size() == 8).?;
1640 switch (reg_or_mem) {
1641 .register => |dst_reg| {
1642 if (dst_reg.size() != reg.size()) {
1643 return error.OperandSizeMismatch;
1644 }
1645 const encoder = try Encoder.init(code, 3);
1646 encoder.rex(.{
1647 .w = setRexWRegister(dst_reg) or setRexWRegister(reg),
1648 .r = reg.isExtended(),
1649 .b = dst_reg.isExtended(),
1650 });
1651 opc.encode(encoder);
1652 encoder.modRm_direct(reg.lowId(), dst_reg.lowId());
1653 },
1654 .memory => |dst_mem| {
1655 if (dst_mem.ptr_size.size() != reg.size()) {
1656 return error.OperandSizeMismatch;
1657 }
1658 const encoder = try Encoder.init(code, 9);
1659 if (reg.size() == 16) {
1660 encoder.prefix16BitMode();
1661 }
1662 if (dst_mem.base) |base| {
1663 if (base.size() != 64) {
1664 return error.OperandSizeMismatch;
1665 }
1666 encoder.rex(.{
1667 .w = dst_mem.ptr_size == .qword_ptr or setRexWRegister(reg),
1668 .r = reg.isExtended(),
1669 .b = base.isExtended(),
1670 });
1671 } else {
1672 encoder.rex(.{
1673 .w = dst_mem.ptr_size == .qword_ptr or setRexWRegister(reg),
1674 .r = reg.isExtended(),
1675 });
1676 }
1677 opc.encode(encoder);
1678 dst_mem.encode(encoder, reg.lowId());
1679 },
1680 }
1681}
1682
1683fn lowerToRmiEnc(
1684 tag: Tag,
1685 reg: Register,
1686 reg_or_mem: RegisterOrMemory,
1687 imm: u32,
1688 code: *std.ArrayList(u8),
1689) LoweringError!void {
1690 if (reg.size() == 8) {
1691 return error.OperandSizeMismatch;
1692 }
1693 const opc = getOpCode(tag, .rmi, false).?;
1694 const encoder = try Encoder.init(code, 13);
1695 if (reg.size() == 16) {
1696 encoder.prefix16BitMode();
1697 }
1698 switch (reg_or_mem) {
1699 .register => |src_reg| {
1700 if (reg.size() != src_reg.size()) {
1701 return error.OperandSizeMismatch;
1702 }
1703 encoder.rex(.{
1704 .w = setRexWRegister(reg) or setRexWRegister(src_reg),
1705 .r = reg.isExtended(),
1706 .b = src_reg.isExtended(),
1707 });
1708 opc.encode(encoder);
1709 encoder.modRm_direct(reg.lowId(), src_reg.lowId());
1710 },
1711 .memory => |src_mem| {
1712 if (src_mem.base) |base| {
1713 // TODO handle 32-bit base register - requires prefix 0x67
1714 // Intel Manual, Vol 1, chapter 3.6 and 3.6.1
1715 if (base.size() != 64) {
1716 return error.OperandSizeMismatch;
1717 }
1718 if (src_mem.ptr_size == .byte_ptr) {
1719 return error.OperandSizeMismatch;
1720 }
1721 encoder.rex(.{
1722 .w = setRexWRegister(reg),
1723 .r = reg.isExtended(),
1724 .b = base.isExtended(),
1725 });
1726 } else {
1727 encoder.rex(.{
1728 .w = setRexWRegister(reg),
1729 .r = reg.isExtended(),
1730 });
1731 }
1732 opc.encode(encoder);
1733 src_mem.encode(encoder, reg.lowId());
1734 },
1735 }
1736 encodeImm(encoder, imm, reg.size());
1737}
1738
1739fn expectEqualHexStrings(expected: []const u8, given: []const u8, assembly: []const u8) !void {
1740 assert(expected.len > 0);
1741 if (mem.eql(u8, expected, given)) return;
1742 const expected_fmt = try std.fmt.allocPrint(testing.allocator, "{x}", .{std.fmt.fmtSliceHexLower(expected)});
1743 defer testing.allocator.free(expected_fmt);
1744 const given_fmt = try std.fmt.allocPrint(testing.allocator, "{x}", .{std.fmt.fmtSliceHexLower(given)});
1745 defer testing.allocator.free(given_fmt);
1746 const idx = mem.indexOfDiff(u8, expected_fmt, given_fmt).?;
1747 var padding = try testing.allocator.alloc(u8, idx + 5);
1748 defer testing.allocator.free(padding);
1749 mem.set(u8, padding, ' ');
1750 std.debug.print("\nASM: {s}\nEXP: {s}\nGIV: {s}\n{s}^ -- first differing byte\n", .{
1751 assembly,
1752 expected_fmt,
1753 given_fmt,
1754 padding,
1755 });
1756 return error.TestFailed;
1757}
1758
1759const TestEmit = struct {
1760 code_buffer: std.ArrayList(u8),
1761 next: usize = 0,
1762
1763 fn init() TestEmit {
1764 return .{
1765 .code_buffer = std.ArrayList(u8).init(testing.allocator),
1766 };
1767 }
1768
1769 fn deinit(emit: *TestEmit) void {
1770 emit.code_buffer.deinit();
1771 emit.next = undefined;
1772 }
1773
1774 fn code(emit: *TestEmit) *std.ArrayList(u8) {
1775 emit.next = emit.code_buffer.items.len;
1776 return &emit.code_buffer;
1777 }
1778
1779 fn lowered(emit: TestEmit) []const u8 {
1780 return emit.code_buffer.items[emit.next..];
1781 }
1782};
1783
1784test "lower MI encoding" {
1785 var emit = TestEmit.init();
1786 defer emit.deinit();
1787 try lowerToMiEnc(.mov, RegisterOrMemory.reg(.rax), 0x10, emit.code());
1788 try expectEqualHexStrings("\x48\xc7\xc0\x10\x00\x00\x00", emit.lowered(), "mov rax, 0x10");
1789 try lowerToMiEnc(.mov, RegisterOrMemory.mem(.dword_ptr, .{ .disp = 0, .base = .r11 }), 0x10, emit.code());
1790 try expectEqualHexStrings("\x41\xc7\x03\x10\x00\x00\x00", emit.lowered(), "mov dword ptr [r11 + 0], 0x10");
1791 try lowerToMiEnc(.add, RegisterOrMemory.mem(.dword_ptr, .{
1792 .disp = @bitCast(u32, @as(i32, -8)),
1793 .base = .rdx,
1794 }), 0x10, emit.code());
1795 try expectEqualHexStrings("\x81\x42\xF8\x10\x00\x00\x00", emit.lowered(), "add dword ptr [rdx - 8], 0x10");
1796 try lowerToMiEnc(.sub, RegisterOrMemory.mem(.dword_ptr, .{
1797 .disp = 0x10000000,
1798 .base = .r11,
1799 }), 0x10, emit.code());
1800 try expectEqualHexStrings(
1801 "\x41\x81\xab\x00\x00\x00\x10\x10\x00\x00\x00",
1802 emit.lowered(),
1803 "sub dword ptr [r11 + 0x10000000], 0x10",
1804 );
1805 try lowerToMiEnc(.@"and", RegisterOrMemory.mem(.dword_ptr, .{ .disp = 0x10000000 }), 0x10, emit.code());
1806 try expectEqualHexStrings(
1807 "\x81\x24\x25\x00\x00\x00\x10\x10\x00\x00\x00",
1808 emit.lowered(),
1809 "and dword ptr [ds:0x10000000], 0x10",
1810 );
1811 try lowerToMiEnc(.@"and", RegisterOrMemory.mem(.dword_ptr, .{
1812 .disp = 0x10000000,
1813 .base = .r12,
1814 }), 0x10, emit.code());
1815 try expectEqualHexStrings(
1816 "\x41\x81\xA4\x24\x00\x00\x00\x10\x10\x00\x00\x00",
1817 emit.lowered(),
1818 "and dword ptr [r12 + 0x10000000], 0x10",
1819 );
1820 try lowerToMiEnc(.mov, RegisterOrMemory.rip(.qword_ptr, 0x10), 0x10, emit.code());
1821 try expectEqualHexStrings(
1822 "\x48\xC7\x05\x10\x00\x00\x00\x10\x00\x00\x00",
1823 emit.lowered(),
1824 "mov qword ptr [rip + 0x10], 0x10",
1825 );
1826 try lowerToMiEnc(.mov, RegisterOrMemory.mem(.qword_ptr, .{
1827 .disp = @bitCast(u32, @as(i32, -8)),
1828 .base = .rbp,
1829 }), 0x10, emit.code());
1830 try expectEqualHexStrings(
1831 "\x48\xc7\x45\xf8\x10\x00\x00\x00",
1832 emit.lowered(),
1833 "mov qword ptr [rbp - 8], 0x10",
1834 );
1835 try lowerToMiEnc(.mov, RegisterOrMemory.mem(.word_ptr, .{
1836 .disp = @bitCast(u32, @as(i32, -2)),
1837 .base = .rbp,
1838 }), 0x10, emit.code());
1839 try expectEqualHexStrings("\x66\xC7\x45\xFE\x10\x00", emit.lowered(), "mov word ptr [rbp - 2], 0x10");
1840 try lowerToMiEnc(.mov, RegisterOrMemory.mem(.byte_ptr, .{
1841 .disp = @bitCast(u32, @as(i32, -1)),
1842 .base = .rbp,
1843 }), 0x10, emit.code());
1844 try expectEqualHexStrings("\xC6\x45\xFF\x10", emit.lowered(), "mov byte ptr [rbp - 1], 0x10");
1845 try lowerToMiEnc(.mov, RegisterOrMemory.mem(.qword_ptr, .{
1846 .disp = 0x10000000,
1847 .scale_index = .{
1848 .scale = 1,
1849 .index = .rcx,
1850 },
1851 }), 0x10, emit.code());
1852 try expectEqualHexStrings(
1853 "\x48\xC7\x04\x4D\x00\x00\x00\x10\x10\x00\x00\x00",
1854 emit.lowered(),
1855 "mov qword ptr [rcx*2 + 0x10000000], 0x10",
1856 );
1857}
1858
1859test "lower RM encoding" {
1860 var emit = TestEmit.init();
1861 defer emit.deinit();
1862 try lowerToRmEnc(.mov, .rax, RegisterOrMemory.reg(.rbx), emit.code());
1863 try expectEqualHexStrings("\x48\x8b\xc3", emit.lowered(), "mov rax, rbx");
1864 try lowerToRmEnc(.mov, .rax, RegisterOrMemory.mem(.qword_ptr, .{ .disp = 0, .base = .r11 }), emit.code());
1865 try expectEqualHexStrings("\x49\x8b\x03", emit.lowered(), "mov rax, qword ptr [r11 + 0]");
1866 try lowerToRmEnc(.add, .r11, RegisterOrMemory.mem(.qword_ptr, .{ .disp = 0x10000000 }), emit.code());
1867 try expectEqualHexStrings(
1868 "\x4C\x03\x1C\x25\x00\x00\x00\x10",
1869 emit.lowered(),
1870 "add r11, qword ptr [ds:0x10000000]",
1871 );
1872 try lowerToRmEnc(.add, .r12b, RegisterOrMemory.mem(.byte_ptr, .{ .disp = 0x10000000 }), emit.code());
1873 try expectEqualHexStrings(
1874 "\x44\x02\x24\x25\x00\x00\x00\x10",
1875 emit.lowered(),
1876 "add r11b, byte ptr [ds:0x10000000]",
1877 );
1878 try lowerToRmEnc(.sub, .r11, RegisterOrMemory.mem(.qword_ptr, .{
1879 .disp = 0x10000000,
1880 .base = .r13,
1881 }), emit.code());
1882 try expectEqualHexStrings(
1883 "\x4D\x2B\x9D\x00\x00\x00\x10",
1884 emit.lowered(),
1885 "sub r11, qword ptr [r13 + 0x10000000]",
1886 );
1887 try lowerToRmEnc(.sub, .r11, RegisterOrMemory.mem(.qword_ptr, .{
1888 .disp = 0x10000000,
1889 .base = .r12,
1890 }), emit.code());
1891 try expectEqualHexStrings(
1892 "\x4D\x2B\x9C\x24\x00\x00\x00\x10",
1893 emit.lowered(),
1894 "sub r11, qword ptr [r12 + 0x10000000]",
1895 );
1896 try lowerToRmEnc(.mov, .rax, RegisterOrMemory.mem(.qword_ptr, .{
1897 .disp = @bitCast(u32, @as(i32, -4)),
1898 .base = .rbp,
1899 }), emit.code());
1900 try expectEqualHexStrings("\x48\x8B\x45\xFC", emit.lowered(), "mov rax, qword ptr [rbp - 4]");
1901 try lowerToRmEnc(.lea, .rax, RegisterOrMemory.rip(.qword_ptr, 0x10), emit.code());
1902 try expectEqualHexStrings("\x48\x8D\x05\x10\x00\x00\x00", emit.lowered(), "lea rax, [rip + 0x10]");
1903 try lowerToRmEnc(.mov, .rax, RegisterOrMemory.mem(.qword_ptr, .{
1904 .disp = @bitCast(u32, @as(i32, -8)),
1905 .base = .rbp,
1906 .scale_index = .{
1907 .scale = 0,
1908 .index = .rcx,
1909 },
1910 }), emit.code());
1911 try expectEqualHexStrings("\x48\x8B\x44\x0D\xF8", emit.lowered(), "mov rax, qword ptr [rbp + rcx*1 - 8]");
1912 try lowerToRmEnc(.mov, .eax, RegisterOrMemory.mem(.dword_ptr, .{
1913 .disp = @bitCast(u32, @as(i32, -4)),
1914 .base = .rbp,
1915 .scale_index = .{
1916 .scale = 2,
1917 .index = .rdx,
1918 },
1919 }), emit.code());
1920 try expectEqualHexStrings("\x8B\x44\x95\xFC", emit.lowered(), "mov eax, dword ptr [rbp + rdx*4 - 4]");
1921 try lowerToRmEnc(.mov, .rax, RegisterOrMemory.mem(.qword_ptr, .{
1922 .disp = @bitCast(u32, @as(i32, -8)),
1923 .base = .rbp,
1924 .scale_index = .{
1925 .scale = 3,
1926 .index = .rcx,
1927 },
1928 }), emit.code());
1929 try expectEqualHexStrings("\x48\x8B\x44\xCD\xF8", emit.lowered(), "mov rax, qword ptr [rbp + rcx*8 - 8]");
1930 try lowerToRmEnc(.mov, .r8b, RegisterOrMemory.mem(.byte_ptr, .{
1931 .disp = @bitCast(u32, @as(i32, -24)),
1932 .base = .rsi,
1933 .scale_index = .{
1934 .scale = 0,
1935 .index = .rcx,
1936 },
1937 }), emit.code());
1938 try expectEqualHexStrings("\x44\x8A\x44\x0E\xE8", emit.lowered(), "mov r8b, byte ptr [rsi + rcx*1 - 24]");
1939 try lowerToRmEnc(.lea, .rsi, RegisterOrMemory.mem(.qword_ptr, .{
1940 .disp = 0,
1941 .base = .rbp,
1942 .scale_index = .{
1943 .scale = 0,
1944 .index = .rcx,
1945 },
1946 }), emit.code());
1947 try expectEqualHexStrings("\x48\x8D\x74\x0D\x00", emit.lowered(), "lea rsi, qword ptr [rbp + rcx*1 + 0]");
1948}
1949
1950test "lower MR encoding" {
1951 var emit = TestEmit.init();
1952 defer emit.deinit();
1953 try lowerToMrEnc(.mov, RegisterOrMemory.reg(.rax), .rbx, emit.code());
1954 try expectEqualHexStrings("\x48\x89\xd8", emit.lowered(), "mov rax, rbx");
1955 try lowerToMrEnc(.mov, RegisterOrMemory.mem(.qword_ptr, .{
1956 .disp = @bitCast(u32, @as(i32, -4)),
1957 .base = .rbp,
1958 }), .r11, emit.code());
1959 try expectEqualHexStrings("\x4c\x89\x5d\xfc", emit.lowered(), "mov qword ptr [rbp - 4], r11");
1960 try lowerToMrEnc(.add, RegisterOrMemory.mem(.byte_ptr, .{ .disp = 0x10000000 }), .r12b, emit.code());
1961 try expectEqualHexStrings(
1962 "\x44\x00\x24\x25\x00\x00\x00\x10",
1963 emit.lowered(),
1964 "add byte ptr [ds:0x10000000], r12b",
1965 );
1966 try lowerToMrEnc(.add, RegisterOrMemory.mem(.dword_ptr, .{ .disp = 0x10000000 }), .r12d, emit.code());
1967 try expectEqualHexStrings(
1968 "\x44\x01\x24\x25\x00\x00\x00\x10",
1969 emit.lowered(),
1970 "add dword ptr [ds:0x10000000], r12d",
1971 );
1972 try lowerToMrEnc(.sub, RegisterOrMemory.mem(.qword_ptr, .{
1973 .disp = 0x10000000,
1974 .base = .r11,
1975 }), .r12, emit.code());
1976 try expectEqualHexStrings(
1977 "\x4D\x29\xA3\x00\x00\x00\x10",
1978 emit.lowered(),
1979 "sub qword ptr [r11 + 0x10000000], r12",
1980 );
1981 try lowerToMrEnc(.mov, RegisterOrMemory.rip(.qword_ptr, 0x10), .r12, emit.code());
1982 try expectEqualHexStrings("\x4C\x89\x25\x10\x00\x00\x00", emit.lowered(), "mov qword ptr [rip + 0x10], r12");
1983}
1984
1985test "lower OI encoding" {
1986 var emit = TestEmit.init();
1987 defer emit.deinit();
1988 try lowerToOiEnc(.mov, .rax, 0x1000000000000000, emit.code());
1989 try expectEqualHexStrings(
1990 "\x48\xB8\x00\x00\x00\x00\x00\x00\x00\x10",
1991 emit.lowered(),
1992 "movabs rax, 0x1000000000000000",
1993 );
1994 try lowerToOiEnc(.mov, .r11, 0x1000000000000000, emit.code());
1995 try expectEqualHexStrings(
1996 "\x49\xBB\x00\x00\x00\x00\x00\x00\x00\x10",
1997 emit.lowered(),
1998 "movabs r11, 0x1000000000000000",
1999 );
2000 try lowerToOiEnc(.mov, .r11d, 0x10000000, emit.code());
2001 try expectEqualHexStrings("\x41\xBB\x00\x00\x00\x10", emit.lowered(), "mov r11d, 0x10000000");
2002 try lowerToOiEnc(.mov, .r11w, 0x1000, emit.code());
2003 try expectEqualHexStrings("\x66\x41\xBB\x00\x10", emit.lowered(), "mov r11w, 0x1000");
2004 try lowerToOiEnc(.mov, .r11b, 0x10, emit.code());
2005 try expectEqualHexStrings("\x41\xB3\x10", emit.lowered(), "mov r11b, 0x10");
2006}
2007
2008test "lower FD/TD encoding" {
2009 var emit = TestEmit.init();
2010 defer emit.deinit();
2011 try lowerToFdEnc(.mov, .rax, 0x1000000000000000, emit.code());
2012 try expectEqualHexStrings(
2013 "\x48\xa1\x00\x00\x00\x00\x00\x00\x00\x10",
2014 emit.lowered(),
2015 "mov rax, ds:0x1000000000000000",
2016 );
2017 try lowerToFdEnc(.mov, .eax, 0x10000000, emit.code());
2018 try expectEqualHexStrings("\xa1\x00\x00\x00\x10", emit.lowered(), "mov eax, ds:0x10000000");
2019 try lowerToFdEnc(.mov, .ax, 0x1000, emit.code());
2020 try expectEqualHexStrings("\x66\xa1\x00\x10", emit.lowered(), "mov ax, ds:0x1000");
2021 try lowerToFdEnc(.mov, .al, 0x10, emit.code());
2022 try expectEqualHexStrings("\xa0\x10", emit.lowered(), "mov al, ds:0x10");
2023}
2024
2025test "lower M encoding" {
2026 var emit = TestEmit.init();
2027 defer emit.deinit();
2028 try lowerToMEnc(.jmp_near, RegisterOrMemory.reg(.r12), emit.code());
2029 try expectEqualHexStrings("\x41\xFF\xE4", emit.lowered(), "jmp r12");
2030 try lowerToMEnc(.jmp_near, RegisterOrMemory.reg(.r12w), emit.code());
2031 try expectEqualHexStrings("\x66\x41\xFF\xE4", emit.lowered(), "jmp r12w");
2032 try lowerToMEnc(.jmp_near, RegisterOrMemory.mem(.qword_ptr, .{ .disp = 0, .base = .r12 }), emit.code());
2033 try expectEqualHexStrings("\x41\xFF\x24\x24", emit.lowered(), "jmp qword ptr [r12]");
2034 try lowerToMEnc(.jmp_near, RegisterOrMemory.mem(.word_ptr, .{ .disp = 0, .base = .r12 }), emit.code());
2035 try expectEqualHexStrings("\x66\x41\xFF\x24\x24", emit.lowered(), "jmp word ptr [r12]");
2036 try lowerToMEnc(.jmp_near, RegisterOrMemory.mem(.qword_ptr, .{ .disp = 0x10, .base = .r12 }), emit.code());
2037 try expectEqualHexStrings("\x41\xFF\x64\x24\x10", emit.lowered(), "jmp qword ptr [r12 + 0x10]");
2038 try lowerToMEnc(.jmp_near, RegisterOrMemory.mem(.qword_ptr, .{
2039 .disp = 0x1000,
2040 .base = .r12,
2041 }), emit.code());
2042 try expectEqualHexStrings(
2043 "\x41\xFF\xA4\x24\x00\x10\x00\x00",
2044 emit.lowered(),
2045 "jmp qword ptr [r12 + 0x1000]",
2046 );
2047 try lowerToMEnc(.jmp_near, RegisterOrMemory.rip(.qword_ptr, 0x10), emit.code());
2048 try expectEqualHexStrings("\xFF\x25\x10\x00\x00\x00", emit.lowered(), "jmp qword ptr [rip + 0x10]");
2049 try lowerToMEnc(.jmp_near, RegisterOrMemory.mem(.qword_ptr, .{ .disp = 0x10 }), emit.code());
2050 try expectEqualHexStrings("\xFF\x24\x25\x10\x00\x00\x00", emit.lowered(), "jmp qword ptr [ds:0x10]");
2051 try lowerToMEnc(.seta, RegisterOrMemory.reg(.r11b), emit.code());
2052 try expectEqualHexStrings("\x41\x0F\x97\xC3", emit.lowered(), "seta r11b");
2053}
2054
2055test "lower O encoding" {
2056 var emit = TestEmit.init();
2057 defer emit.deinit();
2058 try lowerToOEnc(.pop, .r12, emit.code());
2059 try expectEqualHexStrings("\x41\x5c", emit.lowered(), "pop r12");
2060 try lowerToOEnc(.push, .r12w, emit.code());
2061 try expectEqualHexStrings("\x66\x41\x54", emit.lowered(), "push r12w");
2062}
2063
2064test "lower RMI encoding" {
2065 var emit = TestEmit.init();
2066 defer emit.deinit();
2067 try lowerToRmiEnc(.imul, .rax, RegisterOrMemory.mem(.qword_ptr, .{
2068 .disp = @bitCast(u32, @as(i32, -8)),
2069 .base = .rbp,
2070 }), 0x10, emit.code());
2071 try expectEqualHexStrings(
2072 "\x48\x69\x45\xF8\x10\x00\x00\x00",
2073 emit.lowered(),
2074 "imul rax, qword ptr [rbp - 8], 0x10",
2075 );
2076 try lowerToRmiEnc(.imul, .eax, RegisterOrMemory.mem(.dword_ptr, .{
2077 .disp = @bitCast(u32, @as(i32, -4)),
2078 .base = .rbp,
2079 }), 0x10, emit.code());
2080 try expectEqualHexStrings("\x69\x45\xFC\x10\x00\x00\x00", emit.lowered(), "imul eax, dword ptr [rbp - 4], 0x10");
2081 try lowerToRmiEnc(.imul, .ax, RegisterOrMemory.mem(.word_ptr, .{
2082 .disp = @bitCast(u32, @as(i32, -2)),
2083 .base = .rbp,
2084 }), 0x10, emit.code());
2085 try expectEqualHexStrings("\x66\x69\x45\xFE\x10\x00", emit.lowered(), "imul ax, word ptr [rbp - 2], 0x10");
2086 try lowerToRmiEnc(.imul, .r12, RegisterOrMemory.reg(.r12), 0x10, emit.code());
2087 try expectEqualHexStrings("\x4D\x69\xE4\x10\x00\x00\x00", emit.lowered(), "imul r12, r12, 0x10");
2088 try lowerToRmiEnc(.imul, .r12w, RegisterOrMemory.reg(.r12w), 0x10, emit.code());
2089 try expectEqualHexStrings("\x66\x45\x69\xE4\x10\x00", emit.lowered(), "imul r12w, r12w, 0x10");
2090}
src/arch/x86_64/Isel.zig deleted-2093
......@@ -1,2093 +0,0 @@
1//! This file contains the functionality for lowering x86_64 MIR into
2//! machine code
3
4const Isel = @This();
5
6const std = @import("std");
7const assert = std.debug.assert;
8const bits = @import("bits.zig");
9const leb128 = std.leb;
10const link = @import("../../link.zig");
11const log = std.log.scoped(.codegen);
12const math = std.math;
13const mem = std.mem;
14const testing = std.testing;
15
16const Air = @import("../../Air.zig");
17const Allocator = mem.Allocator;
18const DebugInfoOutput = @import("../../codegen.zig").DebugInfoOutput;
19const DW = std.dwarf;
20const Encoder = bits.Encoder;
21const ErrorMsg = Module.ErrorMsg;
22const MCValue = @import("CodeGen.zig").MCValue;
23const Mir = @import("Mir.zig");
24const Module = @import("../../Module.zig");
25const Instruction = bits.Instruction;
26const Register = bits.Register;
27const Type = @import("../../type.zig").Type;
28
29mir: Mir,
30bin_file: *link.File,
31debug_output: DebugInfoOutput,
32target: *const std.Target,
33err_msg: ?*ErrorMsg = null,
34src_loc: Module.SrcLoc,
35code: *std.ArrayList(u8),
36
37prev_di_line: u32,
38prev_di_column: u32,
39/// Relative to the beginning of `code`.
40prev_di_pc: usize,
41
42code_offset_mapping: std.AutoHashMapUnmanaged(Mir.Inst.Index, usize) = .{},
43relocs: std.ArrayListUnmanaged(Reloc) = .{},
44
45const InnerError = error{
46 OutOfMemory,
47 Overflow,
48 IselFail,
49};
50
51const Reloc = struct {
52 /// Offset of the instruction.
53 source: u64,
54 /// Target of the relocation.
55 target: Mir.Inst.Index,
56 /// Offset of the relocation within the instruction.
57 offset: u64,
58 /// Length of the instruction.
59 length: u5,
60};
61
62pub fn lowerMir(isel: *Isel) InnerError!void {
63 const mir_tags = isel.mir.instructions.items(.tag);
64
65 for (mir_tags) |tag, index| {
66 const inst = @intCast(u32, index);
67 try isel.code_offset_mapping.putNoClobber(isel.bin_file.allocator, inst, isel.code.items.len);
68 switch (tag) {
69 .adc => try isel.mirArith(.adc, inst),
70 .add => try isel.mirArith(.add, inst),
71 .sub => try isel.mirArith(.sub, inst),
72 .xor => try isel.mirArith(.xor, inst),
73 .@"and" => try isel.mirArith(.@"and", inst),
74 .@"or" => try isel.mirArith(.@"or", inst),
75 .sbb => try isel.mirArith(.sbb, inst),
76 .cmp => try isel.mirArith(.cmp, inst),
77 .mov => try isel.mirArith(.mov, inst),
78
79 .adc_mem_imm => try isel.mirArithMemImm(.adc, inst),
80 .add_mem_imm => try isel.mirArithMemImm(.add, inst),
81 .sub_mem_imm => try isel.mirArithMemImm(.sub, inst),
82 .xor_mem_imm => try isel.mirArithMemImm(.xor, inst),
83 .and_mem_imm => try isel.mirArithMemImm(.@"and", inst),
84 .or_mem_imm => try isel.mirArithMemImm(.@"or", inst),
85 .sbb_mem_imm => try isel.mirArithMemImm(.sbb, inst),
86 .cmp_mem_imm => try isel.mirArithMemImm(.cmp, inst),
87 .mov_mem_imm => try isel.mirArithMemImm(.mov, inst),
88
89 .adc_scale_src => try isel.mirArithScaleSrc(.adc, inst),
90 .add_scale_src => try isel.mirArithScaleSrc(.add, inst),
91 .sub_scale_src => try isel.mirArithScaleSrc(.sub, inst),
92 .xor_scale_src => try isel.mirArithScaleSrc(.xor, inst),
93 .and_scale_src => try isel.mirArithScaleSrc(.@"and", inst),
94 .or_scale_src => try isel.mirArithScaleSrc(.@"or", inst),
95 .sbb_scale_src => try isel.mirArithScaleSrc(.sbb, inst),
96 .cmp_scale_src => try isel.mirArithScaleSrc(.cmp, inst),
97 .mov_scale_src => try isel.mirArithScaleSrc(.mov, inst),
98
99 .adc_scale_dst => try isel.mirArithScaleDst(.adc, inst),
100 .add_scale_dst => try isel.mirArithScaleDst(.add, inst),
101 .sub_scale_dst => try isel.mirArithScaleDst(.sub, inst),
102 .xor_scale_dst => try isel.mirArithScaleDst(.xor, inst),
103 .and_scale_dst => try isel.mirArithScaleDst(.@"and", inst),
104 .or_scale_dst => try isel.mirArithScaleDst(.@"or", inst),
105 .sbb_scale_dst => try isel.mirArithScaleDst(.sbb, inst),
106 .cmp_scale_dst => try isel.mirArithScaleDst(.cmp, inst),
107 .mov_scale_dst => try isel.mirArithScaleDst(.mov, inst),
108
109 .adc_scale_imm => try isel.mirArithScaleImm(.adc, inst),
110 .add_scale_imm => try isel.mirArithScaleImm(.add, inst),
111 .sub_scale_imm => try isel.mirArithScaleImm(.sub, inst),
112 .xor_scale_imm => try isel.mirArithScaleImm(.xor, inst),
113 .and_scale_imm => try isel.mirArithScaleImm(.@"and", inst),
114 .or_scale_imm => try isel.mirArithScaleImm(.@"or", inst),
115 .sbb_scale_imm => try isel.mirArithScaleImm(.sbb, inst),
116 .cmp_scale_imm => try isel.mirArithScaleImm(.cmp, inst),
117 .mov_scale_imm => try isel.mirArithScaleImm(.mov, inst),
118
119 .movabs => try isel.mirMovabs(inst),
120
121 .lea => try isel.mirLea(inst),
122
123 .imul_complex => try isel.mirIMulComplex(inst),
124
125 .push => try isel.mirPushPop(.push, inst),
126 .pop => try isel.mirPushPop(.pop, inst),
127
128 .jmp => try isel.mirJmpCall(.jmp_near, inst),
129 .call => try isel.mirJmpCall(.call_near, inst),
130
131 .cond_jmp_greater_less,
132 .cond_jmp_above_below,
133 .cond_jmp_eq_ne,
134 => try isel.mirCondJmp(tag, inst),
135
136 .cond_set_byte_greater_less,
137 .cond_set_byte_above_below,
138 .cond_set_byte_eq_ne,
139 => try isel.mirCondSetByte(tag, inst),
140
141 .ret => try isel.mirRet(inst),
142
143 .syscall => try isel.mirSyscall(),
144
145 .@"test" => try isel.mirTest(inst),
146
147 .brk => try isel.mirBrk(),
148 .nop => try isel.mirNop(),
149
150 .call_extern => try isel.mirCallExtern(inst),
151
152 .dbg_line => try isel.mirDbgLine(inst),
153 .dbg_prologue_end => try isel.mirDbgPrologueEnd(inst),
154 .dbg_epilogue_begin => try isel.mirDbgEpilogueBegin(inst),
155 .arg_dbg_info => try isel.mirArgDbgInfo(inst),
156
157 .push_regs_from_callee_preserved_regs => try isel.mirPushPopRegsFromCalleePreservedRegs(.push, inst),
158 .pop_regs_from_callee_preserved_regs => try isel.mirPushPopRegsFromCalleePreservedRegs(.pop, inst),
159
160 else => {
161 return isel.fail("Implement MIR->Isel lowering for x86_64 for pseudo-inst: {s}", .{tag});
162 },
163 }
164 }
165
166 try isel.fixupRelocs();
167}
168
169pub fn deinit(isel: *Isel) void {
170 isel.relocs.deinit(isel.bin_file.allocator);
171 isel.code_offset_mapping.deinit(isel.bin_file.allocator);
172 isel.* = undefined;
173}
174
175fn fail(isel: *Isel, comptime format: []const u8, args: anytype) InnerError {
176 @setCold(true);
177 assert(isel.err_msg == null);
178 isel.err_msg = try ErrorMsg.create(isel.bin_file.allocator, isel.src_loc, format, args);
179 return error.IselFail;
180}
181
182fn failWithLoweringError(isel: *Isel, err: LoweringError) InnerError {
183 return switch (err) {
184 error.RaxOperandExpected => isel.fail("Register.rax expected as destination operand", .{}),
185 error.OperandSizeMismatch => isel.fail("operand size mismatch", .{}),
186 else => |e| e,
187 };
188}
189
190fn fixupRelocs(isel: *Isel) InnerError!void {
191 // TODO this function currently assumes all relocs via JMP/CALL instructions are 32bit in size.
192 // This should be reversed like it is done in aarch64 MIR emit code: start with the smallest
193 // possible resolution, i.e., 8bit, and iteratively converge on the minimum required resolution
194 // until the entire decl is correctly emitted with all JMP/CALL instructions within range.
195 for (isel.relocs.items) |reloc| {
196 const offset = try math.cast(usize, reloc.offset);
197 const target = isel.code_offset_mapping.get(reloc.target) orelse
198 return isel.fail("JMP/CALL relocation target not found!", .{});
199 const disp = @intCast(i32, @intCast(i64, target) - @intCast(i64, reloc.source + reloc.length));
200 mem.writeIntLittle(i32, isel.code.items[offset..][0..4], disp);
201 }
202}
203
204fn mirBrk(isel: *Isel) InnerError!void {
205 return lowerToZoEnc(.brk, isel.code) catch |err| isel.failWithLoweringError(err);
206}
207
208fn mirNop(isel: *Isel) InnerError!void {
209 return lowerToZoEnc(.nop, isel.code) catch |err| isel.failWithLoweringError(err);
210}
211
212fn mirSyscall(isel: *Isel) InnerError!void {
213 return lowerToZoEnc(.syscall, isel.code) catch |err| isel.failWithLoweringError(err);
214}
215
216fn mirPushPop(isel: *Isel, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
217 const ops = Mir.Ops.decode(isel.mir.instructions.items(.ops)[inst]);
218 switch (ops.flags) {
219 0b00 => {
220 // PUSH/POP reg
221 return lowerToOEnc(tag, ops.reg1, isel.code) catch |err| isel.failWithLoweringError(err);
222 },
223 0b01 => {
224 // PUSH/POP r/m64
225 const imm = isel.mir.instructions.items(.data)[inst].imm;
226 const ptr_size: Memory.PtrSize = switch (immOpSize(imm)) {
227 16 => .word_ptr,
228 else => .qword_ptr,
229 };
230 return lowerToMEnc(tag, RegisterOrMemory.mem(ptr_size, .{
231 .disp = imm,
232 .base = ops.reg1,
233 }), isel.code) catch |err| isel.failWithLoweringError(err);
234 },
235 0b10 => {
236 // PUSH imm32
237 assert(tag == .push);
238 const imm = isel.mir.instructions.items(.data)[inst].imm;
239 return lowerToIEnc(.push, imm, isel.code) catch |err|
240 isel.failWithLoweringError(err);
241 },
242 0b11 => unreachable,
243 }
244}
245fn mirPushPopRegsFromCalleePreservedRegs(isel: *Isel, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
246 const callee_preserved_regs = bits.callee_preserved_regs;
247 const regs = isel.mir.instructions.items(.data)[inst].regs_to_push_or_pop;
248 if (tag == .push) {
249 for (callee_preserved_regs) |reg, i| {
250 if ((regs >> @intCast(u5, i)) & 1 == 0) continue;
251 lowerToOEnc(.push, reg, isel.code) catch |err|
252 return isel.failWithLoweringError(err);
253 }
254 } else {
255 // pop in the reverse direction
256 var i = callee_preserved_regs.len;
257 while (i > 0) : (i -= 1) {
258 const reg = callee_preserved_regs[i - 1];
259 if ((regs >> @intCast(u5, i - 1)) & 1 == 0) continue;
260 lowerToOEnc(.pop, reg, isel.code) catch |err|
261 return isel.failWithLoweringError(err);
262 }
263 }
264}
265
266fn mirJmpCall(isel: *Isel, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
267 const ops = Mir.Ops.decode(isel.mir.instructions.items(.ops)[inst]);
268 switch (ops.flags) {
269 0b00 => {
270 const target = isel.mir.instructions.items(.data)[inst].inst;
271 const source = isel.code.items.len;
272 lowerToDEnc(tag, 0, isel.code) catch |err|
273 return isel.failWithLoweringError(err);
274 try isel.relocs.append(isel.bin_file.allocator, .{
275 .source = source,
276 .target = target,
277 .offset = isel.code.items.len - 4,
278 .length = 5,
279 });
280 },
281 0b01 => {
282 if (ops.reg1 == .none) {
283 // JMP/CALL [imm]
284 const imm = isel.mir.instructions.items(.data)[inst].imm;
285 const ptr_size: Memory.PtrSize = switch (immOpSize(imm)) {
286 16 => .word_ptr,
287 else => .qword_ptr,
288 };
289 return lowerToMEnc(tag, RegisterOrMemory.mem(ptr_size, .{ .disp = imm }), isel.code) catch |err|
290 isel.failWithLoweringError(err);
291 }
292 // JMP/CALL reg
293 return lowerToMEnc(tag, RegisterOrMemory.reg(ops.reg1), isel.code) catch |err| isel.failWithLoweringError(err);
294 },
295 0b10 => {
296 // JMP/CALL r/m64
297 const imm = isel.mir.instructions.items(.data)[inst].imm;
298 return lowerToMEnc(tag, RegisterOrMemory.mem(Memory.PtrSize.fromBits(ops.reg1.size()), .{
299 .disp = imm,
300 .base = ops.reg1,
301 }), isel.code) catch |err| isel.failWithLoweringError(err);
302 },
303 0b11 => return isel.fail("TODO unused JMP/CALL variant 0b11", .{}),
304 }
305}
306
307fn mirCondJmp(isel: *Isel, mir_tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerError!void {
308 const ops = Mir.Ops.decode(isel.mir.instructions.items(.ops)[inst]);
309 const target = isel.mir.instructions.items(.data)[inst].inst;
310 const tag = switch (mir_tag) {
311 .cond_jmp_greater_less => switch (ops.flags) {
312 0b00 => Tag.jge,
313 0b01 => Tag.jg,
314 0b10 => Tag.jl,
315 0b11 => Tag.jle,
316 },
317 .cond_jmp_above_below => switch (ops.flags) {
318 0b00 => Tag.jae,
319 0b01 => Tag.ja,
320 0b10 => Tag.jb,
321 0b11 => Tag.jbe,
322 },
323 .cond_jmp_eq_ne => switch (@truncate(u1, ops.flags)) {
324 0b0 => Tag.jne,
325 0b1 => Tag.je,
326 },
327 else => unreachable,
328 };
329 const source = isel.code.items.len;
330 lowerToDEnc(tag, 0, isel.code) catch |err|
331 return isel.failWithLoweringError(err);
332 try isel.relocs.append(isel.bin_file.allocator, .{
333 .source = source,
334 .target = target,
335 .offset = isel.code.items.len - 4,
336 .length = 6,
337 });
338}
339
340fn mirCondSetByte(isel: *Isel, mir_tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerError!void {
341 const ops = Mir.Ops.decode(isel.mir.instructions.items(.ops)[inst]);
342 const tag = switch (mir_tag) {
343 .cond_set_byte_greater_less => switch (ops.flags) {
344 0b00 => Tag.setge,
345 0b01 => Tag.setg,
346 0b10 => Tag.setl,
347 0b11 => Tag.setle,
348 },
349 .cond_set_byte_above_below => switch (ops.flags) {
350 0b00 => Tag.setae,
351 0b01 => Tag.seta,
352 0b10 => Tag.setb,
353 0b11 => Tag.setbe,
354 },
355 .cond_set_byte_eq_ne => switch (@truncate(u1, ops.flags)) {
356 0b0 => Tag.setne,
357 0b1 => Tag.sete,
358 },
359 else => unreachable,
360 };
361 return lowerToMEnc(tag, RegisterOrMemory.reg(ops.reg1.to8()), isel.code) catch |err|
362 isel.failWithLoweringError(err);
363}
364
365fn mirTest(isel: *Isel, inst: Mir.Inst.Index) InnerError!void {
366 const tag = isel.mir.instructions.items(.tag)[inst];
367 assert(tag == .@"test");
368 const ops = Mir.Ops.decode(isel.mir.instructions.items(.ops)[inst]);
369 switch (ops.flags) {
370 0b00 => {
371 if (ops.reg2 == .none) {
372 // TEST r/m64, imm32
373 // MI
374 const imm = isel.mir.instructions.items(.data)[inst].imm;
375 if (ops.reg1.to64() == .rax) {
376 // TEST rax, imm32
377 // I
378 return lowerToIEnc(.@"test", imm, isel.code) catch |err|
379 isel.failWithLoweringError(err);
380 }
381 return lowerToMiEnc(.@"test", RegisterOrMemory.reg(ops.reg1), imm, isel.code) catch |err|
382 isel.failWithLoweringError(err);
383 }
384 // TEST r/m64, r64
385 return isel.fail("TODO TEST r/m64, r64", .{});
386 },
387 else => return isel.fail("TODO more TEST alternatives", .{}),
388 }
389}
390
391fn mirRet(isel: *Isel, inst: Mir.Inst.Index) InnerError!void {
392 const tag = isel.mir.instructions.items(.tag)[inst];
393 assert(tag == .ret);
394 const ops = Mir.Ops.decode(isel.mir.instructions.items(.ops)[inst]);
395 switch (ops.flags) {
396 0b00 => {
397 // RETF imm16
398 // I
399 const imm = isel.mir.instructions.items(.data)[inst].imm;
400 return lowerToIEnc(.ret_far, imm, isel.code) catch |err| isel.failWithLoweringError(err);
401 },
402 0b01 => {
403 return lowerToZoEnc(.ret_far, isel.code) catch |err| isel.failWithLoweringError(err);
404 },
405 0b10 => {
406 // RET imm16
407 // I
408 const imm = isel.mir.instructions.items(.data)[inst].imm;
409 return lowerToIEnc(.ret_near, imm, isel.code) catch |err| isel.failWithLoweringError(err);
410 },
411 0b11 => {
412 return lowerToZoEnc(.ret_near, isel.code) catch |err| isel.failWithLoweringError(err);
413 },
414 }
415}
416
417fn mirArith(isel: *Isel, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
418 const ops = Mir.Ops.decode(isel.mir.instructions.items(.ops)[inst]);
419 switch (ops.flags) {
420 0b00 => {
421 if (ops.reg2 == .none) {
422 // mov reg1, imm32
423 // MI
424 const imm = isel.mir.instructions.items(.data)[inst].imm;
425 return lowerToMiEnc(tag, RegisterOrMemory.reg(ops.reg1), imm, isel.code) catch |err|
426 isel.failWithLoweringError(err);
427 }
428 // mov reg1, reg2
429 // RM
430 return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.reg(ops.reg2), isel.code) catch |err|
431 isel.failWithLoweringError(err);
432 },
433 0b01 => {
434 // mov reg1, [reg2 + imm32]
435 // RM
436 const imm = isel.mir.instructions.items(.data)[inst].imm;
437 const src_reg: ?Register = if (ops.reg2 == .none) null else ops.reg2;
438 return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.mem(Memory.PtrSize.fromBits(ops.reg1.size()), .{
439 .disp = imm,
440 .base = src_reg,
441 }), isel.code) catch |err| isel.failWithLoweringError(err);
442 },
443 0b10 => {
444 if (ops.reg2 == .none) {
445 return isel.fail("TODO unused variant: mov reg1, none, 0b10", .{});
446 }
447 // mov [reg1 + imm32], reg2
448 // MR
449 const imm = isel.mir.instructions.items(.data)[inst].imm;
450 return lowerToMrEnc(tag, RegisterOrMemory.mem(Memory.PtrSize.fromBits(ops.reg2.size()), .{
451 .disp = imm,
452 .base = ops.reg1,
453 }), ops.reg2, isel.code) catch |err| isel.failWithLoweringError(err);
454 },
455 0b11 => {
456 return isel.fail("TODO unused variant: mov reg1, reg2, 0b11", .{});
457 },
458 }
459}
460
461fn mirArithMemImm(isel: *Isel, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
462 const ops = Mir.Ops.decode(isel.mir.instructions.items(.ops)[inst]);
463 assert(ops.reg2 == .none);
464 const payload = isel.mir.instructions.items(.data)[inst].payload;
465 const imm_pair = isel.mir.extraData(Mir.ImmPair, payload).data;
466 const ptr_size: Memory.PtrSize = switch (ops.flags) {
467 0b00 => .byte_ptr,
468 0b01 => .word_ptr,
469 0b10 => .dword_ptr,
470 0b11 => .qword_ptr,
471 };
472 return lowerToMiEnc(tag, RegisterOrMemory.mem(ptr_size, .{
473 .disp = imm_pair.dest_off,
474 .base = ops.reg1,
475 }), imm_pair.operand, isel.code) catch |err| isel.failWithLoweringError(err);
476}
477
478inline fn setRexWRegister(reg: Register) bool {
479 if (reg.size() == 64) return true;
480 return switch (reg) {
481 .ah, .bh, .ch, .dh => true,
482 else => false,
483 };
484}
485
486inline fn immOpSize(u_imm: u32) u8 {
487 const imm = @bitCast(i32, u_imm);
488 if (math.minInt(i8) <= imm and imm <= math.maxInt(i8)) {
489 return 8;
490 }
491 if (math.minInt(i16) <= imm and imm <= math.maxInt(i16)) {
492 return 16;
493 }
494 return 32;
495}
496
497inline fn imm64OpSize(u_imm: u64) u8 {
498 const imm = @bitCast(i64, u_imm);
499 if (math.minInt(i8) <= imm and imm <= math.maxInt(i8)) {
500 return 8;
501 }
502 if (math.minInt(i16) <= imm and imm <= math.maxInt(i16)) {
503 return 16;
504 }
505 if (math.minInt(i32) <= imm and imm <= math.maxInt(i32)) {
506 return 32;
507 }
508 return 64;
509}
510
511fn mirArithScaleSrc(isel: *Isel, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
512 const ops = Mir.Ops.decode(isel.mir.instructions.items(.ops)[inst]);
513 const scale = ops.flags;
514 const imm = isel.mir.instructions.items(.data)[inst].imm;
515 // OP reg1, [reg2 + scale*rcx + imm32]
516 const scale_index = ScaleIndex{
517 .scale = scale,
518 .index = .rcx,
519 };
520 return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.mem(Memory.PtrSize.fromBits(ops.reg1.size()), .{
521 .disp = imm,
522 .base = ops.reg2,
523 .scale_index = scale_index,
524 }), isel.code) catch |err| isel.failWithLoweringError(err);
525}
526
527fn mirArithScaleDst(isel: *Isel, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
528 const ops = Mir.Ops.decode(isel.mir.instructions.items(.ops)[inst]);
529 const scale = ops.flags;
530 const imm = isel.mir.instructions.items(.data)[inst].imm;
531 const scale_index = ScaleIndex{
532 .scale = scale,
533 .index = .rax,
534 };
535 if (ops.reg2 == .none) {
536 // OP qword ptr [reg1 + scale*rax + 0], imm32
537 return lowerToMiEnc(tag, RegisterOrMemory.mem(.qword_ptr, .{
538 .disp = 0,
539 .base = ops.reg1,
540 .scale_index = scale_index,
541 }), imm, isel.code) catch |err| isel.failWithLoweringError(err);
542 }
543 // OP [reg1 + scale*rax + imm32], reg2
544 return lowerToMrEnc(tag, RegisterOrMemory.mem(Memory.PtrSize.fromBits(ops.reg2.size()), .{
545 .disp = imm,
546 .base = ops.reg1,
547 .scale_index = scale_index,
548 }), ops.reg2, isel.code) catch |err| isel.failWithLoweringError(err);
549}
550
551fn mirArithScaleImm(isel: *Isel, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
552 const ops = Mir.Ops.decode(isel.mir.instructions.items(.ops)[inst]);
553 const scale = ops.flags;
554 const payload = isel.mir.instructions.items(.data)[inst].payload;
555 const imm_pair = isel.mir.extraData(Mir.ImmPair, payload).data;
556 const scale_index = ScaleIndex{
557 .scale = scale,
558 .index = .rax,
559 };
560 // OP qword ptr [reg1 + scale*rax + imm32], imm32
561 return lowerToMiEnc(tag, RegisterOrMemory.mem(.qword_ptr, .{
562 .disp = imm_pair.dest_off,
563 .base = ops.reg1,
564 .scale_index = scale_index,
565 }), imm_pair.operand, isel.code) catch |err| isel.failWithLoweringError(err);
566}
567
568fn mirMovabs(isel: *Isel, inst: Mir.Inst.Index) InnerError!void {
569 const tag = isel.mir.instructions.items(.tag)[inst];
570 assert(tag == .movabs);
571 const ops = Mir.Ops.decode(isel.mir.instructions.items(.ops)[inst]);
572 const imm: u64 = if (ops.reg1.size() == 64) blk: {
573 const payload = isel.mir.instructions.items(.data)[inst].payload;
574 const imm = isel.mir.extraData(Mir.Imm64, payload).data;
575 break :blk imm.decode();
576 } else isel.mir.instructions.items(.data)[inst].imm;
577 if (ops.flags == 0b00) {
578 // movabs reg, imm64
579 // OI
580 return lowerToOiEnc(.mov, ops.reg1, imm, isel.code) catch |err| isel.failWithLoweringError(err);
581 }
582 if (ops.reg1 == .none) {
583 // movabs moffs64, rax
584 // TD
585 return lowerToTdEnc(.mov, imm, ops.reg2, isel.code) catch |err| isel.failWithLoweringError(err);
586 }
587 // movabs rax, moffs64
588 // FD
589 return lowerToFdEnc(.mov, ops.reg1, imm, isel.code) catch |err| isel.failWithLoweringError(err);
590}
591
592fn mirIMulComplex(isel: *Isel, inst: Mir.Inst.Index) InnerError!void {
593 const tag = isel.mir.instructions.items(.tag)[inst];
594 assert(tag == .imul_complex);
595 const ops = Mir.Ops.decode(isel.mir.instructions.items(.ops)[inst]);
596 switch (ops.flags) {
597 0b00 => {
598 return lowerToRmEnc(.imul, ops.reg1, RegisterOrMemory.reg(ops.reg2), isel.code) catch |err|
599 isel.failWithLoweringError(err);
600 },
601 0b10 => {
602 const imm = isel.mir.instructions.items(.data)[inst].imm;
603 return lowerToRmiEnc(.imul, ops.reg1, RegisterOrMemory.reg(ops.reg2), imm, isel.code) catch |err|
604 isel.failWithLoweringError(err);
605 },
606 else => return isel.fail("TODO implement imul", .{}),
607 }
608}
609
610fn mirLea(isel: *Isel, inst: Mir.Inst.Index) InnerError!void {
611 const tag = isel.mir.instructions.items(.tag)[inst];
612 assert(tag == .lea);
613 const ops = Mir.Ops.decode(isel.mir.instructions.items(.ops)[inst]);
614 switch (ops.flags) {
615 0b00 => {
616 // lea reg1, [reg2 + imm32]
617 // RM
618 const imm = isel.mir.instructions.items(.data)[inst].imm;
619 const src_reg: ?Register = if (ops.reg2 == .none) null else ops.reg2;
620 return lowerToRmEnc(
621 .lea,
622 ops.reg1,
623 RegisterOrMemory.mem(Memory.PtrSize.fromBits(ops.reg1.size()), .{
624 .disp = imm,
625 .base = src_reg,
626 }),
627 isel.code,
628 ) catch |err| isel.failWithLoweringError(err);
629 },
630 0b01 => {
631 // lea reg1, [rip + imm32]
632 // RM
633 const start_offset = isel.code.items.len;
634 lowerToRmEnc(
635 .lea,
636 ops.reg1,
637 RegisterOrMemory.rip(Memory.PtrSize.fromBits(ops.reg1.size()), 0),
638 isel.code,
639 ) catch |err| return isel.failWithLoweringError(err);
640 const end_offset = isel.code.items.len;
641 // Backpatch the displacement
642 const payload = isel.mir.instructions.items(.data)[inst].payload;
643 const imm = isel.mir.extraData(Mir.Imm64, payload).data.decode();
644 const disp = @intCast(i32, @intCast(i64, imm) - @intCast(i64, end_offset - start_offset));
645 mem.writeIntLittle(i32, isel.code.items[end_offset - 4 ..][0..4], disp);
646 },
647 0b10 => {
648 // lea reg1, [rip + reloc]
649 // RM
650 lowerToRmEnc(
651 .lea,
652 ops.reg1,
653 RegisterOrMemory.rip(Memory.PtrSize.fromBits(ops.reg1.size()), 0),
654 isel.code,
655 ) catch |err| return isel.failWithLoweringError(err);
656 const end_offset = isel.code.items.len;
657 const got_entry = isel.mir.instructions.items(.data)[inst].got_entry;
658 if (isel.bin_file.cast(link.File.MachO)) |macho_file| {
659 // TODO I think the reloc might be in the wrong place.
660 const decl = macho_file.active_decl.?;
661 try decl.link.macho.relocs.append(isel.bin_file.allocator, .{
662 .offset = @intCast(u32, end_offset - 4),
663 .target = .{ .local = got_entry },
664 .addend = 0,
665 .subtractor = null,
666 .pcrel = true,
667 .length = 2,
668 .@"type" = @enumToInt(std.macho.reloc_type_x86_64.X86_64_RELOC_GOT),
669 });
670 } else {
671 return isel.fail(
672 "TODO implement lea reg, [rip + reloc] for linking backends different than MachO",
673 .{},
674 );
675 }
676 },
677 0b11 => {
678 // lea reg, [rbp + rcx + imm32]
679 const imm = isel.mir.instructions.items(.data)[inst].imm;
680 const src_reg: ?Register = if (ops.reg2 == .none) null else ops.reg2;
681 const scale_index = ScaleIndex{
682 .scale = 0,
683 .index = .rcx,
684 };
685 return lowerToRmEnc(
686 .lea,
687 ops.reg1,
688 RegisterOrMemory.mem(Memory.PtrSize.fromBits(ops.reg1.size()), .{
689 .disp = imm,
690 .base = src_reg,
691 .scale_index = scale_index,
692 }),
693 isel.code,
694 ) catch |err| isel.failWithLoweringError(err);
695 },
696 }
697}
698
699fn mirCallExtern(isel: *Isel, inst: Mir.Inst.Index) InnerError!void {
700 const tag = isel.mir.instructions.items(.tag)[inst];
701 assert(tag == .call_extern);
702 const n_strx = isel.mir.instructions.items(.data)[inst].extern_fn;
703 const offset = blk: {
704 // callq
705 lowerToDEnc(.call_near, 0, isel.code) catch |err|
706 return isel.failWithLoweringError(err);
707 break :blk @intCast(u32, isel.code.items.len) - 4;
708 };
709 if (isel.bin_file.cast(link.File.MachO)) |macho_file| {
710 // Add relocation to the decl.
711 try macho_file.active_decl.?.link.macho.relocs.append(isel.bin_file.allocator, .{
712 .offset = offset,
713 .target = .{ .global = n_strx },
714 .addend = 0,
715 .subtractor = null,
716 .pcrel = true,
717 .length = 2,
718 .@"type" = @enumToInt(std.macho.reloc_type_x86_64.X86_64_RELOC_BRANCH),
719 });
720 } else {
721 return isel.fail("TODO implement call_extern for linking backends different than MachO", .{});
722 }
723}
724
725fn mirDbgLine(isel: *Isel, inst: Mir.Inst.Index) InnerError!void {
726 const tag = isel.mir.instructions.items(.tag)[inst];
727 assert(tag == .dbg_line);
728 const payload = isel.mir.instructions.items(.data)[inst].payload;
729 const dbg_line_column = isel.mir.extraData(Mir.DbgLineColumn, payload).data;
730 try isel.dbgAdvancePCAndLine(dbg_line_column.line, dbg_line_column.column);
731}
732
733fn dbgAdvancePCAndLine(isel: *Isel, line: u32, column: u32) InnerError!void {
734 const delta_line = @intCast(i32, line) - @intCast(i32, isel.prev_di_line);
735 const delta_pc: usize = isel.code.items.len - isel.prev_di_pc;
736 switch (isel.debug_output) {
737 .dwarf => |dbg_out| {
738 // TODO Look into using the DWARF special opcodes to compress this data.
739 // It lets you emit single-byte opcodes that add different numbers to
740 // both the PC and the line number at the same time.
741 try dbg_out.dbg_line.ensureUnusedCapacity(11);
742 dbg_out.dbg_line.appendAssumeCapacity(DW.LNS.advance_pc);
743 leb128.writeULEB128(dbg_out.dbg_line.writer(), delta_pc) catch unreachable;
744 if (delta_line != 0) {
745 dbg_out.dbg_line.appendAssumeCapacity(DW.LNS.advance_line);
746 leb128.writeILEB128(dbg_out.dbg_line.writer(), delta_line) catch unreachable;
747 }
748 dbg_out.dbg_line.appendAssumeCapacity(DW.LNS.copy);
749 isel.prev_di_pc = isel.code.items.len;
750 isel.prev_di_line = line;
751 isel.prev_di_column = column;
752 isel.prev_di_pc = isel.code.items.len;
753 },
754 .plan9 => |dbg_out| {
755 if (delta_pc <= 0) return; // only do this when the pc changes
756 // we have already checked the target in the linker to make sure it is compatable
757 const quant = @import("../../link/Plan9/aout.zig").getPCQuant(isel.target.cpu.arch) catch unreachable;
758
759 // increasing the line number
760 try @import("../../link/Plan9.zig").changeLine(dbg_out.dbg_line, delta_line);
761 // increasing the pc
762 const d_pc_p9 = @intCast(i64, delta_pc) - quant;
763 if (d_pc_p9 > 0) {
764 // minus one because if its the last one, we want to leave space to change the line which is one quanta
765 var diff = @divExact(d_pc_p9, quant) - quant;
766 while (diff > 0) {
767 if (diff < 64) {
768 try dbg_out.dbg_line.append(@intCast(u8, diff + 128));
769 diff = 0;
770 } else {
771 try dbg_out.dbg_line.append(@intCast(u8, 64 + 128));
772 diff -= 64;
773 }
774 }
775 if (dbg_out.pcop_change_index.*) |pci|
776 dbg_out.dbg_line.items[pci] += 1;
777 dbg_out.pcop_change_index.* = @intCast(u32, dbg_out.dbg_line.items.len - 1);
778 } else if (d_pc_p9 == 0) {
779 // we don't need to do anything, because adding the quant does it for us
780 } else unreachable;
781 if (dbg_out.start_line.* == null)
782 dbg_out.start_line.* = isel.prev_di_line;
783 dbg_out.end_line.* = line;
784 // only do this if the pc changed
785 isel.prev_di_line = line;
786 isel.prev_di_column = column;
787 isel.prev_di_pc = isel.code.items.len;
788 },
789 .none => {},
790 }
791}
792
793fn mirDbgPrologueEnd(isel: *Isel, inst: Mir.Inst.Index) InnerError!void {
794 const tag = isel.mir.instructions.items(.tag)[inst];
795 assert(tag == .dbg_prologue_end);
796 switch (isel.debug_output) {
797 .dwarf => |dbg_out| {
798 try dbg_out.dbg_line.append(DW.LNS.set_prologue_end);
799 try isel.dbgAdvancePCAndLine(isel.prev_di_line, isel.prev_di_column);
800 },
801 .plan9 => {},
802 .none => {},
803 }
804}
805
806fn mirDbgEpilogueBegin(isel: *Isel, inst: Mir.Inst.Index) InnerError!void {
807 const tag = isel.mir.instructions.items(.tag)[inst];
808 assert(tag == .dbg_epilogue_begin);
809 switch (isel.debug_output) {
810 .dwarf => |dbg_out| {
811 try dbg_out.dbg_line.append(DW.LNS.set_epilogue_begin);
812 try isel.dbgAdvancePCAndLine(isel.prev_di_line, isel.prev_di_column);
813 },
814 .plan9 => {},
815 .none => {},
816 }
817}
818
819fn mirArgDbgInfo(isel: *Isel, inst: Mir.Inst.Index) InnerError!void {
820 const tag = isel.mir.instructions.items(.tag)[inst];
821 assert(tag == .arg_dbg_info);
822 const payload = isel.mir.instructions.items(.data)[inst].payload;
823 const arg_dbg_info = isel.mir.extraData(Mir.ArgDbgInfo, payload).data;
824 const mcv = isel.mir.function.args[arg_dbg_info.arg_index];
825 try isel.genArgDbgInfo(arg_dbg_info.air_inst, mcv);
826}
827
828fn genArgDbgInfo(isel: *Isel, inst: Air.Inst.Index, mcv: MCValue) !void {
829 const ty_str = isel.mir.function.air.instructions.items(.data)[inst].ty_str;
830 const zir = &isel.mir.function.mod_fn.owner_decl.getFileScope().zir;
831 const name = zir.nullTerminatedString(ty_str.str);
832 const name_with_null = name.ptr[0 .. name.len + 1];
833 const ty = isel.mir.function.air.getRefType(ty_str.ty);
834
835 switch (mcv) {
836 .register => |reg| {
837 switch (isel.debug_output) {
838 .dwarf => |dbg_out| {
839 try dbg_out.dbg_info.ensureUnusedCapacity(3);
840 dbg_out.dbg_info.appendAssumeCapacity(link.File.Elf.abbrev_parameter);
841 dbg_out.dbg_info.appendSliceAssumeCapacity(&[2]u8{ // DW.AT.location, DW.FORM.exprloc
842 1, // ULEB128 dwarf expression length
843 reg.dwarfLocOp(),
844 });
845 try dbg_out.dbg_info.ensureUnusedCapacity(5 + name_with_null.len);
846 try isel.addDbgInfoTypeReloc(ty); // DW.AT.type, DW.FORM.ref4
847 dbg_out.dbg_info.appendSliceAssumeCapacity(name_with_null); // DW.AT.name, DW.FORM.string
848 },
849 .plan9 => {},
850 .none => {},
851 }
852 },
853 .stack_offset => {
854 switch (isel.debug_output) {
855 .dwarf => {},
856 .plan9 => {},
857 .none => {},
858 }
859 },
860 else => {},
861 }
862}
863
864/// Adds a Type to the .debug_info at the current position. The bytes will be populated later,
865/// after codegen for this symbol is done.
866fn addDbgInfoTypeReloc(isel: *Isel, ty: Type) !void {
867 switch (isel.debug_output) {
868 .dwarf => |dbg_out| {
869 assert(ty.hasCodeGenBits());
870 const index = dbg_out.dbg_info.items.len;
871 try dbg_out.dbg_info.resize(index + 4); // DW.AT.type, DW.FORM.ref4
872
873 const gop = try dbg_out.dbg_info_type_relocs.getOrPut(isel.bin_file.allocator, ty);
874 if (!gop.found_existing) {
875 gop.value_ptr.* = .{
876 .off = undefined,
877 .relocs = .{},
878 };
879 }
880 try gop.value_ptr.relocs.append(isel.bin_file.allocator, @intCast(u32, index));
881 },
882 .plan9 => {},
883 .none => {},
884 }
885}
886
887const Tag = enum {
888 adc,
889 add,
890 sub,
891 xor,
892 @"and",
893 @"or",
894 sbb,
895 cmp,
896 mov,
897 lea,
898 jmp_near,
899 call_near,
900 push,
901 pop,
902 @"test",
903 brk,
904 nop,
905 imul,
906 syscall,
907 ret_near,
908 ret_far,
909 jo,
910 jno,
911 jb,
912 jbe,
913 jc,
914 jnae,
915 jnc,
916 jae,
917 je,
918 jz,
919 jne,
920 jnz,
921 jna,
922 jnb,
923 jnbe,
924 ja,
925 js,
926 jns,
927 jpe,
928 jp,
929 jpo,
930 jnp,
931 jnge,
932 jl,
933 jge,
934 jnl,
935 jle,
936 jng,
937 jg,
938 jnle,
939 seto,
940 setno,
941 setb,
942 setc,
943 setnae,
944 setnb,
945 setnc,
946 setae,
947 sete,
948 setz,
949 setne,
950 setnz,
951 setbe,
952 setna,
953 seta,
954 setnbe,
955 sets,
956 setns,
957 setp,
958 setpe,
959 setnp,
960 setop,
961 setl,
962 setnge,
963 setnl,
964 setge,
965 setle,
966 setng,
967 setnle,
968 setg,
969
970 fn isSetCC(tag: Tag) bool {
971 return switch (tag) {
972 .seto,
973 .setno,
974 .setb,
975 .setc,
976 .setnae,
977 .setnb,
978 .setnc,
979 .setae,
980 .sete,
981 .setz,
982 .setne,
983 .setnz,
984 .setbe,
985 .setna,
986 .seta,
987 .setnbe,
988 .sets,
989 .setns,
990 .setp,
991 .setpe,
992 .setnp,
993 .setop,
994 .setl,
995 .setnge,
996 .setnl,
997 .setge,
998 .setle,
999 .setng,
1000 .setnle,
1001 .setg,
1002 => true,
1003 else => false,
1004 };
1005 }
1006};
1007
1008const Encoding = enum {
1009 /// OP
1010 zo,
1011
1012 /// OP rel32
1013 d,
1014
1015 /// OP r/m64
1016 m,
1017
1018 /// OP r64
1019 o,
1020
1021 /// OP imm32
1022 i,
1023
1024 /// OP r/m64, imm32
1025 mi,
1026
1027 /// OP r/m64, r64
1028 mr,
1029
1030 /// OP r64, r/m64
1031 rm,
1032
1033 /// OP r64, imm64
1034 oi,
1035
1036 /// OP al/ax/eax/rax, moffs
1037 fd,
1038
1039 /// OP moffs, al/ax/eax/rax
1040 td,
1041
1042 /// OP r64, r/m64, imm32
1043 rmi,
1044};
1045
1046const OpCode = union(enum) {
1047 one_byte: u8,
1048 two_byte: struct { _1: u8, _2: u8 },
1049
1050 fn oneByte(opc: u8) OpCode {
1051 return .{ .one_byte = opc };
1052 }
1053
1054 fn twoByte(opc1: u8, opc2: u8) OpCode {
1055 return .{ .two_byte = .{ ._1 = opc1, ._2 = opc2 } };
1056 }
1057
1058 fn encode(opc: OpCode, encoder: Encoder) void {
1059 switch (opc) {
1060 .one_byte => |v| encoder.opcode_1byte(v),
1061 .two_byte => |v| encoder.opcode_2byte(v._1, v._2),
1062 }
1063 }
1064
1065 fn encodeWithReg(opc: OpCode, encoder: Encoder, reg: Register) void {
1066 assert(opc == .one_byte);
1067 encoder.opcode_withReg(opc.one_byte, reg.lowId());
1068 }
1069};
1070
1071inline fn getOpCode(tag: Tag, enc: Encoding, is_one_byte: bool) ?OpCode {
1072 switch (enc) {
1073 .zo => return switch (tag) {
1074 .ret_near => OpCode.oneByte(0xc3),
1075 .ret_far => OpCode.oneByte(0xcb),
1076 .brk => OpCode.oneByte(0xcc),
1077 .nop => OpCode.oneByte(0x90),
1078 .syscall => OpCode.twoByte(0x0f, 0x05),
1079 else => null,
1080 },
1081 .d => return switch (tag) {
1082 .jmp_near => OpCode.oneByte(0xe9),
1083 .call_near => OpCode.oneByte(0xe8),
1084 .jo => if (is_one_byte) OpCode.oneByte(0x70) else OpCode.twoByte(0x0f, 0x80),
1085 .jno => if (is_one_byte) OpCode.oneByte(0x71) else OpCode.twoByte(0x0f, 0x81),
1086 .jb, .jc, .jnae => if (is_one_byte) OpCode.oneByte(0x72) else OpCode.twoByte(0x0f, 0x82),
1087 .jnb, .jnc, .jae => if (is_one_byte) OpCode.oneByte(0x73) else OpCode.twoByte(0x0f, 0x83),
1088 .je, .jz => if (is_one_byte) OpCode.oneByte(0x74) else OpCode.twoByte(0x0f, 0x84),
1089 .jne, .jnz => if (is_one_byte) OpCode.oneByte(0x75) else OpCode.twoByte(0x0f, 0x85),
1090 .jna, .jbe => if (is_one_byte) OpCode.oneByte(0x76) else OpCode.twoByte(0x0f, 0x86),
1091 .jnbe, .ja => if (is_one_byte) OpCode.oneByte(0x77) else OpCode.twoByte(0x0f, 0x87),
1092 .js => if (is_one_byte) OpCode.oneByte(0x78) else OpCode.twoByte(0x0f, 0x88),
1093 .jns => if (is_one_byte) OpCode.oneByte(0x79) else OpCode.twoByte(0x0f, 0x89),
1094 .jpe, .jp => if (is_one_byte) OpCode.oneByte(0x7a) else OpCode.twoByte(0x0f, 0x8a),
1095 .jpo, .jnp => if (is_one_byte) OpCode.oneByte(0x7b) else OpCode.twoByte(0x0f, 0x8b),
1096 .jnge, .jl => if (is_one_byte) OpCode.oneByte(0x7c) else OpCode.twoByte(0x0f, 0x8c),
1097 .jge, .jnl => if (is_one_byte) OpCode.oneByte(0x7d) else OpCode.twoByte(0x0f, 0x8d),
1098 .jle, .jng => if (is_one_byte) OpCode.oneByte(0x7e) else OpCode.twoByte(0x0f, 0x8e),
1099 .jg, .jnle => if (is_one_byte) OpCode.oneByte(0x7f) else OpCode.twoByte(0x0f, 0x8f),
1100 else => null,
1101 },
1102 .m => return switch (tag) {
1103 .jmp_near, .call_near, .push => OpCode.oneByte(0xff),
1104 .pop => OpCode.oneByte(0x8f),
1105 .seto => OpCode.twoByte(0x0f, 0x90),
1106 .setno => OpCode.twoByte(0x0f, 0x91),
1107 .setb, .setc, .setnae => OpCode.twoByte(0x0f, 0x92),
1108 .setnb, .setnc, .setae => OpCode.twoByte(0x0f, 0x93),
1109 .sete, .setz => OpCode.twoByte(0x0f, 0x94),
1110 .setne, .setnz => OpCode.twoByte(0x0f, 0x95),
1111 .setbe, .setna => OpCode.twoByte(0x0f, 0x96),
1112 .seta, .setnbe => OpCode.twoByte(0x0f, 0x97),
1113 .sets => OpCode.twoByte(0x0f, 0x98),
1114 .setns => OpCode.twoByte(0x0f, 0x99),
1115 .setp, .setpe => OpCode.twoByte(0x0f, 0x9a),
1116 .setnp, .setop => OpCode.twoByte(0x0f, 0x9b),
1117 .setl, .setnge => OpCode.twoByte(0x0f, 0x9c),
1118 .setnl, .setge => OpCode.twoByte(0x0f, 0x9d),
1119 .setle, .setng => OpCode.twoByte(0x0f, 0x9e),
1120 .setnle, .setg => OpCode.twoByte(0x0f, 0x9f),
1121 else => null,
1122 },
1123 .o => return switch (tag) {
1124 .push => OpCode.oneByte(0x50),
1125 .pop => OpCode.oneByte(0x58),
1126 else => null,
1127 },
1128 .i => return switch (tag) {
1129 .push => OpCode.oneByte(if (is_one_byte) 0x6a else 0x68),
1130 .@"test" => OpCode.oneByte(if (is_one_byte) 0xa8 else 0xa9),
1131 .ret_near => OpCode.oneByte(0xc2),
1132 .ret_far => OpCode.oneByte(0xca),
1133 else => null,
1134 },
1135 .mi => return switch (tag) {
1136 .adc, .add, .sub, .xor, .@"and", .@"or", .sbb, .cmp => OpCode.oneByte(if (is_one_byte) 0x80 else 0x81),
1137 .mov => OpCode.oneByte(if (is_one_byte) 0xc6 else 0xc7),
1138 .@"test" => OpCode.oneByte(if (is_one_byte) 0xf6 else 0xf7),
1139 else => null,
1140 },
1141 .mr => return switch (tag) {
1142 .adc => OpCode.oneByte(if (is_one_byte) 0x10 else 0x11),
1143 .add => OpCode.oneByte(if (is_one_byte) 0x00 else 0x01),
1144 .sub => OpCode.oneByte(if (is_one_byte) 0x28 else 0x29),
1145 .xor => OpCode.oneByte(if (is_one_byte) 0x30 else 0x31),
1146 .@"and" => OpCode.oneByte(if (is_one_byte) 0x20 else 0x21),
1147 .@"or" => OpCode.oneByte(if (is_one_byte) 0x08 else 0x09),
1148 .sbb => OpCode.oneByte(if (is_one_byte) 0x18 else 0x19),
1149 .cmp => OpCode.oneByte(if (is_one_byte) 0x38 else 0x39),
1150 .mov => OpCode.oneByte(if (is_one_byte) 0x88 else 0x89),
1151 else => null,
1152 },
1153 .rm => return switch (tag) {
1154 .adc => OpCode.oneByte(if (is_one_byte) 0x12 else 0x13),
1155 .add => OpCode.oneByte(if (is_one_byte) 0x02 else 0x03),
1156 .sub => OpCode.oneByte(if (is_one_byte) 0x2a else 0x2b),
1157 .xor => OpCode.oneByte(if (is_one_byte) 0x32 else 0x33),
1158 .@"and" => OpCode.oneByte(if (is_one_byte) 0x22 else 0x23),
1159 .@"or" => OpCode.oneByte(if (is_one_byte) 0x0b else 0x0b),
1160 .sbb => OpCode.oneByte(if (is_one_byte) 0x1a else 0x1b),
1161 .cmp => OpCode.oneByte(if (is_one_byte) 0x3a else 0x3b),
1162 .mov => OpCode.oneByte(if (is_one_byte) 0x8a else 0x8b),
1163 .lea => OpCode.oneByte(if (is_one_byte) 0x8c else 0x8d),
1164 .imul => OpCode.twoByte(0x0f, 0xaf),
1165 else => null,
1166 },
1167 .oi => return switch (tag) {
1168 .mov => OpCode.oneByte(if (is_one_byte) 0xb0 else 0xb8),
1169 else => null,
1170 },
1171 .fd => return switch (tag) {
1172 .mov => OpCode.oneByte(if (is_one_byte) 0xa0 else 0xa1),
1173 else => null,
1174 },
1175 .td => return switch (tag) {
1176 .mov => OpCode.oneByte(if (is_one_byte) 0xa2 else 0xa3),
1177 else => null,
1178 },
1179 .rmi => return switch (tag) {
1180 .imul => OpCode.oneByte(if (is_one_byte) 0x6b else 0x69),
1181 else => null,
1182 },
1183 }
1184}
1185
1186inline fn getModRmExt(tag: Tag) ?u3 {
1187 return switch (tag) {
1188 .adc => 0x2,
1189 .add => 0x0,
1190 .sub => 0x5,
1191 .xor => 0x6,
1192 .@"and" => 0x4,
1193 .@"or" => 0x1,
1194 .sbb => 0x3,
1195 .cmp => 0x7,
1196 .mov => 0x0,
1197 .jmp_near => 0x4,
1198 .call_near => 0x2,
1199 .push => 0x6,
1200 .pop => 0x0,
1201 .@"test" => 0x0,
1202 .seto,
1203 .setno,
1204 .setb,
1205 .setc,
1206 .setnae,
1207 .setnb,
1208 .setnc,
1209 .setae,
1210 .sete,
1211 .setz,
1212 .setne,
1213 .setnz,
1214 .setbe,
1215 .setna,
1216 .seta,
1217 .setnbe,
1218 .sets,
1219 .setns,
1220 .setp,
1221 .setpe,
1222 .setnp,
1223 .setop,
1224 .setl,
1225 .setnge,
1226 .setnl,
1227 .setge,
1228 .setle,
1229 .setng,
1230 .setnle,
1231 .setg,
1232 => 0x0,
1233 else => null,
1234 };
1235}
1236
1237const ScaleIndex = struct {
1238 scale: u2,
1239 index: Register,
1240};
1241
1242const Memory = struct {
1243 base: ?Register,
1244 rip: bool = false,
1245 disp: u32,
1246 ptr_size: PtrSize,
1247 scale_index: ?ScaleIndex = null,
1248
1249 const PtrSize = enum {
1250 byte_ptr,
1251 word_ptr,
1252 dword_ptr,
1253 qword_ptr,
1254
1255 fn fromBits(in_bits: u64) PtrSize {
1256 return switch (in_bits) {
1257 8 => .byte_ptr,
1258 16 => .word_ptr,
1259 32 => .dword_ptr,
1260 64 => .qword_ptr,
1261 else => unreachable,
1262 };
1263 }
1264
1265 /// Returns size in bits.
1266 fn size(ptr_size: PtrSize) u64 {
1267 return switch (ptr_size) {
1268 .byte_ptr => 8,
1269 .word_ptr => 16,
1270 .dword_ptr => 32,
1271 .qword_ptr => 64,
1272 };
1273 }
1274 };
1275
1276 fn encode(mem_op: Memory, encoder: Encoder, operand: u3) void {
1277 if (mem_op.base) |base| {
1278 const dst = base.lowId();
1279 const src = operand;
1280 if (dst == 4 or mem_op.scale_index != null) {
1281 if (mem_op.disp == 0 and dst != 5) {
1282 encoder.modRm_SIBDisp0(src);
1283 if (mem_op.scale_index) |si| {
1284 encoder.sib_scaleIndexBase(si.scale, si.index.lowId(), dst);
1285 } else {
1286 encoder.sib_base(dst);
1287 }
1288 } else if (immOpSize(mem_op.disp) == 8) {
1289 encoder.modRm_SIBDisp8(src);
1290 if (mem_op.scale_index) |si| {
1291 encoder.sib_scaleIndexBaseDisp8(si.scale, si.index.lowId(), dst);
1292 } else {
1293 encoder.sib_baseDisp8(dst);
1294 }
1295 encoder.disp8(@bitCast(i8, @truncate(u8, mem_op.disp)));
1296 } else {
1297 encoder.modRm_SIBDisp32(src);
1298 if (mem_op.scale_index) |si| {
1299 encoder.sib_scaleIndexBaseDisp32(si.scale, si.index.lowId(), dst);
1300 } else {
1301 encoder.sib_baseDisp32(dst);
1302 }
1303 encoder.disp32(@bitCast(i32, mem_op.disp));
1304 }
1305 } else {
1306 if (mem_op.disp == 0) {
1307 encoder.modRm_indirectDisp0(src, dst);
1308 } else if (immOpSize(mem_op.disp) == 8) {
1309 encoder.modRm_indirectDisp8(src, dst);
1310 encoder.disp8(@bitCast(i8, @truncate(u8, mem_op.disp)));
1311 } else {
1312 encoder.modRm_indirectDisp32(src, dst);
1313 encoder.disp32(@bitCast(i32, mem_op.disp));
1314 }
1315 }
1316 } else {
1317 if (mem_op.rip) {
1318 encoder.modRm_RIPDisp32(operand);
1319 } else {
1320 encoder.modRm_SIBDisp0(operand);
1321 if (mem_op.scale_index) |si| {
1322 encoder.sib_scaleIndexDisp32(si.scale, si.index.lowId());
1323 } else {
1324 encoder.sib_disp32();
1325 }
1326 }
1327 encoder.disp32(@bitCast(i32, mem_op.disp));
1328 }
1329 }
1330};
1331
1332fn encodeImm(encoder: Encoder, imm: u32, size: u64) void {
1333 switch (size) {
1334 8 => encoder.imm8(@bitCast(i8, @truncate(u8, imm))),
1335 16 => encoder.imm16(@bitCast(i16, @truncate(u16, imm))),
1336 32, 64 => encoder.imm32(@bitCast(i32, imm)),
1337 else => unreachable,
1338 }
1339}
1340
1341const RegisterOrMemory = union(enum) {
1342 register: Register,
1343 memory: Memory,
1344
1345 fn reg(register: Register) RegisterOrMemory {
1346 return .{ .register = register };
1347 }
1348
1349 fn mem(ptr_size: Memory.PtrSize, args: struct {
1350 disp: u32,
1351 base: ?Register = null,
1352 scale_index: ?ScaleIndex = null,
1353 }) RegisterOrMemory {
1354 return .{
1355 .memory = .{
1356 .base = args.base,
1357 .disp = args.disp,
1358 .ptr_size = ptr_size,
1359 .scale_index = args.scale_index,
1360 },
1361 };
1362 }
1363
1364 fn rip(ptr_size: Memory.PtrSize, disp: u32) RegisterOrMemory {
1365 return .{
1366 .memory = .{
1367 .base = null,
1368 .rip = true,
1369 .disp = disp,
1370 .ptr_size = ptr_size,
1371 },
1372 };
1373 }
1374};
1375
1376const LoweringError = error{
1377 OutOfMemory,
1378 Overflow,
1379 OperandSizeMismatch,
1380 RaxOperandExpected,
1381};
1382
1383fn lowerToZoEnc(tag: Tag, code: *std.ArrayList(u8)) LoweringError!void {
1384 const opc = getOpCode(tag, .zo, false).?;
1385 const encoder = try Encoder.init(code, 1);
1386 opc.encode(encoder);
1387}
1388
1389fn lowerToIEnc(tag: Tag, imm: u32, code: *std.ArrayList(u8)) LoweringError!void {
1390 if (tag == .ret_far or tag == .ret_near) {
1391 const encoder = try Encoder.init(code, 3);
1392 const opc = getOpCode(tag, .i, false).?;
1393 opc.encode(encoder);
1394 encoder.imm16(@bitCast(i16, @truncate(u16, imm)));
1395 return;
1396 }
1397 const opc = getOpCode(tag, .i, immOpSize(imm) == 8).?;
1398 const encoder = try Encoder.init(code, 5);
1399 if (immOpSize(imm) == 16) {
1400 encoder.prefix16BitMode();
1401 }
1402 opc.encode(encoder);
1403 encodeImm(encoder, imm, immOpSize(imm));
1404}
1405
1406fn lowerToOEnc(tag: Tag, reg: Register, code: *std.ArrayList(u8)) LoweringError!void {
1407 if (reg.size() != 16 and reg.size() != 64) {
1408 return error.OperandSizeMismatch; // TODO correct for push/pop, but is it universal?
1409 }
1410 const opc = getOpCode(tag, .o, false).?;
1411 const encoder = try Encoder.init(code, 3);
1412 if (reg.size() == 16) {
1413 encoder.prefix16BitMode();
1414 }
1415 encoder.rex(.{
1416 .w = false,
1417 .b = reg.isExtended(),
1418 });
1419 opc.encodeWithReg(encoder, reg);
1420}
1421
1422fn lowerToDEnc(tag: Tag, imm: u32, code: *std.ArrayList(u8)) LoweringError!void {
1423 const opc = getOpCode(tag, .d, false).?;
1424 const encoder = try Encoder.init(code, 6);
1425 opc.encode(encoder);
1426 encoder.imm32(@bitCast(i32, imm));
1427}
1428
1429fn lowerToMEnc(tag: Tag, reg_or_mem: RegisterOrMemory, code: *std.ArrayList(u8)) LoweringError!void {
1430 const opc = getOpCode(tag, .m, false).?;
1431 const modrm_ext = getModRmExt(tag).?;
1432 switch (reg_or_mem) {
1433 .register => |reg| {
1434 const op_size_mismatch = blk: {
1435 if (tag.isSetCC() and reg.size() == 8)
1436 break :blk false;
1437 break :blk reg.size() != 64 and reg.size() != 16;
1438 };
1439 if (op_size_mismatch) {
1440 return error.OperandSizeMismatch;
1441 }
1442 const encoder = try Encoder.init(code, 4);
1443 if (reg.size() == 16) {
1444 encoder.prefix16BitMode();
1445 }
1446 encoder.rex(.{
1447 .w = switch (reg) {
1448 .ah, .bh, .ch, .dh => true,
1449 else => false,
1450 },
1451 .b = reg.isExtended(),
1452 });
1453 opc.encode(encoder);
1454 encoder.modRm_direct(modrm_ext, reg.lowId());
1455 },
1456 .memory => |mem_op| {
1457 if (mem_op.ptr_size != .qword_ptr and mem_op.ptr_size != .word_ptr) {
1458 return error.OperandSizeMismatch;
1459 }
1460 const encoder = try Encoder.init(code, 8);
1461 if (mem_op.ptr_size == .word_ptr) {
1462 encoder.prefix16BitMode();
1463 }
1464 if (mem_op.base) |base| {
1465 if (base.size() != 64) {
1466 return error.OperandSizeMismatch;
1467 }
1468 encoder.rex(.{
1469 .w = false,
1470 .b = base.isExtended(),
1471 });
1472 }
1473 opc.encode(encoder);
1474 mem_op.encode(encoder, modrm_ext);
1475 },
1476 }
1477}
1478
1479fn lowerToTdEnc(tag: Tag, moffs: u64, reg: Register, code: *std.ArrayList(u8)) LoweringError!void {
1480 return lowerToTdFdEnc(tag, reg, moffs, code, true);
1481}
1482
1483fn lowerToFdEnc(tag: Tag, reg: Register, moffs: u64, code: *std.ArrayList(u8)) LoweringError!void {
1484 return lowerToTdFdEnc(tag, reg, moffs, code, false);
1485}
1486
1487fn lowerToTdFdEnc(tag: Tag, reg: Register, moffs: u64, code: *std.ArrayList(u8), td: bool) LoweringError!void {
1488 if (reg.lowId() != Register.rax.lowId()) {
1489 return error.RaxOperandExpected;
1490 }
1491 if (reg.size() != imm64OpSize(moffs)) {
1492 return error.OperandSizeMismatch;
1493 }
1494 const opc = if (td)
1495 getOpCode(tag, .td, reg.size() == 8).?
1496 else
1497 getOpCode(tag, .fd, reg.size() == 8).?;
1498 const encoder = try Encoder.init(code, 10);
1499 if (reg.size() == 16) {
1500 encoder.prefix16BitMode();
1501 }
1502 encoder.rex(.{
1503 .w = setRexWRegister(reg),
1504 });
1505 opc.encode(encoder);
1506 switch (reg.size()) {
1507 8 => encoder.imm8(@bitCast(i8, @truncate(u8, moffs))),
1508 16 => encoder.imm16(@bitCast(i16, @truncate(u16, moffs))),
1509 32 => encoder.imm32(@bitCast(i32, @truncate(u32, moffs))),
1510 64 => encoder.imm64(moffs),
1511 else => unreachable,
1512 }
1513}
1514
1515fn lowerToOiEnc(tag: Tag, reg: Register, imm: u64, code: *std.ArrayList(u8)) LoweringError!void {
1516 if (reg.size() != imm64OpSize(imm)) {
1517 return error.OperandSizeMismatch;
1518 }
1519 const opc = getOpCode(tag, .oi, reg.size() == 8).?;
1520 const encoder = try Encoder.init(code, 10);
1521 if (reg.size() == 16) {
1522 encoder.prefix16BitMode();
1523 }
1524 encoder.rex(.{
1525 .w = setRexWRegister(reg),
1526 .b = reg.isExtended(),
1527 });
1528 opc.encodeWithReg(encoder, reg);
1529 switch (reg.size()) {
1530 8 => encoder.imm8(@bitCast(i8, @truncate(u8, imm))),
1531 16 => encoder.imm16(@bitCast(i16, @truncate(u16, imm))),
1532 32 => encoder.imm32(@bitCast(i32, @truncate(u32, imm))),
1533 64 => encoder.imm64(imm),
1534 else => unreachable,
1535 }
1536}
1537
1538fn lowerToMiEnc(tag: Tag, reg_or_mem: RegisterOrMemory, imm: u32, code: *std.ArrayList(u8)) LoweringError!void {
1539 const modrm_ext = getModRmExt(tag).?;
1540 switch (reg_or_mem) {
1541 .register => |dst_reg| {
1542 const opc = getOpCode(tag, .mi, dst_reg.size() == 8).?;
1543 const encoder = try Encoder.init(code, 7);
1544 if (dst_reg.size() == 16) {
1545 // 0x66 prefix switches to the non-default size; here we assume a switch from
1546 // the default 32bits to 16bits operand-size.
1547 // More info: https://www.cs.uni-potsdam.de/desn/lehre/ss15/64-ia-32-architectures-software-developer-instruction-set-reference-manual-325383.pdf#page=32&zoom=auto,-159,773
1548 encoder.prefix16BitMode();
1549 }
1550 encoder.rex(.{
1551 .w = setRexWRegister(dst_reg),
1552 .b = dst_reg.isExtended(),
1553 });
1554 opc.encode(encoder);
1555 encoder.modRm_direct(modrm_ext, dst_reg.lowId());
1556 encodeImm(encoder, imm, dst_reg.size());
1557 },
1558 .memory => |dst_mem| {
1559 const opc = getOpCode(tag, .mi, dst_mem.ptr_size == .byte_ptr).?;
1560 const encoder = try Encoder.init(code, 12);
1561 if (dst_mem.ptr_size == .word_ptr) {
1562 encoder.prefix16BitMode();
1563 }
1564 if (dst_mem.base) |base| {
1565 if (base.size() != 64) {
1566 return error.OperandSizeMismatch;
1567 }
1568 encoder.rex(.{
1569 .w = dst_mem.ptr_size == .qword_ptr,
1570 .b = base.isExtended(),
1571 });
1572 } else {
1573 encoder.rex(.{
1574 .w = dst_mem.ptr_size == .qword_ptr,
1575 });
1576 }
1577 opc.encode(encoder);
1578 dst_mem.encode(encoder, modrm_ext);
1579 encodeImm(encoder, imm, dst_mem.ptr_size.size());
1580 },
1581 }
1582}
1583
1584fn lowerToRmEnc(
1585 tag: Tag,
1586 reg: Register,
1587 reg_or_mem: RegisterOrMemory,
1588 code: *std.ArrayList(u8),
1589) LoweringError!void {
1590 const opc = getOpCode(tag, .rm, reg.size() == 8).?;
1591 switch (reg_or_mem) {
1592 .register => |src_reg| {
1593 if (reg.size() != src_reg.size()) {
1594 return error.OperandSizeMismatch;
1595 }
1596 const encoder = try Encoder.init(code, 3);
1597 encoder.rex(.{
1598 .w = setRexWRegister(reg) or setRexWRegister(src_reg),
1599 .r = reg.isExtended(),
1600 .b = src_reg.isExtended(),
1601 });
1602 opc.encode(encoder);
1603 encoder.modRm_direct(reg.lowId(), src_reg.lowId());
1604 },
1605 .memory => |src_mem| {
1606 if (reg.size() != src_mem.ptr_size.size()) {
1607 return error.OperandSizeMismatch;
1608 }
1609 const encoder = try Encoder.init(code, 9);
1610 if (reg.size() == 16) {
1611 encoder.prefix16BitMode();
1612 }
1613 if (src_mem.base) |base| {
1614 // TODO handle 32-bit base register - requires prefix 0x67
1615 // Intel Manual, Vol 1, chapter 3.6 and 3.6.1
1616 if (base.size() != 64) {
1617 return error.OperandSizeMismatch;
1618 }
1619 encoder.rex(.{
1620 .w = setRexWRegister(reg),
1621 .r = reg.isExtended(),
1622 .b = base.isExtended(),
1623 });
1624 } else {
1625 encoder.rex(.{
1626 .w = setRexWRegister(reg),
1627 .r = reg.isExtended(),
1628 });
1629 }
1630 opc.encode(encoder);
1631 src_mem.encode(encoder, reg.lowId());
1632 },
1633 }
1634}
1635
1636fn lowerToMrEnc(
1637 tag: Tag,
1638 reg_or_mem: RegisterOrMemory,
1639 reg: Register,
1640 code: *std.ArrayList(u8),
1641) LoweringError!void {
1642 const opc = getOpCode(tag, .mr, reg.size() == 8).?;
1643 switch (reg_or_mem) {
1644 .register => |dst_reg| {
1645 if (dst_reg.size() != reg.size()) {
1646 return error.OperandSizeMismatch;
1647 }
1648 const encoder = try Encoder.init(code, 3);
1649 encoder.rex(.{
1650 .w = setRexWRegister(dst_reg) or setRexWRegister(reg),
1651 .r = reg.isExtended(),
1652 .b = dst_reg.isExtended(),
1653 });
1654 opc.encode(encoder);
1655 encoder.modRm_direct(reg.lowId(), dst_reg.lowId());
1656 },
1657 .memory => |dst_mem| {
1658 if (dst_mem.ptr_size.size() != reg.size()) {
1659 return error.OperandSizeMismatch;
1660 }
1661 const encoder = try Encoder.init(code, 9);
1662 if (reg.size() == 16) {
1663 encoder.prefix16BitMode();
1664 }
1665 if (dst_mem.base) |base| {
1666 if (base.size() != 64) {
1667 return error.OperandSizeMismatch;
1668 }
1669 encoder.rex(.{
1670 .w = dst_mem.ptr_size == .qword_ptr or setRexWRegister(reg),
1671 .r = reg.isExtended(),
1672 .b = base.isExtended(),
1673 });
1674 } else {
1675 encoder.rex(.{
1676 .w = dst_mem.ptr_size == .qword_ptr or setRexWRegister(reg),
1677 .r = reg.isExtended(),
1678 });
1679 }
1680 opc.encode(encoder);
1681 dst_mem.encode(encoder, reg.lowId());
1682 },
1683 }
1684}
1685
1686fn lowerToRmiEnc(
1687 tag: Tag,
1688 reg: Register,
1689 reg_or_mem: RegisterOrMemory,
1690 imm: u32,
1691 code: *std.ArrayList(u8),
1692) LoweringError!void {
1693 if (reg.size() == 8) {
1694 return error.OperandSizeMismatch;
1695 }
1696 const opc = getOpCode(tag, .rmi, false).?;
1697 const encoder = try Encoder.init(code, 13);
1698 if (reg.size() == 16) {
1699 encoder.prefix16BitMode();
1700 }
1701 switch (reg_or_mem) {
1702 .register => |src_reg| {
1703 if (reg.size() != src_reg.size()) {
1704 return error.OperandSizeMismatch;
1705 }
1706 encoder.rex(.{
1707 .w = setRexWRegister(reg) or setRexWRegister(src_reg),
1708 .r = reg.isExtended(),
1709 .b = src_reg.isExtended(),
1710 });
1711 opc.encode(encoder);
1712 encoder.modRm_direct(reg.lowId(), src_reg.lowId());
1713 },
1714 .memory => |src_mem| {
1715 if (src_mem.base) |base| {
1716 // TODO handle 32-bit base register - requires prefix 0x67
1717 // Intel Manual, Vol 1, chapter 3.6 and 3.6.1
1718 if (base.size() != 64) {
1719 return error.OperandSizeMismatch;
1720 }
1721 if (src_mem.ptr_size == .byte_ptr) {
1722 return error.OperandSizeMismatch;
1723 }
1724 encoder.rex(.{
1725 .w = setRexWRegister(reg),
1726 .r = reg.isExtended(),
1727 .b = base.isExtended(),
1728 });
1729 } else {
1730 encoder.rex(.{
1731 .w = setRexWRegister(reg),
1732 .r = reg.isExtended(),
1733 });
1734 }
1735 opc.encode(encoder);
1736 src_mem.encode(encoder, reg.lowId());
1737 },
1738 }
1739 encodeImm(encoder, imm, reg.size());
1740}
1741
1742fn expectEqualHexStrings(expected: []const u8, given: []const u8, assembly: []const u8) !void {
1743 assert(expected.len > 0);
1744 if (mem.eql(u8, expected, given)) return;
1745 const expected_fmt = try std.fmt.allocPrint(testing.allocator, "{x}", .{std.fmt.fmtSliceHexLower(expected)});
1746 defer testing.allocator.free(expected_fmt);
1747 const given_fmt = try std.fmt.allocPrint(testing.allocator, "{x}", .{std.fmt.fmtSliceHexLower(given)});
1748 defer testing.allocator.free(given_fmt);
1749 const idx = mem.indexOfDiff(u8, expected_fmt, given_fmt).?;
1750 var padding = try testing.allocator.alloc(u8, idx + 5);
1751 defer testing.allocator.free(padding);
1752 mem.set(u8, padding, ' ');
1753 std.debug.print("\nASM: {s}\nEXP: {s}\nGIV: {s}\n{s}^ -- first differing byte\n", .{
1754 assembly,
1755 expected_fmt,
1756 given_fmt,
1757 padding,
1758 });
1759 return error.TestFailed;
1760}
1761
1762const TestIsel = struct {
1763 code_buffer: std.ArrayList(u8),
1764 next: usize = 0,
1765
1766 fn init() TestIsel {
1767 return .{
1768 .code_buffer = std.ArrayList(u8).init(testing.allocator),
1769 };
1770 }
1771
1772 fn deinit(isel: *TestIsel) void {
1773 isel.code_buffer.deinit();
1774 isel.next = undefined;
1775 }
1776
1777 fn code(isel: *TestIsel) *std.ArrayList(u8) {
1778 isel.next = isel.code_buffer.items.len;
1779 return &isel.code_buffer;
1780 }
1781
1782 fn lowered(isel: TestIsel) []const u8 {
1783 return isel.code_buffer.items[isel.next..];
1784 }
1785};
1786
1787test "lower MI encoding" {
1788 var isel = TestIsel.init();
1789 defer isel.deinit();
1790 try lowerToMiEnc(.mov, RegisterOrMemory.reg(.rax), 0x10, isel.code());
1791 try expectEqualHexStrings("\x48\xc7\xc0\x10\x00\x00\x00", isel.lowered(), "mov rax, 0x10");
1792 try lowerToMiEnc(.mov, RegisterOrMemory.mem(.dword_ptr, .{ .disp = 0, .base = .r11 }), 0x10, isel.code());
1793 try expectEqualHexStrings("\x41\xc7\x03\x10\x00\x00\x00", isel.lowered(), "mov dword ptr [r11 + 0], 0x10");
1794 try lowerToMiEnc(.add, RegisterOrMemory.mem(.dword_ptr, .{
1795 .disp = @bitCast(u32, @as(i32, -8)),
1796 .base = .rdx,
1797 }), 0x10, isel.code());
1798 try expectEqualHexStrings("\x81\x42\xF8\x10\x00\x00\x00", isel.lowered(), "add dword ptr [rdx - 8], 0x10");
1799 try lowerToMiEnc(.sub, RegisterOrMemory.mem(.dword_ptr, .{
1800 .disp = 0x10000000,
1801 .base = .r11,
1802 }), 0x10, isel.code());
1803 try expectEqualHexStrings(
1804 "\x41\x81\xab\x00\x00\x00\x10\x10\x00\x00\x00",
1805 isel.lowered(),
1806 "sub dword ptr [r11 + 0x10000000], 0x10",
1807 );
1808 try lowerToMiEnc(.@"and", RegisterOrMemory.mem(.dword_ptr, .{ .disp = 0x10000000 }), 0x10, isel.code());
1809 try expectEqualHexStrings(
1810 "\x81\x24\x25\x00\x00\x00\x10\x10\x00\x00\x00",
1811 isel.lowered(),
1812 "and dword ptr [ds:0x10000000], 0x10",
1813 );
1814 try lowerToMiEnc(.@"and", RegisterOrMemory.mem(.dword_ptr, .{
1815 .disp = 0x10000000,
1816 .base = .r12,
1817 }), 0x10, isel.code());
1818 try expectEqualHexStrings(
1819 "\x41\x81\xA4\x24\x00\x00\x00\x10\x10\x00\x00\x00",
1820 isel.lowered(),
1821 "and dword ptr [r12 + 0x10000000], 0x10",
1822 );
1823 try lowerToMiEnc(.mov, RegisterOrMemory.rip(.qword_ptr, 0x10), 0x10, isel.code());
1824 try expectEqualHexStrings(
1825 "\x48\xC7\x05\x10\x00\x00\x00\x10\x00\x00\x00",
1826 isel.lowered(),
1827 "mov qword ptr [rip + 0x10], 0x10",
1828 );
1829 try lowerToMiEnc(.mov, RegisterOrMemory.mem(.qword_ptr, .{
1830 .disp = @bitCast(u32, @as(i32, -8)),
1831 .base = .rbp,
1832 }), 0x10, isel.code());
1833 try expectEqualHexStrings(
1834 "\x48\xc7\x45\xf8\x10\x00\x00\x00",
1835 isel.lowered(),
1836 "mov qword ptr [rbp - 8], 0x10",
1837 );
1838 try lowerToMiEnc(.mov, RegisterOrMemory.mem(.word_ptr, .{
1839 .disp = @bitCast(u32, @as(i32, -2)),
1840 .base = .rbp,
1841 }), 0x10, isel.code());
1842 try expectEqualHexStrings("\x66\xC7\x45\xFE\x10\x00", isel.lowered(), "mov word ptr [rbp - 2], 0x10");
1843 try lowerToMiEnc(.mov, RegisterOrMemory.mem(.byte_ptr, .{
1844 .disp = @bitCast(u32, @as(i32, -1)),
1845 .base = .rbp,
1846 }), 0x10, isel.code());
1847 try expectEqualHexStrings("\xC6\x45\xFF\x10", isel.lowered(), "mov byte ptr [rbp - 1], 0x10");
1848 try lowerToMiEnc(.mov, RegisterOrMemory.mem(.qword_ptr, .{
1849 .disp = 0x10000000,
1850 .scale_index = .{
1851 .scale = 1,
1852 .index = .rcx,
1853 },
1854 }), 0x10, isel.code());
1855 try expectEqualHexStrings(
1856 "\x48\xC7\x04\x4D\x00\x00\x00\x10\x10\x00\x00\x00",
1857 isel.lowered(),
1858 "mov qword ptr [rcx*2 + 0x10000000], 0x10",
1859 );
1860}
1861
1862test "lower RM encoding" {
1863 var isel = TestIsel.init();
1864 defer isel.deinit();
1865 try lowerToRmEnc(.mov, .rax, RegisterOrMemory.reg(.rbx), isel.code());
1866 try expectEqualHexStrings("\x48\x8b\xc3", isel.lowered(), "mov rax, rbx");
1867 try lowerToRmEnc(.mov, .rax, RegisterOrMemory.mem(.qword_ptr, .{ .disp = 0, .base = .r11 }), isel.code());
1868 try expectEqualHexStrings("\x49\x8b\x03", isel.lowered(), "mov rax, qword ptr [r11 + 0]");
1869 try lowerToRmEnc(.add, .r11, RegisterOrMemory.mem(.qword_ptr, .{ .disp = 0x10000000 }), isel.code());
1870 try expectEqualHexStrings(
1871 "\x4C\x03\x1C\x25\x00\x00\x00\x10",
1872 isel.lowered(),
1873 "add r11, qword ptr [ds:0x10000000]",
1874 );
1875 try lowerToRmEnc(.add, .r12b, RegisterOrMemory.mem(.byte_ptr, .{ .disp = 0x10000000 }), isel.code());
1876 try expectEqualHexStrings(
1877 "\x44\x02\x24\x25\x00\x00\x00\x10",
1878 isel.lowered(),
1879 "add r11b, byte ptr [ds:0x10000000]",
1880 );
1881 try lowerToRmEnc(.sub, .r11, RegisterOrMemory.mem(.qword_ptr, .{
1882 .disp = 0x10000000,
1883 .base = .r13,
1884 }), isel.code());
1885 try expectEqualHexStrings(
1886 "\x4D\x2B\x9D\x00\x00\x00\x10",
1887 isel.lowered(),
1888 "sub r11, qword ptr [r13 + 0x10000000]",
1889 );
1890 try lowerToRmEnc(.sub, .r11, RegisterOrMemory.mem(.qword_ptr, .{
1891 .disp = 0x10000000,
1892 .base = .r12,
1893 }), isel.code());
1894 try expectEqualHexStrings(
1895 "\x4D\x2B\x9C\x24\x00\x00\x00\x10",
1896 isel.lowered(),
1897 "sub r11, qword ptr [r12 + 0x10000000]",
1898 );
1899 try lowerToRmEnc(.mov, .rax, RegisterOrMemory.mem(.qword_ptr, .{
1900 .disp = @bitCast(u32, @as(i32, -4)),
1901 .base = .rbp,
1902 }), isel.code());
1903 try expectEqualHexStrings("\x48\x8B\x45\xFC", isel.lowered(), "mov rax, qword ptr [rbp - 4]");
1904 try lowerToRmEnc(.lea, .rax, RegisterOrMemory.rip(.qword_ptr, 0x10), isel.code());
1905 try expectEqualHexStrings("\x48\x8D\x05\x10\x00\x00\x00", isel.lowered(), "lea rax, [rip + 0x10]");
1906 try lowerToRmEnc(.mov, .rax, RegisterOrMemory.mem(.qword_ptr, .{
1907 .disp = @bitCast(u32, @as(i32, -8)),
1908 .base = .rbp,
1909 .scale_index = .{
1910 .scale = 0,
1911 .index = .rcx,
1912 },
1913 }), isel.code());
1914 try expectEqualHexStrings("\x48\x8B\x44\x0D\xF8", isel.lowered(), "mov rax, qword ptr [rbp + rcx*1 - 8]");
1915 try lowerToRmEnc(.mov, .eax, RegisterOrMemory.mem(.dword_ptr, .{
1916 .disp = @bitCast(u32, @as(i32, -4)),
1917 .base = .rbp,
1918 .scale_index = .{
1919 .scale = 2,
1920 .index = .rdx,
1921 },
1922 }), isel.code());
1923 try expectEqualHexStrings("\x8B\x44\x95\xFC", isel.lowered(), "mov eax, dword ptr [rbp + rdx*4 - 4]");
1924 try lowerToRmEnc(.mov, .rax, RegisterOrMemory.mem(.qword_ptr, .{
1925 .disp = @bitCast(u32, @as(i32, -8)),
1926 .base = .rbp,
1927 .scale_index = .{
1928 .scale = 3,
1929 .index = .rcx,
1930 },
1931 }), isel.code());
1932 try expectEqualHexStrings("\x48\x8B\x44\xCD\xF8", isel.lowered(), "mov rax, qword ptr [rbp + rcx*8 - 8]");
1933 try lowerToRmEnc(.mov, .r8b, RegisterOrMemory.mem(.byte_ptr, .{
1934 .disp = @bitCast(u32, @as(i32, -24)),
1935 .base = .rsi,
1936 .scale_index = .{
1937 .scale = 0,
1938 .index = .rcx,
1939 },
1940 }), isel.code());
1941 try expectEqualHexStrings("\x44\x8A\x44\x0E\xE8", isel.lowered(), "mov r8b, byte ptr [rsi + rcx*1 - 24]");
1942 try lowerToRmEnc(.lea, .rsi, RegisterOrMemory.mem(.qword_ptr, .{
1943 .disp = 0,
1944 .base = .rbp,
1945 .scale_index = .{
1946 .scale = 0,
1947 .index = .rcx,
1948 },
1949 }), isel.code());
1950 try expectEqualHexStrings("\x48\x8D\x74\x0D\x00", isel.lowered(), "lea rsi, qword ptr [rbp + rcx*1 + 0]");
1951}
1952
1953test "lower MR encoding" {
1954 var isel = TestIsel.init();
1955 defer isel.deinit();
1956 try lowerToMrEnc(.mov, RegisterOrMemory.reg(.rax), .rbx, isel.code());
1957 try expectEqualHexStrings("\x48\x89\xd8", isel.lowered(), "mov rax, rbx");
1958 try lowerToMrEnc(.mov, RegisterOrMemory.mem(.qword_ptr, .{
1959 .disp = @bitCast(u32, @as(i32, -4)),
1960 .base = .rbp,
1961 }), .r11, isel.code());
1962 try expectEqualHexStrings("\x4c\x89\x5d\xfc", isel.lowered(), "mov qword ptr [rbp - 4], r11");
1963 try lowerToMrEnc(.add, RegisterOrMemory.mem(.byte_ptr, .{ .disp = 0x10000000 }), .r12b, isel.code());
1964 try expectEqualHexStrings(
1965 "\x44\x00\x24\x25\x00\x00\x00\x10",
1966 isel.lowered(),
1967 "add byte ptr [ds:0x10000000], r12b",
1968 );
1969 try lowerToMrEnc(.add, RegisterOrMemory.mem(.dword_ptr, .{ .disp = 0x10000000 }), .r12d, isel.code());
1970 try expectEqualHexStrings(
1971 "\x44\x01\x24\x25\x00\x00\x00\x10",
1972 isel.lowered(),
1973 "add dword ptr [ds:0x10000000], r12d",
1974 );
1975 try lowerToMrEnc(.sub, RegisterOrMemory.mem(.qword_ptr, .{
1976 .disp = 0x10000000,
1977 .base = .r11,
1978 }), .r12, isel.code());
1979 try expectEqualHexStrings(
1980 "\x4D\x29\xA3\x00\x00\x00\x10",
1981 isel.lowered(),
1982 "sub qword ptr [r11 + 0x10000000], r12",
1983 );
1984 try lowerToMrEnc(.mov, RegisterOrMemory.rip(.qword_ptr, 0x10), .r12, isel.code());
1985 try expectEqualHexStrings("\x4C\x89\x25\x10\x00\x00\x00", isel.lowered(), "mov qword ptr [rip + 0x10], r12");
1986}
1987
1988test "lower OI encoding" {
1989 var isel = TestIsel.init();
1990 defer isel.deinit();
1991 try lowerToOiEnc(.mov, .rax, 0x1000000000000000, isel.code());
1992 try expectEqualHexStrings(
1993 "\x48\xB8\x00\x00\x00\x00\x00\x00\x00\x10",
1994 isel.lowered(),
1995 "movabs rax, 0x1000000000000000",
1996 );
1997 try lowerToOiEnc(.mov, .r11, 0x1000000000000000, isel.code());
1998 try expectEqualHexStrings(
1999 "\x49\xBB\x00\x00\x00\x00\x00\x00\x00\x10",
2000 isel.lowered(),
2001 "movabs r11, 0x1000000000000000",
2002 );
2003 try lowerToOiEnc(.mov, .r11d, 0x10000000, isel.code());
2004 try expectEqualHexStrings("\x41\xBB\x00\x00\x00\x10", isel.lowered(), "mov r11d, 0x10000000");
2005 try lowerToOiEnc(.mov, .r11w, 0x1000, isel.code());
2006 try expectEqualHexStrings("\x66\x41\xBB\x00\x10", isel.lowered(), "mov r11w, 0x1000");
2007 try lowerToOiEnc(.mov, .r11b, 0x10, isel.code());
2008 try expectEqualHexStrings("\x41\xB3\x10", isel.lowered(), "mov r11b, 0x10");
2009}
2010
2011test "lower FD/TD encoding" {
2012 var isel = TestIsel.init();
2013 defer isel.deinit();
2014 try lowerToFdEnc(.mov, .rax, 0x1000000000000000, isel.code());
2015 try expectEqualHexStrings(
2016 "\x48\xa1\x00\x00\x00\x00\x00\x00\x00\x10",
2017 isel.lowered(),
2018 "mov rax, ds:0x1000000000000000",
2019 );
2020 try lowerToFdEnc(.mov, .eax, 0x10000000, isel.code());
2021 try expectEqualHexStrings("\xa1\x00\x00\x00\x10", isel.lowered(), "mov eax, ds:0x10000000");
2022 try lowerToFdEnc(.mov, .ax, 0x1000, isel.code());
2023 try expectEqualHexStrings("\x66\xa1\x00\x10", isel.lowered(), "mov ax, ds:0x1000");
2024 try lowerToFdEnc(.mov, .al, 0x10, isel.code());
2025 try expectEqualHexStrings("\xa0\x10", isel.lowered(), "mov al, ds:0x10");
2026}
2027
2028test "lower M encoding" {
2029 var isel = TestIsel.init();
2030 defer isel.deinit();
2031 try lowerToMEnc(.jmp_near, RegisterOrMemory.reg(.r12), isel.code());
2032 try expectEqualHexStrings("\x41\xFF\xE4", isel.lowered(), "jmp r12");
2033 try lowerToMEnc(.jmp_near, RegisterOrMemory.reg(.r12w), isel.code());
2034 try expectEqualHexStrings("\x66\x41\xFF\xE4", isel.lowered(), "jmp r12w");
2035 try lowerToMEnc(.jmp_near, RegisterOrMemory.mem(.qword_ptr, .{ .disp = 0, .base = .r12 }), isel.code());
2036 try expectEqualHexStrings("\x41\xFF\x24\x24", isel.lowered(), "jmp qword ptr [r12]");
2037 try lowerToMEnc(.jmp_near, RegisterOrMemory.mem(.word_ptr, .{ .disp = 0, .base = .r12 }), isel.code());
2038 try expectEqualHexStrings("\x66\x41\xFF\x24\x24", isel.lowered(), "jmp word ptr [r12]");
2039 try lowerToMEnc(.jmp_near, RegisterOrMemory.mem(.qword_ptr, .{ .disp = 0x10, .base = .r12 }), isel.code());
2040 try expectEqualHexStrings("\x41\xFF\x64\x24\x10", isel.lowered(), "jmp qword ptr [r12 + 0x10]");
2041 try lowerToMEnc(.jmp_near, RegisterOrMemory.mem(.qword_ptr, .{
2042 .disp = 0x1000,
2043 .base = .r12,
2044 }), isel.code());
2045 try expectEqualHexStrings(
2046 "\x41\xFF\xA4\x24\x00\x10\x00\x00",
2047 isel.lowered(),
2048 "jmp qword ptr [r12 + 0x1000]",
2049 );
2050 try lowerToMEnc(.jmp_near, RegisterOrMemory.rip(.qword_ptr, 0x10), isel.code());
2051 try expectEqualHexStrings("\xFF\x25\x10\x00\x00\x00", isel.lowered(), "jmp qword ptr [rip + 0x10]");
2052 try lowerToMEnc(.jmp_near, RegisterOrMemory.mem(.qword_ptr, .{ .disp = 0x10 }), isel.code());
2053 try expectEqualHexStrings("\xFF\x24\x25\x10\x00\x00\x00", isel.lowered(), "jmp qword ptr [ds:0x10]");
2054 try lowerToMEnc(.seta, RegisterOrMemory.reg(.r11b), isel.code());
2055 try expectEqualHexStrings("\x41\x0F\x97\xC3", isel.lowered(), "seta r11b");
2056}
2057
2058test "lower O encoding" {
2059 var isel = TestIsel.init();
2060 defer isel.deinit();
2061 try lowerToOEnc(.pop, .r12, isel.code());
2062 try expectEqualHexStrings("\x41\x5c", isel.lowered(), "pop r12");
2063 try lowerToOEnc(.push, .r12w, isel.code());
2064 try expectEqualHexStrings("\x66\x41\x54", isel.lowered(), "push r12w");
2065}
2066
2067test "lower RMI encoding" {
2068 var isel = TestIsel.init();
2069 defer isel.deinit();
2070 try lowerToRmiEnc(.imul, .rax, RegisterOrMemory.mem(.qword_ptr, .{
2071 .disp = @bitCast(u32, @as(i32, -8)),
2072 .base = .rbp,
2073 }), 0x10, isel.code());
2074 try expectEqualHexStrings(
2075 "\x48\x69\x45\xF8\x10\x00\x00\x00",
2076 isel.lowered(),
2077 "imul rax, qword ptr [rbp - 8], 0x10",
2078 );
2079 try lowerToRmiEnc(.imul, .eax, RegisterOrMemory.mem(.dword_ptr, .{
2080 .disp = @bitCast(u32, @as(i32, -4)),
2081 .base = .rbp,
2082 }), 0x10, isel.code());
2083 try expectEqualHexStrings("\x69\x45\xFC\x10\x00\x00\x00", isel.lowered(), "imul eax, dword ptr [rbp - 4], 0x10");
2084 try lowerToRmiEnc(.imul, .ax, RegisterOrMemory.mem(.word_ptr, .{
2085 .disp = @bitCast(u32, @as(i32, -2)),
2086 .base = .rbp,
2087 }), 0x10, isel.code());
2088 try expectEqualHexStrings("\x66\x69\x45\xFE\x10\x00", isel.lowered(), "imul ax, word ptr [rbp - 2], 0x10");
2089 try lowerToRmiEnc(.imul, .r12, RegisterOrMemory.reg(.r12), 0x10, isel.code());
2090 try expectEqualHexStrings("\x4D\x69\xE4\x10\x00\x00\x00", isel.lowered(), "imul r12, r12, 0x10");
2091 try lowerToRmiEnc(.imul, .r12w, RegisterOrMemory.reg(.r12w), 0x10, isel.code());
2092 try expectEqualHexStrings("\x66\x45\x69\xE4\x10\x00", isel.lowered(), "imul r12w, r12w, 0x10");
2093}