1//! This file contains the functionality for lowering x86_64 MIR to Instructions
2
3target: *const std.Target,
4allocator: std.mem.Allocator,
5mir: Mir,
6cc: std.lang.CallingConvention,
7err_msg: ?*Zcu.ErrorMsg = null,
8src_loc: Zcu.LazySrcLoc,
9result_insts_len: ResultInstIndex = undefined,
10result_insts: [max_result_insts]Instruction = undefined,
11result_relocs_len: ResultRelocIndex = undefined,
12result_relocs: [max_result_relocs]Reloc = undefined,
13
14const max_result_insts = @max(
15 1, // non-pseudo instructions
16 2, // cmovcc: cmovcc \ cmovcc
17 3, // setcc: setcc \ setcc \ logicop
18 2, // jcc: jcc \ jcc
19 pseudo_probe_align_insts,
20 pseudo_probe_adjust_unrolled_max_insts,
21 pseudo_probe_adjust_setup_insts,
22 pseudo_probe_adjust_loop_insts,
23 abi.zigcc.callee_preserved_regs.len * 2, // push_regs/pop_regs
24 abi.Win64.callee_preserved_regs.len * 2, // push_regs/pop_regs
25 abi.SysV.callee_preserved_regs.len * 2, // push_regs/pop_regs
26);
27const max_result_relocs = @max(
28 1, // jmp/jcc/call/mov/lea: jmp/jcc/call/mov/lea
29 2, // jcc: jcc \ jcc
30 2, // test \ jcc \ probe \ sub \ jmp
31 1, // probe \ sub \ jcc
32);
33
34const ResultInstIndex = std.math.IntFittingRange(0, max_result_insts);
35const ResultRelocIndex = std.math.IntFittingRange(0, max_result_relocs);
36pub const InstOpIndex = std.math.IntFittingRange(
37 0,
38 @typeInfo(@FieldType(Instruction, "ops")).array.len,
39);
40
41pub const pseudo_probe_align_insts = 5; // test \ jcc \ probe \ sub \ jmp
42pub const pseudo_probe_adjust_unrolled_max_insts =
43 pseudo_probe_adjust_setup_insts + pseudo_probe_adjust_loop_insts;
44pub const pseudo_probe_adjust_setup_insts = 2; // mov \ sub
45pub const pseudo_probe_adjust_loop_insts = 3; // probe \ sub \ jcc
46
47pub const Error = error{
48 OutOfMemory,
49 LowerFail,
50 InvalidInstruction,
51 CannotEncode,
52 AlreadyReported,
53} || link.Error;
54
55pub const Reloc = struct {
56 lowered_inst_index: ResultInstIndex,
57 op_index: InstOpIndex,
58 target: Target,
59 off: i32,
60
61 const Target = union(enum) {
62 inst: Mir.Inst.Index,
63 table,
64 nav: InternPool.Nav.Index,
65 uav: InternPool.Key.Ptr.BaseAddr.Uav,
66 lazy_sym: link.File.LazySymbol,
67 extern_func: Mir.NullTerminatedString,
68 };
69};
70
71const Options = struct { allow_frame_locs: bool };
72
73/// The returned slice is overwritten by the next call to lowerMir.
74pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct {
75 insts: []Instruction,
76 relocs: []const Reloc,
77} {
78 lower.result_insts = undefined;
79 lower.result_relocs = undefined;
80 errdefer lower.result_insts = undefined;
81 errdefer lower.result_relocs = undefined;
82 lower.result_insts_len = 0;
83 lower.result_relocs_len = 0;
84 defer lower.result_insts_len = undefined;
85 defer lower.result_relocs_len = undefined;
86
87 const inst = lower.mir.instructions.get(index);
88 switch (inst.tag) {
89 else => try lower.generic(inst),
90 .pseudo => switch (inst.ops) {
91 .pseudo_cmov_z_and_np_rr => {
92 assert(inst.data.rr.fixes == ._);
93 try lower.encode(.none, .cmovnz, &.{
94 .{ .reg = inst.data.rr.r2 },
95 .{ .reg = inst.data.rr.r1 },
96 });
97 try lower.encode(.none, .cmovnp, &.{
98 .{ .reg = inst.data.rr.r1 },
99 .{ .reg = inst.data.rr.r2 },
100 });
101 },
102 .pseudo_cmov_nz_or_p_rr => {
103 assert(inst.data.rr.fixes == ._);
104 try lower.encode(.none, .cmovnz, &.{
105 .{ .reg = inst.data.rr.r1 },
106 .{ .reg = inst.data.rr.r2 },
107 });
108 try lower.encode(.none, .cmovp, &.{
109 .{ .reg = inst.data.rr.r1 },
110 .{ .reg = inst.data.rr.r2 },
111 });
112 },
113 .pseudo_cmov_nz_or_p_rm => {
114 assert(inst.data.rx.fixes == ._);
115 try lower.encode(.none, .cmovnz, &.{
116 .{ .reg = inst.data.rx.r1 },
117 .{ .mem = lower.mem(1, inst.data.rx.payload) },
118 });
119 try lower.encode(.none, .cmovp, &.{
120 .{ .reg = inst.data.rx.r1 },
121 .{ .mem = lower.mem(1, inst.data.rx.payload) },
122 });
123 },
124 .pseudo_set_z_and_np_r => {
125 assert(inst.data.rr.fixes == ._);
126 try lower.encode(.none, .setz, &.{
127 .{ .reg = inst.data.rr.r1 },
128 });
129 try lower.encode(.none, .setnp, &.{
130 .{ .reg = inst.data.rr.r2 },
131 });
132 try lower.encode(.none, .@"and", &.{
133 .{ .reg = inst.data.rr.r1 },
134 .{ .reg = inst.data.rr.r2 },
135 });
136 },
137 .pseudo_set_z_and_np_m => {
138 assert(inst.data.rx.fixes == ._);
139 try lower.encode(.none, .setz, &.{
140 .{ .mem = lower.mem(0, inst.data.rx.payload) },
141 });
142 try lower.encode(.none, .setnp, &.{
143 .{ .reg = inst.data.rx.r1 },
144 });
145 try lower.encode(.none, .@"and", &.{
146 .{ .mem = lower.mem(0, inst.data.rx.payload) },
147 .{ .reg = inst.data.rx.r1 },
148 });
149 },
150 .pseudo_set_nz_or_p_r => {
151 assert(inst.data.rr.fixes == ._);
152 try lower.encode(.none, .setnz, &.{
153 .{ .reg = inst.data.rr.r1 },
154 });
155 try lower.encode(.none, .setp, &.{
156 .{ .reg = inst.data.rr.r2 },
157 });
158 try lower.encode(.none, .@"or", &.{
159 .{ .reg = inst.data.rr.r1 },
160 .{ .reg = inst.data.rr.r2 },
161 });
162 },
163 .pseudo_set_nz_or_p_m => {
164 assert(inst.data.rx.fixes == ._);
165 try lower.encode(.none, .setnz, &.{
166 .{ .mem = lower.mem(0, inst.data.rx.payload) },
167 });
168 try lower.encode(.none, .setp, &.{
169 .{ .reg = inst.data.rx.r1 },
170 });
171 try lower.encode(.none, .@"or", &.{
172 .{ .mem = lower.mem(0, inst.data.rx.payload) },
173 .{ .reg = inst.data.rx.r1 },
174 });
175 },
176 .pseudo_j_z_and_np_inst => {
177 assert(inst.data.inst.fixes == ._);
178 try lower.encode(.none, .jnz, &.{
179 .{ .imm = lower.reloc(0, .{ .inst = index + 1 }, 0) },
180 });
181 try lower.encode(.none, .jnp, &.{
182 .{ .imm = lower.reloc(0, .{ .inst = inst.data.inst.inst }, 0) },
183 });
184 },
185 .pseudo_j_nz_or_p_inst => {
186 assert(inst.data.inst.fixes == ._);
187 try lower.encode(.none, .jnz, &.{
188 .{ .imm = lower.reloc(0, .{ .inst = inst.data.inst.inst }, 0) },
189 });
190 try lower.encode(.none, .jp, &.{
191 .{ .imm = lower.reloc(0, .{ .inst = inst.data.inst.inst }, 0) },
192 });
193 },
194
195 .pseudo_probe_align_ri_s => {
196 try lower.encode(.none, .@"test", &.{
197 .{ .reg = inst.data.ri.r1 },
198 .{ .imm = .s(@bitCast(inst.data.ri.i)) },
199 });
200 try lower.encode(.none, .jz, &.{
201 .{ .imm = lower.reloc(0, .{ .inst = index + 1 }, 0) },
202 });
203 try lower.encode(.none, .lea, &.{
204 .{ .reg = inst.data.ri.r1 },
205 .{ .mem = Memory.initSib(.qword, .{
206 .base = .{ .reg = inst.data.ri.r1 },
207 .disp = -page_size,
208 }) },
209 });
210 try lower.encode(.none, .@"test", &.{
211 .{ .mem = Memory.initSib(.dword, .{
212 .base = .{ .reg = inst.data.ri.r1 },
213 }) },
214 .{ .reg = inst.data.ri.r1.to32() },
215 });
216 try lower.encode(.none, .jmp, &.{
217 .{ .imm = lower.reloc(0, .{ .inst = index }, 0) },
218 });
219 assert(lower.result_insts_len == pseudo_probe_align_insts);
220 },
221 .pseudo_probe_adjust_unrolled_ri_s => {
222 var offset = page_size;
223 while (offset < @as(i32, @bitCast(inst.data.ri.i))) : (offset += page_size) {
224 try lower.encode(.none, .@"test", &.{
225 .{ .mem = Memory.initSib(.dword, .{
226 .base = .{ .reg = inst.data.ri.r1 },
227 .disp = -offset,
228 }) },
229 .{ .reg = inst.data.ri.r1.to32() },
230 });
231 }
232 try lower.encode(.none, .sub, &.{
233 .{ .reg = inst.data.ri.r1 },
234 .{ .imm = .s(@bitCast(inst.data.ri.i)) },
235 });
236 assert(lower.result_insts_len <= pseudo_probe_adjust_unrolled_max_insts);
237 },
238 .pseudo_probe_adjust_setup_rri_s => {
239 try lower.encode(.none, .mov, &.{
240 .{ .reg = inst.data.rri.r2.to32() },
241 .{ .imm = .s(@bitCast(inst.data.rri.i)) },
242 });
243 try lower.encode(.none, .sub, &.{
244 .{ .reg = inst.data.rri.r1 },
245 .{ .reg = inst.data.rri.r2 },
246 });
247 assert(lower.result_insts_len == pseudo_probe_adjust_setup_insts);
248 },
249 .pseudo_probe_adjust_loop_rr => {
250 try lower.encode(.none, .@"test", &.{
251 .{ .mem = Memory.initSib(.dword, .{
252 .base = .{ .reg = inst.data.rr.r1 },
253 .scale_index = .{ .scale = 1, .index = inst.data.rr.r2 },
254 .disp = -page_size,
255 }) },
256 .{ .reg = inst.data.rr.r1.to32() },
257 });
258 try lower.encode(.none, .sub, &.{
259 .{ .reg = inst.data.rr.r2 },
260 .{ .imm = .s(page_size) },
261 });
262 try lower.encode(.none, .jae, &.{
263 .{ .imm = lower.reloc(0, .{ .inst = index }, 0) },
264 });
265 assert(lower.result_insts_len == pseudo_probe_adjust_loop_insts);
266 },
267 .pseudo_push_reg_list => try lower.pushPopRegList(.push, inst),
268 .pseudo_pop_reg_list => try lower.pushPopRegList(.pop, inst),
269
270 .pseudo_cfi_def_cfa_ri_s => try lower.encode(.directive, .@".cfi_def_cfa", &.{
271 .{ .reg = inst.data.ri.r1 },
272 .{ .imm = lower.imm(.ri_s, inst.data.ri.i) },
273 }),
274 .pseudo_cfi_def_cfa_register_r => try lower.encode(.directive, .@".cfi_def_cfa_register", &.{
275 .{ .reg = inst.data.r.r1 },
276 }),
277 .pseudo_cfi_def_cfa_offset_i_s => try lower.encode(.directive, .@".cfi_def_cfa_offset", &.{
278 .{ .imm = lower.imm(.i_s, inst.data.i.i) },
279 }),
280 .pseudo_cfi_adjust_cfa_offset_i_s => try lower.encode(.directive, .@".cfi_adjust_cfa_offset", &.{
281 .{ .imm = lower.imm(.i_s, inst.data.i.i) },
282 }),
283 .pseudo_cfi_offset_ri_s => try lower.encode(.directive, .@".cfi_offset", &.{
284 .{ .reg = inst.data.ri.r1 },
285 .{ .imm = lower.imm(.ri_s, inst.data.ri.i) },
286 }),
287 .pseudo_cfi_val_offset_ri_s => try lower.encode(.directive, .@".cfi_val_offset", &.{
288 .{ .reg = inst.data.ri.r1 },
289 .{ .imm = lower.imm(.ri_s, inst.data.ri.i) },
290 }),
291 .pseudo_cfi_rel_offset_ri_s => try lower.encode(.directive, .@".cfi_rel_offset", &.{
292 .{ .reg = inst.data.ri.r1 },
293 .{ .imm = lower.imm(.ri_s, inst.data.ri.i) },
294 }),
295 .pseudo_cfi_register_rr => try lower.encode(.directive, .@".cfi_register", &.{
296 .{ .reg = inst.data.rr.r1 },
297 .{ .reg = inst.data.rr.r2 },
298 }),
299 .pseudo_cfi_restore_r => try lower.encode(.directive, .@".cfi_restore", &.{
300 .{ .reg = inst.data.r.r1 },
301 }),
302 .pseudo_cfi_undefined_r => try lower.encode(.directive, .@".cfi_undefined", &.{
303 .{ .reg = inst.data.r.r1 },
304 }),
305 .pseudo_cfi_same_value_r => try lower.encode(.directive, .@".cfi_same_value", &.{
306 .{ .reg = inst.data.r.r1 },
307 }),
308 .pseudo_cfi_remember_state_none => try lower.encode(.directive, .@".cfi_remember_state", &.{}),
309 .pseudo_cfi_restore_state_none => try lower.encode(.directive, .@".cfi_restore_state", &.{}),
310 .pseudo_cfi_escape_bytes => try lower.encode(.directive, .@".cfi_escape", &.{
311 .{ .bytes = inst.data.bytes.get(lower.mir) },
312 }),
313
314 .pseudo_dbg_prologue_end_none,
315 .pseudo_dbg_line_stmt_line_column,
316 .pseudo_dbg_line_line_column,
317 .pseudo_dbg_epilogue_begin_line_column,
318 .pseudo_dbg_enter_block_none,
319 .pseudo_dbg_leave_block_none,
320 .pseudo_dbg_enter_inline_func,
321 .pseudo_dbg_leave_inline_func,
322 .pseudo_dbg_end_none,
323 .pseudo_dbg_arg_none,
324 .pseudo_dbg_arg_i_s,
325 .pseudo_dbg_arg_i_u,
326 .pseudo_dbg_arg_i_64,
327 .pseudo_dbg_arg_ro,
328 .pseudo_dbg_arg_fa,
329 .pseudo_dbg_arg_m,
330 .pseudo_dbg_arg_val,
331 .pseudo_dbg_var_args_none,
332 .pseudo_dbg_var_none,
333 .pseudo_dbg_var_i_s,
334 .pseudo_dbg_var_i_u,
335 .pseudo_dbg_var_i_64,
336 .pseudo_dbg_var_ro,
337 .pseudo_dbg_var_fa,
338 .pseudo_dbg_var_m,
339 .pseudo_dbg_var_val,
340
341 .pseudo_dead_none,
342 => {},
343 else => unreachable,
344 },
345 }
346
347 return .{
348 .insts = lower.result_insts[0..lower.result_insts_len],
349 .relocs = lower.result_relocs[0..lower.result_relocs_len],
350 };
351}
352
353pub fn fail(lower: *Lower, comptime format: []const u8, args: anytype) Error {
354 @branchHint(.cold);
355 assert(lower.err_msg == null);
356 lower.err_msg = try .create(lower.allocator, lower.src_loc, format, args);
357 return error.LowerFail;
358}
359
360pub fn imm(lower: *const Lower, ops: Mir.Inst.Ops, i: u32) Immediate {
361 return switch (ops) {
362 .rri_s,
363 .ri_s,
364 .i_s,
365 .mi_s,
366 .rmi_s,
367 .pseudo_dbg_arg_i_s,
368 .pseudo_dbg_var_i_s,
369 => .s(@bitCast(i)),
370
371 .ii,
372 .ir,
373 .rrri,
374 .rri_u,
375 .ri_u,
376 .i_u,
377 .mi_u,
378 .rmi_u,
379 .rrm,
380 .rrmi,
381 .pseudo_dbg_arg_i_u,
382 .pseudo_dbg_var_i_u,
383 => .u(i),
384
385 .ri_64,
386 => .u(lower.mir.extraData(Mir.Imm64, i).data.decode()),
387
388 .pseudo_dbg_arg_i_64,
389 .pseudo_dbg_var_i_64,
390 => unreachable,
391
392 else => unreachable,
393 };
394}
395
396fn mem(lower: *Lower, op_index: InstOpIndex, payload: u32) Memory {
397 var m = lower.mir.resolveMemoryExtra(payload).decode();
398 switch (m) {
399 .sib => |*sib| switch (sib.base) {
400 .none, .reg, .frame => {},
401 .table => sib.disp = lower.reloc(op_index, .table, sib.disp).signed,
402 .rip_inst => |inst_index| sib.disp = lower.reloc(op_index, .{ .inst = inst_index }, sib.disp).signed,
403 .nav => |nav| sib.disp = lower.reloc(op_index, .{ .nav = nav }, sib.disp).signed,
404 .uav => |uav| sib.disp = lower.reloc(op_index, .{ .uav = uav }, sib.disp).signed,
405 .lazy_sym => |lazy_sym| sib.disp = lower.reloc(op_index, .{ .lazy_sym = lazy_sym }, sib.disp).signed,
406 .extern_func => |extern_func| sib.disp = lower.reloc(op_index, .{ .extern_func = extern_func }, sib.disp).signed,
407 },
408 else => {},
409 }
410 return m;
411}
412
413fn reloc(lower: *Lower, op_index: InstOpIndex, target: Reloc.Target, off: i32) Immediate {
414 lower.result_relocs[lower.result_relocs_len] = .{
415 .lowered_inst_index = lower.result_insts_len,
416 .op_index = op_index,
417 .target = target,
418 .off = off,
419 };
420 lower.result_relocs_len += 1;
421 return .s(0);
422}
423
424fn encode(lower: *Lower, prefix: Prefix, mnemonic: Mnemonic, ops: []const Operand) Error!void {
425 lower.result_insts[lower.result_insts_len] = try .new(prefix, mnemonic, ops, lower.target);
426 lower.result_insts_len += 1;
427}
428
429const inst_tags_len = @typeInfo(Mir.Inst.Tag).@"enum".field_names.len;
430const inst_fixes_len = @typeInfo(Mir.Inst.Fixes).@"enum".field_names.len;
431/// Lookup table, indexed by `@intFromEnum(inst.tag) * inst_fixes_len + @intFromEnum(fixes)`.
432/// The value is the resulting `Mnemonic`, or `null` if the combination is not valid.
433const mnemonic_table: [inst_tags_len * inst_fixes_len]?Mnemonic = table: {
434 @setEvalBranchQuota(80_000);
435 var table: [inst_tags_len * inst_fixes_len]?Mnemonic = undefined;
436 for (0..inst_fixes_len) |fixes_i| {
437 const fixes: Mir.Inst.Fixes = @fromBackingInt(@intCast(fixes_i));
438 const prefix, const suffix = affix: {
439 const pattern = if (std.mem.findScalar(u8, @tagName(fixes), ' ')) |i|
440 @tagName(fixes)[i + 1 ..]
441 else
442 @tagName(fixes);
443 const wildcard_idx = std.mem.findScalar(u8, pattern, '_').?;
444 break :affix .{ pattern[0..wildcard_idx], pattern[wildcard_idx + 1 ..] };
445 };
446 for (0..inst_tags_len) |inst_tag_i| {
447 const inst_tag: Mir.Inst.Tag = @fromBackingInt(@intCast(inst_tag_i));
448 const name = prefix ++ @tagName(inst_tag) ++ suffix;
449 const idx = inst_tag_i * inst_fixes_len + fixes_i;
450 table[idx] = if (@hasField(Mnemonic, name)) @field(Mnemonic, name) else null;
451 }
452 }
453 break :table table;
454};
455
456fn generic(lower: *Lower, inst: Mir.Inst) Error!void {
457 @setEvalBranchQuota(2_000);
458 const fixes = switch (inst.ops) {
459 .none => inst.data.none.fixes,
460 .inst => inst.data.inst.fixes,
461 .i_s, .i_u => inst.data.i.fixes,
462 .ii => inst.data.ii.fixes,
463 .r => inst.data.r.fixes,
464 .rr => inst.data.rr.fixes,
465 .rrr => inst.data.rrr.fixes,
466 .rrrr => inst.data.rrrr.fixes,
467 .rrri => inst.data.rrri.fixes,
468 .rri_s, .rri_u => inst.data.rri.fixes,
469 .ri_s, .ri_u, .ri_64, .ir => inst.data.ri.fixes,
470 .rm, .rmi_s, .rmi_u, .mr => inst.data.rx.fixes,
471 .mrr, .rrm, .rmr => inst.data.rrx.fixes,
472 .rmi, .mri => inst.data.rix.fixes,
473 .rrmr => inst.data.rrrx.fixes,
474 .rrmi => inst.data.rrix.fixes,
475 .mi_u, .mi_s => inst.data.x.fixes,
476 .m => inst.data.x.fixes,
477 .nav, .uav, .lazy_sym, .extern_func => ._,
478 else => return lower.fail("TODO lower .{s}", .{@tagName(inst.ops)}),
479 };
480 try lower.encode(switch (fixes) {
481 inline else => |tag| comptime if (std.mem.findScalar(u8, @tagName(tag), ' ')) |space|
482 @field(Prefix, @tagName(tag)[0..space])
483 else
484 .none,
485 }, mnemonic: {
486 if (mnemonic_table[@backingInt(inst.tag) * inst_fixes_len + @backingInt(fixes)]) |mnemonic| {
487 break :mnemonic mnemonic;
488 }
489 // This combination is invalid; make the theoretical mnemonic name and emit an error with it.
490 const fixes_name = @tagName(fixes);
491 const pattern = fixes_name[if (std.mem.findScalar(u8, fixes_name, ' ')) |i| i + " ".len else 0..];
492 const wildcard_index = std.mem.findScalar(u8, pattern, '_').?;
493 return lower.fail("unsupported mnemonic: '{s}{s}{s}'", .{
494 pattern[0..wildcard_index],
495 @tagName(inst.tag),
496 pattern[wildcard_index + "_".len ..],
497 });
498 }, switch (inst.ops) {
499 .none => &.{},
500 .inst => &.{
501 .{ .imm = lower.reloc(0, .{ .inst = inst.data.inst.inst }, 0) },
502 },
503 .i_s, .i_u => &.{
504 .{ .imm = lower.imm(inst.ops, inst.data.i.i) },
505 },
506 .ii => &.{
507 .{ .imm = lower.imm(inst.ops, inst.data.ii.i1) },
508 .{ .imm = lower.imm(inst.ops, inst.data.ii.i2) },
509 },
510 .ir => &.{
511 .{ .imm = lower.imm(inst.ops, inst.data.ri.i) },
512 .{ .reg = inst.data.ri.r1 },
513 },
514 .r => &.{
515 .{ .reg = inst.data.r.r1 },
516 },
517 .rr => &.{
518 .{ .reg = inst.data.rr.r1 },
519 .{ .reg = inst.data.rr.r2 },
520 },
521 .rrr => &.{
522 .{ .reg = inst.data.rrr.r1 },
523 .{ .reg = inst.data.rrr.r2 },
524 .{ .reg = inst.data.rrr.r3 },
525 },
526 .rrrr => &.{
527 .{ .reg = inst.data.rrrr.r1 },
528 .{ .reg = inst.data.rrrr.r2 },
529 .{ .reg = inst.data.rrrr.r3 },
530 .{ .reg = inst.data.rrrr.r4 },
531 },
532 .rrri => &.{
533 .{ .reg = inst.data.rrri.r1 },
534 .{ .reg = inst.data.rrri.r2 },
535 .{ .reg = inst.data.rrri.r3 },
536 .{ .imm = lower.imm(inst.ops, inst.data.rrri.i) },
537 },
538 .ri_s, .ri_u, .ri_64 => &.{
539 .{ .reg = inst.data.ri.r1 },
540 .{ .imm = lower.imm(inst.ops, inst.data.ri.i) },
541 },
542 .rri_s, .rri_u => &.{
543 .{ .reg = inst.data.rri.r1 },
544 .{ .reg = inst.data.rri.r2 },
545 .{ .imm = lower.imm(inst.ops, inst.data.rri.i) },
546 },
547 .m => &.{
548 .{ .mem = lower.mem(0, inst.data.x.payload) },
549 },
550 .mi_s, .mi_u => &.{
551 .{ .mem = lower.mem(0, inst.data.x.payload + 1) },
552 .{ .imm = lower.imm(
553 inst.ops,
554 lower.mir.extraData(Mir.Imm32, inst.data.x.payload).data.imm,
555 ) },
556 },
557 .rm => &.{
558 .{ .reg = inst.data.rx.r1 },
559 .{ .mem = lower.mem(1, inst.data.rx.payload) },
560 },
561 .rmr => &.{
562 .{ .reg = inst.data.rrx.r1 },
563 .{ .mem = lower.mem(1, inst.data.rrx.payload) },
564 .{ .reg = inst.data.rrx.r2 },
565 },
566 .rmi => &.{
567 .{ .reg = inst.data.rix.r1 },
568 .{ .mem = lower.mem(1, inst.data.rix.payload) },
569 .{ .imm = .s(inst.data.rix.i) },
570 },
571 .rmi_s, .rmi_u => &.{
572 .{ .reg = inst.data.rx.r1 },
573 .{ .mem = lower.mem(1, inst.data.rx.payload + 1) },
574 .{ .imm = lower.imm(
575 inst.ops,
576 lower.mir.extraData(Mir.Imm32, inst.data.rx.payload).data.imm,
577 ) },
578 },
579 .mr => &.{
580 .{ .mem = lower.mem(0, inst.data.rx.payload) },
581 .{ .reg = inst.data.rx.r1 },
582 },
583 .mrr => &.{
584 .{ .mem = lower.mem(0, inst.data.rrx.payload) },
585 .{ .reg = inst.data.rrx.r1 },
586 .{ .reg = inst.data.rrx.r2 },
587 },
588 .mri => &.{
589 .{ .mem = lower.mem(0, inst.data.rix.payload) },
590 .{ .reg = inst.data.rix.r1 },
591 .{ .imm = .u(@as(u8, @intCast(inst.data.rix.i))) },
592 },
593 .rrm => &.{
594 .{ .reg = inst.data.rrx.r1 },
595 .{ .reg = inst.data.rrx.r2 },
596 .{ .mem = lower.mem(2, inst.data.rrx.payload) },
597 },
598 .rrmr => &.{
599 .{ .reg = inst.data.rrrx.r1 },
600 .{ .reg = inst.data.rrrx.r2 },
601 .{ .mem = lower.mem(2, inst.data.rrrx.payload) },
602 .{ .reg = inst.data.rrrx.r3 },
603 },
604 .rrmi => &.{
605 .{ .reg = inst.data.rrix.r1 },
606 .{ .reg = inst.data.rrix.r2 },
607 .{ .mem = lower.mem(2, inst.data.rrix.payload) },
608 .{ .imm = lower.imm(inst.ops, inst.data.rrix.i) },
609 },
610 .nav => &.{
611 .{ .imm = lower.reloc(0, .{ .nav = inst.data.nav.index }, inst.data.nav.off) },
612 },
613 .uav => &.{
614 .{ .imm = lower.reloc(0, .{ .uav = inst.data.uav }, 0) },
615 },
616 .lazy_sym => &.{
617 .{ .imm = lower.reloc(0, .{ .lazy_sym = inst.data.lazy_sym }, 0) },
618 },
619 .extern_func => &.{
620 .{ .imm = lower.reloc(0, .{ .extern_func = inst.data.extern_func }, 0) },
621 },
622 else => return lower.fail("TODO lower {s} {s}", .{ @tagName(inst.tag), @tagName(inst.ops) }),
623 });
624}
625
626fn pushPopRegList(lower: *Lower, comptime mnemonic: Mnemonic, inst: Mir.Inst) Error!void {
627 const callee_preserved_regs = abi.getCalleePreservedRegs(lower.cc);
628 var off: i32 = switch (mnemonic) {
629 .push => 0,
630 .pop => undefined,
631 else => unreachable,
632 };
633 {
634 var it = inst.data.reg_list.iterator(.{ .direction = switch (mnemonic) {
635 .push => .reverse,
636 .pop => .forward,
637 else => unreachable,
638 } });
639 while (it.next()) |i| {
640 try lower.encode(.none, mnemonic, &.{.{
641 .reg = callee_preserved_regs[i],
642 }});
643 switch (mnemonic) {
644 .push => off -= 8,
645 .pop => {},
646 else => unreachable,
647 }
648 }
649 }
650 switch (mnemonic) {
651 .push => {
652 var it = inst.data.reg_list.iterator(.{});
653 while (it.next()) |i| {
654 try lower.encode(.directive, .@".cfi_rel_offset", &.{
655 .{ .reg = callee_preserved_regs[i] },
656 .{ .imm = .s(off) },
657 });
658 off += 8;
659 }
660 assert(off == 0);
661 },
662 .pop => {},
663 else => unreachable,
664 }
665}
666
667const page_size: i32 = 1 << 12;
668
669const abi = @import("abi.zig");
670const assert = std.debug.assert;
671const bits = @import("bits.zig");
672const codegen = @import("../../codegen.zig");
673const encoder = @import("encoder.zig");
674const link = @import("../../link.zig");
675const std = @import("std");
676
677const Immediate = Instruction.Immediate;
678const Instruction = encoder.Instruction;
679const InternPool = @import("../../InternPool.zig");
680const Lower = @This();
681const Memory = Instruction.Memory;
682const Mir = @import("Mir.zig");
683const Mnemonic = Instruction.Mnemonic;
684const Zcu = @import("../../Zcu.zig");
685const Operand = Instruction.Operand;
686const Prefix = Instruction.Prefix;
687const Register = bits.Register;
688const Type = @import("../../Type.zig");