authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-07-03 04:57:25+02:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2025-07-03 04:57:25+02:00
log31bc6d5a9ddaf09511d8e5dc6017957adec0564b
treef0de91fd97b736c2e069abaeec58a35cb9a0fc37
parentedf785db0f30842b958f540a3aaf7205b8b82493
parente126e5592d205c2e7e8e4946af22f033ee1db79a
signaturebadge-check Signed by PGP key B5690EEEBB952194

Merge pull request #24322 from ziglang/delete-dead-backends

delete abandoned backends

22 files changed, 498 insertions(+), 18029 deletions(-)

CMakeLists.txt-39
...@@ -549,45 +549,6 @@ set(ZIG_STAGE2_SOURCES...@@ -549,45 +549,6 @@ set(ZIG_STAGE2_SOURCES
549 src/Value.zig549 src/Value.zig
550 src/Zcu.zig550 src/Zcu.zig
551 src/Zcu/PerThread.zig551 src/Zcu/PerThread.zig
552 src/arch/aarch64/CodeGen.zig
553 src/arch/aarch64/Emit.zig
554 src/arch/aarch64/Mir.zig
555 src/arch/aarch64/abi.zig
556 src/arch/aarch64/bits.zig
557 src/arch/arm/CodeGen.zig
558 src/arch/arm/Emit.zig
559 src/arch/arm/Mir.zig
560 src/arch/arm/abi.zig
561 src/arch/arm/bits.zig
562 src/arch/powerpc/CodeGen.zig
563 src/arch/riscv64/abi.zig
564 src/arch/riscv64/bits.zig
565 src/arch/riscv64/CodeGen.zig
566 src/arch/riscv64/Emit.zig
567 src/arch/riscv64/encoding.zig
568 src/arch/riscv64/Lower.zig
569 src/arch/riscv64/Mir.zig
570 src/arch/riscv64/mnem.zig
571 src/arch/sparc64/CodeGen.zig
572 src/arch/sparc64/Emit.zig
573 src/arch/sparc64/Mir.zig
574 src/arch/sparc64/abi.zig
575 src/arch/sparc64/bits.zig
576 src/arch/wasm/CodeGen.zig
577 src/arch/wasm/Emit.zig
578 src/arch/wasm/Mir.zig
579 src/arch/wasm/abi.zig
580 src/arch/x86/bits.zig
581 src/arch/x86_64/CodeGen.zig
582 src/arch/x86_64/Disassembler.zig
583 src/arch/x86_64/Emit.zig
584 src/arch/x86_64/Encoding.zig
585 src/arch/x86_64/Lower.zig
586 src/arch/x86_64/Mir.zig
587 src/arch/x86_64/abi.zig
588 src/arch/x86_64/bits.zig
589 src/arch/x86_64/encoder.zig
590 src/arch/x86_64/encodings.zon
591 src/clang.zig552 src/clang.zig
592 src/clang_options.zig553 src/clang_options.zig
593 src/clang_options_data.zig554 src/clang_options_data.zig
src/arch/aarch64/CodeGen.zig deleted-6401
...@@ -1,6401 +0,0 @@
1const std = @import("std");
2const builtin = @import("builtin");
3const mem = std.mem;
4const math = std.math;
5const assert = std.debug.assert;
6const codegen = @import("../../codegen.zig");
7const Air = @import("../../Air.zig");
8const Mir = @import("Mir.zig");
9const Emit = @import("Emit.zig");
10const Type = @import("../../Type.zig");
11const Value = @import("../../Value.zig");
12const link = @import("../../link.zig");
13const Zcu = @import("../../Zcu.zig");
14const InternPool = @import("../../InternPool.zig");
15const Compilation = @import("../../Compilation.zig");
16const ErrorMsg = Zcu.ErrorMsg;
17const Target = std.Target;
18const Allocator = mem.Allocator;
19const trace = @import("../../tracy.zig").trace;
20const leb128 = std.leb;
21const log = std.log.scoped(.codegen);
22const build_options = @import("build_options");
23const Alignment = InternPool.Alignment;
24
25const CodeGenError = codegen.CodeGenError;
26
27const bits = @import("bits.zig");
28const abi = @import("abi.zig");
29const errUnionPayloadOffset = codegen.errUnionPayloadOffset;
30const errUnionErrorOffset = codegen.errUnionErrorOffset;
31const RegisterManager = abi.RegisterManager;
32const RegisterLock = RegisterManager.RegisterLock;
33const Register = bits.Register;
34const Instruction = bits.Instruction;
35const Condition = bits.Instruction.Condition;
36const callee_preserved_regs = abi.callee_preserved_regs;
37const c_abi_int_param_regs = abi.c_abi_int_param_regs;
38const c_abi_int_return_regs = abi.c_abi_int_return_regs;
39const gp = abi.RegisterClass.gp;
40
41const InnerError = CodeGenError || error{OutOfRegisters};
42
43pub fn legalizeFeatures(_: *const std.Target) ?*const Air.Legalize.Features {
44 return null;
45}
46
47gpa: Allocator,
48pt: Zcu.PerThread,
49air: Air,
50liveness: Air.Liveness,
51bin_file: *link.File,
52target: *const std.Target,
53func_index: InternPool.Index,
54owner_nav: InternPool.Nav.Index,
55args: []MCValue,
56ret_mcv: MCValue,
57fn_type: Type,
58arg_index: u32,
59src_loc: Zcu.LazySrcLoc,
60stack_align: u32,
61
62/// MIR Instructions
63mir_instructions: std.MultiArrayList(Mir.Inst) = .{},
64/// MIR extra data
65mir_extra: std.ArrayListUnmanaged(u32) = .empty,
66
67/// Byte offset within the source file of the ending curly.
68end_di_line: u32,
69end_di_column: u32,
70
71/// The value is an offset into the `Function` `code` from the beginning.
72/// To perform the reloc, write 32-bit signed little-endian integer
73/// which is a relative jump, based on the address following the reloc.
74exitlude_jump_relocs: std.ArrayListUnmanaged(usize) = .empty,
75
76reused_operands: std.StaticBitSet(Air.Liveness.bpi - 1) = undefined,
77
78/// We postpone the creation of debug info for function args and locals
79/// until after all Mir instructions have been generated. Only then we
80/// will know saved_regs_stack_space which is necessary in order to
81/// calculate the right stack offsest with respect to the `.fp` register.
82dbg_info_relocs: std.ArrayListUnmanaged(DbgInfoReloc) = .empty,
83
84/// Whenever there is a runtime branch, we push a Branch onto this stack,
85/// and pop it off when the runtime branch joins. This provides an "overlay"
86/// of the table of mappings from instructions to `MCValue` from within the branch.
87/// This way we can modify the `MCValue` for an instruction in different ways
88/// within different branches. Special consideration is needed when a branch
89/// joins with its parent, to make sure all instructions have the same MCValue
90/// across each runtime branch upon joining.
91branch_stack: *std.ArrayList(Branch),
92
93// Key is the block instruction
94blocks: std.AutoHashMapUnmanaged(Air.Inst.Index, BlockData) = .empty,
95
96register_manager: RegisterManager = .{},
97/// Maps offset to what is stored there.
98stack: std.AutoHashMapUnmanaged(u32, StackAllocation) = .empty,
99/// Tracks the current instruction allocated to the compare flags
100compare_flags_inst: ?Air.Inst.Index = null,
101
102/// Offset from the stack base, representing the end of the stack frame.
103max_end_stack: u32 = 0,
104/// Represents the current end stack offset. If there is no existing slot
105/// to place a new stack allocation, it goes here, and then bumps `max_end_stack`.
106next_stack_offset: u32 = 0,
107
108saved_regs_stack_space: u32 = 0,
109
110/// Debug field, used to find bugs in the compiler.
111air_bookkeeping: @TypeOf(air_bookkeeping_init) = air_bookkeeping_init,
112
113const air_bookkeeping_init = if (std.debug.runtime_safety) @as(usize, 0) else {};
114
115const MCValue = union(enum) {
116 /// No runtime bits. `void` types, empty structs, u0, enums with 1
117 /// tag, etc.
118 ///
119 /// TODO Look into deleting this tag and using `dead` instead,
120 /// since every use of MCValue.none should be instead looking at
121 /// the type and noticing it is 0 bits.
122 none,
123 /// Control flow will not allow this value to be observed.
124 unreach,
125 /// No more references to this value remain.
126 dead,
127 /// The value is undefined.
128 undef,
129 /// A pointer-sized integer that fits in a register.
130 ///
131 /// If the type is a pointer, this is the pointer address in
132 /// virtual address space.
133 immediate: u64,
134 /// The value is in a target-specific register.
135 register: Register,
136 /// The value is a tuple { wrapped: u32, overflow: u1 } where
137 /// wrapped is stored in the register and the overflow bit is
138 /// stored in the C (signed) or V (unsigned) flag of the CPSR.
139 ///
140 /// This MCValue is only generated by a add_with_overflow or
141 /// sub_with_overflow instruction operating on 32- or 64-bit values.
142 register_with_overflow: struct { reg: Register, flag: bits.Instruction.Condition },
143 /// The value is in memory at a hard-coded address.
144 ///
145 /// If the type is a pointer, it means the pointer address is at
146 /// this memory location.
147 memory: u64,
148 /// The value is in memory but requires a linker relocation fixup.
149 linker_load: codegen.LinkerLoad,
150 /// The value is one of the stack variables.
151 ///
152 /// If the type is a pointer, it means the pointer address is in
153 /// the stack at this offset.
154 stack_offset: u32,
155 /// The value is a pointer to one of the stack variables (payload
156 /// is stack offset).
157 ptr_stack_offset: u32,
158 /// The value resides in the N, Z, C, V flags. The value is 1 (if
159 /// the type is u1) or true (if the type in bool) iff the
160 /// specified condition is true.
161 compare_flags: Condition,
162 /// The value is a function argument passed via the stack.
163 stack_argument_offset: u32,
164};
165
166const DbgInfoReloc = struct {
167 tag: Air.Inst.Tag,
168 ty: Type,
169 name: [:0]const u8,
170 mcv: MCValue,
171
172 fn genDbgInfo(reloc: DbgInfoReloc, function: Self) !void {
173 switch (reloc.tag) {
174 .arg,
175 .dbg_arg_inline,
176 => try reloc.genArgDbgInfo(function),
177
178 .dbg_var_ptr,
179 .dbg_var_val,
180 => try reloc.genVarDbgInfo(function),
181
182 else => unreachable,
183 }
184 }
185
186 fn genArgDbgInfo(reloc: DbgInfoReloc, function: Self) !void {
187 // TODO: Add a pseudo-instruction or something to defer this work until Emit.
188 // We aren't allowed to interact with linker state here.
189 if (true) return;
190 switch (function.debug_output) {
191 .dwarf => |dw| {
192 const loc: link.File.Dwarf.Loc = switch (reloc.mcv) {
193 .register => |reg| .{ .reg = reg.dwarfNum() },
194 .stack_offset,
195 .stack_argument_offset,
196 => |offset| blk: {
197 const adjusted_offset = switch (reloc.mcv) {
198 .stack_offset => -@as(i32, @intCast(offset)),
199 .stack_argument_offset => @as(i32, @intCast(function.saved_regs_stack_space + offset)),
200 else => unreachable,
201 };
202 break :blk .{ .plus = .{
203 &.{ .breg = Register.x29.dwarfNum() },
204 &.{ .consts = adjusted_offset },
205 } };
206 },
207 else => unreachable, // not a possible argument
208
209 };
210 try dw.genLocalDebugInfo(.local_arg, reloc.name, reloc.ty, loc);
211 },
212 .plan9 => {},
213 .none => {},
214 }
215 }
216
217 fn genVarDbgInfo(reloc: DbgInfoReloc, function: Self) !void {
218 // TODO: Add a pseudo-instruction or something to defer this work until Emit.
219 // We aren't allowed to interact with linker state here.
220 if (true) return;
221 switch (function.debug_output) {
222 .dwarf => |dwarf| {
223 const loc: link.File.Dwarf.Loc = switch (reloc.mcv) {
224 .register => |reg| .{ .reg = reg.dwarfNum() },
225 .ptr_stack_offset,
226 .stack_offset,
227 .stack_argument_offset,
228 => |offset| blk: {
229 const adjusted_offset = switch (reloc.mcv) {
230 .ptr_stack_offset,
231 .stack_offset,
232 => -@as(i32, @intCast(offset)),
233 .stack_argument_offset => @as(i32, @intCast(function.saved_regs_stack_space + offset)),
234 else => unreachable,
235 };
236 break :blk .{ .plus = .{
237 &.{ .reg = Register.x29.dwarfNum() },
238 &.{ .consts = adjusted_offset },
239 } };
240 },
241 .memory => |address| .{ .constu = address },
242 .immediate => |x| .{ .constu = x },
243 .none => .empty,
244 else => blk: {
245 log.debug("TODO generate debug info for {}", .{reloc.mcv});
246 break :blk .empty;
247 },
248 };
249 try dwarf.genLocalDebugInfo(.local_var, reloc.name, reloc.ty, loc);
250 },
251 .plan9 => {},
252 .none => {},
253 }
254 }
255};
256
257const Branch = struct {
258 inst_table: std.AutoArrayHashMapUnmanaged(Air.Inst.Index, MCValue) = .empty,
259
260 fn deinit(self: *Branch, gpa: Allocator) void {
261 self.inst_table.deinit(gpa);
262 self.* = undefined;
263 }
264};
265
266const StackAllocation = struct {
267 inst: Air.Inst.Index,
268 /// TODO do we need size? should be determined by inst.ty.abiSize()
269 size: u32,
270};
271
272const BlockData = struct {
273 relocs: std.ArrayListUnmanaged(Mir.Inst.Index),
274 /// The first break instruction encounters `null` here and chooses a
275 /// machine code value for the block result, populating this field.
276 /// Following break instructions encounter that value and use it for
277 /// the location to store their block results.
278 mcv: MCValue,
279};
280
281const BigTomb = struct {
282 function: *Self,
283 inst: Air.Inst.Index,
284 lbt: Air.Liveness.BigTomb,
285
286 fn feed(bt: *BigTomb, op_ref: Air.Inst.Ref) void {
287 const dies = bt.lbt.feed();
288 const op_index = op_ref.toIndex() orelse return;
289 if (!dies) return;
290 bt.function.processDeath(op_index);
291 }
292
293 fn finishAir(bt: *BigTomb, result: MCValue) void {
294 const is_used = !bt.function.liveness.isUnused(bt.inst);
295 if (is_used) {
296 log.debug("%{d} => {}", .{ bt.inst, result });
297 const branch = &bt.function.branch_stack.items[bt.function.branch_stack.items.len - 1];
298 branch.inst_table.putAssumeCapacityNoClobber(bt.inst, result);
299
300 switch (result) {
301 .register => |reg| {
302 // In some cases (such as bitcast), an operand
303 // may be the same MCValue as the result. If
304 // that operand died and was a register, it
305 // was freed by processDeath. We have to
306 // "re-allocate" the register.
307 if (bt.function.register_manager.isRegFree(reg)) {
308 bt.function.register_manager.getRegAssumeFree(reg, bt.inst);
309 }
310 },
311 .register_with_overflow => |rwo| {
312 if (bt.function.register_manager.isRegFree(rwo.reg)) {
313 bt.function.register_manager.getRegAssumeFree(rwo.reg, bt.inst);
314 }
315 bt.function.compare_flags_inst = bt.inst;
316 },
317 .compare_flags => |_| {
318 bt.function.compare_flags_inst = bt.inst;
319 },
320 else => {},
321 }
322 }
323 bt.function.finishAirBookkeeping();
324 }
325};
326
327const Self = @This();
328
329pub fn generate(
330 lf: *link.File,
331 pt: Zcu.PerThread,
332 src_loc: Zcu.LazySrcLoc,
333 func_index: InternPool.Index,
334 air: *const Air,
335 liveness: *const Air.Liveness,
336) CodeGenError!Mir {
337 const zcu = pt.zcu;
338 const gpa = zcu.gpa;
339 const func = zcu.funcInfo(func_index);
340 const fn_type = Type.fromInterned(func.ty);
341 const file_scope = zcu.navFileScope(func.owner_nav);
342 const target = &file_scope.mod.?.resolved_target.result;
343
344 var branch_stack = std.ArrayList(Branch).init(gpa);
345 defer {
346 assert(branch_stack.items.len == 1);
347 branch_stack.items[0].deinit(gpa);
348 branch_stack.deinit();
349 }
350 try branch_stack.append(.{});
351
352 var function: Self = .{
353 .gpa = gpa,
354 .pt = pt,
355 .air = air.*,
356 .liveness = liveness.*,
357 .target = target,
358 .bin_file = lf,
359 .func_index = func_index,
360 .owner_nav = func.owner_nav,
361 .args = undefined, // populated after `resolveCallingConventionValues`
362 .ret_mcv = undefined, // populated after `resolveCallingConventionValues`
363 .fn_type = fn_type,
364 .arg_index = 0,
365 .branch_stack = &branch_stack,
366 .src_loc = src_loc,
367 .stack_align = undefined,
368 .end_di_line = func.rbrace_line,
369 .end_di_column = func.rbrace_column,
370 };
371 defer function.stack.deinit(gpa);
372 defer function.blocks.deinit(gpa);
373 defer function.exitlude_jump_relocs.deinit(gpa);
374 defer function.dbg_info_relocs.deinit(gpa);
375
376 var call_info = function.resolveCallingConventionValues(fn_type) catch |err| switch (err) {
377 error.CodegenFail => return error.CodegenFail,
378 else => |e| return e,
379 };
380 defer call_info.deinit(&function);
381
382 function.args = call_info.args;
383 function.ret_mcv = call_info.return_value;
384 function.stack_align = call_info.stack_align;
385 function.max_end_stack = call_info.stack_byte_count;
386
387 function.gen() catch |err| switch (err) {
388 error.CodegenFail => return error.CodegenFail,
389 error.OutOfRegisters => return function.fail("ran out of registers (Zig compiler bug)", .{}),
390 else => |e| return e,
391 };
392
393 for (function.dbg_info_relocs.items) |reloc| {
394 reloc.genDbgInfo(function) catch |err|
395 return function.fail("failed to generate debug info: {s}", .{@errorName(err)});
396 }
397
398 var mir: Mir = .{
399 .instructions = function.mir_instructions.toOwnedSlice(),
400 .extra = &.{}, // fallible, so assign after errdefer
401 .max_end_stack = function.max_end_stack,
402 .saved_regs_stack_space = function.saved_regs_stack_space,
403 };
404 errdefer mir.deinit(gpa);
405 mir.extra = try function.mir_extra.toOwnedSlice(gpa);
406 return mir;
407}
408
409fn addInst(self: *Self, inst: Mir.Inst) error{OutOfMemory}!Mir.Inst.Index {
410 const gpa = self.gpa;
411
412 try self.mir_instructions.ensureUnusedCapacity(gpa, 1);
413
414 const result_index: Mir.Inst.Index = @intCast(self.mir_instructions.len);
415 self.mir_instructions.appendAssumeCapacity(inst);
416 return result_index;
417}
418
419fn addNop(self: *Self) error{OutOfMemory}!Mir.Inst.Index {
420 return try self.addInst(.{
421 .tag = .nop,
422 .data = .{ .nop = {} },
423 });
424}
425
426pub fn addExtra(self: *Self, extra: anytype) Allocator.Error!u32 {
427 const fields = std.meta.fields(@TypeOf(extra));
428 try self.mir_extra.ensureUnusedCapacity(self.gpa, fields.len);
429 return self.addExtraAssumeCapacity(extra);
430}
431
432pub fn addExtraAssumeCapacity(self: *Self, extra: anytype) u32 {
433 const fields = std.meta.fields(@TypeOf(extra));
434 const result = @as(u32, @intCast(self.mir_extra.items.len));
435 inline for (fields) |field| {
436 self.mir_extra.appendAssumeCapacity(switch (field.type) {
437 u32 => @field(extra, field.name),
438 i32 => @as(u32, @bitCast(@field(extra, field.name))),
439 else => @compileError("bad field type"),
440 });
441 }
442 return result;
443}
444
445fn gen(self: *Self) !void {
446 const pt = self.pt;
447 const zcu = pt.zcu;
448 const cc = self.fn_type.fnCallingConvention(zcu);
449 if (cc != .naked) {
450 // stp fp, lr, [sp, #-16]!
451 _ = try self.addInst(.{
452 .tag = .stp,
453 .data = .{ .load_store_register_pair = .{
454 .rt = .x29,
455 .rt2 = .x30,
456 .rn = .sp,
457 .offset = Instruction.LoadStorePairOffset.pre_index(-16),
458 } },
459 });
460
461 // <store other registers>
462 const backpatch_save_registers = try self.addNop();
463
464 // mov fp, sp
465 _ = try self.addInst(.{
466 .tag = .mov_to_from_sp,
467 .data = .{ .rr = .{ .rd = .x29, .rn = .sp } },
468 });
469
470 // sub sp, sp, #reloc
471 const backpatch_reloc = try self.addNop();
472
473 if (self.ret_mcv == .stack_offset) {
474 // The address of where to store the return value is in x0
475 // (or w0 when pointer size is 32 bits). As this register
476 // might get overwritten along the way, save the address
477 // to the stack.
478 const ret_ptr_reg = self.registerAlias(.x0, Type.usize);
479
480 const stack_offset = try self.allocMem(8, .@"8", null);
481
482 try self.genSetStack(Type.usize, stack_offset, MCValue{ .register = ret_ptr_reg });
483 self.ret_mcv = MCValue{ .stack_offset = stack_offset };
484 }
485
486 for (self.args, 0..) |*arg, arg_index| {
487 // Copy register arguments to the stack
488 switch (arg.*) {
489 .register => |reg| {
490 // The first AIR instructions of the main body are guaranteed
491 // to be the functions arguments
492 const inst = self.air.getMainBody()[arg_index];
493 assert(self.air.instructions.items(.tag)[@intFromEnum(inst)] == .arg);
494
495 const ty = self.typeOfIndex(inst);
496
497 const abi_size = @as(u32, @intCast(ty.abiSize(zcu)));
498 const abi_align = ty.abiAlignment(zcu);
499 const stack_offset = try self.allocMem(abi_size, abi_align, inst);
500 try self.genSetStack(ty, stack_offset, MCValue{ .register = reg });
501
502 arg.* = MCValue{ .stack_offset = stack_offset };
503 },
504 else => {},
505 }
506 }
507
508 _ = try self.addInst(.{
509 .tag = .dbg_prologue_end,
510 .data = .{ .nop = {} },
511 });
512
513 try self.genBody(self.air.getMainBody());
514
515 // Backpatch push callee saved regs
516 var saved_regs: u32 = 0;
517 self.saved_regs_stack_space = 16;
518 inline for (callee_preserved_regs) |reg| {
519 if (self.register_manager.isRegAllocated(reg)) {
520 saved_regs |= @as(u32, 1) << @as(u5, @intCast(reg.id()));
521 self.saved_regs_stack_space += 8;
522 }
523 }
524
525 // Emit.mirPopPushRegs automatically adds extra empty space so
526 // that sp is always aligned to 16
527 if (!std.mem.isAlignedGeneric(u32, self.saved_regs_stack_space, 16)) {
528 self.saved_regs_stack_space += 8;
529 }
530 assert(std.mem.isAlignedGeneric(u32, self.saved_regs_stack_space, 16));
531
532 self.mir_instructions.set(backpatch_save_registers, .{
533 .tag = .push_regs,
534 .data = .{ .reg_list = saved_regs },
535 });
536
537 // Backpatch stack offset
538 const total_stack_size = self.max_end_stack + self.saved_regs_stack_space;
539 const aligned_total_stack_end = mem.alignForward(u32, total_stack_size, self.stack_align);
540 const stack_size = aligned_total_stack_end - self.saved_regs_stack_space;
541 self.max_end_stack = stack_size;
542 if (math.cast(u12, stack_size)) |size| {
543 self.mir_instructions.set(backpatch_reloc, .{
544 .tag = .sub_immediate,
545 .data = .{ .rr_imm12_sh = .{ .rd = .sp, .rn = .sp, .imm12 = size } },
546 });
547 } else {
548 @panic("TODO AArch64: allow larger stacks");
549 }
550
551 _ = try self.addInst(.{
552 .tag = .dbg_epilogue_begin,
553 .data = .{ .nop = {} },
554 });
555
556 // exitlude jumps
557 if (self.exitlude_jump_relocs.items.len > 0 and
558 self.exitlude_jump_relocs.items[self.exitlude_jump_relocs.items.len - 1] == self.mir_instructions.len - 2)
559 {
560 // If the last Mir instruction (apart from the
561 // dbg_epilogue_begin) is the last exitlude jump
562 // relocation (which would just jump one instruction
563 // further), it can be safely removed
564 self.mir_instructions.orderedRemove(self.exitlude_jump_relocs.pop().?);
565 }
566
567 for (self.exitlude_jump_relocs.items) |jmp_reloc| {
568 self.mir_instructions.set(jmp_reloc, .{
569 .tag = .b,
570 .data = .{ .inst = @as(u32, @intCast(self.mir_instructions.len)) },
571 });
572 }
573
574 // add sp, sp, #stack_size
575 _ = try self.addInst(.{
576 .tag = .add_immediate,
577 .data = .{ .rr_imm12_sh = .{ .rd = .sp, .rn = .sp, .imm12 = @as(u12, @intCast(stack_size)) } },
578 });
579
580 // <load other registers>
581 _ = try self.addInst(.{
582 .tag = .pop_regs,
583 .data = .{ .reg_list = saved_regs },
584 });
585
586 // ldp fp, lr, [sp], #16
587 _ = try self.addInst(.{
588 .tag = .ldp,
589 .data = .{ .load_store_register_pair = .{
590 .rt = .x29,
591 .rt2 = .x30,
592 .rn = .sp,
593 .offset = Instruction.LoadStorePairOffset.post_index(16),
594 } },
595 });
596
597 // ret lr
598 _ = try self.addInst(.{
599 .tag = .ret,
600 .data = .{ .reg = .x30 },
601 });
602 } else {
603 _ = try self.addInst(.{
604 .tag = .dbg_prologue_end,
605 .data = .{ .nop = {} },
606 });
607
608 try self.genBody(self.air.getMainBody());
609
610 _ = try self.addInst(.{
611 .tag = .dbg_epilogue_begin,
612 .data = .{ .nop = {} },
613 });
614 }
615
616 // Drop them off at the rbrace.
617 _ = try self.addInst(.{
618 .tag = .dbg_line,
619 .data = .{ .dbg_line_column = .{
620 .line = self.end_di_line,
621 .column = self.end_di_column,
622 } },
623 });
624}
625
626fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
627 const pt = self.pt;
628 const zcu = pt.zcu;
629 const ip = &zcu.intern_pool;
630 const air_tags = self.air.instructions.items(.tag);
631
632 for (body) |inst| {
633 // TODO: remove now-redundant isUnused calls from AIR handler functions
634 if (self.liveness.isUnused(inst) and !self.air.mustLower(inst, ip))
635 continue;
636
637 const old_air_bookkeeping = self.air_bookkeeping;
638 try self.ensureProcessDeathCapacity(Air.Liveness.bpi);
639
640 self.reused_operands = @TypeOf(self.reused_operands).initEmpty();
641 switch (air_tags[@intFromEnum(inst)]) {
642 // zig fmt: off
643 .add => try self.airBinOp(inst, .add),
644 .add_wrap => try self.airBinOp(inst, .add_wrap),
645 .sub => try self.airBinOp(inst, .sub),
646 .sub_wrap => try self.airBinOp(inst, .sub_wrap),
647 .mul => try self.airBinOp(inst, .mul),
648 .mul_wrap => try self.airBinOp(inst, .mul_wrap),
649 .shl => try self.airBinOp(inst, .shl),
650 .shl_exact => try self.airBinOp(inst, .shl_exact),
651 .bool_and => try self.airBinOp(inst, .bool_and),
652 .bool_or => try self.airBinOp(inst, .bool_or),
653 .bit_and => try self.airBinOp(inst, .bit_and),
654 .bit_or => try self.airBinOp(inst, .bit_or),
655 .xor => try self.airBinOp(inst, .xor),
656 .shr => try self.airBinOp(inst, .shr),
657 .shr_exact => try self.airBinOp(inst, .shr_exact),
658 .div_float => try self.airBinOp(inst, .div_float),
659 .div_trunc => try self.airBinOp(inst, .div_trunc),
660 .div_floor => try self.airBinOp(inst, .div_floor),
661 .div_exact => try self.airBinOp(inst, .div_exact),
662 .rem => try self.airBinOp(inst, .rem),
663 .mod => try self.airBinOp(inst, .mod),
664
665 .ptr_add => try self.airPtrArithmetic(inst, .ptr_add),
666 .ptr_sub => try self.airPtrArithmetic(inst, .ptr_sub),
667
668 .min => try self.airMinMax(inst),
669 .max => try self.airMinMax(inst),
670
671 .add_sat => try self.airAddSat(inst),
672 .sub_sat => try self.airSubSat(inst),
673 .mul_sat => try self.airMulSat(inst),
674 .shl_sat => try self.airShlSat(inst),
675 .slice => try self.airSlice(inst),
676
677 .sqrt,
678 .sin,
679 .cos,
680 .tan,
681 .exp,
682 .exp2,
683 .log,
684 .log2,
685 .log10,
686 .floor,
687 .ceil,
688 .round,
689 .trunc_float,
690 .neg,
691 => try self.airUnaryMath(inst),
692
693 .add_with_overflow => try self.airOverflow(inst),
694 .sub_with_overflow => try self.airOverflow(inst),
695 .mul_with_overflow => try self.airMulWithOverflow(inst),
696 .shl_with_overflow => try self.airShlWithOverflow(inst),
697
698 .cmp_lt => try self.airCmp(inst, .lt),
699 .cmp_lte => try self.airCmp(inst, .lte),
700 .cmp_eq => try self.airCmp(inst, .eq),
701 .cmp_gte => try self.airCmp(inst, .gte),
702 .cmp_gt => try self.airCmp(inst, .gt),
703 .cmp_neq => try self.airCmp(inst, .neq),
704
705 .cmp_vector => try self.airCmpVector(inst),
706 .cmp_lt_errors_len => try self.airCmpLtErrorsLen(inst),
707
708 .alloc => try self.airAlloc(inst),
709 .ret_ptr => try self.airRetPtr(inst),
710 .arg => try self.airArg(inst),
711 .assembly => try self.airAsm(inst),
712 .bitcast => try self.airBitCast(inst),
713 .block => try self.airBlock(inst),
714 .br => try self.airBr(inst),
715 .repeat => return self.fail("TODO implement `repeat`", .{}),
716 .switch_dispatch => return self.fail("TODO implement `switch_dispatch`", .{}),
717 .trap => try self.airTrap(),
718 .breakpoint => try self.airBreakpoint(),
719 .ret_addr => try self.airRetAddr(inst),
720 .frame_addr => try self.airFrameAddress(inst),
721 .cond_br => try self.airCondBr(inst),
722 .fptrunc => try self.airFptrunc(inst),
723 .fpext => try self.airFpext(inst),
724 .intcast => try self.airIntCast(inst),
725 .trunc => try self.airTrunc(inst),
726 .is_non_null => try self.airIsNonNull(inst),
727 .is_non_null_ptr => try self.airIsNonNullPtr(inst),
728 .is_null => try self.airIsNull(inst),
729 .is_null_ptr => try self.airIsNullPtr(inst),
730 .is_non_err => try self.airIsNonErr(inst),
731 .is_non_err_ptr => try self.airIsNonErrPtr(inst),
732 .is_err => try self.airIsErr(inst),
733 .is_err_ptr => try self.airIsErrPtr(inst),
734 .load => try self.airLoad(inst),
735 .loop => try self.airLoop(inst),
736 .not => try self.airNot(inst),
737 .ret => try self.airRet(inst),
738 .ret_safe => try self.airRet(inst), // TODO
739 .ret_load => try self.airRetLoad(inst),
740 .store => try self.airStore(inst, false),
741 .store_safe => try self.airStore(inst, true),
742 .struct_field_ptr=> try self.airStructFieldPtr(inst),
743 .struct_field_val=> try self.airStructFieldVal(inst),
744 .array_to_slice => try self.airArrayToSlice(inst),
745 .float_from_int => try self.airFloatFromInt(inst),
746 .int_from_float => try self.airIntFromFloat(inst),
747 .cmpxchg_strong => try self.airCmpxchg(inst),
748 .cmpxchg_weak => try self.airCmpxchg(inst),
749 .atomic_rmw => try self.airAtomicRmw(inst),
750 .atomic_load => try self.airAtomicLoad(inst),
751 .memcpy => try self.airMemcpy(inst),
752 .memmove => try self.airMemmove(inst),
753 .memset => try self.airMemset(inst, false),
754 .memset_safe => try self.airMemset(inst, true),
755 .set_union_tag => try self.airSetUnionTag(inst),
756 .get_union_tag => try self.airGetUnionTag(inst),
757 .clz => try self.airClz(inst),
758 .ctz => try self.airCtz(inst),
759 .popcount => try self.airPopcount(inst),
760 .abs => try self.airAbs(inst),
761 .byte_swap => try self.airByteSwap(inst),
762 .bit_reverse => try self.airBitReverse(inst),
763 .tag_name => try self.airTagName(inst),
764 .error_name => try self.airErrorName(inst),
765 .splat => try self.airSplat(inst),
766 .select => try self.airSelect(inst),
767 .shuffle_one => try self.airShuffleOne(inst),
768 .shuffle_two => try self.airShuffleTwo(inst),
769 .reduce => try self.airReduce(inst),
770 .aggregate_init => try self.airAggregateInit(inst),
771 .union_init => try self.airUnionInit(inst),
772 .prefetch => try self.airPrefetch(inst),
773 .mul_add => try self.airMulAdd(inst),
774 .addrspace_cast => return self.fail("TODO implement addrspace_cast", .{}),
775
776 .@"try" => try self.airTry(inst),
777 .try_cold => try self.airTry(inst),
778 .try_ptr => try self.airTryPtr(inst),
779 .try_ptr_cold => try self.airTryPtr(inst),
780
781 .dbg_stmt => try self.airDbgStmt(inst),
782 .dbg_empty_stmt => self.finishAirBookkeeping(),
783 .dbg_inline_block => try self.airDbgInlineBlock(inst),
784 .dbg_var_ptr,
785 .dbg_var_val,
786 .dbg_arg_inline,
787 => try self.airDbgVar(inst),
788
789 .call => try self.airCall(inst, .auto),
790 .call_always_tail => try self.airCall(inst, .always_tail),
791 .call_never_tail => try self.airCall(inst, .never_tail),
792 .call_never_inline => try self.airCall(inst, .never_inline),
793
794 .atomic_store_unordered => try self.airAtomicStore(inst, .unordered),
795 .atomic_store_monotonic => try self.airAtomicStore(inst, .monotonic),
796 .atomic_store_release => try self.airAtomicStore(inst, .release),
797 .atomic_store_seq_cst => try self.airAtomicStore(inst, .seq_cst),
798
799 .struct_field_ptr_index_0 => try self.airStructFieldPtrIndex(inst, 0),
800 .struct_field_ptr_index_1 => try self.airStructFieldPtrIndex(inst, 1),
801 .struct_field_ptr_index_2 => try self.airStructFieldPtrIndex(inst, 2),
802 .struct_field_ptr_index_3 => try self.airStructFieldPtrIndex(inst, 3),
803
804 .field_parent_ptr => try self.airFieldParentPtr(inst),
805
806 .switch_br => try self.airSwitch(inst),
807 .loop_switch_br => return self.fail("TODO implement `loop_switch_br`", .{}),
808 .slice_ptr => try self.airSlicePtr(inst),
809 .slice_len => try self.airSliceLen(inst),
810
811 .ptr_slice_len_ptr => try self.airPtrSliceLenPtr(inst),
812 .ptr_slice_ptr_ptr => try self.airPtrSlicePtrPtr(inst),
813
814 .array_elem_val => try self.airArrayElemVal(inst),
815 .slice_elem_val => try self.airSliceElemVal(inst),
816 .slice_elem_ptr => try self.airSliceElemPtr(inst),
817 .ptr_elem_val => try self.airPtrElemVal(inst),
818 .ptr_elem_ptr => try self.airPtrElemPtr(inst),
819
820 .inferred_alloc, .inferred_alloc_comptime => unreachable,
821 .unreach => self.finishAirBookkeeping(),
822
823 .optional_payload => try self.airOptionalPayload(inst),
824 .optional_payload_ptr => try self.airOptionalPayloadPtr(inst),
825 .optional_payload_ptr_set => try self.airOptionalPayloadPtrSet(inst),
826 .unwrap_errunion_err => try self.airUnwrapErrErr(inst),
827 .unwrap_errunion_payload => try self.airUnwrapErrPayload(inst),
828 .unwrap_errunion_err_ptr => try self.airUnwrapErrErrPtr(inst),
829 .unwrap_errunion_payload_ptr=> try self.airUnwrapErrPayloadPtr(inst),
830 .errunion_payload_ptr_set => try self.airErrUnionPayloadPtrSet(inst),
831 .err_return_trace => try self.airErrReturnTrace(inst),
832 .set_err_return_trace => try self.airSetErrReturnTrace(inst),
833 .save_err_return_trace_index=> try self.airSaveErrReturnTraceIndex(inst),
834
835 .wrap_optional => try self.airWrapOptional(inst),
836 .wrap_errunion_payload => try self.airWrapErrUnionPayload(inst),
837 .wrap_errunion_err => try self.airWrapErrUnionErr(inst),
838
839 .add_optimized,
840 .sub_optimized,
841 .mul_optimized,
842 .div_float_optimized,
843 .div_trunc_optimized,
844 .div_floor_optimized,
845 .div_exact_optimized,
846 .rem_optimized,
847 .mod_optimized,
848 .neg_optimized,
849 .cmp_lt_optimized,
850 .cmp_lte_optimized,
851 .cmp_eq_optimized,
852 .cmp_gte_optimized,
853 .cmp_gt_optimized,
854 .cmp_neq_optimized,
855 .cmp_vector_optimized,
856 .reduce_optimized,
857 .int_from_float_optimized,
858 => return self.fail("TODO implement optimized float mode", .{}),
859
860 .add_safe,
861 .sub_safe,
862 .mul_safe,
863 .intcast_safe,
864 .int_from_float_safe,
865 .int_from_float_optimized_safe,
866 => return self.fail("TODO implement safety_checked_instructions", .{}),
867
868 .is_named_enum_value => return self.fail("TODO implement is_named_enum_value", .{}),
869 .error_set_has_value => return self.fail("TODO implement error_set_has_value", .{}),
870 .vector_store_elem => return self.fail("TODO implement vector_store_elem", .{}),
871 .runtime_nav_ptr => return self.fail("TODO implement runtime_nav_ptr", .{}),
872
873 .c_va_arg => return self.fail("TODO implement c_va_arg", .{}),
874 .c_va_copy => return self.fail("TODO implement c_va_copy", .{}),
875 .c_va_end => return self.fail("TODO implement c_va_end", .{}),
876 .c_va_start => return self.fail("TODO implement c_va_start", .{}),
877
878 .wasm_memory_size => unreachable,
879 .wasm_memory_grow => unreachable,
880
881 .work_item_id => unreachable,
882 .work_group_size => unreachable,
883 .work_group_id => unreachable,
884 // zig fmt: on
885 }
886
887 assert(!self.register_manager.lockedRegsExist());
888
889 if (std.debug.runtime_safety) {
890 if (self.air_bookkeeping < old_air_bookkeeping + 1) {
891 std.debug.panic("in codegen.zig, handling of AIR instruction %{d} ('{}') did not do proper bookkeeping. Look for a missing call to finishAir.", .{ inst, air_tags[@intFromEnum(inst)] });
892 }
893 }
894 }
895}
896
897/// Asserts there is already capacity to insert into top branch inst_table.
898fn processDeath(self: *Self, inst: Air.Inst.Index) void {
899 // When editing this function, note that the logic must synchronize with `reuseOperand`.
900 const prev_value = self.getResolvedInstValue(inst);
901 const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
902 branch.inst_table.putAssumeCapacity(inst, .dead);
903 switch (prev_value) {
904 .register => |reg| {
905 self.register_manager.freeReg(reg);
906 },
907 .register_with_overflow => |rwo| {
908 self.register_manager.freeReg(rwo.reg);
909 self.compare_flags_inst = null;
910 },
911 .compare_flags => {
912 self.compare_flags_inst = null;
913 },
914 else => {}, // TODO process stack allocation death
915 }
916}
917
918/// Called when there are no operands, and the instruction is always unreferenced.
919fn finishAirBookkeeping(self: *Self) void {
920 if (std.debug.runtime_safety) {
921 self.air_bookkeeping += 1;
922 }
923}
924
925fn finishAir(self: *Self, inst: Air.Inst.Index, result: MCValue, operands: [Air.Liveness.bpi - 1]Air.Inst.Ref) void {
926 const tomb_bits = self.liveness.getTombBits(inst);
927 for (0.., operands) |op_index, op| {
928 if (tomb_bits & @as(Air.Liveness.Bpi, 1) << @intCast(op_index) == 0) continue;
929 if (self.reused_operands.isSet(op_index)) continue;
930 self.processDeath(op.toIndexAllowNone() orelse continue);
931 }
932 if (tomb_bits & 1 << (Air.Liveness.bpi - 1) == 0) {
933 log.debug("%{d} => {}", .{ inst, result });
934 const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
935 branch.inst_table.putAssumeCapacityNoClobber(inst, result);
936
937 switch (result) {
938 .register => |reg| {
939 // In some cases (such as bitcast), an operand
940 // may be the same MCValue as the result. If
941 // that operand died and was a register, it
942 // was freed by processDeath. We have to
943 // "re-allocate" the register.
944 if (self.register_manager.isRegFree(reg)) {
945 self.register_manager.getRegAssumeFree(reg, inst);
946 }
947 },
948 .register_with_overflow => |rwo| {
949 if (self.register_manager.isRegFree(rwo.reg)) {
950 self.register_manager.getRegAssumeFree(rwo.reg, inst);
951 }
952 self.compare_flags_inst = inst;
953 },
954 .compare_flags => |_| {
955 self.compare_flags_inst = inst;
956 },
957 else => {},
958 }
959 }
960 self.finishAirBookkeeping();
961}
962
963fn ensureProcessDeathCapacity(self: *Self, additional_count: usize) !void {
964 const table = &self.branch_stack.items[self.branch_stack.items.len - 1].inst_table;
965 try table.ensureUnusedCapacity(self.gpa, additional_count);
966}
967
968fn allocMem(
969 self: *Self,
970 abi_size: u32,
971 abi_align: Alignment,
972 maybe_inst: ?Air.Inst.Index,
973) !u32 {
974 assert(abi_size > 0);
975 assert(abi_align != .none);
976
977 // In order to efficiently load and store stack items that fit
978 // into registers, we bump up the alignment to the next power of
979 // two.
980 const adjusted_align = if (abi_size > 8)
981 abi_align
982 else
983 Alignment.fromNonzeroByteUnits(std.math.ceilPowerOfTwoAssert(u64, abi_size));
984
985 // TODO find a free slot instead of always appending
986 const offset: u32 = @intCast(adjusted_align.forward(self.next_stack_offset) + abi_size);
987 self.next_stack_offset = offset;
988 self.max_end_stack = @max(self.max_end_stack, self.next_stack_offset);
989
990 if (maybe_inst) |inst| {
991 try self.stack.putNoClobber(self.gpa, offset, .{
992 .inst = inst,
993 .size = abi_size,
994 });
995 }
996
997 return offset;
998}
999
1000/// Use a pointer instruction as the basis for allocating stack memory.
1001fn allocMemPtr(self: *Self, inst: Air.Inst.Index) !u32 {
1002 const pt = self.pt;
1003 const zcu = pt.zcu;
1004 const elem_ty = self.typeOfIndex(inst).childType(zcu);
1005
1006 if (!elem_ty.hasRuntimeBits(zcu)) {
1007 // return the stack offset 0. Stack offset 0 will be where all
1008 // zero-sized stack allocations live as non-zero-sized
1009 // allocations will always have an offset > 0.
1010 return @as(u32, 0);
1011 }
1012
1013 const abi_size = math.cast(u32, elem_ty.abiSize(zcu)) orelse {
1014 return self.fail("type '{}' too big to fit into stack frame", .{elem_ty.fmt(pt)});
1015 };
1016 // TODO swap this for inst.ty.ptrAlign
1017 const abi_align = elem_ty.abiAlignment(zcu);
1018
1019 return self.allocMem(abi_size, abi_align, inst);
1020}
1021
1022fn allocRegOrMem(self: *Self, elem_ty: Type, reg_ok: bool, maybe_inst: ?Air.Inst.Index) !MCValue {
1023 const pt = self.pt;
1024 const abi_size = math.cast(u32, elem_ty.abiSize(pt.zcu)) orelse {
1025 return self.fail("type '{}' too big to fit into stack frame", .{elem_ty.fmt(pt)});
1026 };
1027 const abi_align = elem_ty.abiAlignment(pt.zcu);
1028
1029 if (reg_ok) {
1030 // Make sure the type can fit in a register before we try to allocate one.
1031 if (abi_size <= 8) {
1032 if (self.register_manager.tryAllocReg(maybe_inst, gp)) |reg| {
1033 return MCValue{ .register = self.registerAlias(reg, elem_ty) };
1034 }
1035 }
1036 }
1037
1038 const stack_offset = try self.allocMem(abi_size, abi_align, maybe_inst);
1039 return MCValue{ .stack_offset = stack_offset };
1040}
1041
1042pub fn spillInstruction(self: *Self, reg: Register, inst: Air.Inst.Index) !void {
1043 const stack_mcv = try self.allocRegOrMem(self.typeOfIndex(inst), false, inst);
1044 log.debug("spilling {d} to stack mcv {any}", .{ inst, stack_mcv });
1045
1046 const reg_mcv = self.getResolvedInstValue(inst);
1047 switch (reg_mcv) {
1048 .register => |r| assert(reg.id() == r.id()),
1049 .register_with_overflow => |rwo| assert(rwo.reg.id() == reg.id()),
1050 else => unreachable, // not a register
1051 }
1052
1053 const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
1054 try branch.inst_table.put(self.gpa, inst, stack_mcv);
1055 try self.genSetStack(self.typeOfIndex(inst), stack_mcv.stack_offset, reg_mcv);
1056}
1057
1058/// Save the current instruction stored in the compare flags if
1059/// occupied
1060fn spillCompareFlagsIfOccupied(self: *Self) !void {
1061 if (self.compare_flags_inst) |inst_to_save| {
1062 const ty = self.typeOfIndex(inst_to_save);
1063 const mcv = self.getResolvedInstValue(inst_to_save);
1064 const new_mcv = switch (mcv) {
1065 .compare_flags => try self.allocRegOrMem(ty, true, inst_to_save),
1066 .register_with_overflow => try self.allocRegOrMem(ty, false, inst_to_save),
1067 else => unreachable, // mcv doesn't occupy the compare flags
1068 };
1069
1070 try self.setRegOrMem(self.typeOfIndex(inst_to_save), new_mcv, mcv);
1071 log.debug("spilling {d} to mcv {any}", .{ inst_to_save, new_mcv });
1072
1073 const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
1074 try branch.inst_table.put(self.gpa, inst_to_save, new_mcv);
1075
1076 self.compare_flags_inst = null;
1077
1078 // TODO consolidate with register manager and spillInstruction
1079 // this call should really belong in the register manager!
1080 switch (mcv) {
1081 .register_with_overflow => |rwo| self.register_manager.freeReg(rwo.reg),
1082 else => {},
1083 }
1084 }
1085}
1086
1087/// Copies a value to a register without tracking the register. The register is not considered
1088/// allocated. A second call to `copyToTmpRegister` may return the same register.
1089/// This can have a side effect of spilling instructions to the stack to free up a register.
1090fn copyToTmpRegister(self: *Self, ty: Type, mcv: MCValue) InnerError!Register {
1091 const raw_reg = try self.register_manager.allocReg(null, gp);
1092 const reg = self.registerAlias(raw_reg, ty);
1093 try self.genSetReg(ty, reg, mcv);
1094 return reg;
1095}
1096
1097/// Allocates a new register and copies `mcv` into it.
1098/// `reg_owner` is the instruction that gets associated with the register in the register table.
1099/// This can have a side effect of spilling instructions to the stack to free up a register.
1100fn copyToNewRegister(self: *Self, reg_owner: Air.Inst.Index, mcv: MCValue) !MCValue {
1101 const raw_reg = try self.register_manager.allocReg(reg_owner, gp);
1102 const ty = self.typeOfIndex(reg_owner);
1103 const reg = self.registerAlias(raw_reg, ty);
1104 try self.genSetReg(self.typeOfIndex(reg_owner), reg, mcv);
1105 return MCValue{ .register = reg };
1106}
1107
1108fn airAlloc(self: *Self, inst: Air.Inst.Index) InnerError!void {
1109 const stack_offset = try self.allocMemPtr(inst);
1110 return self.finishAir(inst, .{ .ptr_stack_offset = stack_offset }, .{ .none, .none, .none });
1111}
1112
1113fn airRetPtr(self: *Self, inst: Air.Inst.Index) InnerError!void {
1114 const pt = self.pt;
1115 const zcu = pt.zcu;
1116 const result: MCValue = switch (self.ret_mcv) {
1117 .none, .register => .{ .ptr_stack_offset = try self.allocMemPtr(inst) },
1118 .stack_offset => blk: {
1119 // self.ret_mcv is an address to where this function
1120 // should store its result into
1121 const ret_ty = self.fn_type.fnReturnType(zcu);
1122 const ptr_ty = try pt.singleMutPtrType(ret_ty);
1123
1124 // addr_reg will contain the address of where to store the
1125 // result into
1126 const addr_reg = try self.copyToTmpRegister(ptr_ty, self.ret_mcv);
1127 break :blk .{ .register = addr_reg };
1128 },
1129 else => unreachable, // invalid return result
1130 };
1131
1132 return self.finishAir(inst, result, .{ .none, .none, .none });
1133}
1134
1135fn airFptrunc(self: *Self, inst: Air.Inst.Index) InnerError!void {
1136 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
1137 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement airFptrunc for {}", .{self.target.cpu.arch});
1138 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1139}
1140
1141fn airFpext(self: *Self, inst: Air.Inst.Index) InnerError!void {
1142 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
1143 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement airFpext for {}", .{self.target.cpu.arch});
1144 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1145}
1146
1147fn airIntCast(self: *Self, inst: Air.Inst.Index) InnerError!void {
1148 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
1149 if (self.liveness.isUnused(inst))
1150 return self.finishAir(inst, .dead, .{ ty_op.operand, .none, .none });
1151
1152 const pt = self.pt;
1153 const zcu = pt.zcu;
1154 const operand = ty_op.operand;
1155 const operand_mcv = try self.resolveInst(operand);
1156 const operand_ty = self.typeOf(operand);
1157 const operand_info = operand_ty.intInfo(zcu);
1158
1159 const dest_ty = self.typeOfIndex(inst);
1160 const dest_info = dest_ty.intInfo(zcu);
1161
1162 const result: MCValue = result: {
1163 const operand_lock: ?RegisterLock = switch (operand_mcv) {
1164 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
1165 else => null,
1166 };
1167 defer if (operand_lock) |lock| self.register_manager.unlockReg(lock);
1168
1169 const truncated: MCValue = switch (operand_mcv) {
1170 .register => |r| MCValue{ .register = self.registerAlias(r, dest_ty) },
1171 else => operand_mcv,
1172 };
1173
1174 if (dest_info.bits > operand_info.bits) {
1175 const dest_mcv = try self.allocRegOrMem(dest_ty, true, inst);
1176 try self.setRegOrMem(self.typeOfIndex(inst), dest_mcv, truncated);
1177 break :result dest_mcv;
1178 } else {
1179 if (self.reuseOperand(inst, operand, 0, truncated)) {
1180 break :result truncated;
1181 } else {
1182 const dest_mcv = try self.allocRegOrMem(dest_ty, true, inst);
1183 try self.setRegOrMem(self.typeOfIndex(inst), dest_mcv, truncated);
1184 break :result dest_mcv;
1185 }
1186 }
1187 };
1188
1189 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1190}
1191
1192fn truncRegister(
1193 self: *Self,
1194 operand_reg: Register,
1195 dest_reg: Register,
1196 int_signedness: std.builtin.Signedness,
1197 int_bits: u16,
1198) !void {
1199 switch (int_bits) {
1200 1...31, 33...63 => {
1201 _ = try self.addInst(.{
1202 .tag = switch (int_signedness) {
1203 .signed => .sbfx,
1204 .unsigned => .ubfx,
1205 },
1206 .data = .{ .rr_lsb_width = .{
1207 .rd = dest_reg,
1208 .rn = operand_reg,
1209 .lsb = 0,
1210 .width = @as(u6, @intCast(int_bits)),
1211 } },
1212 });
1213 },
1214 32, 64 => {
1215 _ = try self.addInst(.{
1216 .tag = .mov_register,
1217 .data = .{ .rr = .{
1218 .rd = if (int_bits == 32) dest_reg.toW() else dest_reg.toX(),
1219 .rn = if (int_bits == 32) operand_reg.toW() else operand_reg.toX(),
1220 } },
1221 });
1222 },
1223 else => unreachable,
1224 }
1225}
1226
1227fn trunc(
1228 self: *Self,
1229 maybe_inst: ?Air.Inst.Index,
1230 operand: MCValue,
1231 operand_ty: Type,
1232 dest_ty: Type,
1233) !MCValue {
1234 const pt = self.pt;
1235 const zcu = pt.zcu;
1236 const info_a = operand_ty.intInfo(zcu);
1237 const info_b = dest_ty.intInfo(zcu);
1238
1239 if (info_b.bits <= 64) {
1240 const operand_reg = switch (operand) {
1241 .register => |r| r,
1242 else => operand_reg: {
1243 if (info_a.bits <= 64) {
1244 const raw_reg = try self.copyToTmpRegister(operand_ty, operand);
1245 break :operand_reg self.registerAlias(raw_reg, operand_ty);
1246 } else {
1247 return self.fail("TODO load least significant word into register", .{});
1248 }
1249 },
1250 };
1251 const lock = self.register_manager.lockReg(operand_reg);
1252 defer if (lock) |reg| self.register_manager.unlockReg(reg);
1253
1254 const dest_reg = if (maybe_inst) |inst| blk: {
1255 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
1256
1257 if (operand == .register and self.reuseOperand(inst, ty_op.operand, 0, operand)) {
1258 break :blk self.registerAlias(operand_reg, dest_ty);
1259 } else {
1260 const raw_reg = try self.register_manager.allocReg(inst, gp);
1261 break :blk self.registerAlias(raw_reg, dest_ty);
1262 }
1263 } else blk: {
1264 const raw_reg = try self.register_manager.allocReg(null, gp);
1265 break :blk self.registerAlias(raw_reg, dest_ty);
1266 };
1267
1268 try self.truncRegister(operand_reg, dest_reg, info_b.signedness, info_b.bits);
1269
1270 return MCValue{ .register = dest_reg };
1271 } else {
1272 return self.fail("TODO: truncate to ints > 64 bits", .{});
1273 }
1274}
1275
1276fn airTrunc(self: *Self, inst: Air.Inst.Index) InnerError!void {
1277 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
1278 const operand = try self.resolveInst(ty_op.operand);
1279 const operand_ty = self.typeOf(ty_op.operand);
1280 const dest_ty = self.typeOfIndex(inst);
1281
1282 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else blk: {
1283 break :blk try self.trunc(inst, operand, operand_ty, dest_ty);
1284 };
1285
1286 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1287}
1288
1289fn airNot(self: *Self, inst: Air.Inst.Index) InnerError!void {
1290 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
1291 const pt = self.pt;
1292 const zcu = pt.zcu;
1293 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
1294 const operand = try self.resolveInst(ty_op.operand);
1295 const operand_ty = self.typeOf(ty_op.operand);
1296 switch (operand) {
1297 .dead => unreachable,
1298 .unreach => unreachable,
1299 .compare_flags => |cond| break :result MCValue{ .compare_flags = cond.negate() },
1300 else => {
1301 switch (operand_ty.zigTypeTag(zcu)) {
1302 .bool => {
1303 // TODO convert this to mvn + and
1304 const op_reg = switch (operand) {
1305 .register => |r| r,
1306 else => try self.copyToTmpRegister(operand_ty, operand),
1307 };
1308 const reg_lock = self.register_manager.lockRegAssumeUnused(op_reg);
1309 defer self.register_manager.unlockReg(reg_lock);
1310
1311 const dest_reg = blk: {
1312 if (operand == .register and self.reuseOperand(inst, ty_op.operand, 0, operand)) {
1313 break :blk op_reg;
1314 }
1315
1316 const raw_reg = try self.register_manager.allocReg(null, gp);
1317 break :blk self.registerAlias(raw_reg, operand_ty);
1318 };
1319
1320 _ = try self.addInst(.{
1321 .tag = .eor_immediate,
1322 .data = .{ .rr_bitmask = .{
1323 .rd = dest_reg,
1324 .rn = op_reg,
1325 .imms = 0b000000,
1326 .immr = 0b000000,
1327 .n = 0b0,
1328 } },
1329 });
1330
1331 break :result MCValue{ .register = dest_reg };
1332 },
1333 .vector => return self.fail("TODO bitwise not for vectors", .{}),
1334 .int => {
1335 const int_info = operand_ty.intInfo(zcu);
1336 if (int_info.bits <= 64) {
1337 const op_reg = switch (operand) {
1338 .register => |r| r,
1339 else => try self.copyToTmpRegister(operand_ty, operand),
1340 };
1341 const reg_lock = self.register_manager.lockRegAssumeUnused(op_reg);
1342 defer self.register_manager.unlockReg(reg_lock);
1343
1344 const dest_reg = blk: {
1345 if (operand == .register and self.reuseOperand(inst, ty_op.operand, 0, operand)) {
1346 break :blk op_reg;
1347 }
1348
1349 const raw_reg = try self.register_manager.allocReg(null, gp);
1350 break :blk self.registerAlias(raw_reg, operand_ty);
1351 };
1352
1353 _ = try self.addInst(.{
1354 .tag = .mvn,
1355 .data = .{ .rr_imm6_logical_shift = .{
1356 .rd = dest_reg,
1357 .rm = op_reg,
1358 .imm6 = 0,
1359 .shift = .lsl,
1360 } },
1361 });
1362
1363 try self.truncRegister(dest_reg, dest_reg, int_info.signedness, int_info.bits);
1364
1365 break :result MCValue{ .register = dest_reg };
1366 } else {
1367 return self.fail("TODO AArch64 not on integers > u64/i64", .{});
1368 }
1369 },
1370 else => unreachable,
1371 }
1372 },
1373 }
1374 };
1375 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1376}
1377
1378fn minMax(
1379 self: *Self,
1380 tag: Air.Inst.Tag,
1381 lhs_bind: ReadArg.Bind,
1382 rhs_bind: ReadArg.Bind,
1383 lhs_ty: Type,
1384 rhs_ty: Type,
1385 maybe_inst: ?Air.Inst.Index,
1386) !MCValue {
1387 const pt = self.pt;
1388 const zcu = pt.zcu;
1389 switch (lhs_ty.zigTypeTag(zcu)) {
1390 .float => return self.fail("TODO ARM min/max on floats", .{}),
1391 .vector => return self.fail("TODO ARM min/max on vectors", .{}),
1392 .int => {
1393 assert(lhs_ty.eql(rhs_ty, zcu));
1394 const int_info = lhs_ty.intInfo(zcu);
1395 if (int_info.bits <= 64) {
1396 var lhs_reg: Register = undefined;
1397 var rhs_reg: Register = undefined;
1398 var dest_reg: Register = undefined;
1399
1400 const read_args = [_]ReadArg{
1401 .{ .ty = lhs_ty, .bind = lhs_bind, .class = gp, .reg = &lhs_reg },
1402 .{ .ty = rhs_ty, .bind = rhs_bind, .class = gp, .reg = &rhs_reg },
1403 };
1404 const write_args = [_]WriteArg{
1405 .{ .ty = lhs_ty, .bind = .none, .class = gp, .reg = &dest_reg },
1406 };
1407 try self.allocRegs(
1408 &read_args,
1409 &write_args,
1410 if (maybe_inst) |inst| .{
1411 .corresponding_inst = inst,
1412 .operand_mapping = &.{ 0, 1 },
1413 } else null,
1414 );
1415
1416 // lhs == reg should have been checked by airMinMax
1417 assert(lhs_reg != rhs_reg); // see note above
1418
1419 _ = try self.addInst(.{
1420 .tag = .cmp_shifted_register,
1421 .data = .{ .rr_imm6_shift = .{
1422 .rn = lhs_reg,
1423 .rm = rhs_reg,
1424 .imm6 = 0,
1425 .shift = .lsl,
1426 } },
1427 });
1428
1429 const cond_choose_lhs: Condition = switch (tag) {
1430 .max => switch (int_info.signedness) {
1431 .signed => Condition.gt,
1432 .unsigned => Condition.hi,
1433 },
1434 .min => switch (int_info.signedness) {
1435 .signed => Condition.lt,
1436 .unsigned => Condition.cc,
1437 },
1438 else => unreachable,
1439 };
1440
1441 _ = try self.addInst(.{
1442 .tag = .csel,
1443 .data = .{ .rrr_cond = .{
1444 .rd = dest_reg,
1445 .rn = lhs_reg,
1446 .rm = rhs_reg,
1447 .cond = cond_choose_lhs,
1448 } },
1449 });
1450
1451 return MCValue{ .register = dest_reg };
1452 } else {
1453 return self.fail("TODO ARM min/max on integers > u32/i32", .{});
1454 }
1455 },
1456 else => unreachable,
1457 }
1458}
1459
1460fn airMinMax(self: *Self, inst: Air.Inst.Index) InnerError!void {
1461 const tag = self.air.instructions.items(.tag)[@intFromEnum(inst)];
1462 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
1463 const lhs_ty = self.typeOf(bin_op.lhs);
1464 const rhs_ty = self.typeOf(bin_op.rhs);
1465
1466 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
1467 const lhs_bind: ReadArg.Bind = .{ .inst = bin_op.lhs };
1468 const rhs_bind: ReadArg.Bind = .{ .inst = bin_op.rhs };
1469
1470 const lhs = try self.resolveInst(bin_op.lhs);
1471 if (bin_op.lhs == bin_op.rhs) break :result lhs;
1472
1473 break :result try self.minMax(tag, lhs_bind, rhs_bind, lhs_ty, rhs_ty, inst);
1474 };
1475 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1476}
1477
1478fn airSlice(self: *Self, inst: Air.Inst.Index) InnerError!void {
1479 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
1480 const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;
1481 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
1482 const ptr = try self.resolveInst(bin_op.lhs);
1483 const ptr_ty = self.typeOf(bin_op.lhs);
1484 const len = try self.resolveInst(bin_op.rhs);
1485 const len_ty = self.typeOf(bin_op.rhs);
1486
1487 const stack_offset = try self.allocMem(16, .@"8", inst);
1488 try self.genSetStack(ptr_ty, stack_offset, ptr);
1489 try self.genSetStack(len_ty, stack_offset - 8, len);
1490 break :result MCValue{ .stack_offset = stack_offset };
1491 };
1492 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1493}
1494
1495/// An argument to a Mir instruction which is read (and possibly also
1496/// written to) by the respective instruction
1497const ReadArg = struct {
1498 ty: Type,
1499 bind: Bind,
1500 class: RegisterManager.RegisterBitSet,
1501 reg: *Register,
1502
1503 const Bind = union(enum) {
1504 inst: Air.Inst.Ref,
1505 mcv: MCValue,
1506
1507 fn resolveToMcv(bind: Bind, function: *Self) InnerError!MCValue {
1508 return switch (bind) {
1509 .inst => |inst| try function.resolveInst(inst),
1510 .mcv => |mcv| mcv,
1511 };
1512 }
1513
1514 fn resolveToImmediate(bind: Bind, function: *Self) InnerError!?u64 {
1515 switch (bind) {
1516 .inst => |inst| {
1517 // TODO resolve independently of inst_table
1518 const mcv = try function.resolveInst(inst);
1519 switch (mcv) {
1520 .immediate => |imm| return imm,
1521 else => return null,
1522 }
1523 },
1524 .mcv => |mcv| {
1525 switch (mcv) {
1526 .immediate => |imm| return imm,
1527 else => return null,
1528 }
1529 },
1530 }
1531 }
1532 };
1533};
1534
1535/// An argument to a Mir instruction which is written to (but not read
1536/// from) by the respective instruction
1537const WriteArg = struct {
1538 ty: Type,
1539 bind: Bind,
1540 class: RegisterManager.RegisterBitSet,
1541 reg: *Register,
1542
1543 const Bind = union(enum) {
1544 reg: Register,
1545 none: void,
1546 };
1547};
1548
1549/// Holds all data necessary for enabling the potential reuse of
1550/// operand registers as destinations
1551const ReuseMetadata = struct {
1552 corresponding_inst: Air.Inst.Index,
1553
1554 /// Maps every element index of read_args to the corresponding
1555 /// index in the Air instruction
1556 ///
1557 /// When the order of read_args corresponds exactly to the order
1558 /// of the inputs of the Air instruction, this would be e.g.
1559 /// &.{ 0, 1 }. However, when the order is not the same or some
1560 /// inputs to the Air instruction are omitted (e.g. when they can
1561 /// be represented as immediates to the Mir instruction),
1562 /// operand_mapping should reflect that fact.
1563 operand_mapping: []const Air.Liveness.OperandInt,
1564};
1565
1566/// Allocate a set of registers for use as arguments for a Mir
1567/// instruction
1568///
1569/// If the Mir instruction these registers are allocated for
1570/// corresponds exactly to a single Air instruction, populate
1571/// reuse_metadata in order to enable potential reuse of an operand as
1572/// the destination (provided that that operand dies in this
1573/// instruction).
1574///
1575/// Reusing an operand register as destination is the only time two
1576/// arguments may share the same register. In all other cases,
1577/// allocRegs guarantees that a register will never be allocated to
1578/// more than one argument.
1579///
1580/// Furthermore, allocReg guarantees that all arguments which are
1581/// already bound to registers before calling allocRegs will not
1582/// change their register binding. This is done by locking these
1583/// registers.
1584fn allocRegs(
1585 self: *Self,
1586 read_args: []const ReadArg,
1587 write_args: []const WriteArg,
1588 reuse_metadata: ?ReuseMetadata,
1589) InnerError!void {
1590 // Air instructions have exactly one output
1591 assert(!(reuse_metadata != null and write_args.len != 1)); // see note above
1592
1593 // The operand mapping is a 1:1 mapping of read args to their
1594 // corresponding operand index in the Air instruction
1595 assert(!(reuse_metadata != null and reuse_metadata.?.operand_mapping.len != read_args.len)); // see note above
1596
1597 const locks = try self.gpa.alloc(?RegisterLock, read_args.len + write_args.len);
1598 defer self.gpa.free(locks);
1599 const read_locks = locks[0..read_args.len];
1600 const write_locks = locks[read_args.len..];
1601
1602 @memset(locks, null);
1603 defer for (locks) |lock| {
1604 if (lock) |locked_reg| self.register_manager.unlockReg(locked_reg);
1605 };
1606
1607 // When we reuse a read_arg as a destination, the corresponding
1608 // MCValue of the read_arg will be set to .dead. In that case, we
1609 // skip allocating this read_arg.
1610 var reused_read_arg: ?usize = null;
1611
1612 // Lock all args which are already allocated to registers
1613 for (read_args, 0..) |arg, i| {
1614 const mcv = try arg.bind.resolveToMcv(self);
1615 if (mcv == .register) {
1616 read_locks[i] = self.register_manager.lockReg(mcv.register);
1617 }
1618 }
1619
1620 for (write_args, 0..) |arg, i| {
1621 if (arg.bind == .reg) {
1622 write_locks[i] = self.register_manager.lockReg(arg.bind.reg);
1623 }
1624 }
1625
1626 // Allocate registers for all args which aren't allocated to
1627 // registers yet
1628 for (read_args, 0..) |arg, i| {
1629 const mcv = try arg.bind.resolveToMcv(self);
1630 if (mcv == .register) {
1631 const raw_reg = mcv.register;
1632 arg.reg.* = self.registerAlias(raw_reg, arg.ty);
1633 } else {
1634 const track_inst: ?Air.Inst.Index = switch (arg.bind) {
1635 .inst => |inst| inst.toIndex().?,
1636 else => null,
1637 };
1638 const raw_reg = try self.register_manager.allocReg(track_inst, gp);
1639 arg.reg.* = self.registerAlias(raw_reg, arg.ty);
1640 read_locks[i] = self.register_manager.lockRegAssumeUnused(arg.reg.*);
1641 }
1642 }
1643
1644 if (reuse_metadata != null) {
1645 const inst = reuse_metadata.?.corresponding_inst;
1646 const operand_mapping = reuse_metadata.?.operand_mapping;
1647 const arg = write_args[0];
1648 if (arg.bind == .reg) {
1649 const raw_reg = arg.bind.reg;
1650 arg.reg.* = self.registerAlias(raw_reg, arg.ty);
1651 } else {
1652 reuse_operand: for (read_args, 0..) |read_arg, i| {
1653 if (read_arg.bind == .inst) {
1654 const operand = read_arg.bind.inst;
1655 const mcv = try self.resolveInst(operand);
1656 if (mcv == .register and
1657 std.meta.eql(arg.class, read_arg.class) and
1658 self.reuseOperand(inst, operand, operand_mapping[i], mcv))
1659 {
1660 const raw_reg = mcv.register;
1661 arg.reg.* = self.registerAlias(raw_reg, arg.ty);
1662 write_locks[0] = null;
1663 reused_read_arg = i;
1664 break :reuse_operand;
1665 }
1666 }
1667 } else {
1668 const raw_reg = try self.register_manager.allocReg(inst, arg.class);
1669 arg.reg.* = self.registerAlias(raw_reg, arg.ty);
1670 write_locks[0] = self.register_manager.lockReg(arg.reg.*);
1671 }
1672 }
1673 } else {
1674 for (write_args, 0..) |arg, i| {
1675 if (arg.bind == .reg) {
1676 const raw_reg = arg.bind.reg;
1677 arg.reg.* = self.registerAlias(raw_reg, arg.ty);
1678 } else {
1679 const raw_reg = try self.register_manager.allocReg(null, arg.class);
1680 arg.reg.* = self.registerAlias(raw_reg, arg.ty);
1681 write_locks[i] = self.register_manager.lockReg(arg.reg.*);
1682 }
1683 }
1684 }
1685
1686 // For all read_args which need to be moved from non-register to
1687 // register, perform the move
1688 for (read_args, 0..) |arg, i| {
1689 if (reused_read_arg) |j| {
1690 // Check whether this read_arg was reused
1691 if (i == j) continue;
1692 }
1693
1694 const mcv = try arg.bind.resolveToMcv(self);
1695 if (mcv != .register) {
1696 if (arg.bind == .inst) {
1697 const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
1698 const inst = arg.bind.inst.toIndex().?;
1699
1700 // Overwrite the MCValue associated with this inst
1701 branch.inst_table.putAssumeCapacity(inst, .{ .register = arg.reg.* });
1702
1703 // If the previous MCValue occupied some space we track, we
1704 // need to make sure it is marked as free now.
1705 switch (mcv) {
1706 .compare_flags => {
1707 assert(self.compare_flags_inst.? == inst);
1708 self.compare_flags_inst = null;
1709 },
1710 .register => |prev_reg| {
1711 assert(!self.register_manager.isRegFree(prev_reg));
1712 self.register_manager.freeReg(prev_reg);
1713 },
1714 else => {},
1715 }
1716 }
1717
1718 try self.genSetReg(arg.ty, arg.reg.*, mcv);
1719 }
1720 }
1721}
1722
1723/// Wrapper around allocRegs and addInst tailored for specific Mir
1724/// instructions which are binary operations acting on two registers
1725///
1726/// Returns the destination register
1727fn binOpRegister(
1728 self: *Self,
1729 mir_tag: Mir.Inst.Tag,
1730 lhs_bind: ReadArg.Bind,
1731 rhs_bind: ReadArg.Bind,
1732 lhs_ty: Type,
1733 rhs_ty: Type,
1734 maybe_inst: ?Air.Inst.Index,
1735) !MCValue {
1736 var lhs_reg: Register = undefined;
1737 var rhs_reg: Register = undefined;
1738 var dest_reg: Register = undefined;
1739
1740 const read_args = [_]ReadArg{
1741 .{ .ty = lhs_ty, .bind = lhs_bind, .class = gp, .reg = &lhs_reg },
1742 .{ .ty = rhs_ty, .bind = rhs_bind, .class = gp, .reg = &rhs_reg },
1743 };
1744 const write_args = [_]WriteArg{
1745 .{ .ty = lhs_ty, .bind = .none, .class = gp, .reg = &dest_reg },
1746 };
1747 try self.allocRegs(
1748 &read_args,
1749 &write_args,
1750 if (maybe_inst) |inst| .{
1751 .corresponding_inst = inst,
1752 .operand_mapping = &.{ 0, 1 },
1753 } else null,
1754 );
1755
1756 const mir_data: Mir.Inst.Data = switch (mir_tag) {
1757 .add_shifted_register,
1758 .adds_shifted_register,
1759 .sub_shifted_register,
1760 .subs_shifted_register,
1761 => .{ .rrr_imm6_shift = .{
1762 .rd = dest_reg,
1763 .rn = lhs_reg,
1764 .rm = rhs_reg,
1765 .imm6 = 0,
1766 .shift = .lsl,
1767 } },
1768 .mul,
1769 .lsl_register,
1770 .asr_register,
1771 .lsr_register,
1772 .sdiv,
1773 .udiv,
1774 => .{ .rrr = .{
1775 .rd = dest_reg,
1776 .rn = lhs_reg,
1777 .rm = rhs_reg,
1778 } },
1779 .smull,
1780 .umull,
1781 => .{ .rrr = .{
1782 .rd = dest_reg.toX(),
1783 .rn = lhs_reg,
1784 .rm = rhs_reg,
1785 } },
1786 .and_shifted_register,
1787 .orr_shifted_register,
1788 .eor_shifted_register,
1789 => .{ .rrr_imm6_logical_shift = .{
1790 .rd = dest_reg,
1791 .rn = lhs_reg,
1792 .rm = rhs_reg,
1793 .imm6 = 0,
1794 .shift = .lsl,
1795 } },
1796 else => unreachable,
1797 };
1798
1799 _ = try self.addInst(.{
1800 .tag = mir_tag,
1801 .data = mir_data,
1802 });
1803
1804 return MCValue{ .register = dest_reg };
1805}
1806
1807/// Wrapper around allocRegs and addInst tailored for specific Mir
1808/// instructions which are binary operations acting on a register and
1809/// an immediate
1810///
1811/// Returns the destination register
1812fn binOpImmediate(
1813 self: *Self,
1814 mir_tag: Mir.Inst.Tag,
1815 lhs_bind: ReadArg.Bind,
1816 rhs_immediate: u64,
1817 lhs_ty: Type,
1818 lhs_and_rhs_swapped: bool,
1819 maybe_inst: ?Air.Inst.Index,
1820) !MCValue {
1821 var lhs_reg: Register = undefined;
1822 var dest_reg: Register = undefined;
1823
1824 const read_args = [_]ReadArg{
1825 .{ .ty = lhs_ty, .bind = lhs_bind, .class = gp, .reg = &lhs_reg },
1826 };
1827 const write_args = [_]WriteArg{
1828 .{ .ty = lhs_ty, .bind = .none, .class = gp, .reg = &dest_reg },
1829 };
1830 const operand_mapping: []const Air.Liveness.OperandInt = if (lhs_and_rhs_swapped) &.{1} else &.{0};
1831 try self.allocRegs(
1832 &read_args,
1833 &write_args,
1834 if (maybe_inst) |inst| .{
1835 .corresponding_inst = inst,
1836 .operand_mapping = operand_mapping,
1837 } else null,
1838 );
1839
1840 const mir_data: Mir.Inst.Data = switch (mir_tag) {
1841 .add_immediate,
1842 .adds_immediate,
1843 .sub_immediate,
1844 .subs_immediate,
1845 => .{ .rr_imm12_sh = .{
1846 .rd = dest_reg,
1847 .rn = lhs_reg,
1848 .imm12 = @as(u12, @intCast(rhs_immediate)),
1849 } },
1850 .lsl_immediate,
1851 .asr_immediate,
1852 .lsr_immediate,
1853 => .{ .rr_shift = .{
1854 .rd = dest_reg,
1855 .rn = lhs_reg,
1856 .shift = @as(u6, @intCast(rhs_immediate)),
1857 } },
1858 else => unreachable,
1859 };
1860
1861 _ = try self.addInst(.{
1862 .tag = mir_tag,
1863 .data = mir_data,
1864 });
1865
1866 return MCValue{ .register = dest_reg };
1867}
1868
1869fn addSub(
1870 self: *Self,
1871 tag: Air.Inst.Tag,
1872 lhs_bind: ReadArg.Bind,
1873 rhs_bind: ReadArg.Bind,
1874 lhs_ty: Type,
1875 rhs_ty: Type,
1876 maybe_inst: ?Air.Inst.Index,
1877) InnerError!MCValue {
1878 const pt = self.pt;
1879 const zcu = pt.zcu;
1880 switch (lhs_ty.zigTypeTag(zcu)) {
1881 .float => return self.fail("TODO binary operations on floats", .{}),
1882 .vector => return self.fail("TODO binary operations on vectors", .{}),
1883 .int => {
1884 assert(lhs_ty.eql(rhs_ty, zcu));
1885 const int_info = lhs_ty.intInfo(zcu);
1886 if (int_info.bits <= 64) {
1887 const lhs_immediate = try lhs_bind.resolveToImmediate(self);
1888 const rhs_immediate = try rhs_bind.resolveToImmediate(self);
1889
1890 // Only say yes if the operation is
1891 // commutative, i.e. we can swap both of the
1892 // operands
1893 const lhs_immediate_ok = switch (tag) {
1894 .add => if (lhs_immediate) |imm| imm <= std.math.maxInt(u12) else false,
1895 .sub => false,
1896 else => unreachable,
1897 };
1898 const rhs_immediate_ok = switch (tag) {
1899 .add,
1900 .sub,
1901 => if (rhs_immediate) |imm| imm <= std.math.maxInt(u12) else false,
1902 else => unreachable,
1903 };
1904
1905 const mir_tag_register: Mir.Inst.Tag = switch (tag) {
1906 .add => .add_shifted_register,
1907 .sub => .sub_shifted_register,
1908 else => unreachable,
1909 };
1910 const mir_tag_immediate: Mir.Inst.Tag = switch (tag) {
1911 .add => .add_immediate,
1912 .sub => .sub_immediate,
1913 else => unreachable,
1914 };
1915
1916 if (rhs_immediate_ok) {
1917 return try self.binOpImmediate(mir_tag_immediate, lhs_bind, rhs_immediate.?, lhs_ty, false, maybe_inst);
1918 } else if (lhs_immediate_ok) {
1919 // swap lhs and rhs
1920 return try self.binOpImmediate(mir_tag_immediate, rhs_bind, lhs_immediate.?, rhs_ty, true, maybe_inst);
1921 } else {
1922 return try self.binOpRegister(mir_tag_register, lhs_bind, rhs_bind, lhs_ty, rhs_ty, maybe_inst);
1923 }
1924 } else {
1925 return self.fail("TODO binary operations on int with bits > 64", .{});
1926 }
1927 },
1928 else => unreachable,
1929 }
1930}
1931
1932fn mul(
1933 self: *Self,
1934 lhs_bind: ReadArg.Bind,
1935 rhs_bind: ReadArg.Bind,
1936 lhs_ty: Type,
1937 rhs_ty: Type,
1938 maybe_inst: ?Air.Inst.Index,
1939) InnerError!MCValue {
1940 const pt = self.pt;
1941 const zcu = pt.zcu;
1942 switch (lhs_ty.zigTypeTag(zcu)) {
1943 .vector => return self.fail("TODO binary operations on vectors", .{}),
1944 .int => {
1945 assert(lhs_ty.eql(rhs_ty, zcu));
1946 const int_info = lhs_ty.intInfo(zcu);
1947 if (int_info.bits <= 64) {
1948 // TODO add optimisations for multiplication
1949 // with immediates, for example a * 2 can be
1950 // lowered to a << 1
1951 return try self.binOpRegister(.mul, lhs_bind, rhs_bind, lhs_ty, rhs_ty, maybe_inst);
1952 } else {
1953 return self.fail("TODO binary operations on int with bits > 64", .{});
1954 }
1955 },
1956 else => unreachable,
1957 }
1958}
1959
1960fn divFloat(
1961 self: *Self,
1962 lhs_bind: ReadArg.Bind,
1963 rhs_bind: ReadArg.Bind,
1964 lhs_ty: Type,
1965 rhs_ty: Type,
1966 maybe_inst: ?Air.Inst.Index,
1967) InnerError!MCValue {
1968 _ = lhs_bind;
1969 _ = rhs_bind;
1970 _ = rhs_ty;
1971 _ = maybe_inst;
1972
1973 const pt = self.pt;
1974 const zcu = pt.zcu;
1975 switch (lhs_ty.zigTypeTag(zcu)) {
1976 .float => return self.fail("TODO div_float", .{}),
1977 .vector => return self.fail("TODO div_float on vectors", .{}),
1978 else => unreachable,
1979 }
1980}
1981
1982fn divTrunc(
1983 self: *Self,
1984 lhs_bind: ReadArg.Bind,
1985 rhs_bind: ReadArg.Bind,
1986 lhs_ty: Type,
1987 rhs_ty: Type,
1988 maybe_inst: ?Air.Inst.Index,
1989) InnerError!MCValue {
1990 const pt = self.pt;
1991 const zcu = pt.zcu;
1992 switch (lhs_ty.zigTypeTag(zcu)) {
1993 .float => return self.fail("TODO div on floats", .{}),
1994 .vector => return self.fail("TODO div on vectors", .{}),
1995 .int => {
1996 assert(lhs_ty.eql(rhs_ty, zcu));
1997 const int_info = lhs_ty.intInfo(zcu);
1998 if (int_info.bits <= 64) {
1999 switch (int_info.signedness) {
2000 .signed => {
2001 // TODO optimize integer division by constants
2002 return try self.binOpRegister(.sdiv, lhs_bind, rhs_bind, lhs_ty, rhs_ty, maybe_inst);
2003 },
2004 .unsigned => {
2005 // TODO optimize integer division by constants
2006 return try self.binOpRegister(.udiv, lhs_bind, rhs_bind, lhs_ty, rhs_ty, maybe_inst);
2007 },
2008 }
2009 } else {
2010 return self.fail("TODO integer division for ints with bits > 64", .{});
2011 }
2012 },
2013 else => unreachable,
2014 }
2015}
2016
2017fn divFloor(
2018 self: *Self,
2019 lhs_bind: ReadArg.Bind,
2020 rhs_bind: ReadArg.Bind,
2021 lhs_ty: Type,
2022 rhs_ty: Type,
2023 maybe_inst: ?Air.Inst.Index,
2024) InnerError!MCValue {
2025 const pt = self.pt;
2026 const zcu = pt.zcu;
2027 switch (lhs_ty.zigTypeTag(zcu)) {
2028 .float => return self.fail("TODO div on floats", .{}),
2029 .vector => return self.fail("TODO div on vectors", .{}),
2030 .int => {
2031 assert(lhs_ty.eql(rhs_ty, zcu));
2032 const int_info = lhs_ty.intInfo(zcu);
2033 if (int_info.bits <= 64) {
2034 switch (int_info.signedness) {
2035 .signed => {
2036 return self.fail("TODO div_floor on signed integers", .{});
2037 },
2038 .unsigned => {
2039 // TODO optimize integer division by constants
2040 return try self.binOpRegister(.udiv, lhs_bind, rhs_bind, lhs_ty, rhs_ty, maybe_inst);
2041 },
2042 }
2043 } else {
2044 return self.fail("TODO integer division for ints with bits > 64", .{});
2045 }
2046 },
2047 else => unreachable,
2048 }
2049}
2050
2051fn divExact(
2052 self: *Self,
2053 lhs_bind: ReadArg.Bind,
2054 rhs_bind: ReadArg.Bind,
2055 lhs_ty: Type,
2056 rhs_ty: Type,
2057 maybe_inst: ?Air.Inst.Index,
2058) InnerError!MCValue {
2059 const pt = self.pt;
2060 const zcu = pt.zcu;
2061 switch (lhs_ty.zigTypeTag(zcu)) {
2062 .float => return self.fail("TODO div on floats", .{}),
2063 .vector => return self.fail("TODO div on vectors", .{}),
2064 .int => {
2065 assert(lhs_ty.eql(rhs_ty, zcu));
2066 const int_info = lhs_ty.intInfo(zcu);
2067 if (int_info.bits <= 64) {
2068 switch (int_info.signedness) {
2069 .signed => {
2070 // TODO optimize integer division by constants
2071 return try self.binOpRegister(.sdiv, lhs_bind, rhs_bind, lhs_ty, rhs_ty, maybe_inst);
2072 },
2073 .unsigned => {
2074 // TODO optimize integer division by constants
2075 return try self.binOpRegister(.udiv, lhs_bind, rhs_bind, lhs_ty, rhs_ty, maybe_inst);
2076 },
2077 }
2078 } else {
2079 return self.fail("TODO integer division for ints with bits > 64", .{});
2080 }
2081 },
2082 else => unreachable,
2083 }
2084}
2085
2086fn rem(
2087 self: *Self,
2088 lhs_bind: ReadArg.Bind,
2089 rhs_bind: ReadArg.Bind,
2090 lhs_ty: Type,
2091 rhs_ty: Type,
2092 maybe_inst: ?Air.Inst.Index,
2093) InnerError!MCValue {
2094 _ = maybe_inst;
2095
2096 const pt = self.pt;
2097 const zcu = pt.zcu;
2098 switch (lhs_ty.zigTypeTag(zcu)) {
2099 .float => return self.fail("TODO rem/zcu on floats", .{}),
2100 .vector => return self.fail("TODO rem/zcu on vectors", .{}),
2101 .int => {
2102 assert(lhs_ty.eql(rhs_ty, zcu));
2103 const int_info = lhs_ty.intInfo(zcu);
2104 if (int_info.bits <= 64) {
2105 var lhs_reg: Register = undefined;
2106 var rhs_reg: Register = undefined;
2107 var quotient_reg: Register = undefined;
2108 var remainder_reg: Register = undefined;
2109
2110 const read_args = [_]ReadArg{
2111 .{ .ty = lhs_ty, .bind = lhs_bind, .class = gp, .reg = &lhs_reg },
2112 .{ .ty = rhs_ty, .bind = rhs_bind, .class = gp, .reg = &rhs_reg },
2113 };
2114 const write_args = [_]WriteArg{
2115 .{ .ty = lhs_ty, .bind = .none, .class = gp, .reg = &quotient_reg },
2116 .{ .ty = lhs_ty, .bind = .none, .class = gp, .reg = &remainder_reg },
2117 };
2118 try self.allocRegs(
2119 &read_args,
2120 &write_args,
2121 null,
2122 );
2123
2124 _ = try self.addInst(.{
2125 .tag = switch (int_info.signedness) {
2126 .signed => .sdiv,
2127 .unsigned => .udiv,
2128 },
2129 .data = .{ .rrr = .{
2130 .rd = quotient_reg,
2131 .rn = lhs_reg,
2132 .rm = rhs_reg,
2133 } },
2134 });
2135
2136 _ = try self.addInst(.{
2137 .tag = .msub,
2138 .data = .{ .rrrr = .{
2139 .rd = remainder_reg,
2140 .rn = quotient_reg,
2141 .rm = rhs_reg,
2142 .ra = lhs_reg,
2143 } },
2144 });
2145
2146 return MCValue{ .register = remainder_reg };
2147 } else {
2148 return self.fail("TODO rem/zcu for integers with bits > 64", .{});
2149 }
2150 },
2151 else => unreachable,
2152 }
2153}
2154
2155fn modulo(
2156 self: *Self,
2157 lhs_bind: ReadArg.Bind,
2158 rhs_bind: ReadArg.Bind,
2159 lhs_ty: Type,
2160 rhs_ty: Type,
2161 maybe_inst: ?Air.Inst.Index,
2162) InnerError!MCValue {
2163 _ = lhs_bind;
2164 _ = rhs_bind;
2165 _ = rhs_ty;
2166 _ = maybe_inst;
2167
2168 const pt = self.pt;
2169 const zcu = pt.zcu;
2170 switch (lhs_ty.zigTypeTag(zcu)) {
2171 .float => return self.fail("TODO zcu on floats", .{}),
2172 .vector => return self.fail("TODO zcu on vectors", .{}),
2173 .int => return self.fail("TODO zcu on ints", .{}),
2174 else => unreachable,
2175 }
2176}
2177
2178fn wrappingArithmetic(
2179 self: *Self,
2180 tag: Air.Inst.Tag,
2181 lhs_bind: ReadArg.Bind,
2182 rhs_bind: ReadArg.Bind,
2183 lhs_ty: Type,
2184 rhs_ty: Type,
2185 maybe_inst: ?Air.Inst.Index,
2186) InnerError!MCValue {
2187 const pt = self.pt;
2188 const zcu = pt.zcu;
2189 switch (lhs_ty.zigTypeTag(zcu)) {
2190 .vector => return self.fail("TODO binary operations on vectors", .{}),
2191 .int => {
2192 const int_info = lhs_ty.intInfo(zcu);
2193 if (int_info.bits <= 64) {
2194 // Generate an add/sub/mul
2195 const result: MCValue = switch (tag) {
2196 .add_wrap => try self.addSub(.add, lhs_bind, rhs_bind, lhs_ty, rhs_ty, maybe_inst),
2197 .sub_wrap => try self.addSub(.sub, lhs_bind, rhs_bind, lhs_ty, rhs_ty, maybe_inst),
2198 .mul_wrap => try self.mul(lhs_bind, rhs_bind, lhs_ty, rhs_ty, maybe_inst),
2199 else => unreachable,
2200 };
2201
2202 // Truncate if necessary
2203 const result_reg = result.register;
2204 try self.truncRegister(result_reg, result_reg, int_info.signedness, int_info.bits);
2205 return result;
2206 } else {
2207 return self.fail("TODO binary operations on integers > u64/i64", .{});
2208 }
2209 },
2210 else => unreachable,
2211 }
2212}
2213
2214fn bitwise(
2215 self: *Self,
2216 tag: Air.Inst.Tag,
2217 lhs_bind: ReadArg.Bind,
2218 rhs_bind: ReadArg.Bind,
2219 lhs_ty: Type,
2220 rhs_ty: Type,
2221 maybe_inst: ?Air.Inst.Index,
2222) InnerError!MCValue {
2223 const pt = self.pt;
2224 const zcu = pt.zcu;
2225 switch (lhs_ty.zigTypeTag(zcu)) {
2226 .vector => return self.fail("TODO binary operations on vectors", .{}),
2227 .int => {
2228 assert(lhs_ty.eql(rhs_ty, zcu));
2229 const int_info = lhs_ty.intInfo(zcu);
2230 if (int_info.bits <= 64) {
2231 // TODO implement bitwise operations with immediates
2232 const mir_tag: Mir.Inst.Tag = switch (tag) {
2233 .bit_and => .and_shifted_register,
2234 .bit_or => .orr_shifted_register,
2235 .xor => .eor_shifted_register,
2236 else => unreachable,
2237 };
2238
2239 return try self.binOpRegister(mir_tag, lhs_bind, rhs_bind, lhs_ty, rhs_ty, maybe_inst);
2240 } else {
2241 return self.fail("TODO binary operations on int with bits > 64", .{});
2242 }
2243 },
2244 else => unreachable,
2245 }
2246}
2247
2248fn shiftExact(
2249 self: *Self,
2250 tag: Air.Inst.Tag,
2251 lhs_bind: ReadArg.Bind,
2252 rhs_bind: ReadArg.Bind,
2253 lhs_ty: Type,
2254 rhs_ty: Type,
2255 maybe_inst: ?Air.Inst.Index,
2256) InnerError!MCValue {
2257 const pt = self.pt;
2258 const zcu = pt.zcu;
2259 switch (lhs_ty.zigTypeTag(zcu)) {
2260 .vector => if (!rhs_ty.isVector(zcu))
2261 return self.fail("TODO vector shift with scalar rhs", .{})
2262 else
2263 return self.fail("TODO binary operations on vectors", .{}),
2264 .int => {
2265 const int_info = lhs_ty.intInfo(zcu);
2266 if (int_info.bits <= 64) {
2267 const rhs_immediate = try rhs_bind.resolveToImmediate(self);
2268
2269 const mir_tag_register: Mir.Inst.Tag = switch (tag) {
2270 .shl_exact => .lsl_register,
2271 .shr_exact => switch (int_info.signedness) {
2272 .signed => Mir.Inst.Tag.asr_register,
2273 .unsigned => Mir.Inst.Tag.lsr_register,
2274 },
2275 else => unreachable,
2276 };
2277 const mir_tag_immediate: Mir.Inst.Tag = switch (tag) {
2278 .shl_exact => .lsl_immediate,
2279 .shr_exact => switch (int_info.signedness) {
2280 .signed => Mir.Inst.Tag.asr_immediate,
2281 .unsigned => Mir.Inst.Tag.lsr_immediate,
2282 },
2283 else => unreachable,
2284 };
2285
2286 if (rhs_immediate) |imm| {
2287 return try self.binOpImmediate(mir_tag_immediate, lhs_bind, imm, lhs_ty, false, maybe_inst);
2288 } else {
2289 // We intentionally pass lhs_ty here in order to
2290 // prevent using the 32-bit register alias when
2291 // lhs_ty is > 32 bits.
2292 return try self.binOpRegister(mir_tag_register, lhs_bind, rhs_bind, lhs_ty, lhs_ty, maybe_inst);
2293 }
2294 } else {
2295 return self.fail("TODO binary operations on int with bits > 64", .{});
2296 }
2297 },
2298 else => unreachable,
2299 }
2300}
2301
2302fn shiftNormal(
2303 self: *Self,
2304 tag: Air.Inst.Tag,
2305 lhs_bind: ReadArg.Bind,
2306 rhs_bind: ReadArg.Bind,
2307 lhs_ty: Type,
2308 rhs_ty: Type,
2309 maybe_inst: ?Air.Inst.Index,
2310) InnerError!MCValue {
2311 const pt = self.pt;
2312 const zcu = pt.zcu;
2313 switch (lhs_ty.zigTypeTag(zcu)) {
2314 .vector => if (!rhs_ty.isVector(zcu))
2315 return self.fail("TODO vector shift with scalar rhs", .{})
2316 else
2317 return self.fail("TODO binary operations on vectors", .{}),
2318 .int => {
2319 const int_info = lhs_ty.intInfo(zcu);
2320 if (int_info.bits <= 64) {
2321 // Generate a shl_exact/shr_exact
2322 const result: MCValue = switch (tag) {
2323 .shl => try self.shiftExact(.shl_exact, lhs_bind, rhs_bind, lhs_ty, rhs_ty, maybe_inst),
2324 .shr => try self.shiftExact(.shr_exact, lhs_bind, rhs_bind, lhs_ty, rhs_ty, maybe_inst),
2325 else => unreachable,
2326 };
2327
2328 // Truncate if necessary
2329 switch (tag) {
2330 .shr => return result,
2331 .shl => {
2332 const result_reg = result.register;
2333 try self.truncRegister(result_reg, result_reg, int_info.signedness, int_info.bits);
2334 return result;
2335 },
2336 else => unreachable,
2337 }
2338 } else {
2339 return self.fail("TODO binary operations on integers > u64/i64", .{});
2340 }
2341 },
2342 else => unreachable,
2343 }
2344}
2345
2346fn booleanOp(
2347 self: *Self,
2348 tag: Air.Inst.Tag,
2349 lhs_bind: ReadArg.Bind,
2350 rhs_bind: ReadArg.Bind,
2351 lhs_ty: Type,
2352 rhs_ty: Type,
2353 maybe_inst: ?Air.Inst.Index,
2354) InnerError!MCValue {
2355 const pt = self.pt;
2356 const zcu = pt.zcu;
2357 switch (lhs_ty.zigTypeTag(zcu)) {
2358 .bool => {
2359 assert((try lhs_bind.resolveToImmediate(self)) == null); // should have been handled by Sema
2360 assert((try rhs_bind.resolveToImmediate(self)) == null); // should have been handled by Sema
2361
2362 const mir_tag_register: Mir.Inst.Tag = switch (tag) {
2363 .bool_and => .and_shifted_register,
2364 .bool_or => .orr_shifted_register,
2365 else => unreachable,
2366 };
2367
2368 return try self.binOpRegister(mir_tag_register, lhs_bind, rhs_bind, lhs_ty, rhs_ty, maybe_inst);
2369 },
2370 else => unreachable,
2371 }
2372}
2373
2374fn ptrArithmetic(
2375 self: *Self,
2376 tag: Air.Inst.Tag,
2377 lhs_bind: ReadArg.Bind,
2378 rhs_bind: ReadArg.Bind,
2379 lhs_ty: Type,
2380 rhs_ty: Type,
2381 maybe_inst: ?Air.Inst.Index,
2382) InnerError!MCValue {
2383 const pt = self.pt;
2384 const zcu = pt.zcu;
2385 switch (lhs_ty.zigTypeTag(zcu)) {
2386 .pointer => {
2387 assert(rhs_ty.eql(Type.usize, zcu));
2388
2389 const ptr_ty = lhs_ty;
2390 const elem_ty = switch (ptr_ty.ptrSize(zcu)) {
2391 .one => ptr_ty.childType(zcu).childType(zcu), // ptr to array, so get array element type
2392 else => ptr_ty.childType(zcu),
2393 };
2394 const elem_size = elem_ty.abiSize(zcu);
2395
2396 const base_tag: Air.Inst.Tag = switch (tag) {
2397 .ptr_add => .add,
2398 .ptr_sub => .sub,
2399 else => unreachable,
2400 };
2401
2402 if (elem_size == 1) {
2403 return try self.addSub(base_tag, lhs_bind, rhs_bind, Type.usize, Type.usize, maybe_inst);
2404 } else {
2405 // convert the offset into a byte offset by
2406 // multiplying it with elem_size
2407 const imm_bind = ReadArg.Bind{ .mcv = .{ .immediate = elem_size } };
2408
2409 const offset = try self.mul(rhs_bind, imm_bind, Type.usize, Type.usize, null);
2410 const offset_bind = ReadArg.Bind{ .mcv = offset };
2411
2412 const addr = try self.addSub(base_tag, lhs_bind, offset_bind, Type.usize, Type.usize, null);
2413 return addr;
2414 }
2415 },
2416 else => unreachable,
2417 }
2418}
2419
2420fn airBinOp(self: *Self, inst: Air.Inst.Index, tag: Air.Inst.Tag) InnerError!void {
2421 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
2422 const lhs_ty = self.typeOf(bin_op.lhs);
2423 const rhs_ty = self.typeOf(bin_op.rhs);
2424
2425 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
2426 const lhs_bind: ReadArg.Bind = .{ .inst = bin_op.lhs };
2427 const rhs_bind: ReadArg.Bind = .{ .inst = bin_op.rhs };
2428
2429 break :result switch (tag) {
2430 .add => try self.addSub(tag, lhs_bind, rhs_bind, lhs_ty, rhs_ty, inst),
2431 .sub => try self.addSub(tag, lhs_bind, rhs_bind, lhs_ty, rhs_ty, inst),
2432
2433 .mul => try self.mul(lhs_bind, rhs_bind, lhs_ty, rhs_ty, inst),
2434
2435 .div_float => try self.divFloat(lhs_bind, rhs_bind, lhs_ty, rhs_ty, inst),
2436
2437 .div_trunc => try self.divTrunc(lhs_bind, rhs_bind, lhs_ty, rhs_ty, inst),
2438
2439 .div_floor => try self.divFloor(lhs_bind, rhs_bind, lhs_ty, rhs_ty, inst),
2440
2441 .div_exact => try self.divExact(lhs_bind, rhs_bind, lhs_ty, rhs_ty, inst),
2442
2443 .rem => try self.rem(lhs_bind, rhs_bind, lhs_ty, rhs_ty, inst),
2444
2445 .mod => try self.modulo(lhs_bind, rhs_bind, lhs_ty, rhs_ty, inst),
2446
2447 .add_wrap => try self.wrappingArithmetic(tag, lhs_bind, rhs_bind, lhs_ty, rhs_ty, inst),
2448 .sub_wrap => try self.wrappingArithmetic(tag, lhs_bind, rhs_bind, lhs_ty, rhs_ty, inst),
2449 .mul_wrap => try self.wrappingArithmetic(tag, lhs_bind, rhs_bind, lhs_ty, rhs_ty, inst),
2450
2451 .bit_and => try self.bitwise(tag, lhs_bind, rhs_bind, lhs_ty, rhs_ty, inst),
2452 .bit_or => try self.bitwise(tag, lhs_bind, rhs_bind, lhs_ty, rhs_ty, inst),
2453 .xor => try self.bitwise(tag, lhs_bind, rhs_bind, lhs_ty, rhs_ty, inst),
2454
2455 .shl_exact => try self.shiftExact(tag, lhs_bind, rhs_bind, lhs_ty, rhs_ty, inst),
2456 .shr_exact => try self.shiftExact(tag, lhs_bind, rhs_bind, lhs_ty, rhs_ty, inst),
2457
2458 .shl => try self.shiftNormal(tag, lhs_bind, rhs_bind, lhs_ty, rhs_ty, inst),
2459 .shr => try self.shiftNormal(tag, lhs_bind, rhs_bind, lhs_ty, rhs_ty, inst),
2460
2461 .bool_and => try self.booleanOp(tag, lhs_bind, rhs_bind, lhs_ty, rhs_ty, inst),
2462 .bool_or => try self.booleanOp(tag, lhs_bind, rhs_bind, lhs_ty, rhs_ty, inst),
2463
2464 else => unreachable,
2465 };
2466 };
2467 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
2468}
2469
2470fn airPtrArithmetic(self: *Self, inst: Air.Inst.Index, tag: Air.Inst.Tag) InnerError!void {
2471 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
2472 const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;
2473 const lhs_ty = self.typeOf(bin_op.lhs);
2474 const rhs_ty = self.typeOf(bin_op.rhs);
2475
2476 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
2477 const lhs_bind: ReadArg.Bind = .{ .inst = bin_op.lhs };
2478 const rhs_bind: ReadArg.Bind = .{ .inst = bin_op.rhs };
2479
2480 break :result try self.ptrArithmetic(tag, lhs_bind, rhs_bind, lhs_ty, rhs_ty, inst);
2481 };
2482 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
2483}
2484
2485fn airAddSat(self: *Self, inst: Air.Inst.Index) InnerError!void {
2486 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
2487 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement add_sat for {}", .{self.target.cpu.arch});
2488 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
2489}
2490
2491fn airSubSat(self: *Self, inst: Air.Inst.Index) InnerError!void {
2492 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
2493 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement sub_sat for {}", .{self.target.cpu.arch});
2494 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
2495}
2496
2497fn airMulSat(self: *Self, inst: Air.Inst.Index) InnerError!void {
2498 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
2499 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement mul_sat for {}", .{self.target.cpu.arch});
2500 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
2501}
2502
2503fn airOverflow(self: *Self, inst: Air.Inst.Index) InnerError!void {
2504 const tag = self.air.instructions.items(.tag)[@intFromEnum(inst)];
2505 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
2506 const extra = self.air.extraData(Air.Bin, ty_pl.payload).data;
2507 const pt = self.pt;
2508 const zcu = pt.zcu;
2509 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
2510 const lhs_bind: ReadArg.Bind = .{ .inst = extra.lhs };
2511 const rhs_bind: ReadArg.Bind = .{ .inst = extra.rhs };
2512 const lhs_ty = self.typeOf(extra.lhs);
2513 const rhs_ty = self.typeOf(extra.rhs);
2514
2515 const tuple_ty = self.typeOfIndex(inst);
2516 const tuple_size: u32 = @intCast(tuple_ty.abiSize(zcu));
2517 const tuple_align = tuple_ty.abiAlignment(zcu);
2518 const overflow_bit_offset: u32 = @intCast(tuple_ty.structFieldOffset(1, zcu));
2519
2520 switch (lhs_ty.zigTypeTag(zcu)) {
2521 .vector => return self.fail("TODO implement add_with_overflow/sub_with_overflow for vectors", .{}),
2522 .int => {
2523 assert(lhs_ty.eql(rhs_ty, zcu));
2524 const int_info = lhs_ty.intInfo(zcu);
2525 switch (int_info.bits) {
2526 1...31, 33...63 => {
2527 const stack_offset = try self.allocMem(tuple_size, tuple_align, inst);
2528
2529 try self.spillCompareFlagsIfOccupied();
2530 self.compare_flags_inst = null;
2531
2532 const base_tag: Air.Inst.Tag = switch (tag) {
2533 .add_with_overflow => .add,
2534 .sub_with_overflow => .sub,
2535 else => unreachable,
2536 };
2537 const dest = try self.addSub(base_tag, lhs_bind, rhs_bind, lhs_ty, rhs_ty, null);
2538 const dest_reg = dest.register;
2539 const dest_reg_lock = self.register_manager.lockRegAssumeUnused(dest_reg);
2540 defer self.register_manager.unlockReg(dest_reg_lock);
2541
2542 const raw_truncated_reg = try self.register_manager.allocReg(null, gp);
2543 const truncated_reg = self.registerAlias(raw_truncated_reg, lhs_ty);
2544 const truncated_reg_lock = self.register_manager.lockRegAssumeUnused(truncated_reg);
2545 defer self.register_manager.unlockReg(truncated_reg_lock);
2546
2547 // sbfx/ubfx truncated, dest, #0, #bits
2548 try self.truncRegister(dest_reg, truncated_reg, int_info.signedness, int_info.bits);
2549
2550 // cmp dest, truncated
2551 _ = try self.addInst(.{
2552 .tag = .cmp_shifted_register,
2553 .data = .{ .rr_imm6_shift = .{
2554 .rn = dest_reg,
2555 .rm = truncated_reg,
2556 .imm6 = 0,
2557 .shift = .lsl,
2558 } },
2559 });
2560
2561 try self.genSetStack(lhs_ty, stack_offset, .{ .register = truncated_reg });
2562 try self.genSetStack(Type.u1, stack_offset - overflow_bit_offset, .{ .compare_flags = .ne });
2563
2564 break :result MCValue{ .stack_offset = stack_offset };
2565 },
2566 32, 64 => {
2567 const lhs_immediate = try lhs_bind.resolveToImmediate(self);
2568 const rhs_immediate = try rhs_bind.resolveToImmediate(self);
2569
2570 // Only say yes if the operation is
2571 // commutative, i.e. we can swap both of the
2572 // operands
2573 const lhs_immediate_ok = switch (tag) {
2574 .add_with_overflow => if (lhs_immediate) |imm| imm <= std.math.maxInt(u12) else false,
2575 .sub_with_overflow => false,
2576 else => unreachable,
2577 };
2578 const rhs_immediate_ok = switch (tag) {
2579 .add_with_overflow,
2580 .sub_with_overflow,
2581 => if (rhs_immediate) |imm| imm <= std.math.maxInt(u12) else false,
2582 else => unreachable,
2583 };
2584
2585 const mir_tag_register: Mir.Inst.Tag = switch (tag) {
2586 .add_with_overflow => .adds_shifted_register,
2587 .sub_with_overflow => .subs_shifted_register,
2588 else => unreachable,
2589 };
2590 const mir_tag_immediate: Mir.Inst.Tag = switch (tag) {
2591 .add_with_overflow => .adds_immediate,
2592 .sub_with_overflow => .subs_immediate,
2593 else => unreachable,
2594 };
2595
2596 try self.spillCompareFlagsIfOccupied();
2597 self.compare_flags_inst = inst;
2598
2599 const dest = blk: {
2600 if (rhs_immediate_ok) {
2601 break :blk try self.binOpImmediate(mir_tag_immediate, lhs_bind, rhs_immediate.?, lhs_ty, false, null);
2602 } else if (lhs_immediate_ok) {
2603 // swap lhs and rhs
2604 break :blk try self.binOpImmediate(mir_tag_immediate, rhs_bind, lhs_immediate.?, rhs_ty, true, null);
2605 } else {
2606 break :blk try self.binOpRegister(mir_tag_register, lhs_bind, rhs_bind, lhs_ty, rhs_ty, null);
2607 }
2608 };
2609
2610 const flag: bits.Instruction.Condition = switch (int_info.signedness) {
2611 .unsigned => switch (tag) {
2612 .add_with_overflow => bits.Instruction.Condition.cs,
2613 .sub_with_overflow => bits.Instruction.Condition.cc,
2614 else => unreachable,
2615 },
2616 .signed => .vs,
2617 };
2618 break :result MCValue{ .register_with_overflow = .{
2619 .reg = dest.register,
2620 .flag = flag,
2621 } };
2622 },
2623 else => return self.fail("TODO overflow operations on integers > u32/i32", .{}),
2624 }
2625 },
2626 else => unreachable,
2627 }
2628 };
2629 return self.finishAir(inst, result, .{ extra.lhs, extra.rhs, .none });
2630}
2631
2632fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) InnerError!void {
2633 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
2634 const extra = self.air.extraData(Air.Bin, ty_pl.payload).data;
2635 if (self.liveness.isUnused(inst)) return self.finishAir(inst, .dead, .{ extra.lhs, extra.rhs, .none });
2636 const zcu = self.pt.zcu;
2637 const result: MCValue = result: {
2638 const lhs_bind: ReadArg.Bind = .{ .inst = extra.lhs };
2639 const rhs_bind: ReadArg.Bind = .{ .inst = extra.rhs };
2640 const lhs_ty = self.typeOf(extra.lhs);
2641 const rhs_ty = self.typeOf(extra.rhs);
2642
2643 const tuple_ty = self.typeOfIndex(inst);
2644 const tuple_size = @as(u32, @intCast(tuple_ty.abiSize(zcu)));
2645 const tuple_align = tuple_ty.abiAlignment(zcu);
2646 const overflow_bit_offset = @as(u32, @intCast(tuple_ty.structFieldOffset(1, zcu)));
2647
2648 switch (lhs_ty.zigTypeTag(zcu)) {
2649 .vector => return self.fail("TODO implement mul_with_overflow for vectors", .{}),
2650 .int => {
2651 assert(lhs_ty.eql(rhs_ty, zcu));
2652 const int_info = lhs_ty.intInfo(zcu);
2653 if (int_info.bits <= 32) {
2654 const stack_offset = try self.allocMem(tuple_size, tuple_align, inst);
2655
2656 try self.spillCompareFlagsIfOccupied();
2657
2658 const base_tag: Mir.Inst.Tag = switch (int_info.signedness) {
2659 .signed => .smull,
2660 .unsigned => .umull,
2661 };
2662
2663 const dest = try self.binOpRegister(base_tag, lhs_bind, rhs_bind, lhs_ty, rhs_ty, null);
2664 const dest_reg = dest.register;
2665 const dest_reg_lock = self.register_manager.lockRegAssumeUnused(dest_reg);
2666 defer self.register_manager.unlockReg(dest_reg_lock);
2667
2668 const truncated_reg = try self.register_manager.allocReg(null, gp);
2669 const truncated_reg_lock = self.register_manager.lockRegAssumeUnused(truncated_reg);
2670 defer self.register_manager.unlockReg(truncated_reg_lock);
2671
2672 try self.truncRegister(
2673 dest_reg.toW(),
2674 truncated_reg.toW(),
2675 int_info.signedness,
2676 int_info.bits,
2677 );
2678
2679 switch (int_info.signedness) {
2680 .signed => {
2681 _ = try self.addInst(.{
2682 .tag = .cmp_extended_register,
2683 .data = .{ .rr_extend_shift = .{
2684 .rn = dest_reg.toX(),
2685 .rm = truncated_reg.toW(),
2686 .ext_type = .sxtw,
2687 .imm3 = 0,
2688 } },
2689 });
2690 },
2691 .unsigned => {
2692 _ = try self.addInst(.{
2693 .tag = .cmp_extended_register,
2694 .data = .{ .rr_extend_shift = .{
2695 .rn = dest_reg.toX(),
2696 .rm = truncated_reg.toW(),
2697 .ext_type = .uxtw,
2698 .imm3 = 0,
2699 } },
2700 });
2701 },
2702 }
2703
2704 try self.genSetStack(lhs_ty, stack_offset, .{ .register = truncated_reg });
2705 try self.genSetStack(Type.u1, stack_offset - overflow_bit_offset, .{ .compare_flags = .ne });
2706
2707 break :result MCValue{ .stack_offset = stack_offset };
2708 } else if (int_info.bits <= 64) {
2709 const stack_offset = try self.allocMem(tuple_size, tuple_align, inst);
2710
2711 try self.spillCompareFlagsIfOccupied();
2712
2713 var lhs_reg: Register = undefined;
2714 var rhs_reg: Register = undefined;
2715 var dest_reg: Register = undefined;
2716 var dest_high_reg: Register = undefined;
2717 var truncated_reg: Register = undefined;
2718
2719 const read_args = [_]ReadArg{
2720 .{ .ty = lhs_ty, .bind = lhs_bind, .class = gp, .reg = &lhs_reg },
2721 .{ .ty = rhs_ty, .bind = rhs_bind, .class = gp, .reg = &rhs_reg },
2722 };
2723 const write_args = [_]WriteArg{
2724 .{ .ty = lhs_ty, .bind = .none, .class = gp, .reg = &dest_reg },
2725 .{ .ty = lhs_ty, .bind = .none, .class = gp, .reg = &dest_high_reg },
2726 .{ .ty = lhs_ty, .bind = .none, .class = gp, .reg = &truncated_reg },
2727 };
2728 try self.allocRegs(
2729 &read_args,
2730 &write_args,
2731 null,
2732 );
2733
2734 switch (int_info.signedness) {
2735 .signed => {
2736 // mul dest, lhs, rhs
2737 _ = try self.addInst(.{
2738 .tag = .mul,
2739 .data = .{ .rrr = .{
2740 .rd = dest_reg,
2741 .rn = lhs_reg,
2742 .rm = rhs_reg,
2743 } },
2744 });
2745
2746 // smulh dest_high, lhs, rhs
2747 _ = try self.addInst(.{
2748 .tag = .smulh,
2749 .data = .{ .rrr = .{
2750 .rd = dest_high_reg,
2751 .rn = lhs_reg,
2752 .rm = rhs_reg,
2753 } },
2754 });
2755
2756 // cmp dest_high, dest, asr #63
2757 _ = try self.addInst(.{
2758 .tag = .cmp_shifted_register,
2759 .data = .{ .rr_imm6_shift = .{
2760 .rn = dest_high_reg,
2761 .rm = dest_reg,
2762 .imm6 = 63,
2763 .shift = .asr,
2764 } },
2765 });
2766
2767 const shift: u6 = @as(u6, @intCast(@as(u7, 64) - @as(u7, @intCast(int_info.bits))));
2768 if (shift > 0) {
2769 // lsl dest_high, dest, #shift
2770 _ = try self.addInst(.{
2771 .tag = .lsl_immediate,
2772 .data = .{ .rr_shift = .{
2773 .rd = dest_high_reg,
2774 .rn = dest_reg,
2775 .shift = shift,
2776 } },
2777 });
2778
2779 // cmp dest, dest_high, #shift
2780 _ = try self.addInst(.{
2781 .tag = .cmp_shifted_register,
2782 .data = .{ .rr_imm6_shift = .{
2783 .rn = dest_reg,
2784 .rm = dest_high_reg,
2785 .imm6 = shift,
2786 .shift = .asr,
2787 } },
2788 });
2789 }
2790 },
2791 .unsigned => {
2792 // umulh dest_high, lhs, rhs
2793 _ = try self.addInst(.{
2794 .tag = .umulh,
2795 .data = .{ .rrr = .{
2796 .rd = dest_high_reg,
2797 .rn = lhs_reg,
2798 .rm = rhs_reg,
2799 } },
2800 });
2801
2802 // mul dest, lhs, rhs
2803 _ = try self.addInst(.{
2804 .tag = .mul,
2805 .data = .{ .rrr = .{
2806 .rd = dest_reg,
2807 .rn = lhs_reg,
2808 .rm = rhs_reg,
2809 } },
2810 });
2811
2812 _ = try self.addInst(.{
2813 .tag = .cmp_immediate,
2814 .data = .{ .r_imm12_sh = .{
2815 .rn = dest_high_reg,
2816 .imm12 = 0,
2817 } },
2818 });
2819
2820 if (int_info.bits < 64) {
2821 // lsr dest_high, dest, #shift
2822 _ = try self.addInst(.{
2823 .tag = .lsr_immediate,
2824 .data = .{ .rr_shift = .{
2825 .rd = dest_high_reg,
2826 .rn = dest_reg,
2827 .shift = @as(u6, @intCast(int_info.bits)),
2828 } },
2829 });
2830
2831 _ = try self.addInst(.{
2832 .tag = .cmp_immediate,
2833 .data = .{ .r_imm12_sh = .{
2834 .rn = dest_high_reg,
2835 .imm12 = 0,
2836 } },
2837 });
2838 }
2839 },
2840 }
2841
2842 try self.truncRegister(dest_reg, truncated_reg, int_info.signedness, int_info.bits);
2843
2844 try self.genSetStack(lhs_ty, stack_offset, .{ .register = truncated_reg });
2845 try self.genSetStack(Type.u1, stack_offset - overflow_bit_offset, .{ .compare_flags = .ne });
2846
2847 break :result MCValue{ .stack_offset = stack_offset };
2848 } else return self.fail("TODO implement mul_with_overflow for integers > u64/i64", .{});
2849 },
2850 else => unreachable,
2851 }
2852 };
2853 return self.finishAir(inst, result, .{ extra.lhs, extra.rhs, .none });
2854}
2855
2856fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) InnerError!void {
2857 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
2858 const extra = self.air.extraData(Air.Bin, ty_pl.payload).data;
2859 if (self.liveness.isUnused(inst)) return self.finishAir(inst, .dead, .{ extra.lhs, extra.rhs, .none });
2860 const pt = self.pt;
2861 const zcu = pt.zcu;
2862 const result: MCValue = result: {
2863 const lhs_bind: ReadArg.Bind = .{ .inst = extra.lhs };
2864 const rhs_bind: ReadArg.Bind = .{ .inst = extra.rhs };
2865 const lhs_ty = self.typeOf(extra.lhs);
2866 const rhs_ty = self.typeOf(extra.rhs);
2867
2868 const tuple_ty = self.typeOfIndex(inst);
2869 const tuple_size = @as(u32, @intCast(tuple_ty.abiSize(zcu)));
2870 const tuple_align = tuple_ty.abiAlignment(zcu);
2871 const overflow_bit_offset = @as(u32, @intCast(tuple_ty.structFieldOffset(1, zcu)));
2872
2873 switch (lhs_ty.zigTypeTag(zcu)) {
2874 .vector => if (!rhs_ty.isVector(zcu))
2875 return self.fail("TODO implement vector shl_with_overflow with scalar rhs", .{})
2876 else
2877 return self.fail("TODO implement shl_with_overflow for vectors", .{}),
2878 .int => {
2879 const int_info = lhs_ty.intInfo(zcu);
2880 if (int_info.bits <= 64) {
2881 const stack_offset = try self.allocMem(tuple_size, tuple_align, inst);
2882
2883 try self.spillCompareFlagsIfOccupied();
2884
2885 var lhs_reg: Register = undefined;
2886 var rhs_reg: Register = undefined;
2887 var dest_reg: Register = undefined;
2888 var reconstructed_reg: Register = undefined;
2889
2890 const rhs_immediate = try rhs_bind.resolveToImmediate(self);
2891 if (rhs_immediate) |imm| {
2892 const read_args = [_]ReadArg{
2893 .{ .ty = lhs_ty, .bind = lhs_bind, .class = gp, .reg = &lhs_reg },
2894 };
2895 const write_args = [_]WriteArg{
2896 .{ .ty = lhs_ty, .bind = .none, .class = gp, .reg = &dest_reg },
2897 .{ .ty = lhs_ty, .bind = .none, .class = gp, .reg = &reconstructed_reg },
2898 };
2899 try self.allocRegs(
2900 &read_args,
2901 &write_args,
2902 null,
2903 );
2904
2905 // lsl dest, lhs, rhs
2906 _ = try self.addInst(.{
2907 .tag = .lsl_immediate,
2908 .data = .{ .rr_shift = .{
2909 .rd = dest_reg,
2910 .rn = lhs_reg,
2911 .shift = @as(u6, @intCast(imm)),
2912 } },
2913 });
2914
2915 try self.truncRegister(dest_reg, dest_reg, int_info.signedness, int_info.bits);
2916
2917 // asr/lsr reconstructed, dest, rhs
2918 _ = try self.addInst(.{
2919 .tag = switch (int_info.signedness) {
2920 .signed => Mir.Inst.Tag.asr_immediate,
2921 .unsigned => Mir.Inst.Tag.lsr_immediate,
2922 },
2923 .data = .{ .rr_shift = .{
2924 .rd = reconstructed_reg,
2925 .rn = dest_reg,
2926 .shift = @as(u6, @intCast(imm)),
2927 } },
2928 });
2929 } else {
2930 const read_args = [_]ReadArg{
2931 .{ .ty = lhs_ty, .bind = lhs_bind, .class = gp, .reg = &lhs_reg },
2932 .{ .ty = rhs_ty, .bind = rhs_bind, .class = gp, .reg = &rhs_reg },
2933 };
2934 const write_args = [_]WriteArg{
2935 .{ .ty = lhs_ty, .bind = .none, .class = gp, .reg = &dest_reg },
2936 .{ .ty = lhs_ty, .bind = .none, .class = gp, .reg = &reconstructed_reg },
2937 };
2938 try self.allocRegs(
2939 &read_args,
2940 &write_args,
2941 null,
2942 );
2943
2944 // lsl dest, lhs, rhs
2945 _ = try self.addInst(.{
2946 .tag = .lsl_register,
2947 .data = .{ .rrr = .{
2948 .rd = dest_reg,
2949 .rn = lhs_reg,
2950 .rm = rhs_reg,
2951 } },
2952 });
2953
2954 try self.truncRegister(dest_reg, dest_reg, int_info.signedness, int_info.bits);
2955
2956 // asr/lsr reconstructed, dest, rhs
2957 _ = try self.addInst(.{
2958 .tag = switch (int_info.signedness) {
2959 .signed => Mir.Inst.Tag.asr_register,
2960 .unsigned => Mir.Inst.Tag.lsr_register,
2961 },
2962 .data = .{ .rrr = .{
2963 .rd = reconstructed_reg,
2964 .rn = dest_reg,
2965 .rm = rhs_reg,
2966 } },
2967 });
2968 }
2969
2970 // cmp lhs, reconstructed
2971 _ = try self.addInst(.{
2972 .tag = .cmp_shifted_register,
2973 .data = .{ .rr_imm6_shift = .{
2974 .rn = lhs_reg,
2975 .rm = reconstructed_reg,
2976 .imm6 = 0,
2977 .shift = .lsl,
2978 } },
2979 });
2980
2981 try self.genSetStack(lhs_ty, stack_offset, .{ .register = dest_reg });
2982 try self.genSetStack(Type.u1, stack_offset - overflow_bit_offset, .{ .compare_flags = .ne });
2983
2984 break :result MCValue{ .stack_offset = stack_offset };
2985 } else {
2986 return self.fail("TODO ARM overflow operations on integers > u32/i32", .{});
2987 }
2988 },
2989 else => unreachable,
2990 }
2991 };
2992 return self.finishAir(inst, result, .{ extra.lhs, extra.rhs, .none });
2993}
2994
2995fn airShlSat(self: *Self, inst: Air.Inst.Index) InnerError!void {
2996 const zcu = self.pt.zcu;
2997 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
2998 const result: MCValue = if (self.liveness.isUnused(inst))
2999 .dead
3000 else if (self.typeOf(bin_op.lhs).isVector(zcu) and !self.typeOf(bin_op.rhs).isVector(zcu))
3001 return self.fail("TODO implement vector shl_sat with scalar rhs for {}", .{self.target.cpu.arch})
3002 else
3003 return self.fail("TODO implement shl_sat for {}", .{self.target.cpu.arch});
3004 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
3005}
3006
3007fn airOptionalPayload(self: *Self, inst: Air.Inst.Index) InnerError!void {
3008 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
3009 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
3010 const optional_ty = self.typeOf(ty_op.operand);
3011 const mcv = try self.resolveInst(ty_op.operand);
3012 break :result try self.optionalPayload(inst, mcv, optional_ty);
3013 };
3014 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
3015}
3016
3017fn optionalPayload(self: *Self, inst: Air.Inst.Index, mcv: MCValue, optional_ty: Type) !MCValue {
3018 const pt = self.pt;
3019 const zcu = pt.zcu;
3020 const payload_ty = optional_ty.optionalChild(zcu);
3021 if (!payload_ty.hasRuntimeBits(zcu)) return MCValue.none;
3022 if (optional_ty.isPtrLikeOptional(zcu)) {
3023 // TODO should we reuse the operand here?
3024 const raw_reg = try self.register_manager.allocReg(inst, gp);
3025 const reg = self.registerAlias(raw_reg, payload_ty);
3026 try self.genSetReg(payload_ty, reg, mcv);
3027 return MCValue{ .register = reg };
3028 }
3029
3030 switch (mcv) {
3031 .register => {
3032 // TODO should we reuse the operand here?
3033 const raw_reg = try self.register_manager.allocReg(inst, gp);
3034 const dest_reg = raw_reg.toX();
3035
3036 try self.genSetReg(payload_ty, dest_reg, mcv);
3037 return MCValue{ .register = self.registerAlias(dest_reg, payload_ty) };
3038 },
3039 .stack_argument_offset, .stack_offset, .memory => return mcv,
3040 else => unreachable, // invalid MCValue for an error union
3041 }
3042}
3043
3044fn airOptionalPayloadPtr(self: *Self, inst: Air.Inst.Index) InnerError!void {
3045 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
3046 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement .optional_payload_ptr for {}", .{self.target.cpu.arch});
3047 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
3048}
3049
3050fn airOptionalPayloadPtrSet(self: *Self, inst: Air.Inst.Index) InnerError!void {
3051 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
3052 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement .optional_payload_ptr_set for {}", .{self.target.cpu.arch});
3053 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
3054}
3055
3056/// Given an error union, returns the error
3057fn errUnionErr(
3058 self: *Self,
3059 error_union_bind: ReadArg.Bind,
3060 error_union_ty: Type,
3061 maybe_inst: ?Air.Inst.Index,
3062) !MCValue {
3063 const pt = self.pt;
3064 const zcu = pt.zcu;
3065 const err_ty = error_union_ty.errorUnionSet(zcu);
3066 const payload_ty = error_union_ty.errorUnionPayload(zcu);
3067 if (err_ty.errorSetIsEmpty(zcu)) {
3068 return MCValue{ .immediate = 0 };
3069 }
3070 if (!payload_ty.hasRuntimeBitsIgnoreComptime(zcu)) {
3071 return try error_union_bind.resolveToMcv(self);
3072 }
3073
3074 const err_offset: u32 = @intCast(errUnionErrorOffset(payload_ty, zcu));
3075 switch (try error_union_bind.resolveToMcv(self)) {
3076 .register => {
3077 var operand_reg: Register = undefined;
3078 var dest_reg: Register = undefined;
3079
3080 const read_args = [_]ReadArg{
3081 .{ .ty = error_union_ty, .bind = error_union_bind, .class = gp, .reg = &operand_reg },
3082 };
3083 const write_args = [_]WriteArg{
3084 .{ .ty = err_ty, .bind = .none, .class = gp, .reg = &dest_reg },
3085 };
3086 try self.allocRegs(
3087 &read_args,
3088 &write_args,
3089 if (maybe_inst) |inst| .{
3090 .corresponding_inst = inst,
3091 .operand_mapping = &.{0},
3092 } else null,
3093 );
3094
3095 const err_bit_offset = err_offset * 8;
3096 const err_bit_size = @as(u32, @intCast(err_ty.abiSize(zcu))) * 8;
3097
3098 _ = try self.addInst(.{
3099 .tag = .ubfx, // errors are unsigned integers
3100 .data = .{
3101 .rr_lsb_width = .{
3102 // Set both registers to the X variant to get the full width
3103 .rd = dest_reg.toX(),
3104 .rn = operand_reg.toX(),
3105 .lsb = @as(u6, @intCast(err_bit_offset)),
3106 .width = @as(u7, @intCast(err_bit_size)),
3107 },
3108 },
3109 });
3110
3111 return MCValue{ .register = dest_reg };
3112 },
3113 .stack_argument_offset => |off| {
3114 return MCValue{ .stack_argument_offset = off + err_offset };
3115 },
3116 .stack_offset => |off| {
3117 return MCValue{ .stack_offset = off - err_offset };
3118 },
3119 .memory => |addr| {
3120 return MCValue{ .memory = addr + err_offset };
3121 },
3122 else => unreachable, // invalid MCValue for an error union
3123 }
3124}
3125
3126fn airUnwrapErrErr(self: *Self, inst: Air.Inst.Index) InnerError!void {
3127 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
3128 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
3129 const error_union_bind: ReadArg.Bind = .{ .inst = ty_op.operand };
3130 const error_union_ty = self.typeOf(ty_op.operand);
3131
3132 break :result try self.errUnionErr(error_union_bind, error_union_ty, inst);
3133 };
3134 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
3135}
3136
3137/// Given an error union, returns the payload
3138fn errUnionPayload(
3139 self: *Self,
3140 error_union_bind: ReadArg.Bind,
3141 error_union_ty: Type,
3142 maybe_inst: ?Air.Inst.Index,
3143) !MCValue {
3144 const pt = self.pt;
3145 const zcu = pt.zcu;
3146 const err_ty = error_union_ty.errorUnionSet(zcu);
3147 const payload_ty = error_union_ty.errorUnionPayload(zcu);
3148 if (err_ty.errorSetIsEmpty(zcu)) {
3149 return try error_union_bind.resolveToMcv(self);
3150 }
3151 if (!payload_ty.hasRuntimeBitsIgnoreComptime(zcu)) {
3152 return MCValue.none;
3153 }
3154
3155 const payload_offset = @as(u32, @intCast(errUnionPayloadOffset(payload_ty, zcu)));
3156 switch (try error_union_bind.resolveToMcv(self)) {
3157 .register => {
3158 var operand_reg: Register = undefined;
3159 var dest_reg: Register = undefined;
3160
3161 const read_args = [_]ReadArg{
3162 .{ .ty = error_union_ty, .bind = error_union_bind, .class = gp, .reg = &operand_reg },
3163 };
3164 const write_args = [_]WriteArg{
3165 .{ .ty = err_ty, .bind = .none, .class = gp, .reg = &dest_reg },
3166 };
3167 try self.allocRegs(
3168 &read_args,
3169 &write_args,
3170 if (maybe_inst) |inst| .{
3171 .corresponding_inst = inst,
3172 .operand_mapping = &.{0},
3173 } else null,
3174 );
3175
3176 const payload_bit_offset = payload_offset * 8;
3177 const payload_bit_size = @as(u32, @intCast(payload_ty.abiSize(zcu))) * 8;
3178
3179 _ = try self.addInst(.{
3180 .tag = if (payload_ty.isSignedInt(zcu)) Mir.Inst.Tag.sbfx else .ubfx,
3181 .data = .{
3182 .rr_lsb_width = .{
3183 // Set both registers to the X variant to get the full width
3184 .rd = dest_reg.toX(),
3185 .rn = operand_reg.toX(),
3186 .lsb = @as(u5, @intCast(payload_bit_offset)),
3187 .width = @as(u6, @intCast(payload_bit_size)),
3188 },
3189 },
3190 });
3191
3192 return MCValue{ .register = dest_reg };
3193 },
3194 .stack_argument_offset => |off| {
3195 return MCValue{ .stack_argument_offset = off + payload_offset };
3196 },
3197 .stack_offset => |off| {
3198 return MCValue{ .stack_offset = off - payload_offset };
3199 },
3200 .memory => |addr| {
3201 return MCValue{ .memory = addr + payload_offset };
3202 },
3203 else => unreachable, // invalid MCValue for an error union
3204 }
3205}
3206
3207fn airUnwrapErrPayload(self: *Self, inst: Air.Inst.Index) InnerError!void {
3208 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
3209 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
3210 const error_union_bind: ReadArg.Bind = .{ .inst = ty_op.operand };
3211 const error_union_ty = self.typeOf(ty_op.operand);
3212
3213 break :result try self.errUnionPayload(error_union_bind, error_union_ty, inst);
3214 };
3215 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
3216}
3217
3218// *(E!T) -> E
3219fn airUnwrapErrErrPtr(self: *Self, inst: Air.Inst.Index) InnerError!void {
3220 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
3221 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement unwrap error union error ptr for {}", .{self.target.cpu.arch});
3222 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
3223}
3224
3225// *(E!T) -> *T
3226fn airUnwrapErrPayloadPtr(self: *Self, inst: Air.Inst.Index) InnerError!void {
3227 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
3228 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement unwrap error union payload ptr for {}", .{self.target.cpu.arch});
3229 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
3230}
3231
3232fn airErrUnionPayloadPtrSet(self: *Self, inst: Air.Inst.Index) InnerError!void {
3233 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
3234 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement .errunion_payload_ptr_set for {}", .{self.target.cpu.arch});
3235 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
3236}
3237
3238fn airErrReturnTrace(self: *Self, inst: Air.Inst.Index) InnerError!void {
3239 const result: MCValue = if (self.liveness.isUnused(inst))
3240 .dead
3241 else
3242 return self.fail("TODO implement airErrReturnTrace for {}", .{self.target.cpu.arch});
3243 return self.finishAir(inst, result, .{ .none, .none, .none });
3244}
3245
3246fn airSetErrReturnTrace(self: *Self, inst: Air.Inst.Index) InnerError!void {
3247 _ = inst;
3248 return self.fail("TODO implement airSetErrReturnTrace for {}", .{self.target.cpu.arch});
3249}
3250
3251fn airSaveErrReturnTraceIndex(self: *Self, inst: Air.Inst.Index) InnerError!void {
3252 _ = inst;
3253 return self.fail("TODO implement airSaveErrReturnTraceIndex for {}", .{self.target.cpu.arch});
3254}
3255
3256fn airWrapOptional(self: *Self, inst: Air.Inst.Index) InnerError!void {
3257 const pt = self.pt;
3258 const zcu = pt.zcu;
3259 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
3260
3261 if (self.liveness.isUnused(inst)) {
3262 return self.finishAir(inst, .dead, .{ ty_op.operand, .none, .none });
3263 }
3264
3265 const result: MCValue = result: {
3266 const payload_ty = self.typeOf(ty_op.operand);
3267 if (!payload_ty.hasRuntimeBits(zcu)) {
3268 break :result MCValue{ .immediate = 1 };
3269 }
3270
3271 const optional_ty = self.typeOfIndex(inst);
3272 const operand = try self.resolveInst(ty_op.operand);
3273 const operand_lock: ?RegisterLock = switch (operand) {
3274 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
3275 else => null,
3276 };
3277 defer if (operand_lock) |lock| self.register_manager.unlockReg(lock);
3278
3279 if (optional_ty.isPtrLikeOptional(zcu)) {
3280 // TODO should we check if we can reuse the operand?
3281 const raw_reg = try self.register_manager.allocReg(inst, gp);
3282 const reg = self.registerAlias(raw_reg, payload_ty);
3283 try self.genSetReg(payload_ty, raw_reg, operand);
3284 break :result MCValue{ .register = reg };
3285 }
3286
3287 const optional_abi_size: u32 = @intCast(optional_ty.abiSize(zcu));
3288 const optional_abi_align = optional_ty.abiAlignment(zcu);
3289 const offset: u32 = @intCast(payload_ty.abiSize(zcu));
3290
3291 const stack_offset = try self.allocMem(optional_abi_size, optional_abi_align, inst);
3292 try self.genSetStack(payload_ty, stack_offset, operand);
3293 try self.genSetStack(Type.bool, stack_offset - offset, .{ .immediate = 1 });
3294
3295 break :result MCValue{ .stack_offset = stack_offset };
3296 };
3297
3298 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
3299}
3300
3301/// T to E!T
3302fn airWrapErrUnionPayload(self: *Self, inst: Air.Inst.Index) InnerError!void {
3303 const pt = self.pt;
3304 const zcu = pt.zcu;
3305 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
3306 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
3307 const error_union_ty = ty_op.ty.toType();
3308 const error_ty = error_union_ty.errorUnionSet(zcu);
3309 const payload_ty = error_union_ty.errorUnionPayload(zcu);
3310 const operand = try self.resolveInst(ty_op.operand);
3311 if (!payload_ty.hasRuntimeBitsIgnoreComptime(zcu)) break :result operand;
3312
3313 const abi_size = @as(u32, @intCast(error_union_ty.abiSize(zcu)));
3314 const abi_align = error_union_ty.abiAlignment(zcu);
3315 const stack_offset = try self.allocMem(abi_size, abi_align, inst);
3316 const payload_off = errUnionPayloadOffset(payload_ty, zcu);
3317 const err_off = errUnionErrorOffset(payload_ty, zcu);
3318 try self.genSetStack(payload_ty, stack_offset - @as(u32, @intCast(payload_off)), operand);
3319 try self.genSetStack(error_ty, stack_offset - @as(u32, @intCast(err_off)), .{ .immediate = 0 });
3320
3321 break :result MCValue{ .stack_offset = stack_offset };
3322 };
3323 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
3324}
3325
3326/// E to E!T
3327fn airWrapErrUnionErr(self: *Self, inst: Air.Inst.Index) InnerError!void {
3328 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
3329 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
3330 const pt = self.pt;
3331 const zcu = pt.zcu;
3332 const error_union_ty = ty_op.ty.toType();
3333 const error_ty = error_union_ty.errorUnionSet(zcu);
3334 const payload_ty = error_union_ty.errorUnionPayload(zcu);
3335 const operand = try self.resolveInst(ty_op.operand);
3336 if (!payload_ty.hasRuntimeBitsIgnoreComptime(zcu)) break :result operand;
3337
3338 const abi_size = @as(u32, @intCast(error_union_ty.abiSize(zcu)));
3339 const abi_align = error_union_ty.abiAlignment(zcu);
3340 const stack_offset = try self.allocMem(abi_size, abi_align, inst);
3341 const payload_off = errUnionPayloadOffset(payload_ty, zcu);
3342 const err_off = errUnionErrorOffset(payload_ty, zcu);
3343 try self.genSetStack(error_ty, stack_offset - @as(u32, @intCast(err_off)), operand);
3344 try self.genSetStack(payload_ty, stack_offset - @as(u32, @intCast(payload_off)), .undef);
3345
3346 break :result MCValue{ .stack_offset = stack_offset };
3347 };
3348 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
3349}
3350
3351fn slicePtr(mcv: MCValue) MCValue {
3352 switch (mcv) {
3353 .dead, .unreach, .none => unreachable,
3354 .register => unreachable, // a slice doesn't fit in one register
3355 .stack_argument_offset => |off| {
3356 return MCValue{ .stack_argument_offset = off };
3357 },
3358 .stack_offset => |off| {
3359 return MCValue{ .stack_offset = off };
3360 },
3361 .memory => |addr| {
3362 return MCValue{ .memory = addr };
3363 },
3364 else => unreachable, // invalid MCValue for a slice
3365 }
3366}
3367
3368fn airSlicePtr(self: *Self, inst: Air.Inst.Index) InnerError!void {
3369 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
3370 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
3371 const mcv = try self.resolveInst(ty_op.operand);
3372 break :result slicePtr(mcv);
3373 };
3374 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
3375}
3376
3377fn airSliceLen(self: *Self, inst: Air.Inst.Index) InnerError!void {
3378 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
3379 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
3380 const ptr_bits = 64;
3381 const ptr_bytes = @divExact(ptr_bits, 8);
3382 const mcv = try self.resolveInst(ty_op.operand);
3383 switch (mcv) {
3384 .dead, .unreach, .none => unreachable,
3385 .register => unreachable, // a slice doesn't fit in one register
3386 .stack_argument_offset => |off| {
3387 break :result MCValue{ .stack_argument_offset = off + ptr_bytes };
3388 },
3389 .stack_offset => |off| {
3390 break :result MCValue{ .stack_offset = off - ptr_bytes };
3391 },
3392 .memory => |addr| {
3393 break :result MCValue{ .memory = addr + ptr_bytes };
3394 },
3395 else => return self.fail("TODO implement slice_len for {}", .{mcv}),
3396 }
3397 };
3398 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
3399}
3400
3401fn airPtrSliceLenPtr(self: *Self, inst: Air.Inst.Index) InnerError!void {
3402 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
3403 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
3404 const ptr_bits = 64;
3405 const ptr_bytes = @divExact(ptr_bits, 8);
3406 const mcv = try self.resolveInst(ty_op.operand);
3407 switch (mcv) {
3408 .dead, .unreach, .none => unreachable,
3409 .ptr_stack_offset => |off| {
3410 break :result MCValue{ .ptr_stack_offset = off - ptr_bytes };
3411 },
3412 else => return self.fail("TODO implement ptr_slice_len_ptr for {}", .{mcv}),
3413 }
3414 };
3415 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
3416}
3417
3418fn airPtrSlicePtrPtr(self: *Self, inst: Air.Inst.Index) InnerError!void {
3419 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
3420 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
3421 const mcv = try self.resolveInst(ty_op.operand);
3422 switch (mcv) {
3423 .dead, .unreach, .none => unreachable,
3424 .ptr_stack_offset => |off| {
3425 break :result MCValue{ .ptr_stack_offset = off };
3426 },
3427 else => return self.fail("TODO implement ptr_slice_len_ptr for {}", .{mcv}),
3428 }
3429 };
3430 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
3431}
3432
3433fn airSliceElemVal(self: *Self, inst: Air.Inst.Index) InnerError!void {
3434 const pt = self.pt;
3435 const zcu = pt.zcu;
3436 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
3437 const slice_ty = self.typeOf(bin_op.lhs);
3438 const result: MCValue = if (!slice_ty.isVolatilePtr(zcu) and self.liveness.isUnused(inst)) .dead else result: {
3439 const ptr_ty = slice_ty.slicePtrFieldType(zcu);
3440
3441 const slice_mcv = try self.resolveInst(bin_op.lhs);
3442 const base_mcv = slicePtr(slice_mcv);
3443
3444 const base_bind: ReadArg.Bind = .{ .mcv = base_mcv };
3445 const index_bind: ReadArg.Bind = .{ .inst = bin_op.rhs };
3446
3447 break :result try self.ptrElemVal(base_bind, index_bind, ptr_ty, inst);
3448 };
3449 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
3450}
3451
3452fn ptrElemVal(
3453 self: *Self,
3454 ptr_bind: ReadArg.Bind,
3455 index_bind: ReadArg.Bind,
3456 ptr_ty: Type,
3457 maybe_inst: ?Air.Inst.Index,
3458) !MCValue {
3459 const pt = self.pt;
3460 const zcu = pt.zcu;
3461 const elem_ty = ptr_ty.childType(zcu);
3462 const elem_size = @as(u32, @intCast(elem_ty.abiSize(zcu)));
3463
3464 // TODO optimize for elem_sizes of 1, 2, 4, 8
3465 switch (elem_size) {
3466 else => {
3467 const addr = try self.ptrArithmetic(.ptr_add, ptr_bind, index_bind, ptr_ty, Type.usize, null);
3468
3469 const dest = try self.allocRegOrMem(elem_ty, true, maybe_inst);
3470 try self.load(dest, addr, ptr_ty);
3471 return dest;
3472 },
3473 }
3474}
3475
3476fn airSliceElemPtr(self: *Self, inst: Air.Inst.Index) InnerError!void {
3477 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
3478 const extra = self.air.extraData(Air.Bin, ty_pl.payload).data;
3479 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
3480 const slice_mcv = try self.resolveInst(extra.lhs);
3481 const base_mcv = slicePtr(slice_mcv);
3482
3483 const base_bind: ReadArg.Bind = .{ .mcv = base_mcv };
3484 const index_bind: ReadArg.Bind = .{ .inst = extra.rhs };
3485
3486 const slice_ty = self.typeOf(extra.lhs);
3487 const index_ty = self.typeOf(extra.rhs);
3488
3489 const addr = try self.ptrArithmetic(.ptr_add, base_bind, index_bind, slice_ty, index_ty, null);
3490 break :result addr;
3491 };
3492 return self.finishAir(inst, result, .{ extra.lhs, extra.rhs, .none });
3493}
3494
3495fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) InnerError!void {
3496 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
3497 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement array_elem_val for {}", .{self.target.cpu.arch});
3498 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
3499}
3500
3501fn airPtrElemVal(self: *Self, inst: Air.Inst.Index) InnerError!void {
3502 const pt = self.pt;
3503 const zcu = pt.zcu;
3504 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
3505 const ptr_ty = self.typeOf(bin_op.lhs);
3506 const result: MCValue = if (!ptr_ty.isVolatilePtr(zcu) and self.liveness.isUnused(inst)) .dead else result: {
3507 const base_bind: ReadArg.Bind = .{ .inst = bin_op.lhs };
3508 const index_bind: ReadArg.Bind = .{ .inst = bin_op.rhs };
3509
3510 break :result try self.ptrElemVal(base_bind, index_bind, ptr_ty, inst);
3511 };
3512 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
3513}
3514
3515fn airPtrElemPtr(self: *Self, inst: Air.Inst.Index) InnerError!void {
3516 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
3517 const extra = self.air.extraData(Air.Bin, ty_pl.payload).data;
3518 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
3519 const ptr_bind: ReadArg.Bind = .{ .inst = extra.lhs };
3520 const index_bind: ReadArg.Bind = .{ .inst = extra.rhs };
3521
3522 const ptr_ty = self.typeOf(extra.lhs);
3523 const index_ty = self.typeOf(extra.rhs);
3524
3525 const addr = try self.ptrArithmetic(.ptr_add, ptr_bind, index_bind, ptr_ty, index_ty, null);
3526 break :result addr;
3527 };
3528 return self.finishAir(inst, result, .{ extra.lhs, extra.rhs, .none });
3529}
3530
3531fn airSetUnionTag(self: *Self, inst: Air.Inst.Index) InnerError!void {
3532 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
3533 _ = bin_op;
3534 return self.fail("TODO implement airSetUnionTag for {}", .{self.target.cpu.arch});
3535}
3536
3537fn airGetUnionTag(self: *Self, inst: Air.Inst.Index) InnerError!void {
3538 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
3539 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement airGetUnionTag for {}", .{self.target.cpu.arch});
3540 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
3541}
3542
3543fn airClz(self: *Self, inst: Air.Inst.Index) InnerError!void {
3544 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
3545 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement airClz for {}", .{self.target.cpu.arch});
3546 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
3547}
3548
3549fn airCtz(self: *Self, inst: Air.Inst.Index) InnerError!void {
3550 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
3551 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement airCtz for {}", .{self.target.cpu.arch});
3552 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
3553}
3554
3555fn airPopcount(self: *Self, inst: Air.Inst.Index) InnerError!void {
3556 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
3557 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement airPopcount for {}", .{self.target.cpu.arch});
3558 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
3559}
3560
3561fn airAbs(self: *Self, inst: Air.Inst.Index) InnerError!void {
3562 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
3563 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement airAbs for {}", .{self.target.cpu.arch});
3564 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
3565}
3566
3567fn airByteSwap(self: *Self, inst: Air.Inst.Index) InnerError!void {
3568 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
3569 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement airByteSwap for {}", .{self.target.cpu.arch});
3570 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
3571}
3572
3573fn airBitReverse(self: *Self, inst: Air.Inst.Index) InnerError!void {
3574 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
3575 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement airBitReverse for {}", .{self.target.cpu.arch});
3576 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
3577}
3578
3579fn airUnaryMath(self: *Self, inst: Air.Inst.Index) InnerError!void {
3580 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
3581 const result: MCValue = if (self.liveness.isUnused(inst))
3582 .dead
3583 else
3584 return self.fail("TODO implement airUnaryMath for {}", .{self.target.cpu.arch});
3585 return self.finishAir(inst, result, .{ un_op, .none, .none });
3586}
3587
3588fn reuseOperand(
3589 self: *Self,
3590 inst: Air.Inst.Index,
3591 operand: Air.Inst.Ref,
3592 op_index: Air.Liveness.OperandInt,
3593 mcv: MCValue,
3594) bool {
3595 if (!self.liveness.operandDies(inst, op_index))
3596 return false;
3597
3598 switch (mcv) {
3599 .register => |reg| {
3600 // If it's in the registers table, need to associate the register with the
3601 // new instruction.
3602 if (RegisterManager.indexOfRegIntoTracked(reg)) |index| {
3603 if (!self.register_manager.isRegFree(reg)) {
3604 self.register_manager.registers[index] = inst;
3605 }
3606 }
3607 log.debug("%{d} => {} (reused)", .{ inst, reg });
3608 },
3609 .stack_offset => |off| {
3610 log.debug("%{d} => stack offset {d} (reused)", .{ inst, off });
3611 },
3612 else => return false,
3613 }
3614
3615 // Prevent the operand deaths processing code from deallocating it.
3616 self.reused_operands.set(op_index);
3617
3618 // That makes us responsible for doing the rest of the stuff that processDeath would have done.
3619 const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
3620 branch.inst_table.putAssumeCapacity(operand.toIndex().?, .dead);
3621
3622 return true;
3623}
3624
3625fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!void {
3626 const pt = self.pt;
3627 const zcu = pt.zcu;
3628 const elem_ty = ptr_ty.childType(zcu);
3629 const elem_size = elem_ty.abiSize(zcu);
3630
3631 switch (ptr) {
3632 .none => unreachable,
3633 .undef => unreachable,
3634 .unreach => unreachable,
3635 .dead => unreachable,
3636 .compare_flags,
3637 .register_with_overflow,
3638 => unreachable, // cannot hold an address
3639 .immediate => |imm| try self.setRegOrMem(elem_ty, dst_mcv, .{ .memory = imm }),
3640 .ptr_stack_offset => |off| try self.setRegOrMem(elem_ty, dst_mcv, .{ .stack_offset = off }),
3641 .register => |addr_reg| {
3642 const addr_reg_lock = self.register_manager.lockReg(addr_reg);
3643 defer if (addr_reg_lock) |reg| self.register_manager.unlockReg(reg);
3644
3645 switch (dst_mcv) {
3646 .dead => unreachable,
3647 .undef => unreachable,
3648 .compare_flags => unreachable,
3649 .register => |dst_reg| {
3650 try self.genLdrRegister(dst_reg, addr_reg, elem_ty);
3651 },
3652 .stack_offset => |off| {
3653 if (elem_size <= 8) {
3654 const raw_tmp_reg = try self.register_manager.allocReg(null, gp);
3655 const tmp_reg = self.registerAlias(raw_tmp_reg, elem_ty);
3656 const tmp_reg_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
3657 defer self.register_manager.unlockReg(tmp_reg_lock);
3658
3659 try self.load(.{ .register = tmp_reg }, ptr, ptr_ty);
3660 try self.genSetStack(elem_ty, off, MCValue{ .register = tmp_reg });
3661 } else {
3662 // TODO optimize the register allocation
3663 const regs = try self.register_manager.allocRegs(4, .{ null, null, null, null }, gp);
3664 const regs_locks = self.register_manager.lockRegsAssumeUnused(4, regs);
3665 defer for (regs_locks) |reg| {
3666 self.register_manager.unlockReg(reg);
3667 };
3668
3669 const src_reg = addr_reg;
3670 const dst_reg = regs[0];
3671 const len_reg = regs[1];
3672 const count_reg = regs[2];
3673 const tmp_reg = regs[3];
3674
3675 // sub dst_reg, fp, #off
3676 try self.genSetReg(ptr_ty, dst_reg, .{ .ptr_stack_offset = off });
3677
3678 // mov len, #elem_size
3679 try self.genSetReg(Type.usize, len_reg, .{ .immediate = elem_size });
3680
3681 // memcpy(src, dst, len)
3682 try self.genInlineMemcpy(src_reg, dst_reg, len_reg, count_reg, tmp_reg);
3683 }
3684 },
3685 else => return self.fail("TODO load from register into {}", .{dst_mcv}),
3686 }
3687 },
3688 .memory,
3689 .stack_offset,
3690 .stack_argument_offset,
3691 .linker_load,
3692 => {
3693 const addr_reg = try self.copyToTmpRegister(ptr_ty, ptr);
3694 try self.load(dst_mcv, .{ .register = addr_reg }, ptr_ty);
3695 },
3696 }
3697}
3698
3699fn genInlineMemcpy(
3700 self: *Self,
3701 src: Register,
3702 dst: Register,
3703 len: Register,
3704 count: Register,
3705 tmp: Register,
3706) !void {
3707 // movz count, #0
3708 _ = try self.addInst(.{
3709 .tag = .movz,
3710 .data = .{ .r_imm16_sh = .{
3711 .rd = count,
3712 .imm16 = 0,
3713 } },
3714 });
3715
3716 // loop:
3717 // cmp count, len
3718 _ = try self.addInst(.{
3719 .tag = .cmp_shifted_register,
3720 .data = .{ .rr_imm6_shift = .{
3721 .rn = count,
3722 .rm = len,
3723 .imm6 = 0,
3724 .shift = .lsl,
3725 } },
3726 });
3727
3728 // bge end
3729 _ = try self.addInst(.{
3730 .tag = .b_cond,
3731 .data = .{ .inst_cond = .{
3732 .inst = @as(u32, @intCast(self.mir_instructions.len + 5)),
3733 .cond = .ge,
3734 } },
3735 });
3736
3737 // ldrb tmp, [src, count]
3738 _ = try self.addInst(.{
3739 .tag = .ldrb_register,
3740 .data = .{ .load_store_register_register = .{
3741 .rt = tmp,
3742 .rn = src,
3743 .offset = Instruction.LoadStoreOffset.reg(count).register,
3744 } },
3745 });
3746
3747 // strb tmp, [dest, count]
3748 _ = try self.addInst(.{
3749 .tag = .strb_register,
3750 .data = .{ .load_store_register_register = .{
3751 .rt = tmp,
3752 .rn = dst,
3753 .offset = Instruction.LoadStoreOffset.reg(count).register,
3754 } },
3755 });
3756
3757 // add count, count, #1
3758 _ = try self.addInst(.{
3759 .tag = .add_immediate,
3760 .data = .{ .rr_imm12_sh = .{
3761 .rd = count,
3762 .rn = count,
3763 .imm12 = 1,
3764 } },
3765 });
3766
3767 // b loop
3768 _ = try self.addInst(.{
3769 .tag = .b,
3770 .data = .{ .inst = @as(u32, @intCast(self.mir_instructions.len - 5)) },
3771 });
3772
3773 // end:
3774}
3775
3776fn genInlineMemset(
3777 self: *Self,
3778 dst: MCValue,
3779 val: MCValue,
3780 len: MCValue,
3781) !void {
3782 const dst_reg = switch (dst) {
3783 .register => |r| r,
3784 else => try self.copyToTmpRegister(Type.manyptr_u8, dst),
3785 };
3786 const dst_reg_lock = self.register_manager.lockReg(dst_reg);
3787 defer if (dst_reg_lock) |lock| self.register_manager.unlockReg(lock);
3788
3789 const val_reg = switch (val) {
3790 .register => |r| r,
3791 else => try self.copyToTmpRegister(Type.u8, val),
3792 };
3793 const val_reg_lock = self.register_manager.lockReg(val_reg);
3794 defer if (val_reg_lock) |lock| self.register_manager.unlockReg(lock);
3795
3796 const len_reg = switch (len) {
3797 .register => |r| r,
3798 else => try self.copyToTmpRegister(Type.usize, len),
3799 };
3800 const len_reg_lock = self.register_manager.lockReg(len_reg);
3801 defer if (len_reg_lock) |lock| self.register_manager.unlockReg(lock);
3802
3803 const count_reg = try self.register_manager.allocReg(null, gp);
3804
3805 try self.genInlineMemsetCode(dst_reg, val_reg, len_reg, count_reg);
3806}
3807
3808fn genInlineMemsetCode(
3809 self: *Self,
3810 dst: Register,
3811 val: Register,
3812 len: Register,
3813 count: Register,
3814) !void {
3815 // mov count, #0
3816 _ = try self.addInst(.{
3817 .tag = .movz,
3818 .data = .{ .r_imm16_sh = .{
3819 .rd = count,
3820 .imm16 = 0,
3821 } },
3822 });
3823
3824 // loop:
3825 // cmp count, len
3826 _ = try self.addInst(.{
3827 .tag = .cmp_shifted_register,
3828 .data = .{ .rr_imm6_shift = .{
3829 .rn = count,
3830 .rm = len,
3831 .imm6 = 0,
3832 .shift = .lsl,
3833 } },
3834 });
3835
3836 // bge end
3837 _ = try self.addInst(.{
3838 .tag = .b_cond,
3839 .data = .{ .inst_cond = .{
3840 .inst = @as(u32, @intCast(self.mir_instructions.len + 4)),
3841 .cond = .ge,
3842 } },
3843 });
3844
3845 // strb val, [src, count]
3846 _ = try self.addInst(.{
3847 .tag = .strb_register,
3848 .data = .{ .load_store_register_register = .{
3849 .rt = val,
3850 .rn = dst,
3851 .offset = Instruction.LoadStoreOffset.reg(count).register,
3852 } },
3853 });
3854
3855 // add count, count, #1
3856 _ = try self.addInst(.{
3857 .tag = .add_immediate,
3858 .data = .{ .rr_imm12_sh = .{
3859 .rd = count,
3860 .rn = count,
3861 .imm12 = 1,
3862 } },
3863 });
3864
3865 // b loop
3866 _ = try self.addInst(.{
3867 .tag = .b,
3868 .data = .{ .inst = @as(u32, @intCast(self.mir_instructions.len - 4)) },
3869 });
3870
3871 // end:
3872}
3873
3874fn airLoad(self: *Self, inst: Air.Inst.Index) InnerError!void {
3875 const pt = self.pt;
3876 const zcu = pt.zcu;
3877 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
3878 const elem_ty = self.typeOfIndex(inst);
3879 const elem_size = elem_ty.abiSize(zcu);
3880 const result: MCValue = result: {
3881 if (!elem_ty.hasRuntimeBits(zcu))
3882 break :result MCValue.none;
3883
3884 const ptr = try self.resolveInst(ty_op.operand);
3885 const is_volatile = self.typeOf(ty_op.operand).isVolatilePtr(zcu);
3886 if (self.liveness.isUnused(inst) and !is_volatile)
3887 break :result MCValue.dead;
3888
3889 const dst_mcv: MCValue = blk: {
3890 if (elem_size <= 8 and self.reuseOperand(inst, ty_op.operand, 0, ptr)) {
3891 // The MCValue that holds the pointer can be re-used as the value.
3892 break :blk switch (ptr) {
3893 .register => |reg| MCValue{ .register = self.registerAlias(reg, elem_ty) },
3894 else => ptr,
3895 };
3896 } else {
3897 break :blk try self.allocRegOrMem(elem_ty, true, inst);
3898 }
3899 };
3900 try self.load(dst_mcv, ptr, self.typeOf(ty_op.operand));
3901 break :result dst_mcv;
3902 };
3903 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
3904}
3905
3906fn genLdrRegister(self: *Self, value_reg: Register, addr_reg: Register, ty: Type) !void {
3907 const pt = self.pt;
3908 const zcu = pt.zcu;
3909 const abi_size = ty.abiSize(zcu);
3910
3911 const tag: Mir.Inst.Tag = switch (abi_size) {
3912 1 => if (ty.isSignedInt(zcu)) Mir.Inst.Tag.ldrsb_immediate else .ldrb_immediate,
3913 2 => if (ty.isSignedInt(zcu)) Mir.Inst.Tag.ldrsh_immediate else .ldrh_immediate,
3914 4 => .ldr_immediate,
3915 8 => .ldr_immediate,
3916 3, 5, 6, 7 => return self.fail("TODO: genLdrRegister for more abi_sizes", .{}),
3917 else => unreachable,
3918 };
3919
3920 _ = try self.addInst(.{
3921 .tag = tag,
3922 .data = .{ .load_store_register_immediate = .{
3923 .rt = value_reg,
3924 .rn = addr_reg,
3925 .offset = Instruction.LoadStoreOffset.none.immediate,
3926 } },
3927 });
3928}
3929
3930fn genStrRegister(self: *Self, value_reg: Register, addr_reg: Register, ty: Type) !void {
3931 const pt = self.pt;
3932 const abi_size = ty.abiSize(pt.zcu);
3933
3934 const tag: Mir.Inst.Tag = switch (abi_size) {
3935 1 => .strb_immediate,
3936 2 => .strh_immediate,
3937 4, 8 => .str_immediate,
3938 3, 5, 6, 7 => return self.fail("TODO: genStrRegister for more abi_sizes", .{}),
3939 else => unreachable,
3940 };
3941
3942 _ = try self.addInst(.{
3943 .tag = tag,
3944 .data = .{ .load_store_register_immediate = .{
3945 .rt = value_reg,
3946 .rn = addr_reg,
3947 .offset = Instruction.LoadStoreOffset.none.immediate,
3948 } },
3949 });
3950}
3951
3952fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type) InnerError!void {
3953 const pt = self.pt;
3954 log.debug("store: storing {} to {}", .{ value, ptr });
3955 const abi_size = value_ty.abiSize(pt.zcu);
3956
3957 switch (ptr) {
3958 .none => unreachable,
3959 .undef => unreachable,
3960 .unreach => unreachable,
3961 .dead => unreachable,
3962 .compare_flags,
3963 .register_with_overflow,
3964 => unreachable, // cannot hold an address
3965 .immediate => |imm| {
3966 try self.setRegOrMem(value_ty, .{ .memory = imm }, value);
3967 },
3968 .ptr_stack_offset => |off| {
3969 try self.genSetStack(value_ty, off, value);
3970 },
3971 .register => |addr_reg| {
3972 const addr_reg_lock = self.register_manager.lockReg(addr_reg);
3973 defer if (addr_reg_lock) |reg| self.register_manager.unlockReg(reg);
3974
3975 switch (value) {
3976 .dead => unreachable,
3977 .undef => {
3978 try self.genSetReg(value_ty, addr_reg, value);
3979 },
3980 .register => |value_reg| {
3981 log.debug("store: register {} to {}", .{ value_reg, addr_reg });
3982 try self.genStrRegister(value_reg, addr_reg, value_ty);
3983 },
3984 else => {
3985 if (abi_size <= 8) {
3986 const raw_tmp_reg = try self.register_manager.allocReg(null, gp);
3987 const tmp_reg = self.registerAlias(raw_tmp_reg, value_ty);
3988 const tmp_reg_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
3989 defer self.register_manager.unlockReg(tmp_reg_lock);
3990
3991 try self.genSetReg(value_ty, tmp_reg, value);
3992 try self.store(ptr, .{ .register = tmp_reg }, ptr_ty, value_ty);
3993 } else {
3994 const regs = try self.register_manager.allocRegs(4, .{ null, null, null, null }, gp);
3995 const regs_locks = self.register_manager.lockRegsAssumeUnused(4, regs);
3996 defer for (regs_locks) |reg| {
3997 self.register_manager.unlockReg(reg);
3998 };
3999
4000 const src_reg = regs[0];
4001 const dst_reg = addr_reg;
4002 const len_reg = regs[1];
4003 const count_reg = regs[2];
4004 const tmp_reg = regs[3];
4005
4006 switch (value) {
4007 .stack_offset => |off| {
4008 // sub src_reg, fp, #off
4009 try self.genSetReg(ptr_ty, src_reg, .{ .ptr_stack_offset = off });
4010 },
4011 .stack_argument_offset => |off| {
4012 _ = try self.addInst(.{
4013 .tag = .ldr_ptr_stack_argument,
4014 .data = .{ .load_store_stack = .{
4015 .rt = src_reg,
4016 .offset = off,
4017 } },
4018 });
4019 },
4020 .memory => |addr| try self.genSetReg(Type.usize, src_reg, .{ .immediate = @as(u32, @intCast(addr)) }),
4021 .linker_load => |load_struct| {
4022 const tag: Mir.Inst.Tag = switch (load_struct.type) {
4023 .got => .load_memory_ptr_got,
4024 .direct => .load_memory_ptr_direct,
4025 .import => unreachable,
4026 };
4027 const atom_index = switch (self.bin_file.tag) {
4028 .macho => {
4029 // const macho_file = self.bin_file.cast(link.File.MachO).?;
4030 // const atom = try macho_file.getOrCreateAtomForDecl(self.owner_decl);
4031 // break :blk macho_file.getAtom(atom).getSymbolIndex().?;
4032 @panic("TODO store");
4033 },
4034 .coff => blk: {
4035 const coff_file = self.bin_file.cast(.coff).?;
4036 const atom = try coff_file.getOrCreateAtomForNav(self.owner_nav);
4037 break :blk coff_file.getAtom(atom).getSymbolIndex().?;
4038 },
4039 else => unreachable, // unsupported target format
4040 };
4041 _ = try self.addInst(.{
4042 .tag = tag,
4043 .data = .{
4044 .payload = try self.addExtra(Mir.LoadMemoryPie{
4045 .register = @intFromEnum(src_reg),
4046 .atom_index = atom_index,
4047 .sym_index = load_struct.sym_index,
4048 }),
4049 },
4050 });
4051 },
4052 else => return self.fail("TODO store {} to register", .{value}),
4053 }
4054
4055 // mov len, #abi_size
4056 try self.genSetReg(Type.usize, len_reg, .{ .immediate = abi_size });
4057
4058 // memcpy(src, dst, len)
4059 try self.genInlineMemcpy(src_reg, dst_reg, len_reg, count_reg, tmp_reg);
4060 }
4061 },
4062 }
4063 },
4064 .memory,
4065 .stack_offset,
4066 .stack_argument_offset,
4067 .linker_load,
4068 => {
4069 const addr_reg = try self.copyToTmpRegister(ptr_ty, ptr);
4070 try self.store(.{ .register = addr_reg }, value, ptr_ty, value_ty);
4071 },
4072 }
4073}
4074
4075fn airStore(self: *Self, inst: Air.Inst.Index, safety: bool) InnerError!void {
4076 if (safety) {
4077 // TODO if the value is undef, write 0xaa bytes to dest
4078 } else {
4079 // TODO if the value is undef, don't lower this instruction
4080 }
4081 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
4082 const ptr = try self.resolveInst(bin_op.lhs);
4083 const value = try self.resolveInst(bin_op.rhs);
4084 const ptr_ty = self.typeOf(bin_op.lhs);
4085 const value_ty = self.typeOf(bin_op.rhs);
4086
4087 try self.store(ptr, value, ptr_ty, value_ty);
4088
4089 return self.finishAir(inst, .dead, .{ bin_op.lhs, bin_op.rhs, .none });
4090}
4091
4092fn airStructFieldPtr(self: *Self, inst: Air.Inst.Index) InnerError!void {
4093 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
4094 const extra = self.air.extraData(Air.StructField, ty_pl.payload).data;
4095 const result = try self.structFieldPtr(inst, extra.struct_operand, extra.field_index);
4096 return self.finishAir(inst, result, .{ extra.struct_operand, .none, .none });
4097}
4098
4099fn airStructFieldPtrIndex(self: *Self, inst: Air.Inst.Index, index: u8) InnerError!void {
4100 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
4101 const result = try self.structFieldPtr(inst, ty_op.operand, index);
4102 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
4103}
4104
4105fn structFieldPtr(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, index: u32) !MCValue {
4106 return if (self.liveness.isUnused(inst)) .dead else result: {
4107 const pt = self.pt;
4108 const zcu = pt.zcu;
4109 const mcv = try self.resolveInst(operand);
4110 const ptr_ty = self.typeOf(operand);
4111 const struct_ty = ptr_ty.childType(zcu);
4112 const struct_field_offset = @as(u32, @intCast(struct_ty.structFieldOffset(index, zcu)));
4113 switch (mcv) {
4114 .ptr_stack_offset => |off| {
4115 break :result MCValue{ .ptr_stack_offset = off - struct_field_offset };
4116 },
4117 else => {
4118 const lhs_bind: ReadArg.Bind = .{ .mcv = mcv };
4119 const rhs_bind: ReadArg.Bind = .{ .mcv = .{ .immediate = struct_field_offset } };
4120
4121 break :result try self.addSub(.add, lhs_bind, rhs_bind, Type.usize, Type.usize, null);
4122 },
4123 }
4124 };
4125}
4126
4127fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) InnerError!void {
4128 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
4129 const extra = self.air.extraData(Air.StructField, ty_pl.payload).data;
4130 const operand = extra.struct_operand;
4131 const index = extra.field_index;
4132 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
4133 const pt = self.pt;
4134 const zcu = pt.zcu;
4135 const mcv = try self.resolveInst(operand);
4136 const struct_ty = self.typeOf(operand);
4137 const struct_field_ty = struct_ty.fieldType(index, zcu);
4138 const struct_field_offset = @as(u32, @intCast(struct_ty.structFieldOffset(index, zcu)));
4139
4140 switch (mcv) {
4141 .dead, .unreach => unreachable,
4142 .stack_argument_offset => |off| {
4143 break :result MCValue{ .stack_argument_offset = off + struct_field_offset };
4144 },
4145 .stack_offset => |off| {
4146 break :result MCValue{ .stack_offset = off - struct_field_offset };
4147 },
4148 .memory => |addr| {
4149 break :result MCValue{ .memory = addr + struct_field_offset };
4150 },
4151 .register_with_overflow => |rwo| {
4152 const reg_lock = self.register_manager.lockRegAssumeUnused(rwo.reg);
4153 defer self.register_manager.unlockReg(reg_lock);
4154
4155 const field: MCValue = switch (index) {
4156 // get wrapped value: return register
4157 0 => MCValue{ .register = rwo.reg },
4158
4159 // get overflow bit: return C or V flag
4160 1 => MCValue{ .compare_flags = rwo.flag },
4161
4162 else => unreachable,
4163 };
4164
4165 if (self.reuseOperand(inst, operand, 0, field)) {
4166 break :result field;
4167 } else {
4168 // Copy to new register
4169 const raw_dest_reg = try self.register_manager.allocReg(null, gp);
4170 const dest_reg = self.registerAlias(raw_dest_reg, struct_field_ty);
4171 try self.genSetReg(struct_field_ty, dest_reg, field);
4172
4173 break :result MCValue{ .register = dest_reg };
4174 }
4175 },
4176 else => return self.fail("TODO implement codegen struct_field_val for {}", .{mcv}),
4177 }
4178 };
4179
4180 return self.finishAir(inst, result, .{ extra.struct_operand, .none, .none });
4181}
4182
4183fn airFieldParentPtr(self: *Self, inst: Air.Inst.Index) InnerError!void {
4184 const pt = self.pt;
4185 const zcu = pt.zcu;
4186 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
4187 const extra = self.air.extraData(Air.FieldParentPtr, ty_pl.payload).data;
4188 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
4189 const field_ptr = try self.resolveInst(extra.field_ptr);
4190 const struct_ty = ty_pl.ty.toType().childType(zcu);
4191 const struct_field_offset = @as(u32, @intCast(struct_ty.structFieldOffset(extra.field_index, zcu)));
4192 switch (field_ptr) {
4193 .ptr_stack_offset => |off| {
4194 break :result MCValue{ .ptr_stack_offset = off + struct_field_offset };
4195 },
4196 else => {
4197 const lhs_bind: ReadArg.Bind = .{ .mcv = field_ptr };
4198 const rhs_bind: ReadArg.Bind = .{ .mcv = .{ .immediate = struct_field_offset } };
4199
4200 break :result try self.addSub(.sub, lhs_bind, rhs_bind, Type.usize, Type.usize, null);
4201 },
4202 }
4203 };
4204 return self.finishAir(inst, result, .{ extra.field_ptr, .none, .none });
4205}
4206
4207fn airArg(self: *Self, inst: Air.Inst.Index) InnerError!void {
4208 // skip zero-bit arguments as they don't have a corresponding arg instruction
4209 var arg_index = self.arg_index;
4210 while (self.args[arg_index] == .none) arg_index += 1;
4211 self.arg_index = arg_index + 1;
4212
4213 const zcu = self.pt.zcu;
4214 const func_zir = zcu.funcInfo(self.func_index).zir_body_inst.resolveFull(&zcu.intern_pool).?;
4215 const file = zcu.fileByIndex(func_zir.file);
4216 if (!file.mod.?.strip) {
4217 const tag = self.air.instructions.items(.tag)[@intFromEnum(inst)];
4218 const arg = self.air.instructions.items(.data)[@intFromEnum(inst)].arg;
4219 const ty = self.typeOfIndex(inst);
4220 const zir = &file.zir.?;
4221 const name = zir.nullTerminatedString(zir.getParamName(zir.getParamBody(func_zir.inst)[arg.zir_param_index]).?);
4222 try self.dbg_info_relocs.append(self.gpa, .{
4223 .tag = tag,
4224 .ty = ty,
4225 .name = name,
4226 .mcv = self.args[arg_index],
4227 });
4228 }
4229
4230 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else self.args[arg_index];
4231 return self.finishAir(inst, result, .{ .none, .none, .none });
4232}
4233
4234fn airTrap(self: *Self) InnerError!void {
4235 _ = try self.addInst(.{
4236 .tag = .brk,
4237 .data = .{ .imm16 = 0x0001 },
4238 });
4239 return self.finishAirBookkeeping();
4240}
4241
4242fn airBreakpoint(self: *Self) InnerError!void {
4243 _ = try self.addInst(.{
4244 .tag = .brk,
4245 .data = .{ .imm16 = 0xf000 },
4246 });
4247 return self.finishAirBookkeeping();
4248}
4249
4250fn airRetAddr(self: *Self, inst: Air.Inst.Index) InnerError!void {
4251 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement airRetAddr for aarch64", .{});
4252 return self.finishAir(inst, result, .{ .none, .none, .none });
4253}
4254
4255fn airFrameAddress(self: *Self, inst: Air.Inst.Index) InnerError!void {
4256 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement airFrameAddress for aarch64", .{});
4257 return self.finishAir(inst, result, .{ .none, .none, .none });
4258}
4259
4260fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier) InnerError!void {
4261 if (modifier == .always_tail) return self.fail("TODO implement tail calls for aarch64", .{});
4262 const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
4263 const callee = pl_op.operand;
4264 const extra = self.air.extraData(Air.Call, pl_op.payload);
4265 const args: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[extra.end..][0..extra.data.args_len]);
4266 const ty = self.typeOf(callee);
4267 const pt = self.pt;
4268 const zcu = pt.zcu;
4269 const ip = &zcu.intern_pool;
4270
4271 const fn_ty = switch (ty.zigTypeTag(zcu)) {
4272 .@"fn" => ty,
4273 .pointer => ty.childType(zcu),
4274 else => unreachable,
4275 };
4276
4277 var info = try self.resolveCallingConventionValues(fn_ty);
4278 defer info.deinit(self);
4279
4280 // According to the Procedure Call Standard for the ARM
4281 // Architecture, compare flags are not preserved across
4282 // calls. Therefore, if some value is currently stored there, we
4283 // need to save it.
4284 //
4285 // TODO once caller-saved registers are implemented, save them
4286 // here too, but crucially *after* we save the compare flags as
4287 // saving compare flags may require a new caller-saved register
4288 try self.spillCompareFlagsIfOccupied();
4289
4290 if (info.return_value == .stack_offset) {
4291 log.debug("airCall: return by reference", .{});
4292 const ret_ty = fn_ty.fnReturnType(zcu);
4293 const ret_abi_size: u32 = @intCast(ret_ty.abiSize(zcu));
4294 const ret_abi_align = ret_ty.abiAlignment(zcu);
4295 const stack_offset = try self.allocMem(ret_abi_size, ret_abi_align, inst);
4296
4297 const ret_ptr_reg = self.registerAlias(.x0, Type.usize);
4298
4299 const ptr_ty = try pt.singleMutPtrType(ret_ty);
4300 try self.register_manager.getReg(ret_ptr_reg, null);
4301 try self.genSetReg(ptr_ty, ret_ptr_reg, .{ .ptr_stack_offset = stack_offset });
4302
4303 info.return_value = .{ .stack_offset = stack_offset };
4304 }
4305
4306 // Make space for the arguments passed via the stack
4307 self.max_end_stack += info.stack_byte_count;
4308
4309 for (info.args, 0..) |mc_arg, arg_i| {
4310 const arg = args[arg_i];
4311 const arg_ty = self.typeOf(arg);
4312 const arg_mcv = try self.resolveInst(args[arg_i]);
4313
4314 switch (mc_arg) {
4315 .none => continue,
4316 .register => |reg| {
4317 try self.register_manager.getReg(reg, null);
4318 try self.genSetReg(arg_ty, reg, arg_mcv);
4319 },
4320 .stack_offset => unreachable,
4321 .stack_argument_offset => |offset| try self.genSetStackArgument(
4322 arg_ty,
4323 offset,
4324 arg_mcv,
4325 ),
4326 else => unreachable,
4327 }
4328 }
4329
4330 // Due to incremental compilation, how function calls are generated depends
4331 // on linking.
4332 if (try self.air.value(callee, pt)) |func_value| switch (ip.indexToKey(func_value.toIntern())) {
4333 .func => |func| {
4334 if (self.bin_file.cast(.elf)) |_| {
4335 return self.fail("TODO implement calling functions for Elf", .{});
4336 } else if (self.bin_file.cast(.macho)) |_| {
4337 return self.fail("TODO implement calling functions for MachO", .{});
4338 } else if (self.bin_file.cast(.coff)) |coff_file| {
4339 const atom = try coff_file.getOrCreateAtomForNav(func.owner_nav);
4340 const sym_index = coff_file.getAtom(atom).getSymbolIndex().?;
4341 try self.genSetReg(Type.u64, .x30, .{
4342 .linker_load = .{
4343 .type = .got,
4344 .sym_index = sym_index,
4345 },
4346 });
4347 } else if (self.bin_file.cast(.plan9)) |p9| {
4348 const atom_index = try p9.seeNav(pt, func.owner_nav);
4349 const atom = p9.getAtom(atom_index);
4350 try self.genSetReg(Type.usize, .x30, .{ .memory = atom.getOffsetTableAddress(p9) });
4351 } else unreachable;
4352
4353 _ = try self.addInst(.{
4354 .tag = .blr,
4355 .data = .{ .reg = .x30 },
4356 });
4357 },
4358 .@"extern" => |@"extern"| {
4359 const nav_name = ip.getNav(@"extern".owner_nav).name.toSlice(ip);
4360 const lib_name = @"extern".lib_name.toSlice(ip);
4361 if (self.bin_file.cast(.macho)) |_| {
4362 return self.fail("TODO implement calling extern functions for MachO", .{});
4363 } else if (self.bin_file.cast(.coff)) |coff_file| {
4364 const sym_index = try coff_file.getGlobalSymbol(nav_name, lib_name);
4365 try self.genSetReg(Type.u64, .x30, .{
4366 .linker_load = .{
4367 .type = .import,
4368 .sym_index = sym_index,
4369 },
4370 });
4371 _ = try self.addInst(.{
4372 .tag = .blr,
4373 .data = .{ .reg = .x30 },
4374 });
4375 } else {
4376 return self.fail("TODO implement calling extern functions", .{});
4377 }
4378 },
4379 else => return self.fail("TODO implement calling bitcasted functions", .{}),
4380 } else {
4381 assert(ty.zigTypeTag(zcu) == .pointer);
4382 const mcv = try self.resolveInst(callee);
4383 try self.genSetReg(ty, .x30, mcv);
4384
4385 _ = try self.addInst(.{
4386 .tag = .blr,
4387 .data = .{ .reg = .x30 },
4388 });
4389 }
4390
4391 const result: MCValue = result: {
4392 switch (info.return_value) {
4393 .register => |reg| {
4394 if (RegisterManager.indexOfReg(&callee_preserved_regs, reg) == null) {
4395 // Save function return value in a callee saved register
4396 break :result try self.copyToNewRegister(inst, info.return_value);
4397 }
4398 },
4399 else => {},
4400 }
4401 break :result info.return_value;
4402 };
4403
4404 if (args.len + 1 <= Air.Liveness.bpi - 1) {
4405 var buf = [1]Air.Inst.Ref{.none} ** (Air.Liveness.bpi - 1);
4406 buf[0] = callee;
4407 @memcpy(buf[1..][0..args.len], args);
4408 return self.finishAir(inst, result, buf);
4409 }
4410 var bt = try self.iterateBigTomb(inst, 1 + args.len);
4411 bt.feed(callee);
4412 for (args) |arg| {
4413 bt.feed(arg);
4414 }
4415 return bt.finishAir(result);
4416}
4417
4418fn airRet(self: *Self, inst: Air.Inst.Index) InnerError!void {
4419 const pt = self.pt;
4420 const zcu = pt.zcu;
4421 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
4422 const operand = try self.resolveInst(un_op);
4423 const ret_ty = self.fn_type.fnReturnType(zcu);
4424
4425 switch (self.ret_mcv) {
4426 .none => {},
4427 .immediate => {
4428 assert(ret_ty.isError(zcu));
4429 },
4430 .register => |reg| {
4431 // Return result by value
4432 try self.genSetReg(ret_ty, reg, operand);
4433 },
4434 .stack_offset => {
4435 // Return result by reference
4436 //
4437 // self.ret_mcv is an address to where this function
4438 // should store its result into
4439 const ptr_ty = try pt.singleMutPtrType(ret_ty);
4440 try self.store(self.ret_mcv, operand, ptr_ty, ret_ty);
4441 },
4442 else => unreachable,
4443 }
4444
4445 // Just add space for an instruction, patch this later
4446 try self.exitlude_jump_relocs.append(self.gpa, try self.addNop());
4447
4448 return self.finishAir(inst, .dead, .{ un_op, .none, .none });
4449}
4450
4451fn airRetLoad(self: *Self, inst: Air.Inst.Index) InnerError!void {
4452 const pt = self.pt;
4453 const zcu = pt.zcu;
4454 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
4455 const ptr = try self.resolveInst(un_op);
4456 const ptr_ty = self.typeOf(un_op);
4457 const ret_ty = self.fn_type.fnReturnType(zcu);
4458
4459 switch (self.ret_mcv) {
4460 .none => {},
4461 .register => {
4462 // Return result by value
4463 try self.load(self.ret_mcv, ptr, ptr_ty);
4464 },
4465 .stack_offset => {
4466 // Return result by reference
4467 //
4468 // self.ret_mcv is an address to where this function
4469 // should store its result into
4470 //
4471 // If the operand is a ret_ptr instruction, we are done
4472 // here. Else we need to load the result from the location
4473 // pointed to by the operand and store it to the result
4474 // location.
4475 const op_inst = un_op.toIndex().?;
4476 if (self.air.instructions.items(.tag)[@intFromEnum(op_inst)] != .ret_ptr) {
4477 const abi_size = @as(u32, @intCast(ret_ty.abiSize(zcu)));
4478 const abi_align = ret_ty.abiAlignment(zcu);
4479
4480 const offset = try self.allocMem(abi_size, abi_align, null);
4481
4482 const tmp_mcv = MCValue{ .stack_offset = offset };
4483 try self.load(tmp_mcv, ptr, ptr_ty);
4484 try self.store(self.ret_mcv, tmp_mcv, ptr_ty, ret_ty);
4485 }
4486 },
4487 else => unreachable, // invalid return result
4488 }
4489
4490 try self.exitlude_jump_relocs.append(self.gpa, try self.addNop());
4491
4492 return self.finishAir(inst, .dead, .{ un_op, .none, .none });
4493}
4494
4495fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) InnerError!void {
4496 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
4497 const lhs_ty = self.typeOf(bin_op.lhs);
4498
4499 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else blk: {
4500 break :blk try self.cmp(.{ .inst = bin_op.lhs }, .{ .inst = bin_op.rhs }, lhs_ty, op);
4501 };
4502
4503 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
4504}
4505
4506fn cmp(
4507 self: *Self,
4508 lhs: ReadArg.Bind,
4509 rhs: ReadArg.Bind,
4510 lhs_ty: Type,
4511 op: math.CompareOperator,
4512) !MCValue {
4513 const pt = self.pt;
4514 const zcu = pt.zcu;
4515 const int_ty = switch (lhs_ty.zigTypeTag(zcu)) {
4516 .optional => blk: {
4517 const payload_ty = lhs_ty.optionalChild(zcu);
4518 if (!payload_ty.hasRuntimeBitsIgnoreComptime(zcu)) {
4519 break :blk Type.u1;
4520 } else if (lhs_ty.isPtrLikeOptional(zcu)) {
4521 break :blk Type.usize;
4522 } else {
4523 return self.fail("TODO ARM cmp non-pointer optionals", .{});
4524 }
4525 },
4526 .float => return self.fail("TODO ARM cmp floats", .{}),
4527 .@"enum" => lhs_ty.intTagType(zcu),
4528 .int => lhs_ty,
4529 .bool => Type.u1,
4530 .pointer => Type.usize,
4531 .error_set => Type.u16,
4532 else => unreachable,
4533 };
4534
4535 const int_info = int_ty.intInfo(zcu);
4536 if (int_info.bits <= 64) {
4537 try self.spillCompareFlagsIfOccupied();
4538
4539 var lhs_reg: Register = undefined;
4540 var rhs_reg: Register = undefined;
4541
4542 const rhs_immediate = try rhs.resolveToImmediate(self);
4543 const rhs_immediate_ok = if (rhs_immediate) |imm| imm <= std.math.maxInt(u12) else false;
4544
4545 if (rhs_immediate_ok) {
4546 const read_args = [_]ReadArg{
4547 .{ .ty = int_ty, .bind = lhs, .class = gp, .reg = &lhs_reg },
4548 };
4549 try self.allocRegs(
4550 &read_args,
4551 &.{},
4552 null, // we won't be able to reuse a register as there are no write_regs
4553 );
4554
4555 _ = try self.addInst(.{
4556 .tag = .cmp_immediate,
4557 .data = .{ .r_imm12_sh = .{
4558 .rn = lhs_reg,
4559 .imm12 = @as(u12, @intCast(rhs_immediate.?)),
4560 } },
4561 });
4562 } else {
4563 const read_args = [_]ReadArg{
4564 .{ .ty = int_ty, .bind = lhs, .class = gp, .reg = &lhs_reg },
4565 .{ .ty = int_ty, .bind = rhs, .class = gp, .reg = &rhs_reg },
4566 };
4567 try self.allocRegs(
4568 &read_args,
4569 &.{},
4570 null, // we won't be able to reuse a register as there are no write_regs
4571 );
4572
4573 _ = try self.addInst(.{
4574 .tag = .cmp_shifted_register,
4575 .data = .{ .rr_imm6_shift = .{
4576 .rn = lhs_reg,
4577 .rm = rhs_reg,
4578 .imm6 = 0,
4579 .shift = .lsl,
4580 } },
4581 });
4582 }
4583
4584 return switch (int_info.signedness) {
4585 .signed => MCValue{ .compare_flags = Condition.fromCompareOperatorSigned(op) },
4586 .unsigned => MCValue{ .compare_flags = Condition.fromCompareOperatorUnsigned(op) },
4587 };
4588 } else {
4589 return self.fail("TODO AArch64 cmp for ints > 64 bits", .{});
4590 }
4591}
4592
4593fn airCmpVector(self: *Self, inst: Air.Inst.Index) InnerError!void {
4594 _ = inst;
4595 return self.fail("TODO implement airCmpVector for {}", .{self.target.cpu.arch});
4596}
4597
4598fn airCmpLtErrorsLen(self: *Self, inst: Air.Inst.Index) InnerError!void {
4599 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
4600 const operand = try self.resolveInst(un_op);
4601 _ = operand;
4602 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement airCmpLtErrorsLen for {}", .{self.target.cpu.arch});
4603 return self.finishAir(inst, result, .{ un_op, .none, .none });
4604}
4605
4606fn airDbgStmt(self: *Self, inst: Air.Inst.Index) InnerError!void {
4607 const dbg_stmt = self.air.instructions.items(.data)[@intFromEnum(inst)].dbg_stmt;
4608
4609 _ = try self.addInst(.{
4610 .tag = .dbg_line,
4611 .data = .{ .dbg_line_column = .{
4612 .line = dbg_stmt.line,
4613 .column = dbg_stmt.column,
4614 } },
4615 });
4616
4617 return self.finishAirBookkeeping();
4618}
4619
4620fn airDbgInlineBlock(self: *Self, inst: Air.Inst.Index) InnerError!void {
4621 const pt = self.pt;
4622 const zcu = pt.zcu;
4623 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
4624 const extra = self.air.extraData(Air.DbgInlineBlock, ty_pl.payload);
4625 const func = zcu.funcInfo(extra.data.func);
4626 // TODO emit debug info for function change
4627 _ = func;
4628 try self.lowerBlock(inst, @ptrCast(self.air.extra.items[extra.end..][0..extra.data.body_len]));
4629}
4630
4631fn airDbgVar(self: *Self, inst: Air.Inst.Index) InnerError!void {
4632 const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
4633 const operand = pl_op.operand;
4634 const tag = self.air.instructions.items(.tag)[@intFromEnum(inst)];
4635 const ty = self.typeOf(operand);
4636 const mcv = try self.resolveInst(operand);
4637 const name: Air.NullTerminatedString = @enumFromInt(pl_op.payload);
4638
4639 log.debug("airDbgVar: %{d}: {}, {}", .{ inst, ty.fmtDebug(), mcv });
4640
4641 try self.dbg_info_relocs.append(self.gpa, .{
4642 .tag = tag,
4643 .ty = ty,
4644 .name = name.toSlice(self.air),
4645 .mcv = mcv,
4646 });
4647
4648 return self.finishAir(inst, .dead, .{ operand, .none, .none });
4649}
4650
4651fn condBr(self: *Self, condition: MCValue) !Mir.Inst.Index {
4652 switch (condition) {
4653 .compare_flags => |cond| return try self.addInst(.{
4654 .tag = .b_cond,
4655 .data = .{
4656 .inst_cond = .{
4657 .inst = undefined, // populated later through performReloc
4658 // Here we map to the opposite condition because the jump is to the false branch.
4659 .cond = cond.negate(),
4660 },
4661 },
4662 }),
4663 else => {
4664 const reg = switch (condition) {
4665 .register => |r| r,
4666 else => try self.copyToTmpRegister(Type.bool, condition),
4667 };
4668
4669 return try self.addInst(.{
4670 .tag = .cbz,
4671 .data = .{
4672 .r_inst = .{
4673 .rt = reg,
4674 .inst = undefined, // populated later through performReloc
4675 },
4676 },
4677 });
4678 },
4679 }
4680}
4681
4682fn airCondBr(self: *Self, inst: Air.Inst.Index) InnerError!void {
4683 const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
4684 const cond = try self.resolveInst(pl_op.operand);
4685 const extra = self.air.extraData(Air.CondBr, pl_op.payload);
4686 const then_body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[extra.end..][0..extra.data.then_body_len]);
4687 const else_body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[extra.end + then_body.len ..][0..extra.data.else_body_len]);
4688 const liveness_condbr = self.liveness.getCondBr(inst);
4689
4690 const reloc = try self.condBr(cond);
4691
4692 // If the condition dies here in this condbr instruction, process
4693 // that death now instead of later as this has an effect on
4694 // whether it needs to be spilled in the branches
4695 if (self.liveness.operandDies(inst, 0)) {
4696 if (pl_op.operand.toIndex()) |op_index| {
4697 self.processDeath(op_index);
4698 }
4699 }
4700
4701 // Capture the state of register and stack allocation state so that we can revert to it.
4702 const parent_next_stack_offset = self.next_stack_offset;
4703 const parent_free_registers = self.register_manager.free_registers;
4704 var parent_stack = try self.stack.clone(self.gpa);
4705 defer parent_stack.deinit(self.gpa);
4706 const parent_registers = self.register_manager.registers;
4707 const parent_compare_flags_inst = self.compare_flags_inst;
4708
4709 try self.branch_stack.append(.{});
4710 errdefer {
4711 _ = self.branch_stack.pop().?;
4712 }
4713
4714 try self.ensureProcessDeathCapacity(liveness_condbr.then_deaths.len);
4715 for (liveness_condbr.then_deaths) |operand| {
4716 self.processDeath(operand);
4717 }
4718 try self.genBody(then_body);
4719
4720 // Revert to the previous register and stack allocation state.
4721
4722 var saved_then_branch = self.branch_stack.pop().?;
4723 defer saved_then_branch.deinit(self.gpa);
4724
4725 self.register_manager.registers = parent_registers;
4726 self.compare_flags_inst = parent_compare_flags_inst;
4727
4728 self.stack.deinit(self.gpa);
4729 self.stack = parent_stack;
4730 parent_stack = .{};
4731
4732 self.next_stack_offset = parent_next_stack_offset;
4733 self.register_manager.free_registers = parent_free_registers;
4734
4735 try self.performReloc(reloc);
4736 const else_branch = self.branch_stack.addOneAssumeCapacity();
4737 else_branch.* = .{};
4738
4739 try self.ensureProcessDeathCapacity(liveness_condbr.else_deaths.len);
4740 for (liveness_condbr.else_deaths) |operand| {
4741 self.processDeath(operand);
4742 }
4743 try self.genBody(else_body);
4744
4745 // At this point, each branch will possibly have conflicting values for where
4746 // each instruction is stored. They agree, however, on which instructions are alive/dead.
4747 // We use the first ("then") branch as canonical, and here emit
4748 // instructions into the second ("else") branch to make it conform.
4749 // We continue respect the data structure semantic guarantees of the else_branch so
4750 // that we can use all the code emitting abstractions. This is why at the bottom we
4751 // assert that parent_branch.free_registers equals the saved_then_branch.free_registers
4752 // rather than assigning it.
4753 const parent_branch = &self.branch_stack.items[self.branch_stack.items.len - 2];
4754 try parent_branch.inst_table.ensureUnusedCapacity(self.gpa, else_branch.inst_table.count());
4755
4756 const else_slice = else_branch.inst_table.entries.slice();
4757 const else_keys = else_slice.items(.key);
4758 const else_values = else_slice.items(.value);
4759 for (else_keys, 0..) |else_key, else_idx| {
4760 const else_value = else_values[else_idx];
4761 const canon_mcv = if (saved_then_branch.inst_table.fetchSwapRemove(else_key)) |then_entry| blk: {
4762 // The instruction's MCValue is overridden in both branches.
4763 parent_branch.inst_table.putAssumeCapacity(else_key, then_entry.value);
4764 if (else_value == .dead) {
4765 assert(then_entry.value == .dead);
4766 continue;
4767 }
4768 break :blk then_entry.value;
4769 } else blk: {
4770 if (else_value == .dead)
4771 continue;
4772 // The instruction is only overridden in the else branch.
4773 var i: usize = self.branch_stack.items.len - 1;
4774 while (true) {
4775 i -= 1; // If this overflows, the question is: why wasn't the instruction marked dead?
4776 if (self.branch_stack.items[i].inst_table.get(else_key)) |mcv| {
4777 assert(mcv != .dead);
4778 break :blk mcv;
4779 }
4780 }
4781 };
4782 log.debug("consolidating else_entry {d} {}=>{}", .{ else_key, else_value, canon_mcv });
4783 // TODO make sure the destination stack offset / register does not already have something
4784 // going on there.
4785 try self.setRegOrMem(self.typeOfIndex(else_key), canon_mcv, else_value);
4786 // TODO track the new register / stack allocation
4787 }
4788 try parent_branch.inst_table.ensureUnusedCapacity(self.gpa, saved_then_branch.inst_table.count());
4789 const then_slice = saved_then_branch.inst_table.entries.slice();
4790 const then_keys = then_slice.items(.key);
4791 const then_values = then_slice.items(.value);
4792 for (then_keys, 0..) |then_key, then_idx| {
4793 const then_value = then_values[then_idx];
4794 // We already deleted the items from this table that matched the else_branch.
4795 // So these are all instructions that are only overridden in the then branch.
4796 parent_branch.inst_table.putAssumeCapacity(then_key, then_value);
4797 if (then_value == .dead)
4798 continue;
4799 const parent_mcv = blk: {
4800 var i: usize = self.branch_stack.items.len - 1;
4801 while (true) {
4802 i -= 1;
4803 if (self.branch_stack.items[i].inst_table.get(then_key)) |mcv| {
4804 assert(mcv != .dead);
4805 break :blk mcv;
4806 }
4807 }
4808 };
4809 log.debug("consolidating then_entry {d} {}=>{}", .{ then_key, parent_mcv, then_value });
4810 // TODO make sure the destination stack offset / register does not already have something
4811 // going on there.
4812 try self.setRegOrMem(self.typeOfIndex(then_key), parent_mcv, then_value);
4813 // TODO track the new register / stack allocation
4814 }
4815
4816 {
4817 var item = self.branch_stack.pop().?;
4818 item.deinit(self.gpa);
4819 }
4820
4821 // We already took care of pl_op.operand earlier, so we're going
4822 // to pass .none here
4823 return self.finishAir(inst, .unreach, .{ .none, .none, .none });
4824}
4825
4826fn isNull(self: *Self, operand_bind: ReadArg.Bind, operand_ty: Type) !MCValue {
4827 const pt = self.pt;
4828 const zcu = pt.zcu;
4829 const sentinel: struct { ty: Type, bind: ReadArg.Bind } = if (!operand_ty.isPtrLikeOptional(zcu)) blk: {
4830 const payload_ty = operand_ty.optionalChild(zcu);
4831 if (!payload_ty.hasRuntimeBitsIgnoreComptime(zcu))
4832 break :blk .{ .ty = operand_ty, .bind = operand_bind };
4833
4834 const offset = @as(u32, @intCast(payload_ty.abiSize(zcu)));
4835 const operand_mcv = try operand_bind.resolveToMcv(self);
4836 const new_mcv: MCValue = switch (operand_mcv) {
4837 .register => |source_reg| new: {
4838 // TODO should we reuse the operand here?
4839 const raw_reg = try self.register_manager.allocReg(null, gp);
4840 const dest_reg = raw_reg.toX();
4841
4842 const shift = @as(u6, @intCast(offset * 8));
4843 if (shift == 0) {
4844 try self.genSetReg(payload_ty, dest_reg, operand_mcv);
4845 } else {
4846 _ = try self.addInst(.{
4847 .tag = if (payload_ty.isSignedInt(zcu))
4848 Mir.Inst.Tag.asr_immediate
4849 else
4850 Mir.Inst.Tag.lsr_immediate,
4851 .data = .{ .rr_shift = .{
4852 .rd = dest_reg,
4853 .rn = source_reg.toX(),
4854 .shift = shift,
4855 } },
4856 });
4857 }
4858
4859 break :new .{ .register = self.registerAlias(dest_reg, payload_ty) };
4860 },
4861 .stack_argument_offset => |off| .{ .stack_argument_offset = off + offset },
4862 .stack_offset => |off| .{ .stack_offset = off - offset },
4863 .memory => |addr| .{ .memory = addr + offset },
4864 else => unreachable, // invalid MCValue for an optional
4865 };
4866
4867 break :blk .{ .ty = Type.bool, .bind = .{ .mcv = new_mcv } };
4868 } else .{ .ty = operand_ty, .bind = operand_bind };
4869 const imm_bind: ReadArg.Bind = .{ .mcv = .{ .immediate = 0 } };
4870 return self.cmp(sentinel.bind, imm_bind, sentinel.ty, .eq);
4871}
4872
4873fn isNonNull(self: *Self, operand_bind: ReadArg.Bind, operand_ty: Type) !MCValue {
4874 const is_null_res = try self.isNull(operand_bind, operand_ty);
4875 assert(is_null_res.compare_flags == .eq);
4876 return MCValue{ .compare_flags = is_null_res.compare_flags.negate() };
4877}
4878
4879fn isErr(
4880 self: *Self,
4881 error_union_bind: ReadArg.Bind,
4882 error_union_ty: Type,
4883) !MCValue {
4884 const pt = self.pt;
4885 const zcu = pt.zcu;
4886 const error_type = error_union_ty.errorUnionSet(zcu);
4887
4888 if (error_type.errorSetIsEmpty(zcu)) {
4889 return MCValue{ .immediate = 0 }; // always false
4890 }
4891
4892 const error_mcv = try self.errUnionErr(error_union_bind, error_union_ty, null);
4893 return try self.cmp(.{ .mcv = error_mcv }, .{ .mcv = .{ .immediate = 0 } }, error_type, .gt);
4894}
4895
4896fn isNonErr(
4897 self: *Self,
4898 error_union_bind: ReadArg.Bind,
4899 error_union_ty: Type,
4900) !MCValue {
4901 const is_err_result = try self.isErr(error_union_bind, error_union_ty);
4902 switch (is_err_result) {
4903 .compare_flags => |cond| {
4904 assert(cond == .hi);
4905 return MCValue{ .compare_flags = cond.negate() };
4906 },
4907 .immediate => |imm| {
4908 assert(imm == 0);
4909 return MCValue{ .immediate = 1 };
4910 },
4911 else => unreachable,
4912 }
4913}
4914
4915fn airIsNull(self: *Self, inst: Air.Inst.Index) InnerError!void {
4916 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
4917 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
4918 const operand = try self.resolveInst(un_op);
4919 const operand_ty = self.typeOf(un_op);
4920
4921 break :result try self.isNull(.{ .mcv = operand }, operand_ty);
4922 };
4923 return self.finishAir(inst, result, .{ un_op, .none, .none });
4924}
4925
4926fn airIsNullPtr(self: *Self, inst: Air.Inst.Index) InnerError!void {
4927 const pt = self.pt;
4928 const zcu = pt.zcu;
4929 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
4930 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
4931 const operand_ptr = try self.resolveInst(un_op);
4932 const ptr_ty = self.typeOf(un_op);
4933 const elem_ty = ptr_ty.childType(zcu);
4934
4935 const operand = try self.allocRegOrMem(elem_ty, true, null);
4936 try self.load(operand, operand_ptr, ptr_ty);
4937
4938 break :result try self.isNull(.{ .mcv = operand }, elem_ty);
4939 };
4940 return self.finishAir(inst, result, .{ un_op, .none, .none });
4941}
4942
4943fn airIsNonNull(self: *Self, inst: Air.Inst.Index) InnerError!void {
4944 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
4945 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
4946 const operand = try self.resolveInst(un_op);
4947 const operand_ty = self.typeOf(un_op);
4948
4949 break :result try self.isNonNull(.{ .mcv = operand }, operand_ty);
4950 };
4951 return self.finishAir(inst, result, .{ un_op, .none, .none });
4952}
4953
4954fn airIsNonNullPtr(self: *Self, inst: Air.Inst.Index) InnerError!void {
4955 const pt = self.pt;
4956 const zcu = pt.zcu;
4957 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
4958 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
4959 const operand_ptr = try self.resolveInst(un_op);
4960 const ptr_ty = self.typeOf(un_op);
4961 const elem_ty = ptr_ty.childType(zcu);
4962
4963 const operand = try self.allocRegOrMem(elem_ty, true, null);
4964 try self.load(operand, operand_ptr, ptr_ty);
4965
4966 break :result try self.isNonNull(.{ .mcv = operand }, elem_ty);
4967 };
4968 return self.finishAir(inst, result, .{ un_op, .none, .none });
4969}
4970
4971fn airIsErr(self: *Self, inst: Air.Inst.Index) InnerError!void {
4972 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
4973 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
4974 const error_union_bind: ReadArg.Bind = .{ .inst = un_op };
4975 const error_union_ty = self.typeOf(un_op);
4976
4977 break :result try self.isErr(error_union_bind, error_union_ty);
4978 };
4979 return self.finishAir(inst, result, .{ un_op, .none, .none });
4980}
4981
4982fn airIsErrPtr(self: *Self, inst: Air.Inst.Index) InnerError!void {
4983 const pt = self.pt;
4984 const zcu = pt.zcu;
4985 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
4986 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
4987 const operand_ptr = try self.resolveInst(un_op);
4988 const ptr_ty = self.typeOf(un_op);
4989 const elem_ty = ptr_ty.childType(zcu);
4990
4991 const operand = try self.allocRegOrMem(elem_ty, true, null);
4992 try self.load(operand, operand_ptr, ptr_ty);
4993
4994 break :result try self.isErr(.{ .mcv = operand }, elem_ty);
4995 };
4996 return self.finishAir(inst, result, .{ un_op, .none, .none });
4997}
4998
4999fn airIsNonErr(self: *Self, inst: Air.Inst.Index) InnerError!void {
5000 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
5001 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
5002 const error_union_bind: ReadArg.Bind = .{ .inst = un_op };
5003 const error_union_ty = self.typeOf(un_op);
5004
5005 break :result try self.isNonErr(error_union_bind, error_union_ty);
5006 };
5007 return self.finishAir(inst, result, .{ un_op, .none, .none });
5008}
5009
5010fn airIsNonErrPtr(self: *Self, inst: Air.Inst.Index) InnerError!void {
5011 const pt = self.pt;
5012 const zcu = pt.zcu;
5013 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
5014 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
5015 const operand_ptr = try self.resolveInst(un_op);
5016 const ptr_ty = self.typeOf(un_op);
5017 const elem_ty = ptr_ty.childType(zcu);
5018
5019 const operand = try self.allocRegOrMem(elem_ty, true, null);
5020 try self.load(operand, operand_ptr, ptr_ty);
5021
5022 break :result try self.isNonErr(.{ .mcv = operand }, elem_ty);
5023 };
5024 return self.finishAir(inst, result, .{ un_op, .none, .none });
5025}
5026
5027fn airLoop(self: *Self, inst: Air.Inst.Index) InnerError!void {
5028 // A loop is a setup to be able to jump back to the beginning.
5029 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
5030 const loop = self.air.extraData(Air.Block, ty_pl.payload);
5031 const body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[loop.end..][0..loop.data.body_len]);
5032 const start_index = @as(u32, @intCast(self.mir_instructions.len));
5033
5034 try self.genBody(body);
5035 try self.jump(start_index);
5036
5037 return self.finishAirBookkeeping();
5038}
5039
5040/// Send control flow to `inst`.
5041fn jump(self: *Self, inst: Mir.Inst.Index) !void {
5042 _ = try self.addInst(.{
5043 .tag = .b,
5044 .data = .{ .inst = inst },
5045 });
5046}
5047
5048fn airBlock(self: *Self, inst: Air.Inst.Index) InnerError!void {
5049 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
5050 const extra = self.air.extraData(Air.Block, ty_pl.payload);
5051 try self.lowerBlock(inst, @ptrCast(self.air.extra.items[extra.end..][0..extra.data.body_len]));
5052}
5053
5054fn lowerBlock(self: *Self, inst: Air.Inst.Index, body: []const Air.Inst.Index) !void {
5055 try self.blocks.putNoClobber(self.gpa, inst, .{
5056 // A block is a setup to be able to jump to the end.
5057 .relocs = .{},
5058 // It also acts as a receptacle for break operands.
5059 // Here we use `MCValue.none` to represent a null value so that the first
5060 // break instruction will choose a MCValue for the block result and overwrite
5061 // this field. Following break instructions will use that MCValue to put their
5062 // block results.
5063 .mcv = MCValue{ .none = {} },
5064 });
5065 defer self.blocks.getPtr(inst).?.relocs.deinit(self.gpa);
5066
5067 // TODO emit debug info lexical block
5068 try self.genBody(body);
5069
5070 // relocations for `br` instructions
5071 const relocs = &self.blocks.getPtr(inst).?.relocs;
5072 if (relocs.items.len > 0 and relocs.items[relocs.items.len - 1] == self.mir_instructions.len - 1) {
5073 // If the last Mir instruction is the last relocation (which
5074 // would just jump one instruction further), it can be safely
5075 // removed
5076 self.mir_instructions.orderedRemove(relocs.pop().?);
5077 }
5078 for (relocs.items) |reloc| {
5079 try self.performReloc(reloc);
5080 }
5081
5082 const result = self.blocks.getPtr(inst).?.mcv;
5083 return self.finishAir(inst, result, .{ .none, .none, .none });
5084}
5085
5086fn airSwitch(self: *Self, inst: Air.Inst.Index) InnerError!void {
5087 const switch_br = self.air.unwrapSwitch(inst);
5088 const condition_ty = self.typeOf(switch_br.operand);
5089 const liveness = try self.liveness.getSwitchBr(
5090 self.gpa,
5091 inst,
5092 switch_br.cases_len + 1,
5093 );
5094 defer self.gpa.free(liveness.deaths);
5095
5096 var it = switch_br.iterateCases();
5097 while (it.next()) |case| {
5098 if (case.ranges.len > 0) return self.fail("TODO: switch with ranges", .{});
5099
5100 // For every item, we compare it to condition and branch into
5101 // the prong if they are equal. After we compared to all
5102 // items, we branch into the next prong (or if no other prongs
5103 // exist out of the switch statement).
5104 //
5105 // cmp condition, item1
5106 // beq prong
5107 // cmp condition, item2
5108 // beq prong
5109 // cmp condition, item3
5110 // beq prong
5111 // b out
5112 // prong: ...
5113 // ...
5114 // out: ...
5115 const branch_into_prong_relocs = try self.gpa.alloc(u32, case.items.len);
5116 defer self.gpa.free(branch_into_prong_relocs);
5117
5118 for (case.items, 0..) |item, idx| {
5119 const cmp_result = try self.cmp(.{ .inst = switch_br.operand }, .{ .inst = item }, condition_ty, .neq);
5120 branch_into_prong_relocs[idx] = try self.condBr(cmp_result);
5121 }
5122
5123 const branch_away_from_prong_reloc = try self.addInst(.{
5124 .tag = .b,
5125 .data = .{ .inst = undefined }, // populated later through performReloc
5126 });
5127
5128 for (branch_into_prong_relocs) |reloc| {
5129 try self.performReloc(reloc);
5130 }
5131
5132 // Capture the state of register and stack allocation state so that we can revert to it.
5133 const parent_next_stack_offset = self.next_stack_offset;
5134 const parent_free_registers = self.register_manager.free_registers;
5135 const parent_compare_flags_inst = self.compare_flags_inst;
5136 var parent_stack = try self.stack.clone(self.gpa);
5137 defer parent_stack.deinit(self.gpa);
5138 const parent_registers = self.register_manager.registers;
5139
5140 try self.branch_stack.append(.{});
5141 errdefer {
5142 _ = self.branch_stack.pop().?;
5143 }
5144
5145 try self.ensureProcessDeathCapacity(liveness.deaths[case.idx].len);
5146 for (liveness.deaths[case.idx]) |operand| {
5147 self.processDeath(operand);
5148 }
5149 try self.genBody(case.body);
5150
5151 // Revert to the previous register and stack allocation state.
5152 var saved_case_branch = self.branch_stack.pop().?;
5153 defer saved_case_branch.deinit(self.gpa);
5154
5155 self.register_manager.registers = parent_registers;
5156 self.compare_flags_inst = parent_compare_flags_inst;
5157 self.stack.deinit(self.gpa);
5158 self.stack = parent_stack;
5159 parent_stack = .{};
5160
5161 self.next_stack_offset = parent_next_stack_offset;
5162 self.register_manager.free_registers = parent_free_registers;
5163
5164 try self.performReloc(branch_away_from_prong_reloc);
5165 }
5166
5167 if (switch_br.else_body_len > 0) {
5168 const else_body = it.elseBody();
5169
5170 // Capture the state of register and stack allocation state so that we can revert to it.
5171 const parent_next_stack_offset = self.next_stack_offset;
5172 const parent_free_registers = self.register_manager.free_registers;
5173 const parent_compare_flags_inst = self.compare_flags_inst;
5174 var parent_stack = try self.stack.clone(self.gpa);
5175 defer parent_stack.deinit(self.gpa);
5176 const parent_registers = self.register_manager.registers;
5177
5178 try self.branch_stack.append(.{});
5179 errdefer {
5180 _ = self.branch_stack.pop().?;
5181 }
5182
5183 const else_deaths = liveness.deaths.len - 1;
5184 try self.ensureProcessDeathCapacity(liveness.deaths[else_deaths].len);
5185 for (liveness.deaths[else_deaths]) |operand| {
5186 self.processDeath(operand);
5187 }
5188 try self.genBody(else_body);
5189
5190 // Revert to the previous register and stack allocation state.
5191 var saved_case_branch = self.branch_stack.pop().?;
5192 defer saved_case_branch.deinit(self.gpa);
5193
5194 self.register_manager.registers = parent_registers;
5195 self.compare_flags_inst = parent_compare_flags_inst;
5196 self.stack.deinit(self.gpa);
5197 self.stack = parent_stack;
5198 parent_stack = .{};
5199
5200 self.next_stack_offset = parent_next_stack_offset;
5201 self.register_manager.free_registers = parent_free_registers;
5202
5203 // TODO consolidate returned MCValues between prongs and else branch like we do
5204 // in airCondBr.
5205 }
5206
5207 return self.finishAir(inst, .unreach, .{ switch_br.operand, .none, .none });
5208}
5209
5210fn performReloc(self: *Self, inst: Mir.Inst.Index) !void {
5211 const tag = self.mir_instructions.items(.tag)[inst];
5212 switch (tag) {
5213 .cbz => self.mir_instructions.items(.data)[inst].r_inst.inst = @intCast(self.mir_instructions.len),
5214 .b_cond => self.mir_instructions.items(.data)[inst].inst_cond.inst = @intCast(self.mir_instructions.len),
5215 .b => self.mir_instructions.items(.data)[inst].inst = @intCast(self.mir_instructions.len),
5216 else => unreachable,
5217 }
5218}
5219
5220fn airBr(self: *Self, inst: Air.Inst.Index) InnerError!void {
5221 const branch = self.air.instructions.items(.data)[@intFromEnum(inst)].br;
5222 try self.br(branch.block_inst, branch.operand);
5223 return self.finishAir(inst, .dead, .{ branch.operand, .none, .none });
5224}
5225
5226fn br(self: *Self, block: Air.Inst.Index, operand: Air.Inst.Ref) !void {
5227 const pt = self.pt;
5228 const zcu = pt.zcu;
5229 const block_data = self.blocks.getPtr(block).?;
5230
5231 if (self.typeOf(operand).hasRuntimeBits(zcu)) {
5232 const operand_mcv = try self.resolveInst(operand);
5233 const block_mcv = block_data.mcv;
5234 if (block_mcv == .none) {
5235 block_data.mcv = switch (operand_mcv) {
5236 .none, .dead, .unreach => unreachable,
5237 .register, .stack_offset, .memory => operand_mcv,
5238 .immediate, .stack_argument_offset, .compare_flags => blk: {
5239 const new_mcv = try self.allocRegOrMem(self.typeOfIndex(block), true, block);
5240 try self.setRegOrMem(self.typeOfIndex(block), new_mcv, operand_mcv);
5241 break :blk new_mcv;
5242 },
5243 else => return self.fail("TODO implement block_data.mcv = operand_mcv for {}", .{operand_mcv}),
5244 };
5245 } else {
5246 try self.setRegOrMem(self.typeOfIndex(block), block_mcv, operand_mcv);
5247 }
5248 }
5249 return self.brVoid(block);
5250}
5251
5252fn brVoid(self: *Self, block: Air.Inst.Index) !void {
5253 const block_data = self.blocks.getPtr(block).?;
5254
5255 // Emit a jump with a relocation. It will be patched up after the block ends.
5256 try block_data.relocs.ensureUnusedCapacity(self.gpa, 1);
5257
5258 block_data.relocs.appendAssumeCapacity(try self.addInst(.{
5259 .tag = .b,
5260 .data = .{ .inst = undefined }, // populated later through performReloc
5261 }));
5262}
5263
5264fn airAsm(self: *Self, inst: Air.Inst.Index) InnerError!void {
5265 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
5266 const extra = self.air.extraData(Air.Asm, ty_pl.payload);
5267 const is_volatile = @as(u1, @truncate(extra.data.flags >> 31)) != 0;
5268 const clobbers_len = @as(u31, @truncate(extra.data.flags));
5269 var extra_i: usize = extra.end;
5270 const outputs: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[extra_i..][0..extra.data.outputs_len]);
5271 extra_i += outputs.len;
5272 const inputs: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[extra_i..][0..extra.data.inputs_len]);
5273 extra_i += inputs.len;
5274
5275 const dead = !is_volatile and self.liveness.isUnused(inst);
5276 const result: MCValue = if (dead) .dead else result: {
5277 if (outputs.len > 1) {
5278 return self.fail("TODO implement codegen for asm with more than 1 output", .{});
5279 }
5280
5281 const output_constraint: ?[]const u8 = for (outputs) |output| {
5282 if (output != .none) {
5283 return self.fail("TODO implement codegen for non-expr asm", .{});
5284 }
5285 const extra_bytes = std.mem.sliceAsBytes(self.air.extra.items[extra_i..]);
5286 const constraint = std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra.items[extra_i..]), 0);
5287 const name = std.mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0);
5288 // This equation accounts for the fact that even if we have exactly 4 bytes
5289 // for the string, we still use the next u32 for the null terminator.
5290 extra_i += (constraint.len + name.len + (2 + 3)) / 4;
5291
5292 break constraint;
5293 } else null;
5294
5295 for (inputs) |input| {
5296 const input_bytes = std.mem.sliceAsBytes(self.air.extra.items[extra_i..]);
5297 const constraint = std.mem.sliceTo(input_bytes, 0);
5298 const name = std.mem.sliceTo(input_bytes[constraint.len + 1 ..], 0);
5299 // This equation accounts for the fact that even if we have exactly 4 bytes
5300 // for the string, we still use the next u32 for the null terminator.
5301 extra_i += (constraint.len + name.len + (2 + 3)) / 4;
5302
5303 if (constraint.len < 3 or constraint[0] != '{' or constraint[constraint.len - 1] != '}') {
5304 return self.fail("unrecognized asm input constraint: '{s}'", .{constraint});
5305 }
5306 const reg_name = constraint[1 .. constraint.len - 1];
5307 const reg = parseRegName(reg_name) orelse
5308 return self.fail("unrecognized register: '{s}'", .{reg_name});
5309
5310 const arg_mcv = try self.resolveInst(input);
5311 try self.register_manager.getReg(reg, null);
5312 try self.genSetReg(self.typeOf(input), reg, arg_mcv);
5313 }
5314
5315 {
5316 var clobber_i: u32 = 0;
5317 while (clobber_i < clobbers_len) : (clobber_i += 1) {
5318 const clobber = std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra.items[extra_i..]), 0);
5319 // This equation accounts for the fact that even if we have exactly 4 bytes
5320 // for the string, we still use the next u32 for the null terminator.
5321 extra_i += clobber.len / 4 + 1;
5322
5323 // TODO honor these
5324 }
5325 }
5326
5327 const asm_source = std.mem.sliceAsBytes(self.air.extra.items[extra_i..])[0..extra.data.source_len];
5328
5329 if (mem.eql(u8, asm_source, "svc #0")) {
5330 _ = try self.addInst(.{
5331 .tag = .svc,
5332 .data = .{ .imm16 = 0x0 },
5333 });
5334 } else if (mem.eql(u8, asm_source, "svc #0x80")) {
5335 _ = try self.addInst(.{
5336 .tag = .svc,
5337 .data = .{ .imm16 = 0x80 },
5338 });
5339 } else {
5340 return self.fail("TODO implement support for more aarch64 assembly instructions", .{});
5341 }
5342
5343 if (output_constraint) |output| {
5344 if (output.len < 4 or output[0] != '=' or output[1] != '{' or output[output.len - 1] != '}') {
5345 return self.fail("unrecognized asm output constraint: '{s}'", .{output});
5346 }
5347 const reg_name = output[2 .. output.len - 1];
5348 const reg = parseRegName(reg_name) orelse
5349 return self.fail("unrecognized register: '{s}'", .{reg_name});
5350 break :result MCValue{ .register = reg };
5351 } else {
5352 break :result MCValue{ .none = {} };
5353 }
5354 };
5355
5356 simple: {
5357 var buf = [1]Air.Inst.Ref{.none} ** (Air.Liveness.bpi - 1);
5358 var buf_index: usize = 0;
5359 for (outputs) |output| {
5360 if (output == .none) continue;
5361
5362 if (buf_index >= buf.len) break :simple;
5363 buf[buf_index] = output;
5364 buf_index += 1;
5365 }
5366 if (buf_index + inputs.len > buf.len) break :simple;
5367 @memcpy(buf[buf_index..][0..inputs.len], inputs);
5368 return self.finishAir(inst, result, buf);
5369 }
5370 var bt = try self.iterateBigTomb(inst, outputs.len + inputs.len);
5371 for (outputs) |output| {
5372 if (output == .none) continue;
5373
5374 bt.feed(output);
5375 }
5376 for (inputs) |input| {
5377 bt.feed(input);
5378 }
5379 return bt.finishAir(result);
5380}
5381
5382fn iterateBigTomb(self: *Self, inst: Air.Inst.Index, operand_count: usize) !BigTomb {
5383 try self.ensureProcessDeathCapacity(operand_count + 1);
5384 return BigTomb{
5385 .function = self,
5386 .inst = inst,
5387 .lbt = self.liveness.iterateBigTomb(inst),
5388 };
5389}
5390
5391/// Sets the value without any modifications to register allocation metadata or stack allocation metadata.
5392fn setRegOrMem(self: *Self, ty: Type, loc: MCValue, val: MCValue) !void {
5393 switch (loc) {
5394 .none => return,
5395 .register => |reg| return self.genSetReg(ty, reg, val),
5396 .stack_offset => |off| return self.genSetStack(ty, off, val),
5397 .memory => {
5398 return self.fail("TODO implement setRegOrMem for memory", .{});
5399 },
5400 else => unreachable,
5401 }
5402}
5403
5404fn genSetStack(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerError!void {
5405 const pt = self.pt;
5406 const zcu = pt.zcu;
5407 const abi_size = @as(u32, @intCast(ty.abiSize(zcu)));
5408 switch (mcv) {
5409 .dead => unreachable,
5410 .unreach, .none => return, // Nothing to do.
5411 .undef => {
5412 if (!self.wantSafety())
5413 return; // The already existing value will do just fine.
5414 // TODO Upgrade this to a memset call when we have that available.
5415 switch (abi_size) {
5416 1 => return self.genSetStack(ty, stack_offset, .{ .immediate = 0xaa }),
5417 2 => return self.genSetStack(ty, stack_offset, .{ .immediate = 0xaaaa }),
5418 4 => return self.genSetStack(ty, stack_offset, .{ .immediate = 0xaaaaaaaa }),
5419 8 => return self.genSetStack(ty, stack_offset, .{ .immediate = 0xaaaaaaaaaaaaaaaa }),
5420 else => try self.genInlineMemset(
5421 .{ .ptr_stack_offset = stack_offset },
5422 .{ .immediate = 0xaa },
5423 .{ .immediate = abi_size },
5424 ),
5425 }
5426 },
5427 .compare_flags,
5428 .immediate,
5429 .ptr_stack_offset,
5430 => {
5431 const reg = try self.copyToTmpRegister(ty, mcv);
5432 return self.genSetStack(ty, stack_offset, MCValue{ .register = reg });
5433 },
5434 .register => |reg| {
5435 switch (abi_size) {
5436 1, 2, 4, 8 => {
5437 assert(std.mem.isAlignedGeneric(u32, stack_offset, abi_size));
5438
5439 const tag: Mir.Inst.Tag = switch (abi_size) {
5440 1 => .strb_stack,
5441 2 => .strh_stack,
5442 4, 8 => .str_stack,
5443 else => unreachable, // unexpected abi size
5444 };
5445 const rt = self.registerAlias(reg, ty);
5446
5447 _ = try self.addInst(.{
5448 .tag = tag,
5449 .data = .{ .load_store_stack = .{
5450 .rt = rt,
5451 .offset = stack_offset,
5452 } },
5453 });
5454 },
5455 else => return self.fail("TODO implement storing other types abi_size={}", .{abi_size}),
5456 }
5457 },
5458 .register_with_overflow => |rwo| {
5459 const reg_lock = self.register_manager.lockReg(rwo.reg);
5460 defer if (reg_lock) |locked_reg| self.register_manager.unlockReg(locked_reg);
5461
5462 const wrapped_ty = ty.fieldType(0, zcu);
5463 try self.genSetStack(wrapped_ty, stack_offset, .{ .register = rwo.reg });
5464
5465 const overflow_bit_ty = ty.fieldType(1, zcu);
5466 const overflow_bit_offset = @as(u32, @intCast(ty.structFieldOffset(1, zcu)));
5467 const raw_cond_reg = try self.register_manager.allocReg(null, gp);
5468 const cond_reg = self.registerAlias(raw_cond_reg, overflow_bit_ty);
5469
5470 _ = try self.addInst(.{
5471 .tag = .cset,
5472 .data = .{ .r_cond = .{
5473 .rd = cond_reg,
5474 .cond = rwo.flag,
5475 } },
5476 });
5477
5478 try self.genSetStack(overflow_bit_ty, stack_offset - overflow_bit_offset, .{
5479 .register = cond_reg,
5480 });
5481 },
5482 .linker_load,
5483 .memory,
5484 .stack_argument_offset,
5485 .stack_offset,
5486 => {
5487 switch (mcv) {
5488 .stack_offset => |off| {
5489 if (stack_offset == off)
5490 return; // Copy stack variable to itself; nothing to do.
5491 },
5492 else => {},
5493 }
5494
5495 if (abi_size <= 8) {
5496 const reg = try self.copyToTmpRegister(ty, mcv);
5497 return self.genSetStack(ty, stack_offset, MCValue{ .register = reg });
5498 } else {
5499 const ptr_ty = try pt.singleMutPtrType(ty);
5500
5501 // TODO call extern memcpy
5502 const regs = try self.register_manager.allocRegs(5, .{ null, null, null, null, null }, gp);
5503 const regs_locks = self.register_manager.lockRegsAssumeUnused(5, regs);
5504 defer for (regs_locks) |reg| {
5505 self.register_manager.unlockReg(reg);
5506 };
5507
5508 const src_reg = regs[0];
5509 const dst_reg = regs[1];
5510 const len_reg = regs[2];
5511 const count_reg = regs[3];
5512 const tmp_reg = regs[4];
5513
5514 switch (mcv) {
5515 .stack_offset => |off| {
5516 // sub src_reg, fp, #off
5517 try self.genSetReg(ptr_ty, src_reg, .{ .ptr_stack_offset = off });
5518 },
5519 .stack_argument_offset => |off| {
5520 _ = try self.addInst(.{
5521 .tag = .ldr_ptr_stack_argument,
5522 .data = .{ .load_store_stack = .{
5523 .rt = src_reg,
5524 .offset = off,
5525 } },
5526 });
5527 },
5528 .memory => |addr| try self.genSetReg(Type.usize, src_reg, .{ .immediate = addr }),
5529 .linker_load => |load_struct| {
5530 const tag: Mir.Inst.Tag = switch (load_struct.type) {
5531 .got => .load_memory_ptr_got,
5532 .direct => .load_memory_ptr_direct,
5533 .import => unreachable,
5534 };
5535 const atom_index = switch (self.bin_file.tag) {
5536 .macho => {
5537 // const macho_file = self.bin_file.cast(link.File.MachO).?;
5538 // const atom = try macho_file.getOrCreateAtomForDecl(self.owner_decl);
5539 // break :blk macho_file.getAtom(atom).getSymbolIndex().?;
5540 @panic("TODO genSetStack");
5541 },
5542 .coff => blk: {
5543 const coff_file = self.bin_file.cast(.coff).?;
5544 const atom = try coff_file.getOrCreateAtomForNav(self.owner_nav);
5545 break :blk coff_file.getAtom(atom).getSymbolIndex().?;
5546 },
5547 else => unreachable, // unsupported target format
5548 };
5549 _ = try self.addInst(.{
5550 .tag = tag,
5551 .data = .{
5552 .payload = try self.addExtra(Mir.LoadMemoryPie{
5553 .register = @intFromEnum(src_reg),
5554 .atom_index = atom_index,
5555 .sym_index = load_struct.sym_index,
5556 }),
5557 },
5558 });
5559 },
5560 else => unreachable,
5561 }
5562
5563 // sub dst_reg, fp, #stack_offset
5564 try self.genSetReg(ptr_ty, dst_reg, .{ .ptr_stack_offset = stack_offset });
5565
5566 // mov len, #abi_size
5567 try self.genSetReg(Type.usize, len_reg, .{ .immediate = abi_size });
5568
5569 // memcpy(src, dst, len)
5570 try self.genInlineMemcpy(src_reg, dst_reg, len_reg, count_reg, tmp_reg);
5571 }
5572 },
5573 }
5574}
5575
5576fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void {
5577 const pt = self.pt;
5578 const zcu = pt.zcu;
5579 switch (mcv) {
5580 .dead => unreachable,
5581 .unreach, .none => return, // Nothing to do.
5582 .undef => {
5583 if (!self.wantSafety())
5584 return; // The already existing value will do just fine.
5585 // Write the debug undefined value.
5586 switch (reg.size()) {
5587 32 => return self.genSetReg(ty, reg, .{ .immediate = 0xaaaaaaaa }),
5588 64 => return self.genSetReg(ty, reg, .{ .immediate = 0xaaaaaaaaaaaaaaaa }),
5589 else => unreachable, // unexpected register size
5590 }
5591 },
5592 .ptr_stack_offset => |off| {
5593 _ = try self.addInst(.{
5594 .tag = .ldr_ptr_stack,
5595 .data = .{ .load_store_stack = .{
5596 .rt = reg,
5597 .offset = @intCast(off),
5598 } },
5599 });
5600 },
5601 .compare_flags => |condition| {
5602 _ = try self.addInst(.{
5603 .tag = .cset,
5604 .data = .{ .r_cond = .{
5605 .rd = reg,
5606 .cond = condition,
5607 } },
5608 });
5609 },
5610 .immediate => |x| {
5611 _ = try self.addInst(.{
5612 .tag = .movz,
5613 .data = .{ .r_imm16_sh = .{ .rd = reg, .imm16 = @truncate(x) } },
5614 });
5615
5616 if (x & 0x0000_0000_ffff_0000 != 0) {
5617 _ = try self.addInst(.{
5618 .tag = .movk,
5619 .data = .{ .r_imm16_sh = .{ .rd = reg, .imm16 = @truncate(x >> 16), .hw = 1 } },
5620 });
5621 }
5622
5623 if (reg.size() == 64) {
5624 if (x & 0x0000_ffff_0000_0000 != 0) {
5625 _ = try self.addInst(.{
5626 .tag = .movk,
5627 .data = .{ .r_imm16_sh = .{ .rd = reg, .imm16 = @truncate(x >> 32), .hw = 2 } },
5628 });
5629 }
5630 if (x & 0xffff_0000_0000_0000 != 0) {
5631 _ = try self.addInst(.{
5632 .tag = .movk,
5633 .data = .{ .r_imm16_sh = .{ .rd = reg, .imm16 = @truncate(x >> 48), .hw = 3 } },
5634 });
5635 }
5636 }
5637 },
5638 .register => |src_reg| {
5639 assert(src_reg.size() == reg.size());
5640
5641 // If the registers are the same, nothing to do.
5642 if (src_reg.id() == reg.id())
5643 return;
5644
5645 // mov reg, src_reg
5646 _ = try self.addInst(.{
5647 .tag = .mov_register,
5648 .data = .{ .rr = .{ .rd = reg, .rn = src_reg } },
5649 });
5650 },
5651 .register_with_overflow => unreachable, // doesn't fit into a register
5652 .linker_load => |load_struct| {
5653 const tag: Mir.Inst.Tag = switch (load_struct.type) {
5654 .got => .load_memory_got,
5655 .direct => .load_memory_direct,
5656 .import => .load_memory_import,
5657 };
5658 const atom_index = switch (self.bin_file.tag) {
5659 .macho => {
5660 @panic("TODO genSetReg");
5661 // const macho_file = self.bin_file.cast(link.File.MachO).?;
5662 // const atom = try macho_file.getOrCreateAtomForDecl(self.owner_decl);
5663 // break :blk macho_file.getAtom(atom).getSymbolIndex().?;
5664 },
5665 .coff => blk: {
5666 const coff_file = self.bin_file.cast(.coff).?;
5667 const atom = try coff_file.getOrCreateAtomForNav(self.owner_nav);
5668 break :blk coff_file.getAtom(atom).getSymbolIndex().?;
5669 },
5670 else => unreachable, // unsupported target format
5671 };
5672 _ = try self.addInst(.{
5673 .tag = tag,
5674 .data = .{
5675 .payload = try self.addExtra(Mir.LoadMemoryPie{
5676 .register = @intFromEnum(reg),
5677 .atom_index = atom_index,
5678 .sym_index = load_struct.sym_index,
5679 }),
5680 },
5681 });
5682 },
5683 .memory => |addr| {
5684 // The value is in memory at a hard-coded address.
5685 // If the type is a pointer, it means the pointer address is at this memory location.
5686 try self.genSetReg(ty, reg.toX(), .{ .immediate = addr });
5687 try self.genLdrRegister(reg, reg.toX(), ty);
5688 },
5689 .stack_offset => |off| {
5690 const abi_size = ty.abiSize(zcu);
5691
5692 switch (abi_size) {
5693 1, 2, 4, 8 => {
5694 const tag: Mir.Inst.Tag = switch (abi_size) {
5695 1 => if (ty.isSignedInt(zcu)) Mir.Inst.Tag.ldrsb_stack else .ldrb_stack,
5696 2 => if (ty.isSignedInt(zcu)) Mir.Inst.Tag.ldrsh_stack else .ldrh_stack,
5697 4, 8 => .ldr_stack,
5698 else => unreachable, // unexpected abi size
5699 };
5700
5701 _ = try self.addInst(.{
5702 .tag = tag,
5703 .data = .{ .load_store_stack = .{
5704 .rt = reg,
5705 .offset = @intCast(off),
5706 } },
5707 });
5708 },
5709 3, 5, 6, 7 => return self.fail("TODO implement genSetReg types size {}", .{abi_size}),
5710 else => unreachable,
5711 }
5712 },
5713 .stack_argument_offset => |off| {
5714 const abi_size = ty.abiSize(zcu);
5715
5716 switch (abi_size) {
5717 1, 2, 4, 8 => {
5718 const tag: Mir.Inst.Tag = switch (abi_size) {
5719 1 => if (ty.isSignedInt(zcu)) Mir.Inst.Tag.ldrsb_stack_argument else .ldrb_stack_argument,
5720 2 => if (ty.isSignedInt(zcu)) Mir.Inst.Tag.ldrsh_stack_argument else .ldrh_stack_argument,
5721 4, 8 => .ldr_stack_argument,
5722 else => unreachable, // unexpected abi size
5723 };
5724
5725 _ = try self.addInst(.{
5726 .tag = tag,
5727 .data = .{ .load_store_stack = .{
5728 .rt = reg,
5729 .offset = @intCast(off),
5730 } },
5731 });
5732 },
5733 3, 5, 6, 7 => return self.fail("TODO implement genSetReg types size {}", .{abi_size}),
5734 else => unreachable,
5735 }
5736 },
5737 }
5738}
5739
5740fn genSetStackArgument(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerError!void {
5741 const pt = self.pt;
5742 const zcu = pt.zcu;
5743 const abi_size = @as(u32, @intCast(ty.abiSize(zcu)));
5744 switch (mcv) {
5745 .dead => unreachable,
5746 .none, .unreach => return,
5747 .undef => {
5748 if (!self.wantSafety())
5749 return; // The already existing value will do just fine.
5750 // TODO Upgrade this to a memset call when we have that available.
5751 switch (ty.abiSize(pt.zcu)) {
5752 1 => return self.genSetStack(ty, stack_offset, .{ .immediate = 0xaa }),
5753 2 => return self.genSetStack(ty, stack_offset, .{ .immediate = 0xaaaa }),
5754 4 => return self.genSetStack(ty, stack_offset, .{ .immediate = 0xaaaaaaaa }),
5755 8 => return self.genSetStack(ty, stack_offset, .{ .immediate = 0xaaaaaaaaaaaaaaaa }),
5756 else => return self.fail("TODO implement memset", .{}),
5757 }
5758 },
5759 .register => |reg| {
5760 switch (abi_size) {
5761 1, 2, 4, 8 => {
5762 const tag: Mir.Inst.Tag = switch (abi_size) {
5763 1 => .strb_immediate,
5764 2 => .strh_immediate,
5765 4, 8 => .str_immediate,
5766 else => unreachable, // unexpected abi size
5767 };
5768 const rt = self.registerAlias(reg, ty);
5769 const offset = switch (abi_size) {
5770 1 => blk: {
5771 if (math.cast(u12, stack_offset)) |imm| {
5772 break :blk Instruction.LoadStoreOffset.imm(imm);
5773 } else {
5774 return self.fail("TODO genSetStackArgument byte with larger offset", .{});
5775 }
5776 },
5777 2 => blk: {
5778 assert(std.mem.isAlignedGeneric(u32, stack_offset, 2)); // misaligned stack entry
5779 if (math.cast(u12, @divExact(stack_offset, 2))) |imm| {
5780 break :blk Instruction.LoadStoreOffset.imm(imm);
5781 } else {
5782 return self.fail("TODO getSetStackArgument halfword with larger offset", .{});
5783 }
5784 },
5785 4, 8 => blk: {
5786 const alignment = abi_size;
5787 assert(std.mem.isAlignedGeneric(u32, stack_offset, alignment)); // misaligned stack entry
5788 if (math.cast(u12, @divExact(stack_offset, alignment))) |imm| {
5789 break :blk Instruction.LoadStoreOffset.imm(imm);
5790 } else {
5791 return self.fail("TODO genSetStackArgument with larger offset", .{});
5792 }
5793 },
5794 else => unreachable,
5795 };
5796
5797 _ = try self.addInst(.{
5798 .tag = tag,
5799 .data = .{ .load_store_register_immediate = .{
5800 .rt = rt,
5801 .rn = .sp,
5802 .offset = offset.immediate,
5803 } },
5804 });
5805 },
5806 else => return self.fail("TODO genSetStackArgument other types abi_size={}", .{abi_size}),
5807 }
5808 },
5809 .register_with_overflow => {
5810 return self.fail("TODO implement genSetStackArgument {}", .{mcv});
5811 },
5812 .linker_load,
5813 .memory,
5814 .stack_argument_offset,
5815 .stack_offset,
5816 => {
5817 if (abi_size <= 4) {
5818 const reg = try self.copyToTmpRegister(ty, mcv);
5819 return self.genSetStackArgument(ty, stack_offset, MCValue{ .register = reg });
5820 } else {
5821 const ptr_ty = try pt.singleMutPtrType(ty);
5822
5823 // TODO call extern memcpy
5824 const regs = try self.register_manager.allocRegs(5, .{ null, null, null, null, null }, gp);
5825 const regs_locks = self.register_manager.lockRegsAssumeUnused(5, regs);
5826 defer for (regs_locks) |reg| {
5827 self.register_manager.unlockReg(reg);
5828 };
5829
5830 const src_reg = regs[0];
5831 const dst_reg = regs[1];
5832 const len_reg = regs[2];
5833 const count_reg = regs[3];
5834 const tmp_reg = regs[4];
5835
5836 switch (mcv) {
5837 .stack_offset => |off| {
5838 // sub src_reg, fp, #off
5839 try self.genSetReg(ptr_ty, src_reg, .{ .ptr_stack_offset = off });
5840 },
5841 .stack_argument_offset => |off| {
5842 _ = try self.addInst(.{
5843 .tag = .ldr_ptr_stack_argument,
5844 .data = .{ .load_store_stack = .{
5845 .rt = src_reg,
5846 .offset = off,
5847 } },
5848 });
5849 },
5850 .memory => |addr| try self.genSetReg(ptr_ty, src_reg, .{ .immediate = @as(u32, @intCast(addr)) }),
5851 .linker_load => |load_struct| {
5852 const tag: Mir.Inst.Tag = switch (load_struct.type) {
5853 .got => .load_memory_ptr_got,
5854 .direct => .load_memory_ptr_direct,
5855 .import => unreachable,
5856 };
5857 const atom_index = switch (self.bin_file.tag) {
5858 .macho => {
5859 @panic("TODO genSetStackArgument");
5860 // const macho_file = self.bin_file.cast(link.File.MachO).?;
5861 // const atom = try macho_file.getOrCreateAtomForDecl(self.owner_decl);
5862 // break :blk macho_file.getAtom(atom).getSymbolIndex().?;
5863 },
5864 .coff => blk: {
5865 const coff_file = self.bin_file.cast(.coff).?;
5866 const atom = try coff_file.getOrCreateAtomForNav(self.owner_nav);
5867 break :blk coff_file.getAtom(atom).getSymbolIndex().?;
5868 },
5869 else => unreachable, // unsupported target format
5870 };
5871 _ = try self.addInst(.{
5872 .tag = tag,
5873 .data = .{
5874 .payload = try self.addExtra(Mir.LoadMemoryPie{
5875 .register = @intFromEnum(src_reg),
5876 .atom_index = atom_index,
5877 .sym_index = load_struct.sym_index,
5878 }),
5879 },
5880 });
5881 },
5882 else => unreachable,
5883 }
5884
5885 // add dst_reg, sp, #stack_offset
5886 _ = try self.addInst(.{
5887 .tag = .add_immediate,
5888 .data = .{ .rr_imm12_sh = .{
5889 .rd = dst_reg,
5890 .rn = .sp,
5891 .imm12 = math.cast(u12, stack_offset) orelse {
5892 return self.fail("TODO load: set reg to stack offset with all possible offsets", .{});
5893 },
5894 } },
5895 });
5896
5897 // mov len, #abi_size
5898 try self.genSetReg(Type.usize, len_reg, .{ .immediate = abi_size });
5899
5900 // memcpy(src, dst, len)
5901 try self.genInlineMemcpy(src_reg, dst_reg, len_reg, count_reg, tmp_reg);
5902 }
5903 },
5904 .compare_flags,
5905 .immediate,
5906 .ptr_stack_offset,
5907 => {
5908 const reg = try self.copyToTmpRegister(ty, mcv);
5909 return self.genSetStackArgument(ty, stack_offset, MCValue{ .register = reg });
5910 },
5911 }
5912}
5913
5914fn airBitCast(self: *Self, inst: Air.Inst.Index) InnerError!void {
5915 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
5916 const result = if (self.liveness.isUnused(inst)) .dead else result: {
5917 const operand = try self.resolveInst(ty_op.operand);
5918 if (self.reuseOperand(inst, ty_op.operand, 0, operand)) break :result operand;
5919
5920 const operand_lock = switch (operand) {
5921 .register => |reg| self.register_manager.lockReg(reg),
5922 .register_with_overflow => |rwo| self.register_manager.lockReg(rwo.reg),
5923 else => null,
5924 };
5925 defer if (operand_lock) |lock| self.register_manager.unlockReg(lock);
5926
5927 const dest_ty = self.typeOfIndex(inst);
5928 const dest = try self.allocRegOrMem(dest_ty, true, inst);
5929 try self.setRegOrMem(dest_ty, dest, operand);
5930 break :result dest;
5931 };
5932 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
5933}
5934
5935fn airArrayToSlice(self: *Self, inst: Air.Inst.Index) InnerError!void {
5936 const pt = self.pt;
5937 const zcu = pt.zcu;
5938 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
5939 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
5940 const ptr_ty = self.typeOf(ty_op.operand);
5941 const ptr = try self.resolveInst(ty_op.operand);
5942 const array_ty = ptr_ty.childType(zcu);
5943 const array_len = @as(u32, @intCast(array_ty.arrayLen(zcu)));
5944 const ptr_bytes = 8;
5945 const stack_offset = try self.allocMem(ptr_bytes * 2, .@"8", inst);
5946 try self.genSetStack(ptr_ty, stack_offset, ptr);
5947 try self.genSetStack(Type.usize, stack_offset - ptr_bytes, .{ .immediate = array_len });
5948 break :result MCValue{ .stack_offset = stack_offset };
5949 };
5950 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
5951}
5952
5953fn airFloatFromInt(self: *Self, inst: Air.Inst.Index) InnerError!void {
5954 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
5955 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement airFloatFromInt for {}", .{
5956 self.target.cpu.arch,
5957 });
5958 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
5959}
5960
5961fn airIntFromFloat(self: *Self, inst: Air.Inst.Index) InnerError!void {
5962 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
5963 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement airIntFromFloat for {}", .{
5964 self.target.cpu.arch,
5965 });
5966 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
5967}
5968
5969fn airCmpxchg(self: *Self, inst: Air.Inst.Index) InnerError!void {
5970 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
5971 const extra = self.air.extraData(Air.Block, ty_pl.payload);
5972 _ = extra;
5973
5974 return self.fail("TODO implement airCmpxchg for {}", .{
5975 self.target.cpu.arch,
5976 });
5977}
5978
5979fn airAtomicRmw(self: *Self, inst: Air.Inst.Index) InnerError!void {
5980 _ = inst;
5981 return self.fail("TODO implement airCmpxchg for {}", .{self.target.cpu.arch});
5982}
5983
5984fn airAtomicLoad(self: *Self, inst: Air.Inst.Index) InnerError!void {
5985 _ = inst;
5986 return self.fail("TODO implement airAtomicLoad for {}", .{self.target.cpu.arch});
5987}
5988
5989fn airAtomicStore(self: *Self, inst: Air.Inst.Index, order: std.builtin.AtomicOrder) InnerError!void {
5990 _ = inst;
5991 _ = order;
5992 return self.fail("TODO implement airAtomicStore for {}", .{self.target.cpu.arch});
5993}
5994
5995fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) InnerError!void {
5996 _ = inst;
5997 if (safety) {
5998 // TODO if the value is undef, write 0xaa bytes to dest
5999 } else {
6000 // TODO if the value is undef, don't lower this instruction
6001 }
6002 return self.fail("TODO implement airMemset for {}", .{self.target.cpu.arch});
6003}
6004
6005fn airMemcpy(self: *Self, inst: Air.Inst.Index) InnerError!void {
6006 _ = inst;
6007 return self.fail("TODO implement airMemcpy for {}", .{self.target.cpu.arch});
6008}
6009
6010fn airMemmove(self: *Self, inst: Air.Inst.Index) InnerError!void {
6011 _ = inst;
6012 return self.fail("TODO implement airMemmove for {}", .{self.target.cpu.arch});
6013}
6014
6015fn airTagName(self: *Self, inst: Air.Inst.Index) InnerError!void {
6016 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
6017 const operand = try self.resolveInst(un_op);
6018 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else {
6019 _ = operand;
6020 return self.fail("TODO implement airTagName for aarch64", .{});
6021 };
6022 return self.finishAir(inst, result, .{ un_op, .none, .none });
6023}
6024
6025fn airErrorName(self: *Self, inst: Air.Inst.Index) InnerError!void {
6026 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
6027 const operand = try self.resolveInst(un_op);
6028 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else {
6029 _ = operand;
6030 return self.fail("TODO implement airErrorName for aarch64", .{});
6031 };
6032 return self.finishAir(inst, result, .{ un_op, .none, .none });
6033}
6034
6035fn airSplat(self: *Self, inst: Air.Inst.Index) InnerError!void {
6036 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
6037 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement airSplat for {}", .{self.target.cpu.arch});
6038 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
6039}
6040
6041fn airSelect(self: *Self, inst: Air.Inst.Index) InnerError!void {
6042 const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
6043 const extra = self.air.extraData(Air.Bin, pl_op.payload).data;
6044 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement airSelect for {}", .{self.target.cpu.arch});
6045 return self.finishAir(inst, result, .{ pl_op.operand, extra.lhs, extra.rhs });
6046}
6047
6048fn airShuffleOne(self: *Self, inst: Air.Inst.Index) InnerError!void {
6049 _ = inst;
6050 return self.fail("TODO implement airShuffleOne for {}", .{self.target.cpu.arch});
6051}
6052
6053fn airShuffleTwo(self: *Self, inst: Air.Inst.Index) InnerError!void {
6054 _ = inst;
6055 return self.fail("TODO implement airShuffleTwo for {}", .{self.target.cpu.arch});
6056}
6057
6058fn airReduce(self: *Self, inst: Air.Inst.Index) InnerError!void {
6059 const reduce = self.air.instructions.items(.data)[@intFromEnum(inst)].reduce;
6060 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement airReduce for aarch64", .{});
6061 return self.finishAir(inst, result, .{ reduce.operand, .none, .none });
6062}
6063
6064fn airAggregateInit(self: *Self, inst: Air.Inst.Index) InnerError!void {
6065 const pt = self.pt;
6066 const zcu = pt.zcu;
6067 const vector_ty = self.typeOfIndex(inst);
6068 const len = vector_ty.vectorLen(zcu);
6069 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
6070 const elements: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[ty_pl.payload..][0..len]);
6071 const result: MCValue = res: {
6072 if (self.liveness.isUnused(inst)) break :res MCValue.dead;
6073 return self.fail("TODO implement airAggregateInit for {}", .{self.target.cpu.arch});
6074 };
6075
6076 if (elements.len <= Air.Liveness.bpi - 1) {
6077 var buf = [1]Air.Inst.Ref{.none} ** (Air.Liveness.bpi - 1);
6078 @memcpy(buf[0..elements.len], elements);
6079 return self.finishAir(inst, result, buf);
6080 }
6081 var bt = try self.iterateBigTomb(inst, elements.len);
6082 for (elements) |elem| {
6083 bt.feed(elem);
6084 }
6085 return bt.finishAir(result);
6086}
6087
6088fn airUnionInit(self: *Self, inst: Air.Inst.Index) InnerError!void {
6089 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
6090 const extra = self.air.extraData(Air.UnionInit, ty_pl.payload).data;
6091 _ = extra;
6092 return self.fail("TODO implement airUnionInit for aarch64", .{});
6093}
6094
6095fn airPrefetch(self: *Self, inst: Air.Inst.Index) InnerError!void {
6096 const prefetch = self.air.instructions.items(.data)[@intFromEnum(inst)].prefetch;
6097 return self.finishAir(inst, MCValue.dead, .{ prefetch.ptr, .none, .none });
6098}
6099
6100fn airMulAdd(self: *Self, inst: Air.Inst.Index) InnerError!void {
6101 const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
6102 const extra = self.air.extraData(Air.Bin, pl_op.payload).data;
6103 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else {
6104 return self.fail("TODO implement airMulAdd for aarch64", .{});
6105 };
6106 return self.finishAir(inst, result, .{ extra.lhs, extra.rhs, pl_op.operand });
6107}
6108
6109fn airTry(self: *Self, inst: Air.Inst.Index) InnerError!void {
6110 const pt = self.pt;
6111 const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
6112 const extra = self.air.extraData(Air.Try, pl_op.payload);
6113 const body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[extra.end..][0..extra.data.body_len]);
6114 const result: MCValue = result: {
6115 const error_union_bind: ReadArg.Bind = .{ .inst = pl_op.operand };
6116 const error_union_ty = self.typeOf(pl_op.operand);
6117 const error_union_size = @as(u32, @intCast(error_union_ty.abiSize(pt.zcu)));
6118 const error_union_align = error_union_ty.abiAlignment(pt.zcu);
6119
6120 // The error union will die in the body. However, we need the
6121 // error union after the body in order to extract the payload
6122 // of the error union, so we create a copy of it
6123 const error_union_copy = try self.allocMem(error_union_size, error_union_align, null);
6124 try self.genSetStack(error_union_ty, error_union_copy, try error_union_bind.resolveToMcv(self));
6125
6126 const is_err_result = try self.isErr(error_union_bind, error_union_ty);
6127 const reloc = try self.condBr(is_err_result);
6128
6129 try self.genBody(body);
6130 try self.performReloc(reloc);
6131
6132 break :result try self.errUnionPayload(.{ .mcv = .{ .stack_offset = error_union_copy } }, error_union_ty, null);
6133 };
6134 return self.finishAir(inst, result, .{ pl_op.operand, .none, .none });
6135}
6136
6137fn airTryPtr(self: *Self, inst: Air.Inst.Index) InnerError!void {
6138 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
6139 const extra = self.air.extraData(Air.TryPtr, ty_pl.payload);
6140 const body = self.air.extra.items[extra.end..][0..extra.data.body_len];
6141 _ = body;
6142 return self.fail("TODO implement airTryPtr for arm", .{});
6143 // return self.finishAir(inst, result, .{ extra.data.ptr, .none, .none });
6144}
6145
6146fn resolveInst(self: *Self, inst: Air.Inst.Ref) InnerError!MCValue {
6147 const pt = self.pt;
6148 const zcu = pt.zcu;
6149
6150 // If the type has no codegen bits, no need to store it.
6151 const inst_ty = self.typeOf(inst);
6152 if (!inst_ty.hasRuntimeBitsIgnoreComptime(zcu) and !inst_ty.isError(zcu))
6153 return MCValue{ .none = {} };
6154
6155 const inst_index = inst.toIndex() orelse return self.genTypedValue((try self.air.value(inst, pt)).?);
6156
6157 return self.getResolvedInstValue(inst_index);
6158}
6159
6160fn getResolvedInstValue(self: *Self, inst: Air.Inst.Index) MCValue {
6161 // Treat each stack item as a "layer" on top of the previous one.
6162 var i: usize = self.branch_stack.items.len;
6163 while (true) {
6164 i -= 1;
6165 if (self.branch_stack.items[i].inst_table.get(inst)) |mcv| {
6166 assert(mcv != .dead);
6167 return mcv;
6168 }
6169 }
6170}
6171
6172fn genTypedValue(self: *Self, val: Value) InnerError!MCValue {
6173 const mcv: MCValue = switch (try codegen.genTypedValue(
6174 self.bin_file,
6175 self.pt,
6176 self.src_loc,
6177 val,
6178 self.target,
6179 )) {
6180 .mcv => |mcv| switch (mcv) {
6181 .none => .none,
6182 .undef => .undef,
6183 .immediate => |imm| .{ .immediate = imm },
6184 .memory => |addr| .{ .memory = addr },
6185 .load_got => |sym_index| .{ .linker_load = .{ .type = .got, .sym_index = sym_index } },
6186 .load_direct => |sym_index| .{ .linker_load = .{ .type = .direct, .sym_index = sym_index } },
6187 .load_symbol, .lea_symbol, .lea_direct => unreachable, // TODO
6188 },
6189 .fail => |msg| return self.failMsg(msg),
6190 };
6191 return mcv;
6192}
6193
6194const CallMCValues = struct {
6195 args: []MCValue,
6196 return_value: MCValue,
6197 stack_byte_count: u32,
6198 stack_align: u32,
6199
6200 fn deinit(self: *CallMCValues, func: *Self) void {
6201 func.gpa.free(self.args);
6202 self.* = undefined;
6203 }
6204};
6205
6206/// Caller must call `CallMCValues.deinit`.
6207fn resolveCallingConventionValues(self: *Self, fn_ty: Type) !CallMCValues {
6208 const pt = self.pt;
6209 const zcu = pt.zcu;
6210 const ip = &zcu.intern_pool;
6211 const fn_info = zcu.typeToFunc(fn_ty).?;
6212 const cc = fn_info.cc;
6213 var result: CallMCValues = .{
6214 .args = try self.gpa.alloc(MCValue, fn_info.param_types.len),
6215 // These undefined values must be populated before returning from this function.
6216 .return_value = undefined,
6217 .stack_byte_count = undefined,
6218 .stack_align = undefined,
6219 };
6220 errdefer self.gpa.free(result.args);
6221
6222 const ret_ty = fn_ty.fnReturnType(zcu);
6223
6224 switch (cc) {
6225 .naked => {
6226 assert(result.args.len == 0);
6227 result.return_value = .{ .unreach = {} };
6228 result.stack_byte_count = 0;
6229 result.stack_align = 1;
6230 return result;
6231 },
6232 .aarch64_aapcs, .aarch64_aapcs_darwin, .aarch64_aapcs_win => {
6233 // ARM64 Procedure Call Standard
6234 var ncrn: usize = 0; // Next Core Register Number
6235 var nsaa: u32 = 0; // Next stacked argument address
6236
6237 if (ret_ty.zigTypeTag(zcu) == .noreturn) {
6238 result.return_value = .{ .unreach = {} };
6239 } else if (!ret_ty.hasRuntimeBitsIgnoreComptime(zcu) and !ret_ty.isError(zcu)) {
6240 result.return_value = .{ .none = {} };
6241 } else {
6242 const ret_ty_size: u32 = @intCast(ret_ty.abiSize(zcu));
6243 if (ret_ty_size == 0) {
6244 assert(ret_ty.isError(zcu));
6245 result.return_value = .{ .immediate = 0 };
6246 } else if (ret_ty_size <= 8) {
6247 result.return_value = .{ .register = self.registerAlias(c_abi_int_return_regs[0], ret_ty) };
6248 } else {
6249 return self.fail("TODO support more return types for ARM backend", .{});
6250 }
6251 }
6252
6253 for (fn_info.param_types.get(ip), result.args) |ty, *result_arg| {
6254 const param_size = @as(u32, @intCast(Type.fromInterned(ty).abiSize(zcu)));
6255 if (param_size == 0) {
6256 result_arg.* = .{ .none = {} };
6257 continue;
6258 }
6259
6260 // We round up NCRN only for non-Apple platforms which allow the 16-byte aligned
6261 // values to spread across odd-numbered registers.
6262 if (Type.fromInterned(ty).abiAlignment(zcu) == .@"16" and cc != .aarch64_aapcs_darwin) {
6263 // Round up NCRN to the next even number
6264 ncrn += ncrn % 2;
6265 }
6266
6267 if (std.math.divCeil(u32, param_size, 8) catch unreachable <= 8 - ncrn) {
6268 if (param_size <= 8) {
6269 result_arg.* = .{ .register = self.registerAlias(c_abi_int_param_regs[ncrn], Type.fromInterned(ty)) };
6270 ncrn += 1;
6271 } else {
6272 return self.fail("TODO MCValues with multiple registers", .{});
6273 }
6274 } else if (ncrn < 8 and nsaa == 0) {
6275 return self.fail("TODO MCValues split between registers and stack", .{});
6276 } else {
6277 ncrn = 8;
6278 // TODO Apple allows the arguments on the stack to be non-8-byte aligned provided
6279 // that the entire stack space consumed by the arguments is 8-byte aligned.
6280 if (Type.fromInterned(ty).abiAlignment(zcu) == .@"8") {
6281 if (nsaa % 8 != 0) {
6282 nsaa += 8 - (nsaa % 8);
6283 }
6284 }
6285
6286 result_arg.* = .{ .stack_argument_offset = nsaa };
6287 nsaa += param_size;
6288 }
6289 }
6290
6291 result.stack_byte_count = nsaa;
6292 result.stack_align = 16;
6293 },
6294 .auto => {
6295 if (ret_ty.zigTypeTag(zcu) == .noreturn) {
6296 result.return_value = .{ .unreach = {} };
6297 } else if (!ret_ty.hasRuntimeBitsIgnoreComptime(zcu) and !ret_ty.isError(zcu)) {
6298 result.return_value = .{ .none = {} };
6299 } else {
6300 const ret_ty_size = @as(u32, @intCast(ret_ty.abiSize(zcu)));
6301 if (ret_ty_size == 0) {
6302 assert(ret_ty.isError(zcu));
6303 result.return_value = .{ .immediate = 0 };
6304 } else if (ret_ty_size <= 8) {
6305 result.return_value = .{ .register = self.registerAlias(.x0, ret_ty) };
6306 } else {
6307 // The result is returned by reference, not by
6308 // value. This means that x0 (or w0 when pointer
6309 // size is 32 bits) will contain the address of
6310 // where this function should write the result
6311 // into.
6312 result.return_value = .{ .stack_offset = 0 };
6313 }
6314 }
6315
6316 var stack_offset: u32 = 0;
6317
6318 for (fn_info.param_types.get(ip), result.args) |ty, *result_arg| {
6319 if (Type.fromInterned(ty).abiSize(zcu) > 0) {
6320 const param_size: u32 = @intCast(Type.fromInterned(ty).abiSize(zcu));
6321 const param_alignment = Type.fromInterned(ty).abiAlignment(zcu);
6322
6323 stack_offset = @intCast(param_alignment.forward(stack_offset));
6324 result_arg.* = .{ .stack_argument_offset = stack_offset };
6325 stack_offset += param_size;
6326 } else {
6327 result_arg.* = .{ .none = {} };
6328 }
6329 }
6330
6331 result.stack_byte_count = stack_offset;
6332 result.stack_align = 16;
6333 },
6334 else => return self.fail("TODO implement function parameters for {} on aarch64", .{cc}),
6335 }
6336
6337 return result;
6338}
6339
6340/// TODO support scope overrides. Also note this logic is duplicated with `Zcu.wantSafety`.
6341fn wantSafety(self: *Self) bool {
6342 return switch (self.bin_file.comp.root_mod.optimize_mode) {
6343 .Debug => true,
6344 .ReleaseSafe => true,
6345 .ReleaseFast => false,
6346 .ReleaseSmall => false,
6347 };
6348}
6349
6350fn fail(self: *Self, comptime format: []const u8, args: anytype) error{ OutOfMemory, CodegenFail } {
6351 @branchHint(.cold);
6352 return self.pt.zcu.codegenFail(self.owner_nav, format, args);
6353}
6354
6355fn failMsg(self: *Self, msg: *ErrorMsg) error{ OutOfMemory, CodegenFail } {
6356 @branchHint(.cold);
6357 return self.pt.zcu.codegenFailMsg(self.owner_nav, msg);
6358}
6359
6360fn parseRegName(name: []const u8) ?Register {
6361 if (@hasDecl(Register, "parseRegName")) {
6362 return Register.parseRegName(name);
6363 }
6364 return std.meta.stringToEnum(Register, name);
6365}
6366
6367fn registerAlias(self: *Self, reg: Register, ty: Type) Register {
6368 const abi_size = ty.abiSize(self.pt.zcu);
6369
6370 switch (reg.class()) {
6371 .general_purpose => {
6372 if (abi_size == 0) {
6373 unreachable; // should be comptime-known
6374 } else if (abi_size <= 4) {
6375 return reg.toW();
6376 } else if (abi_size <= 8) {
6377 return reg.toX();
6378 } else unreachable;
6379 },
6380 .stack_pointer => unreachable, // we can't store/load the sp
6381 .floating_point => {
6382 return switch (ty.floatBits(self.target)) {
6383 16 => reg.toH(),
6384 32 => reg.toS(),
6385 64 => reg.toD(),
6386 128 => reg.toQ(),
6387
6388 80 => unreachable, // f80 registers don't exist
6389 else => unreachable,
6390 };
6391 },
6392 }
6393}
6394
6395fn typeOf(self: *Self, inst: Air.Inst.Ref) Type {
6396 return self.air.typeOf(inst, &self.pt.zcu.intern_pool);
6397}
6398
6399fn typeOfIndex(self: *Self, inst: Air.Inst.Index) Type {
6400 return self.air.typeOfIndex(inst, &self.pt.zcu.intern_pool);
6401}
src/arch/aarch64/Emit.zig deleted-1356
...@@ -1,1356 +0,0 @@
1//! This file contains the functionality for lowering AArch64 MIR into
2//! machine code
3
4const Emit = @This();
5const std = @import("std");
6const math = std.math;
7const Mir = @import("Mir.zig");
8const bits = @import("bits.zig");
9const link = @import("../../link.zig");
10const Zcu = @import("../../Zcu.zig");
11const ErrorMsg = Zcu.ErrorMsg;
12const assert = std.debug.assert;
13const Instruction = bits.Instruction;
14const Register = bits.Register;
15const log = std.log.scoped(.aarch64_emit);
16
17mir: Mir,
18bin_file: *link.File,
19debug_output: link.File.DebugInfoOutput,
20target: *const std.Target,
21err_msg: ?*ErrorMsg = null,
22src_loc: Zcu.LazySrcLoc,
23code: *std.ArrayListUnmanaged(u8),
24
25prev_di_line: u32,
26prev_di_column: u32,
27
28/// Relative to the beginning of `code`.
29prev_di_pc: usize,
30
31/// The amount of stack space consumed by the saved callee-saved
32/// registers in bytes
33saved_regs_stack_space: u32,
34
35/// The branch type of every branch
36branch_types: std.AutoHashMapUnmanaged(Mir.Inst.Index, BranchType) = .empty,
37
38/// For every forward branch, maps the target instruction to a list of
39/// branches which branch to this target instruction
40branch_forward_origins: std.AutoHashMapUnmanaged(Mir.Inst.Index, std.ArrayListUnmanaged(Mir.Inst.Index)) = .empty,
41
42/// For backward branches: stores the code offset of the target
43/// instruction
44///
45/// For forward branches: stores the code offset of the branch
46/// instruction
47code_offset_mapping: std.AutoHashMapUnmanaged(Mir.Inst.Index, usize) = .empty,
48
49/// The final stack frame size of the function (already aligned to the
50/// respective stack alignment). Does not include prologue stack space.
51stack_size: u32,
52
53const InnerError = error{
54 OutOfMemory,
55 EmitFail,
56};
57
58const BranchType = enum {
59 cbz,
60 b_cond,
61 unconditional_branch_immediate,
62
63 fn default(tag: Mir.Inst.Tag) BranchType {
64 return switch (tag) {
65 .cbz => .cbz,
66 .b, .bl => .unconditional_branch_immediate,
67 .b_cond => .b_cond,
68 else => unreachable,
69 };
70 }
71};
72
73pub fn emitMir(
74 emit: *Emit,
75) !void {
76 const mir_tags = emit.mir.instructions.items(.tag);
77
78 // Find smallest lowerings for branch instructions
79 try emit.lowerBranches();
80
81 // Emit machine code
82 for (mir_tags, 0..) |tag, index| {
83 const inst = @as(u32, @intCast(index));
84 switch (tag) {
85 .add_immediate => try emit.mirAddSubtractImmediate(inst),
86 .adds_immediate => try emit.mirAddSubtractImmediate(inst),
87 .cmp_immediate => try emit.mirAddSubtractImmediate(inst),
88 .sub_immediate => try emit.mirAddSubtractImmediate(inst),
89 .subs_immediate => try emit.mirAddSubtractImmediate(inst),
90
91 .asr_register => try emit.mirDataProcessing2Source(inst),
92 .lsl_register => try emit.mirDataProcessing2Source(inst),
93 .lsr_register => try emit.mirDataProcessing2Source(inst),
94 .sdiv => try emit.mirDataProcessing2Source(inst),
95 .udiv => try emit.mirDataProcessing2Source(inst),
96
97 .asr_immediate => try emit.mirShiftImmediate(inst),
98 .lsl_immediate => try emit.mirShiftImmediate(inst),
99 .lsr_immediate => try emit.mirShiftImmediate(inst),
100
101 .b_cond => try emit.mirConditionalBranchImmediate(inst),
102
103 .b => try emit.mirBranch(inst),
104 .bl => try emit.mirBranch(inst),
105
106 .cbz => try emit.mirCompareAndBranch(inst),
107
108 .blr => try emit.mirUnconditionalBranchRegister(inst),
109 .ret => try emit.mirUnconditionalBranchRegister(inst),
110
111 .brk => try emit.mirExceptionGeneration(inst),
112 .svc => try emit.mirExceptionGeneration(inst),
113
114 .call_extern => try emit.mirCallExtern(inst),
115
116 .eor_immediate => try emit.mirLogicalImmediate(inst),
117 .tst_immediate => try emit.mirLogicalImmediate(inst),
118
119 .add_shifted_register => try emit.mirAddSubtractShiftedRegister(inst),
120 .adds_shifted_register => try emit.mirAddSubtractShiftedRegister(inst),
121 .cmp_shifted_register => try emit.mirAddSubtractShiftedRegister(inst),
122 .sub_shifted_register => try emit.mirAddSubtractShiftedRegister(inst),
123 .subs_shifted_register => try emit.mirAddSubtractShiftedRegister(inst),
124
125 .add_extended_register => try emit.mirAddSubtractExtendedRegister(inst),
126 .adds_extended_register => try emit.mirAddSubtractExtendedRegister(inst),
127 .sub_extended_register => try emit.mirAddSubtractExtendedRegister(inst),
128 .subs_extended_register => try emit.mirAddSubtractExtendedRegister(inst),
129 .cmp_extended_register => try emit.mirAddSubtractExtendedRegister(inst),
130
131 .csel => try emit.mirConditionalSelect(inst),
132 .cset => try emit.mirConditionalSelect(inst),
133
134 .dbg_line => try emit.mirDbgLine(inst),
135
136 .dbg_prologue_end => try emit.mirDebugPrologueEnd(),
137 .dbg_epilogue_begin => try emit.mirDebugEpilogueBegin(),
138
139 .and_shifted_register => try emit.mirLogicalShiftedRegister(inst),
140 .eor_shifted_register => try emit.mirLogicalShiftedRegister(inst),
141 .orr_shifted_register => try emit.mirLogicalShiftedRegister(inst),
142
143 .load_memory_got => try emit.mirLoadMemoryPie(inst),
144 .load_memory_direct => try emit.mirLoadMemoryPie(inst),
145 .load_memory_import => try emit.mirLoadMemoryPie(inst),
146 .load_memory_ptr_got => try emit.mirLoadMemoryPie(inst),
147 .load_memory_ptr_direct => try emit.mirLoadMemoryPie(inst),
148
149 .ldp => try emit.mirLoadStoreRegisterPair(inst),
150 .stp => try emit.mirLoadStoreRegisterPair(inst),
151
152 .ldr_ptr_stack => try emit.mirLoadStoreStack(inst),
153 .ldr_stack => try emit.mirLoadStoreStack(inst),
154 .ldrb_stack => try emit.mirLoadStoreStack(inst),
155 .ldrh_stack => try emit.mirLoadStoreStack(inst),
156 .ldrsb_stack => try emit.mirLoadStoreStack(inst),
157 .ldrsh_stack => try emit.mirLoadStoreStack(inst),
158 .str_stack => try emit.mirLoadStoreStack(inst),
159 .strb_stack => try emit.mirLoadStoreStack(inst),
160 .strh_stack => try emit.mirLoadStoreStack(inst),
161
162 .ldr_ptr_stack_argument => try emit.mirLoadStackArgument(inst),
163 .ldr_stack_argument => try emit.mirLoadStackArgument(inst),
164 .ldrb_stack_argument => try emit.mirLoadStackArgument(inst),
165 .ldrh_stack_argument => try emit.mirLoadStackArgument(inst),
166 .ldrsb_stack_argument => try emit.mirLoadStackArgument(inst),
167 .ldrsh_stack_argument => try emit.mirLoadStackArgument(inst),
168
169 .ldr_register => try emit.mirLoadStoreRegisterRegister(inst),
170 .ldrb_register => try emit.mirLoadStoreRegisterRegister(inst),
171 .ldrh_register => try emit.mirLoadStoreRegisterRegister(inst),
172 .str_register => try emit.mirLoadStoreRegisterRegister(inst),
173 .strb_register => try emit.mirLoadStoreRegisterRegister(inst),
174 .strh_register => try emit.mirLoadStoreRegisterRegister(inst),
175
176 .ldr_immediate => try emit.mirLoadStoreRegisterImmediate(inst),
177 .ldrb_immediate => try emit.mirLoadStoreRegisterImmediate(inst),
178 .ldrh_immediate => try emit.mirLoadStoreRegisterImmediate(inst),
179 .ldrsb_immediate => try emit.mirLoadStoreRegisterImmediate(inst),
180 .ldrsh_immediate => try emit.mirLoadStoreRegisterImmediate(inst),
181 .ldrsw_immediate => try emit.mirLoadStoreRegisterImmediate(inst),
182 .str_immediate => try emit.mirLoadStoreRegisterImmediate(inst),
183 .strb_immediate => try emit.mirLoadStoreRegisterImmediate(inst),
184 .strh_immediate => try emit.mirLoadStoreRegisterImmediate(inst),
185
186 .mov_register => try emit.mirMoveRegister(inst),
187 .mov_to_from_sp => try emit.mirMoveRegister(inst),
188 .mvn => try emit.mirMoveRegister(inst),
189
190 .movk => try emit.mirMoveWideImmediate(inst),
191 .movz => try emit.mirMoveWideImmediate(inst),
192
193 .msub => try emit.mirDataProcessing3Source(inst),
194 .mul => try emit.mirDataProcessing3Source(inst),
195 .smulh => try emit.mirDataProcessing3Source(inst),
196 .smull => try emit.mirDataProcessing3Source(inst),
197 .umulh => try emit.mirDataProcessing3Source(inst),
198 .umull => try emit.mirDataProcessing3Source(inst),
199
200 .nop => try emit.mirNop(),
201
202 .push_regs => try emit.mirPushPopRegs(inst),
203 .pop_regs => try emit.mirPushPopRegs(inst),
204
205 .sbfx,
206 .ubfx,
207 => try emit.mirBitfieldExtract(inst),
208
209 .sxtb,
210 .sxth,
211 .sxtw,
212 .uxtb,
213 .uxth,
214 => try emit.mirExtend(inst),
215 }
216 }
217}
218
219pub fn deinit(emit: *Emit) void {
220 const comp = emit.bin_file.comp;
221 const gpa = comp.gpa;
222 var iter = emit.branch_forward_origins.valueIterator();
223 while (iter.next()) |origin_list| {
224 origin_list.deinit(gpa);
225 }
226
227 emit.branch_types.deinit(gpa);
228 emit.branch_forward_origins.deinit(gpa);
229 emit.code_offset_mapping.deinit(gpa);
230 emit.* = undefined;
231}
232
233fn optimalBranchType(emit: *Emit, tag: Mir.Inst.Tag, offset: i64) !BranchType {
234 assert(offset & 0b11 == 0);
235
236 switch (tag) {
237 .cbz => {
238 if (std.math.cast(i19, @shrExact(offset, 2))) |_| {
239 return BranchType.cbz;
240 } else {
241 return emit.fail("TODO support cbz branches larger than +-1 MiB", .{});
242 }
243 },
244 .b, .bl => {
245 if (std.math.cast(i26, @shrExact(offset, 2))) |_| {
246 return BranchType.unconditional_branch_immediate;
247 } else {
248 return emit.fail("TODO support unconditional branches larger than +-128 MiB", .{});
249 }
250 },
251 .b_cond => {
252 if (std.math.cast(i19, @shrExact(offset, 2))) |_| {
253 return BranchType.b_cond;
254 } else {
255 return emit.fail("TODO support conditional branches larger than +-1 MiB", .{});
256 }
257 },
258 else => unreachable,
259 }
260}
261
262fn instructionSize(emit: *Emit, inst: Mir.Inst.Index) usize {
263 const tag = emit.mir.instructions.items(.tag)[inst];
264
265 if (isBranch(tag)) {
266 switch (emit.branch_types.get(inst).?) {
267 .cbz,
268 .unconditional_branch_immediate,
269 .b_cond,
270 => return 4,
271 }
272 }
273
274 switch (tag) {
275 .load_memory_direct => return 3 * 4,
276 .load_memory_got,
277 .load_memory_ptr_got,
278 .load_memory_ptr_direct,
279 => return 2 * 4,
280 .pop_regs, .push_regs => {
281 const reg_list = emit.mir.instructions.items(.data)[inst].reg_list;
282 const number_of_regs = @popCount(reg_list);
283 const number_of_insts = std.math.divCeil(u6, number_of_regs, 2) catch unreachable;
284 return number_of_insts * 4;
285 },
286 .call_extern => return 4,
287 .dbg_line,
288 .dbg_epilogue_begin,
289 .dbg_prologue_end,
290 => return 0,
291 else => return 4,
292 }
293}
294
295fn isBranch(tag: Mir.Inst.Tag) bool {
296 return switch (tag) {
297 .cbz,
298 .b,
299 .bl,
300 .b_cond,
301 => true,
302 else => false,
303 };
304}
305
306fn branchTarget(emit: *Emit, inst: Mir.Inst.Index) Mir.Inst.Index {
307 const tag = emit.mir.instructions.items(.tag)[inst];
308
309 switch (tag) {
310 .cbz => return emit.mir.instructions.items(.data)[inst].r_inst.inst,
311 .b, .bl => return emit.mir.instructions.items(.data)[inst].inst,
312 .b_cond => return emit.mir.instructions.items(.data)[inst].inst_cond.inst,
313 else => unreachable,
314 }
315}
316
317fn lowerBranches(emit: *Emit) !void {
318 const comp = emit.bin_file.comp;
319 const gpa = comp.gpa;
320 const mir_tags = emit.mir.instructions.items(.tag);
321
322 // First pass: Note down all branches and their target
323 // instructions, i.e. populate branch_types,
324 // branch_forward_origins, and code_offset_mapping
325 //
326 // TODO optimization opportunity: do this in codegen while
327 // generating MIR
328 for (mir_tags, 0..) |tag, index| {
329 const inst = @as(u32, @intCast(index));
330 if (isBranch(tag)) {
331 const target_inst = emit.branchTarget(inst);
332
333 // Remember this branch instruction
334 try emit.branch_types.put(gpa, inst, BranchType.default(tag));
335
336 // Forward branches require some extra stuff: We only
337 // know their offset once we arrive at the target
338 // instruction. Therefore, we need to be able to
339 // access the branch instruction when we visit the
340 // target instruction in order to manipulate its type
341 // etc.
342 if (target_inst > inst) {
343 // Remember the branch instruction index
344 try emit.code_offset_mapping.put(gpa, inst, 0);
345
346 if (emit.branch_forward_origins.getPtr(target_inst)) |origin_list| {
347 try origin_list.append(gpa, inst);
348 } else {
349 var origin_list: std.ArrayListUnmanaged(Mir.Inst.Index) = .empty;
350 try origin_list.append(gpa, inst);
351 try emit.branch_forward_origins.put(gpa, target_inst, origin_list);
352 }
353 }
354
355 // Remember the target instruction index so that we
356 // update the real code offset in all future passes
357 //
358 // putNoClobber may not be used as the put operation
359 // may clobber the entry when multiple branches branch
360 // to the same target instruction
361 try emit.code_offset_mapping.put(gpa, target_inst, 0);
362 }
363 }
364
365 // Further passes: Until all branches are lowered, interate
366 // through all instructions and calculate new offsets and
367 // potentially new branch types
368 var all_branches_lowered = false;
369 while (!all_branches_lowered) {
370 all_branches_lowered = true;
371 var current_code_offset: usize = 0;
372
373 for (mir_tags, 0..) |tag, index| {
374 const inst = @as(u32, @intCast(index));
375
376 // If this instruction contained in the code offset
377 // mapping (when it is a target of a branch or if it is a
378 // forward branch), update the code offset
379 if (emit.code_offset_mapping.getPtr(inst)) |offset| {
380 offset.* = current_code_offset;
381 }
382
383 // If this instruction is a backward branch, calculate the
384 // offset, which may potentially update the branch type
385 if (isBranch(tag)) {
386 const target_inst = emit.branchTarget(inst);
387 if (target_inst < inst) {
388 const target_offset = emit.code_offset_mapping.get(target_inst).?;
389 const offset = @as(i64, @intCast(target_offset)) - @as(i64, @intCast(current_code_offset));
390 const branch_type = emit.branch_types.getPtr(inst).?;
391 const optimal_branch_type = try emit.optimalBranchType(tag, offset);
392 if (branch_type.* != optimal_branch_type) {
393 branch_type.* = optimal_branch_type;
394 all_branches_lowered = false;
395 }
396
397 log.debug("lowerBranches: branch {} has offset {}", .{ inst, offset });
398 }
399 }
400
401 // If this instruction is the target of one or more
402 // forward branches, calculate the offset, which may
403 // potentially update the branch type
404 if (emit.branch_forward_origins.get(inst)) |origin_list| {
405 for (origin_list.items) |forward_branch_inst| {
406 const branch_tag = emit.mir.instructions.items(.tag)[forward_branch_inst];
407 const forward_branch_inst_offset = emit.code_offset_mapping.get(forward_branch_inst).?;
408 const offset = @as(i64, @intCast(current_code_offset)) - @as(i64, @intCast(forward_branch_inst_offset));
409 const branch_type = emit.branch_types.getPtr(forward_branch_inst).?;
410 const optimal_branch_type = try emit.optimalBranchType(branch_tag, offset);
411 if (branch_type.* != optimal_branch_type) {
412 branch_type.* = optimal_branch_type;
413 all_branches_lowered = false;
414 }
415
416 log.debug("lowerBranches: branch {} has offset {}", .{ forward_branch_inst, offset });
417 }
418 }
419
420 // Increment code offset
421 current_code_offset += emit.instructionSize(inst);
422 }
423 }
424}
425
426fn writeInstruction(emit: *Emit, instruction: Instruction) !void {
427 const comp = emit.bin_file.comp;
428 const gpa = comp.gpa;
429 const endian = emit.target.cpu.arch.endian();
430 std.mem.writeInt(u32, try emit.code.addManyAsArray(gpa, 4), instruction.toU32(), endian);
431}
432
433fn fail(emit: *Emit, comptime format: []const u8, args: anytype) InnerError {
434 @branchHint(.cold);
435 assert(emit.err_msg == null);
436 const comp = emit.bin_file.comp;
437 const gpa = comp.gpa;
438 emit.err_msg = try ErrorMsg.create(gpa, emit.src_loc, format, args);
439 return error.EmitFail;
440}
441
442fn dbgAdvancePCAndLine(emit: *Emit, line: u32, column: u32) InnerError!void {
443 const delta_line = @as(i33, line) - @as(i33, emit.prev_di_line);
444 const delta_pc: usize = emit.code.items.len - emit.prev_di_pc;
445 log.debug(" (advance pc={d} and line={d})", .{ delta_pc, delta_line });
446 switch (emit.debug_output) {
447 .dwarf => |dw| {
448 if (column != emit.prev_di_column) try dw.setColumn(column);
449 try dw.advancePCAndLine(delta_line, delta_pc);
450 emit.prev_di_line = line;
451 emit.prev_di_column = column;
452 emit.prev_di_pc = emit.code.items.len;
453 },
454 .plan9 => |dbg_out| {
455 if (delta_pc <= 0) return; // only do this when the pc changes
456
457 // increasing the line number
458 try link.File.Plan9.changeLine(&dbg_out.dbg_line, @intCast(delta_line));
459 // increasing the pc
460 const d_pc_p9 = @as(i64, @intCast(delta_pc)) - dbg_out.pc_quanta;
461 if (d_pc_p9 > 0) {
462 // minus one because if its the last one, we want to leave space to change the line which is one pc quanta
463 var diff = @divExact(d_pc_p9, dbg_out.pc_quanta) - dbg_out.pc_quanta;
464 while (diff > 0) {
465 if (diff < 64) {
466 try dbg_out.dbg_line.append(@intCast(diff + 128));
467 diff = 0;
468 } else {
469 try dbg_out.dbg_line.append(@intCast(64 + 128));
470 diff -= 64;
471 }
472 }
473 if (dbg_out.pcop_change_index) |pci|
474 dbg_out.dbg_line.items[pci] += 1;
475 dbg_out.pcop_change_index = @intCast(dbg_out.dbg_line.items.len - 1);
476 } else if (d_pc_p9 == 0) {
477 // we don't need to do anything, because adding the pc quanta does it for us
478 } else unreachable;
479 if (dbg_out.start_line == null)
480 dbg_out.start_line = emit.prev_di_line;
481 dbg_out.end_line = line;
482 // only do this if the pc changed
483 emit.prev_di_line = line;
484 emit.prev_di_column = column;
485 emit.prev_di_pc = emit.code.items.len;
486 },
487 .none => {},
488 }
489}
490
491fn mirAddSubtractImmediate(emit: *Emit, inst: Mir.Inst.Index) !void {
492 const tag = emit.mir.instructions.items(.tag)[inst];
493 switch (tag) {
494 .add_immediate,
495 .adds_immediate,
496 .sub_immediate,
497 .subs_immediate,
498 => {
499 const rr_imm12_sh = emit.mir.instructions.items(.data)[inst].rr_imm12_sh;
500 const rd = rr_imm12_sh.rd;
501 const rn = rr_imm12_sh.rn;
502 const imm12 = rr_imm12_sh.imm12;
503 const sh = rr_imm12_sh.sh == 1;
504
505 switch (tag) {
506 .add_immediate => try emit.writeInstruction(Instruction.add(rd, rn, imm12, sh)),
507 .adds_immediate => try emit.writeInstruction(Instruction.adds(rd, rn, imm12, sh)),
508 .sub_immediate => try emit.writeInstruction(Instruction.sub(rd, rn, imm12, sh)),
509 .subs_immediate => try emit.writeInstruction(Instruction.subs(rd, rn, imm12, sh)),
510 else => unreachable,
511 }
512 },
513 .cmp_immediate => {
514 const r_imm12_sh = emit.mir.instructions.items(.data)[inst].r_imm12_sh;
515 const rn = r_imm12_sh.rn;
516 const imm12 = r_imm12_sh.imm12;
517 const sh = r_imm12_sh.sh == 1;
518 const zr: Register = switch (rn.size()) {
519 32 => .wzr,
520 64 => .xzr,
521 else => unreachable,
522 };
523
524 try emit.writeInstruction(Instruction.subs(zr, rn, imm12, sh));
525 },
526 else => unreachable,
527 }
528}
529
530fn mirDataProcessing2Source(emit: *Emit, inst: Mir.Inst.Index) !void {
531 const tag = emit.mir.instructions.items(.tag)[inst];
532 const rrr = emit.mir.instructions.items(.data)[inst].rrr;
533 const rd = rrr.rd;
534 const rn = rrr.rn;
535 const rm = rrr.rm;
536
537 switch (tag) {
538 .asr_register => try emit.writeInstruction(Instruction.asrRegister(rd, rn, rm)),
539 .lsl_register => try emit.writeInstruction(Instruction.lslRegister(rd, rn, rm)),
540 .lsr_register => try emit.writeInstruction(Instruction.lsrRegister(rd, rn, rm)),
541 .sdiv => try emit.writeInstruction(Instruction.sdiv(rd, rn, rm)),
542 .udiv => try emit.writeInstruction(Instruction.udiv(rd, rn, rm)),
543 else => unreachable,
544 }
545}
546
547fn mirShiftImmediate(emit: *Emit, inst: Mir.Inst.Index) !void {
548 const tag = emit.mir.instructions.items(.tag)[inst];
549 const rr_shift = emit.mir.instructions.items(.data)[inst].rr_shift;
550 const rd = rr_shift.rd;
551 const rn = rr_shift.rn;
552 const shift = rr_shift.shift;
553
554 switch (tag) {
555 .asr_immediate => try emit.writeInstruction(Instruction.asrImmediate(rd, rn, shift)),
556 .lsl_immediate => try emit.writeInstruction(Instruction.lslImmediate(rd, rn, shift)),
557 .lsr_immediate => try emit.writeInstruction(Instruction.lsrImmediate(rd, rn, shift)),
558 else => unreachable,
559 }
560}
561
562fn mirConditionalBranchImmediate(emit: *Emit, inst: Mir.Inst.Index) !void {
563 const tag = emit.mir.instructions.items(.tag)[inst];
564 const inst_cond = emit.mir.instructions.items(.data)[inst].inst_cond;
565
566 const offset = @as(i64, @intCast(emit.code_offset_mapping.get(inst_cond.inst).?)) - @as(i64, @intCast(emit.code.items.len));
567 const branch_type = emit.branch_types.get(inst).?;
568 log.debug("mirConditionalBranchImmediate: {} offset={}", .{ inst, offset });
569
570 switch (branch_type) {
571 .b_cond => switch (tag) {
572 .b_cond => try emit.writeInstruction(Instruction.bCond(inst_cond.cond, @as(i21, @intCast(offset)))),
573 else => unreachable,
574 },
575 else => unreachable,
576 }
577}
578
579fn mirBranch(emit: *Emit, inst: Mir.Inst.Index) !void {
580 const tag = emit.mir.instructions.items(.tag)[inst];
581 const target_inst = emit.mir.instructions.items(.data)[inst].inst;
582
583 log.debug("branch {}(tag: {}) -> {}(tag: {})", .{
584 inst,
585 tag,
586 target_inst,
587 emit.mir.instructions.items(.tag)[target_inst],
588 });
589
590 const offset = @as(i64, @intCast(emit.code_offset_mapping.get(target_inst).?)) - @as(i64, @intCast(emit.code.items.len));
591 const branch_type = emit.branch_types.get(inst).?;
592 log.debug("mirBranch: {} offset={}", .{ inst, offset });
593
594 switch (branch_type) {
595 .unconditional_branch_immediate => switch (tag) {
596 .b => try emit.writeInstruction(Instruction.b(@as(i28, @intCast(offset)))),
597 .bl => try emit.writeInstruction(Instruction.bl(@as(i28, @intCast(offset)))),
598 else => unreachable,
599 },
600 else => unreachable,
601 }
602}
603
604fn mirCompareAndBranch(emit: *Emit, inst: Mir.Inst.Index) !void {
605 const tag = emit.mir.instructions.items(.tag)[inst];
606 const r_inst = emit.mir.instructions.items(.data)[inst].r_inst;
607
608 const offset = @as(i64, @intCast(emit.code_offset_mapping.get(r_inst.inst).?)) - @as(i64, @intCast(emit.code.items.len));
609 const branch_type = emit.branch_types.get(inst).?;
610 log.debug("mirCompareAndBranch: {} offset={}", .{ inst, offset });
611
612 switch (branch_type) {
613 .cbz => switch (tag) {
614 .cbz => try emit.writeInstruction(Instruction.cbz(r_inst.rt, @as(i21, @intCast(offset)))),
615 else => unreachable,
616 },
617 else => unreachable,
618 }
619}
620
621fn mirUnconditionalBranchRegister(emit: *Emit, inst: Mir.Inst.Index) !void {
622 const tag = emit.mir.instructions.items(.tag)[inst];
623 const reg = emit.mir.instructions.items(.data)[inst].reg;
624
625 switch (tag) {
626 .blr => try emit.writeInstruction(Instruction.blr(reg)),
627 .ret => try emit.writeInstruction(Instruction.ret(reg)),
628 else => unreachable,
629 }
630}
631
632fn mirExceptionGeneration(emit: *Emit, inst: Mir.Inst.Index) !void {
633 const tag = emit.mir.instructions.items(.tag)[inst];
634 const imm16 = emit.mir.instructions.items(.data)[inst].imm16;
635
636 switch (tag) {
637 .brk => try emit.writeInstruction(Instruction.brk(imm16)),
638 .svc => try emit.writeInstruction(Instruction.svc(imm16)),
639 else => unreachable,
640 }
641}
642
643fn mirDbgLine(emit: *Emit, inst: Mir.Inst.Index) !void {
644 const tag = emit.mir.instructions.items(.tag)[inst];
645 const dbg_line_column = emit.mir.instructions.items(.data)[inst].dbg_line_column;
646
647 switch (tag) {
648 .dbg_line => try emit.dbgAdvancePCAndLine(dbg_line_column.line, dbg_line_column.column),
649 else => unreachable,
650 }
651}
652
653fn mirDebugPrologueEnd(emit: *Emit) !void {
654 switch (emit.debug_output) {
655 .dwarf => |dw| {
656 try dw.setPrologueEnd();
657 log.debug("mirDbgPrologueEnd (line={d}, col={d})", .{
658 emit.prev_di_line, emit.prev_di_column,
659 });
660 try emit.dbgAdvancePCAndLine(emit.prev_di_line, emit.prev_di_column);
661 },
662 .plan9 => {},
663 .none => {},
664 }
665}
666
667fn mirDebugEpilogueBegin(emit: *Emit) !void {
668 switch (emit.debug_output) {
669 .dwarf => |dw| {
670 try dw.setEpilogueBegin();
671 try emit.dbgAdvancePCAndLine(emit.prev_di_line, emit.prev_di_column);
672 },
673 .plan9 => {},
674 .none => {},
675 }
676}
677
678fn mirCallExtern(emit: *Emit, inst: Mir.Inst.Index) !void {
679 assert(emit.mir.instructions.items(.tag)[inst] == .call_extern);
680 const relocation = emit.mir.instructions.items(.data)[inst].relocation;
681 _ = relocation;
682
683 const offset = blk: {
684 const offset = @as(u32, @intCast(emit.code.items.len));
685 // bl
686 try emit.writeInstruction(Instruction.bl(0));
687 break :blk offset;
688 };
689 _ = offset;
690
691 if (emit.bin_file.cast(.macho)) |macho_file| {
692 _ = macho_file;
693 @panic("TODO mirCallExtern");
694 // // Add relocation to the decl.
695 // const atom_index = macho_file.getAtomIndexForSymbol(.{ .sym_index = relocation.atom_index }).?;
696 // const target = macho_file.getGlobalByIndex(relocation.sym_index);
697 // try link.File.MachO.Atom.addRelocation(macho_file, atom_index, .{
698 // .type = .branch,
699 // .target = target,
700 // .offset = offset,
701 // .addend = 0,
702 // .pcrel = true,
703 // .length = 2,
704 // });
705 } else if (emit.bin_file.cast(.coff)) |_| {
706 unreachable; // Calling imports is handled via `.load_memory_import`
707 } else {
708 return emit.fail("Implement call_extern for linking backends != {{ COFF, MachO }}", .{});
709 }
710}
711
712fn mirLogicalImmediate(emit: *Emit, inst: Mir.Inst.Index) !void {
713 const tag = emit.mir.instructions.items(.tag)[inst];
714 const rr_bitmask = emit.mir.instructions.items(.data)[inst].rr_bitmask;
715 const rd = rr_bitmask.rd;
716 const rn = rr_bitmask.rn;
717 const imms = rr_bitmask.imms;
718 const immr = rr_bitmask.immr;
719 const n = rr_bitmask.n;
720
721 switch (tag) {
722 .eor_immediate => try emit.writeInstruction(Instruction.eorImmediate(rd, rn, imms, immr, n)),
723 .tst_immediate => {
724 const zr: Register = switch (rd.size()) {
725 32 => .wzr,
726 64 => .xzr,
727 else => unreachable,
728 };
729 try emit.writeInstruction(Instruction.andsImmediate(zr, rn, imms, immr, n));
730 },
731 else => unreachable,
732 }
733}
734
735fn mirAddSubtractShiftedRegister(emit: *Emit, inst: Mir.Inst.Index) !void {
736 const tag = emit.mir.instructions.items(.tag)[inst];
737 switch (tag) {
738 .add_shifted_register,
739 .adds_shifted_register,
740 .sub_shifted_register,
741 .subs_shifted_register,
742 => {
743 const rrr_imm6_shift = emit.mir.instructions.items(.data)[inst].rrr_imm6_shift;
744 const rd = rrr_imm6_shift.rd;
745 const rn = rrr_imm6_shift.rn;
746 const rm = rrr_imm6_shift.rm;
747 const shift = rrr_imm6_shift.shift;
748 const imm6 = rrr_imm6_shift.imm6;
749
750 switch (tag) {
751 .add_shifted_register => try emit.writeInstruction(Instruction.addShiftedRegister(rd, rn, rm, shift, imm6)),
752 .adds_shifted_register => try emit.writeInstruction(Instruction.addsShiftedRegister(rd, rn, rm, shift, imm6)),
753 .sub_shifted_register => try emit.writeInstruction(Instruction.subShiftedRegister(rd, rn, rm, shift, imm6)),
754 .subs_shifted_register => try emit.writeInstruction(Instruction.subsShiftedRegister(rd, rn, rm, shift, imm6)),
755 else => unreachable,
756 }
757 },
758 .cmp_shifted_register => {
759 const rr_imm6_shift = emit.mir.instructions.items(.data)[inst].rr_imm6_shift;
760 const rn = rr_imm6_shift.rn;
761 const rm = rr_imm6_shift.rm;
762 const shift = rr_imm6_shift.shift;
763 const imm6 = rr_imm6_shift.imm6;
764 const zr: Register = switch (rn.size()) {
765 32 => .wzr,
766 64 => .xzr,
767 else => unreachable,
768 };
769
770 try emit.writeInstruction(Instruction.subsShiftedRegister(zr, rn, rm, shift, imm6));
771 },
772 else => unreachable,
773 }
774}
775
776fn mirAddSubtractExtendedRegister(emit: *Emit, inst: Mir.Inst.Index) !void {
777 const tag = emit.mir.instructions.items(.tag)[inst];
778 switch (tag) {
779 .add_extended_register,
780 .adds_extended_register,
781 .sub_extended_register,
782 .subs_extended_register,
783 => {
784 const rrr_extend_shift = emit.mir.instructions.items(.data)[inst].rrr_extend_shift;
785 const rd = rrr_extend_shift.rd;
786 const rn = rrr_extend_shift.rn;
787 const rm = rrr_extend_shift.rm;
788 const ext_type = rrr_extend_shift.ext_type;
789 const imm3 = rrr_extend_shift.imm3;
790
791 switch (tag) {
792 .add_extended_register => try emit.writeInstruction(Instruction.addExtendedRegister(rd, rn, rm, ext_type, imm3)),
793 .adds_extended_register => try emit.writeInstruction(Instruction.addsExtendedRegister(rd, rn, rm, ext_type, imm3)),
794 .sub_extended_register => try emit.writeInstruction(Instruction.subExtendedRegister(rd, rn, rm, ext_type, imm3)),
795 .subs_extended_register => try emit.writeInstruction(Instruction.subsExtendedRegister(rd, rn, rm, ext_type, imm3)),
796 else => unreachable,
797 }
798 },
799 .cmp_extended_register => {
800 const rr_extend_shift = emit.mir.instructions.items(.data)[inst].rr_extend_shift;
801 const rn = rr_extend_shift.rn;
802 const rm = rr_extend_shift.rm;
803 const ext_type = rr_extend_shift.ext_type;
804 const imm3 = rr_extend_shift.imm3;
805 const zr: Register = switch (rn.size()) {
806 32 => .wzr,
807 64 => .xzr,
808 else => unreachable,
809 };
810
811 try emit.writeInstruction(Instruction.subsExtendedRegister(zr, rn, rm, ext_type, imm3));
812 },
813 else => unreachable,
814 }
815}
816
817fn mirConditionalSelect(emit: *Emit, inst: Mir.Inst.Index) !void {
818 const tag = emit.mir.instructions.items(.tag)[inst];
819 switch (tag) {
820 .csel => {
821 const rrr_cond = emit.mir.instructions.items(.data)[inst].rrr_cond;
822 const rd = rrr_cond.rd;
823 const rn = rrr_cond.rn;
824 const rm = rrr_cond.rm;
825 const cond = rrr_cond.cond;
826 try emit.writeInstruction(Instruction.csel(rd, rn, rm, cond));
827 },
828 .cset => {
829 const r_cond = emit.mir.instructions.items(.data)[inst].r_cond;
830 const zr: Register = switch (r_cond.rd.size()) {
831 32 => .wzr,
832 64 => .xzr,
833 else => unreachable,
834 };
835 try emit.writeInstruction(Instruction.csinc(r_cond.rd, zr, zr, r_cond.cond.negate()));
836 },
837 else => unreachable,
838 }
839}
840
841fn mirLogicalShiftedRegister(emit: *Emit, inst: Mir.Inst.Index) !void {
842 const tag = emit.mir.instructions.items(.tag)[inst];
843 const rrr_imm6_logical_shift = emit.mir.instructions.items(.data)[inst].rrr_imm6_logical_shift;
844 const rd = rrr_imm6_logical_shift.rd;
845 const rn = rrr_imm6_logical_shift.rn;
846 const rm = rrr_imm6_logical_shift.rm;
847 const shift = rrr_imm6_logical_shift.shift;
848 const imm6 = rrr_imm6_logical_shift.imm6;
849
850 switch (tag) {
851 .and_shifted_register => try emit.writeInstruction(Instruction.andShiftedRegister(rd, rn, rm, shift, imm6)),
852 .eor_shifted_register => try emit.writeInstruction(Instruction.eorShiftedRegister(rd, rn, rm, shift, imm6)),
853 .orr_shifted_register => try emit.writeInstruction(Instruction.orrShiftedRegister(rd, rn, rm, shift, imm6)),
854 else => unreachable,
855 }
856}
857
858fn mirLoadMemoryPie(emit: *Emit, inst: Mir.Inst.Index) !void {
859 const tag = emit.mir.instructions.items(.tag)[inst];
860 const payload = emit.mir.instructions.items(.data)[inst].payload;
861 const data = emit.mir.extraData(Mir.LoadMemoryPie, payload).data;
862 const reg = @as(Register, @enumFromInt(data.register));
863
864 // PC-relative displacement to the entry in memory.
865 // adrp
866 const offset = @as(u32, @intCast(emit.code.items.len));
867 try emit.writeInstruction(Instruction.adrp(reg.toX(), 0));
868
869 switch (tag) {
870 .load_memory_got,
871 .load_memory_import,
872 => {
873 // ldr reg, reg, offset
874 try emit.writeInstruction(Instruction.ldr(
875 reg,
876 reg.toX(),
877 Instruction.LoadStoreOffset.imm(0),
878 ));
879 },
880 .load_memory_direct => {
881 // We cannot load the offset directly as it may not be aligned properly.
882 // For example, load for 64bit register will require the target address offset
883 // to be 8-byte aligned, while the value might have non-8-byte natural alignment,
884 // meaning the linker might have put it at a non-8-byte aligned address. To circumvent
885 // this, we use `adrp, add` to form the address value which we then dereference with
886 // `ldr`.
887 // Note that this can potentially be optimised out by the codegen/linker if the
888 // target address is appropriately aligned.
889 // add reg, reg, offset
890 try emit.writeInstruction(Instruction.add(reg.toX(), reg.toX(), 0, false));
891 // ldr reg, reg, offset
892 try emit.writeInstruction(Instruction.ldr(
893 reg,
894 reg.toX(),
895 Instruction.LoadStoreOffset.imm(0),
896 ));
897 },
898 .load_memory_ptr_direct,
899 .load_memory_ptr_got,
900 => {
901 // add reg, reg, offset
902 try emit.writeInstruction(Instruction.add(reg, reg, 0, false));
903 },
904 else => unreachable,
905 }
906
907 if (emit.bin_file.cast(.macho)) |macho_file| {
908 _ = macho_file;
909 @panic("TODO mirLoadMemoryPie");
910 // const Atom = link.File.MachO.Atom;
911 // const Relocation = Atom.Relocation;
912 // const atom_index = macho_file.getAtomIndexForSymbol(.{ .sym_index = data.atom_index }).?;
913 // try Atom.addRelocations(macho_file, atom_index, &[_]Relocation{ .{
914 // .target = .{ .sym_index = data.sym_index },
915 // .offset = offset,
916 // .addend = 0,
917 // .pcrel = true,
918 // .length = 2,
919 // .type = switch (tag) {
920 // .load_memory_got, .load_memory_ptr_got => Relocation.Type.got_page,
921 // .load_memory_direct, .load_memory_ptr_direct => Relocation.Type.page,
922 // else => unreachable,
923 // },
924 // }, .{
925 // .target = .{ .sym_index = data.sym_index },
926 // .offset = offset + 4,
927 // .addend = 0,
928 // .pcrel = false,
929 // .length = 2,
930 // .type = switch (tag) {
931 // .load_memory_got, .load_memory_ptr_got => Relocation.Type.got_pageoff,
932 // .load_memory_direct, .load_memory_ptr_direct => Relocation.Type.pageoff,
933 // else => unreachable,
934 // },
935 // } });
936 } else if (emit.bin_file.cast(.coff)) |coff_file| {
937 const atom_index = coff_file.getAtomIndexForSymbol(.{ .sym_index = data.atom_index, .file = null }).?;
938 const target = switch (tag) {
939 .load_memory_got,
940 .load_memory_ptr_got,
941 .load_memory_direct,
942 .load_memory_ptr_direct,
943 => link.File.Coff.SymbolWithLoc{ .sym_index = data.sym_index, .file = null },
944 .load_memory_import => coff_file.getGlobalByIndex(data.sym_index),
945 else => unreachable,
946 };
947 try coff_file.addRelocation(atom_index, .{
948 .target = target,
949 .offset = offset,
950 .addend = 0,
951 .pcrel = true,
952 .length = 2,
953 .type = switch (tag) {
954 .load_memory_got,
955 .load_memory_ptr_got,
956 => .got_page,
957 .load_memory_direct,
958 .load_memory_ptr_direct,
959 => .page,
960 .load_memory_import => .import_page,
961 else => unreachable,
962 },
963 });
964 try coff_file.addRelocation(atom_index, .{
965 .target = target,
966 .offset = offset + 4,
967 .addend = 0,
968 .pcrel = false,
969 .length = 2,
970 .type = switch (tag) {
971 .load_memory_got,
972 .load_memory_ptr_got,
973 => .got_pageoff,
974 .load_memory_direct,
975 .load_memory_ptr_direct,
976 => .pageoff,
977 .load_memory_import => .import_pageoff,
978 else => unreachable,
979 },
980 });
981 } else {
982 return emit.fail("TODO implement load_memory for PIE GOT indirection on this platform", .{});
983 }
984}
985
986fn mirLoadStoreRegisterPair(emit: *Emit, inst: Mir.Inst.Index) !void {
987 const tag = emit.mir.instructions.items(.tag)[inst];
988 const load_store_register_pair = emit.mir.instructions.items(.data)[inst].load_store_register_pair;
989 const rt = load_store_register_pair.rt;
990 const rt2 = load_store_register_pair.rt2;
991 const rn = load_store_register_pair.rn;
992 const offset = load_store_register_pair.offset;
993
994 switch (tag) {
995 .stp => try emit.writeInstruction(Instruction.stp(rt, rt2, rn, offset)),
996 .ldp => try emit.writeInstruction(Instruction.ldp(rt, rt2, rn, offset)),
997 else => unreachable,
998 }
999}
1000
1001fn mirLoadStackArgument(emit: *Emit, inst: Mir.Inst.Index) !void {
1002 const tag = emit.mir.instructions.items(.tag)[inst];
1003 const load_store_stack = emit.mir.instructions.items(.data)[inst].load_store_stack;
1004 const rt = load_store_stack.rt;
1005
1006 const raw_offset = emit.stack_size + emit.saved_regs_stack_space + load_store_stack.offset;
1007 switch (tag) {
1008 .ldr_ptr_stack_argument => {
1009 const offset = if (math.cast(u12, raw_offset)) |imm| imm else {
1010 return emit.fail("TODO load stack argument ptr with larger offset", .{});
1011 };
1012
1013 switch (tag) {
1014 .ldr_ptr_stack_argument => try emit.writeInstruction(Instruction.add(rt, .sp, offset, false)),
1015 else => unreachable,
1016 }
1017 },
1018 .ldrb_stack_argument, .ldrsb_stack_argument => {
1019 const offset = if (math.cast(u12, raw_offset)) |imm| Instruction.LoadStoreOffset.imm(imm) else {
1020 return emit.fail("TODO load stack argument byte with larger offset", .{});
1021 };
1022
1023 switch (tag) {
1024 .ldrb_stack_argument => try emit.writeInstruction(Instruction.ldrb(rt, .sp, offset)),
1025 .ldrsb_stack_argument => try emit.writeInstruction(Instruction.ldrsb(rt, .sp, offset)),
1026 else => unreachable,
1027 }
1028 },
1029 .ldrh_stack_argument, .ldrsh_stack_argument => {
1030 assert(std.mem.isAlignedGeneric(u32, raw_offset, 2)); // misaligned stack entry
1031 const offset = if (math.cast(u12, @divExact(raw_offset, 2))) |imm| Instruction.LoadStoreOffset.imm(imm) else {
1032 return emit.fail("TODO load stack argument halfword with larger offset", .{});
1033 };
1034
1035 switch (tag) {
1036 .ldrh_stack_argument => try emit.writeInstruction(Instruction.ldrh(rt, .sp, offset)),
1037 .ldrsh_stack_argument => try emit.writeInstruction(Instruction.ldrsh(rt, .sp, offset)),
1038 else => unreachable,
1039 }
1040 },
1041 .ldr_stack_argument => {
1042 const alignment: u32 = switch (rt.size()) {
1043 32 => 4,
1044 64 => 8,
1045 else => unreachable,
1046 };
1047
1048 assert(std.mem.isAlignedGeneric(u32, raw_offset, alignment)); // misaligned stack entry
1049 const offset = if (math.cast(u12, @divExact(raw_offset, alignment))) |imm| Instruction.LoadStoreOffset.imm(imm) else {
1050 return emit.fail("TODO load stack argument with larger offset", .{});
1051 };
1052
1053 switch (tag) {
1054 .ldr_stack_argument => try emit.writeInstruction(Instruction.ldr(rt, .sp, offset)),
1055 else => unreachable,
1056 }
1057 },
1058 else => unreachable,
1059 }
1060}
1061
1062fn mirLoadStoreStack(emit: *Emit, inst: Mir.Inst.Index) !void {
1063 const tag = emit.mir.instructions.items(.tag)[inst];
1064 const load_store_stack = emit.mir.instructions.items(.data)[inst].load_store_stack;
1065 const rt = load_store_stack.rt;
1066
1067 const raw_offset = emit.stack_size - load_store_stack.offset;
1068 switch (tag) {
1069 .ldr_ptr_stack => {
1070 const offset = if (math.cast(u12, raw_offset)) |imm| imm else {
1071 return emit.fail("TODO load stack argument ptr with larger offset", .{});
1072 };
1073
1074 switch (tag) {
1075 .ldr_ptr_stack => try emit.writeInstruction(Instruction.add(rt, .sp, offset, false)),
1076 else => unreachable,
1077 }
1078 },
1079 .ldrb_stack, .ldrsb_stack, .strb_stack => {
1080 const offset = if (math.cast(u12, raw_offset)) |imm| Instruction.LoadStoreOffset.imm(imm) else {
1081 return emit.fail("TODO load/store stack byte with larger offset", .{});
1082 };
1083
1084 switch (tag) {
1085 .ldrb_stack => try emit.writeInstruction(Instruction.ldrb(rt, .sp, offset)),
1086 .ldrsb_stack => try emit.writeInstruction(Instruction.ldrsb(rt, .sp, offset)),
1087 .strb_stack => try emit.writeInstruction(Instruction.strb(rt, .sp, offset)),
1088 else => unreachable,
1089 }
1090 },
1091 .ldrh_stack, .ldrsh_stack, .strh_stack => {
1092 assert(std.mem.isAlignedGeneric(u32, raw_offset, 2)); // misaligned stack entry
1093 const offset = if (math.cast(u12, @divExact(raw_offset, 2))) |imm| Instruction.LoadStoreOffset.imm(imm) else {
1094 return emit.fail("TODO load/store stack halfword with larger offset", .{});
1095 };
1096
1097 switch (tag) {
1098 .ldrh_stack => try emit.writeInstruction(Instruction.ldrh(rt, .sp, offset)),
1099 .ldrsh_stack => try emit.writeInstruction(Instruction.ldrsh(rt, .sp, offset)),
1100 .strh_stack => try emit.writeInstruction(Instruction.strh(rt, .sp, offset)),
1101 else => unreachable,
1102 }
1103 },
1104 .ldr_stack, .str_stack => {
1105 const alignment: u32 = switch (rt.size()) {
1106 32 => 4,
1107 64 => 8,
1108 else => unreachable,
1109 };
1110
1111 assert(std.mem.isAlignedGeneric(u32, raw_offset, alignment)); // misaligned stack entry
1112 const offset = if (math.cast(u12, @divExact(raw_offset, alignment))) |imm| Instruction.LoadStoreOffset.imm(imm) else {
1113 return emit.fail("TODO load/store stack with larger offset", .{});
1114 };
1115
1116 switch (tag) {
1117 .ldr_stack => try emit.writeInstruction(Instruction.ldr(rt, .sp, offset)),
1118 .str_stack => try emit.writeInstruction(Instruction.str(rt, .sp, offset)),
1119 else => unreachable,
1120 }
1121 },
1122 else => unreachable,
1123 }
1124}
1125
1126fn mirLoadStoreRegisterImmediate(emit: *Emit, inst: Mir.Inst.Index) !void {
1127 const tag = emit.mir.instructions.items(.tag)[inst];
1128 const load_store_register_immediate = emit.mir.instructions.items(.data)[inst].load_store_register_immediate;
1129 const rt = load_store_register_immediate.rt;
1130 const rn = load_store_register_immediate.rn;
1131 const offset = Instruction.LoadStoreOffset{ .immediate = load_store_register_immediate.offset };
1132
1133 switch (tag) {
1134 .ldr_immediate => try emit.writeInstruction(Instruction.ldr(rt, rn, offset)),
1135 .ldrb_immediate => try emit.writeInstruction(Instruction.ldrb(rt, rn, offset)),
1136 .ldrh_immediate => try emit.writeInstruction(Instruction.ldrh(rt, rn, offset)),
1137 .ldrsb_immediate => try emit.writeInstruction(Instruction.ldrsb(rt, rn, offset)),
1138 .ldrsh_immediate => try emit.writeInstruction(Instruction.ldrsh(rt, rn, offset)),
1139 .ldrsw_immediate => try emit.writeInstruction(Instruction.ldrsw(rt, rn, offset)),
1140 .str_immediate => try emit.writeInstruction(Instruction.str(rt, rn, offset)),
1141 .strb_immediate => try emit.writeInstruction(Instruction.strb(rt, rn, offset)),
1142 .strh_immediate => try emit.writeInstruction(Instruction.strh(rt, rn, offset)),
1143 else => unreachable,
1144 }
1145}
1146
1147fn mirLoadStoreRegisterRegister(emit: *Emit, inst: Mir.Inst.Index) !void {
1148 const tag = emit.mir.instructions.items(.tag)[inst];
1149 const load_store_register_register = emit.mir.instructions.items(.data)[inst].load_store_register_register;
1150 const rt = load_store_register_register.rt;
1151 const rn = load_store_register_register.rn;
1152 const offset = Instruction.LoadStoreOffset{ .register = load_store_register_register.offset };
1153
1154 switch (tag) {
1155 .ldr_register => try emit.writeInstruction(Instruction.ldr(rt, rn, offset)),
1156 .ldrb_register => try emit.writeInstruction(Instruction.ldrb(rt, rn, offset)),
1157 .ldrh_register => try emit.writeInstruction(Instruction.ldrh(rt, rn, offset)),
1158 .str_register => try emit.writeInstruction(Instruction.str(rt, rn, offset)),
1159 .strb_register => try emit.writeInstruction(Instruction.strb(rt, rn, offset)),
1160 .strh_register => try emit.writeInstruction(Instruction.strh(rt, rn, offset)),
1161 else => unreachable,
1162 }
1163}
1164
1165fn mirMoveRegister(emit: *Emit, inst: Mir.Inst.Index) !void {
1166 const tag = emit.mir.instructions.items(.tag)[inst];
1167 switch (tag) {
1168 .mov_register => {
1169 const rr = emit.mir.instructions.items(.data)[inst].rr;
1170 const zr: Register = switch (rr.rd.size()) {
1171 32 => .wzr,
1172 64 => .xzr,
1173 else => unreachable,
1174 };
1175
1176 try emit.writeInstruction(Instruction.orrShiftedRegister(rr.rd, zr, rr.rn, .lsl, 0));
1177 },
1178 .mov_to_from_sp => {
1179 const rr = emit.mir.instructions.items(.data)[inst].rr;
1180 try emit.writeInstruction(Instruction.add(rr.rd, rr.rn, 0, false));
1181 },
1182 .mvn => {
1183 const rr_imm6_logical_shift = emit.mir.instructions.items(.data)[inst].rr_imm6_logical_shift;
1184 const rd = rr_imm6_logical_shift.rd;
1185 const rm = rr_imm6_logical_shift.rm;
1186 const shift = rr_imm6_logical_shift.shift;
1187 const imm6 = rr_imm6_logical_shift.imm6;
1188 const zr: Register = switch (rd.size()) {
1189 32 => .wzr,
1190 64 => .xzr,
1191 else => unreachable,
1192 };
1193
1194 try emit.writeInstruction(Instruction.ornShiftedRegister(rd, zr, rm, shift, imm6));
1195 },
1196 else => unreachable,
1197 }
1198}
1199
1200fn mirMoveWideImmediate(emit: *Emit, inst: Mir.Inst.Index) !void {
1201 const tag = emit.mir.instructions.items(.tag)[inst];
1202 const r_imm16_sh = emit.mir.instructions.items(.data)[inst].r_imm16_sh;
1203
1204 switch (tag) {
1205 .movz => try emit.writeInstruction(Instruction.movz(r_imm16_sh.rd, r_imm16_sh.imm16, @as(u6, r_imm16_sh.hw) << 4)),
1206 .movk => try emit.writeInstruction(Instruction.movk(r_imm16_sh.rd, r_imm16_sh.imm16, @as(u6, r_imm16_sh.hw) << 4)),
1207 else => unreachable,
1208 }
1209}
1210
1211fn mirDataProcessing3Source(emit: *Emit, inst: Mir.Inst.Index) !void {
1212 const tag = emit.mir.instructions.items(.tag)[inst];
1213
1214 switch (tag) {
1215 .mul,
1216 .smulh,
1217 .smull,
1218 .umulh,
1219 .umull,
1220 => {
1221 const rrr = emit.mir.instructions.items(.data)[inst].rrr;
1222 switch (tag) {
1223 .mul => try emit.writeInstruction(Instruction.mul(rrr.rd, rrr.rn, rrr.rm)),
1224 .smulh => try emit.writeInstruction(Instruction.smulh(rrr.rd, rrr.rn, rrr.rm)),
1225 .smull => try emit.writeInstruction(Instruction.smull(rrr.rd, rrr.rn, rrr.rm)),
1226 .umulh => try emit.writeInstruction(Instruction.umulh(rrr.rd, rrr.rn, rrr.rm)),
1227 .umull => try emit.writeInstruction(Instruction.umull(rrr.rd, rrr.rn, rrr.rm)),
1228 else => unreachable,
1229 }
1230 },
1231 .msub => {
1232 const rrrr = emit.mir.instructions.items(.data)[inst].rrrr;
1233 switch (tag) {
1234 .msub => try emit.writeInstruction(Instruction.msub(rrrr.rd, rrrr.rn, rrrr.rm, rrrr.ra)),
1235 else => unreachable,
1236 }
1237 },
1238 else => unreachable,
1239 }
1240}
1241
1242fn mirNop(emit: *Emit) !void {
1243 try emit.writeInstruction(Instruction.nop());
1244}
1245
1246fn regListIsSet(reg_list: u32, reg: Register) bool {
1247 return reg_list & @as(u32, 1) << @as(u5, @intCast(reg.id())) != 0;
1248}
1249
1250fn mirPushPopRegs(emit: *Emit, inst: Mir.Inst.Index) !void {
1251 const tag = emit.mir.instructions.items(.tag)[inst];
1252 const reg_list = emit.mir.instructions.items(.data)[inst].reg_list;
1253
1254 if (regListIsSet(reg_list, .xzr)) return emit.fail("xzr is not a valid register for {}", .{tag});
1255
1256 // sp must be aligned at all times, so we only use stp and ldp
1257 // instructions for minimal instruction count.
1258 //
1259 // However, if we have an odd number of registers, for pop_regs we
1260 // use one ldr instruction followed by zero or more ldp
1261 // instructions; for push_regs we use zero or more stp
1262 // instructions followed by one str instruction.
1263 const number_of_regs = @popCount(reg_list);
1264 const odd_number_of_regs = number_of_regs % 2 != 0;
1265
1266 switch (tag) {
1267 .pop_regs => {
1268 var i: u6 = 32;
1269 var count: u6 = 0;
1270 var other_reg: ?Register = null;
1271 while (i > 0) : (i -= 1) {
1272 const reg = @as(Register, @enumFromInt(i - 1));
1273 if (regListIsSet(reg_list, reg)) {
1274 if (count == 0 and odd_number_of_regs) {
1275 try emit.writeInstruction(Instruction.ldr(
1276 reg,
1277 .sp,
1278 Instruction.LoadStoreOffset.imm_post_index(16),
1279 ));
1280 } else if (other_reg) |r| {
1281 try emit.writeInstruction(Instruction.ldp(
1282 reg,
1283 r,
1284 .sp,
1285 Instruction.LoadStorePairOffset.post_index(16),
1286 ));
1287 other_reg = null;
1288 } else {
1289 other_reg = reg;
1290 }
1291 count += 1;
1292 }
1293 }
1294 assert(count == number_of_regs);
1295 },
1296 .push_regs => {
1297 var i: u6 = 0;
1298 var count: u6 = 0;
1299 var other_reg: ?Register = null;
1300 while (i < 32) : (i += 1) {
1301 const reg = @as(Register, @enumFromInt(i));
1302 if (regListIsSet(reg_list, reg)) {
1303 if (count == number_of_regs - 1 and odd_number_of_regs) {
1304 try emit.writeInstruction(Instruction.str(
1305 reg,
1306 .sp,
1307 Instruction.LoadStoreOffset.imm_pre_index(-16),
1308 ));
1309 } else if (other_reg) |r| {
1310 try emit.writeInstruction(Instruction.stp(
1311 r,
1312 reg,
1313 .sp,
1314 Instruction.LoadStorePairOffset.pre_index(-16),
1315 ));
1316 other_reg = null;
1317 } else {
1318 other_reg = reg;
1319 }
1320 count += 1;
1321 }
1322 }
1323 assert(count == number_of_regs);
1324 },
1325 else => unreachable,
1326 }
1327}
1328
1329fn mirBitfieldExtract(emit: *Emit, inst: Mir.Inst.Index) !void {
1330 const tag = emit.mir.instructions.items(.tag)[inst];
1331 const rr_lsb_width = emit.mir.instructions.items(.data)[inst].rr_lsb_width;
1332 const rd = rr_lsb_width.rd;
1333 const rn = rr_lsb_width.rn;
1334 const lsb = rr_lsb_width.lsb;
1335 const width = rr_lsb_width.width;
1336
1337 switch (tag) {
1338 .sbfx => try emit.writeInstruction(Instruction.sbfx(rd, rn, lsb, width)),
1339 .ubfx => try emit.writeInstruction(Instruction.ubfx(rd, rn, lsb, width)),
1340 else => unreachable,
1341 }
1342}
1343
1344fn mirExtend(emit: *Emit, inst: Mir.Inst.Index) !void {
1345 const tag = emit.mir.instructions.items(.tag)[inst];
1346 const rr = emit.mir.instructions.items(.data)[inst].rr;
1347
1348 switch (tag) {
1349 .sxtb => try emit.writeInstruction(Instruction.sxtb(rr.rd, rr.rn)),
1350 .sxth => try emit.writeInstruction(Instruction.sxth(rr.rd, rr.rn)),
1351 .sxtw => try emit.writeInstruction(Instruction.sxtw(rr.rd, rr.rn)),
1352 .uxtb => try emit.writeInstruction(Instruction.uxtb(rr.rd, rr.rn)),
1353 .uxth => try emit.writeInstruction(Instruction.uxth(rr.rd, rr.rn)),
1354 else => unreachable,
1355 }
1356}
src/arch/aarch64/Mir.zig deleted-568
...@@ -1,568 +0,0 @@
1//! Machine Intermediate Representation.
2//! This data is produced by AArch64 Codegen or AArch64 assembly parsing
3//! These instructions have a 1:1 correspondence with machine code instructions
4//! for the target. MIR can be lowered to source-annotated textual assembly code
5//! instructions, or it can be lowered to machine code.
6//! The main purpose of MIR is to postpone the assignment of offsets until Isel,
7//! so that, for example, the smaller encodings of jump instructions can be used.
8
9const Mir = @This();
10const std = @import("std");
11const builtin = @import("builtin");
12const assert = std.debug.assert;
13
14const bits = @import("bits.zig");
15const Register = bits.Register;
16const InternPool = @import("../../InternPool.zig");
17const Emit = @import("Emit.zig");
18const codegen = @import("../../codegen.zig");
19const link = @import("../../link.zig");
20const Zcu = @import("../../Zcu.zig");
21
22max_end_stack: u32,
23saved_regs_stack_space: u32,
24
25instructions: std.MultiArrayList(Inst).Slice,
26/// The meaning of this data is determined by `Inst.Tag` value.
27extra: []const u32,
28
29pub const Inst = struct {
30 tag: Tag,
31 /// The meaning of this depends on `tag`.
32 data: Data,
33
34 pub const Tag = enum(u16) {
35 /// Add (immediate)
36 add_immediate,
37 /// Add, update condition flags (immediate)
38 adds_immediate,
39 /// Add (shifted register)
40 add_shifted_register,
41 /// Add, update condition flags (shifted register)
42 adds_shifted_register,
43 /// Add (extended register)
44 add_extended_register,
45 /// Add, update condition flags (extended register)
46 adds_extended_register,
47 /// Bitwise AND (shifted register)
48 and_shifted_register,
49 /// Arithmetic Shift Right (immediate)
50 asr_immediate,
51 /// Arithmetic Shift Right (register)
52 asr_register,
53 /// Branch conditionally
54 b_cond,
55 /// Branch
56 b,
57 /// Branch with Link
58 bl,
59 /// Branch with Link to Register
60 blr,
61 /// Breakpoint
62 brk,
63 /// Pseudo-instruction: Call extern
64 call_extern,
65 /// Compare and Branch on Zero
66 cbz,
67 /// Compare (immediate)
68 cmp_immediate,
69 /// Compare (shifted register)
70 cmp_shifted_register,
71 /// Compare (extended register)
72 cmp_extended_register,
73 /// Conditional Select
74 csel,
75 /// Conditional set
76 cset,
77 /// Pseudo-instruction: End of prologue
78 dbg_prologue_end,
79 /// Pseudo-instruction: Beginning of epilogue
80 dbg_epilogue_begin,
81 /// Pseudo-instruction: Update debug line
82 dbg_line,
83 /// Bitwise Exclusive OR (immediate)
84 eor_immediate,
85 /// Bitwise Exclusive OR (shifted register)
86 eor_shifted_register,
87 /// Loads the contents into a register
88 ///
89 /// Payload is `LoadMemoryPie`
90 load_memory_got,
91 /// Loads the contents into a register
92 ///
93 /// Payload is `LoadMemoryPie`
94 load_memory_direct,
95 /// Loads the contents into a register
96 ///
97 /// Payload is `LoadMemoryPie`
98 load_memory_import,
99 /// Loads the address into a register
100 ///
101 /// Payload is `LoadMemoryPie`
102 load_memory_ptr_got,
103 /// Loads the address into a register
104 ///
105 /// Payload is `LoadMemoryPie`
106 load_memory_ptr_direct,
107 /// Load Pair of Registers
108 ldp,
109 /// Pseudo-instruction: Load pointer to stack item
110 ldr_ptr_stack,
111 /// Pseudo-instruction: Load pointer to stack argument
112 ldr_ptr_stack_argument,
113 /// Pseudo-instruction: Load from stack
114 ldr_stack,
115 /// Pseudo-instruction: Load from stack argument
116 ldr_stack_argument,
117 /// Load Register (immediate)
118 ldr_immediate,
119 /// Load Register (register)
120 ldr_register,
121 /// Pseudo-instruction: Load byte from stack
122 ldrb_stack,
123 /// Pseudo-instruction: Load byte from stack argument
124 ldrb_stack_argument,
125 /// Load Register Byte (immediate)
126 ldrb_immediate,
127 /// Load Register Byte (register)
128 ldrb_register,
129 /// Pseudo-instruction: Load halfword from stack
130 ldrh_stack,
131 /// Pseudo-instruction: Load halfword from stack argument
132 ldrh_stack_argument,
133 /// Load Register Halfword (immediate)
134 ldrh_immediate,
135 /// Load Register Halfword (register)
136 ldrh_register,
137 /// Load Register Signed Byte (immediate)
138 ldrsb_immediate,
139 /// Pseudo-instruction: Load signed byte from stack
140 ldrsb_stack,
141 /// Pseudo-instruction: Load signed byte from stack argument
142 ldrsb_stack_argument,
143 /// Load Register Signed Halfword (immediate)
144 ldrsh_immediate,
145 /// Pseudo-instruction: Load signed halfword from stack
146 ldrsh_stack,
147 /// Pseudo-instruction: Load signed halfword from stack argument
148 ldrsh_stack_argument,
149 /// Load Register Signed Word (immediate)
150 ldrsw_immediate,
151 /// Logical Shift Left (immediate)
152 lsl_immediate,
153 /// Logical Shift Left (register)
154 lsl_register,
155 /// Logical Shift Right (immediate)
156 lsr_immediate,
157 /// Logical Shift Right (register)
158 lsr_register,
159 /// Move (to/from SP)
160 mov_to_from_sp,
161 /// Move (register)
162 mov_register,
163 /// Move wide with keep
164 movk,
165 /// Move wide with zero
166 movz,
167 /// Multiply-subtract
168 msub,
169 /// Multiply
170 mul,
171 /// Bitwise NOT
172 mvn,
173 /// No Operation
174 nop,
175 /// Bitwise inclusive OR (shifted register)
176 orr_shifted_register,
177 /// Pseudo-instruction: Pop multiple registers
178 pop_regs,
179 /// Pseudo-instruction: Push multiple registers
180 push_regs,
181 /// Return from subroutine
182 ret,
183 /// Signed bitfield extract
184 sbfx,
185 /// Signed divide
186 sdiv,
187 /// Signed multiply high
188 smulh,
189 /// Signed multiply long
190 smull,
191 /// Signed extend byte
192 sxtb,
193 /// Signed extend halfword
194 sxth,
195 /// Signed extend word
196 sxtw,
197 /// Store Pair of Registers
198 stp,
199 /// Pseudo-instruction: Store to stack
200 str_stack,
201 /// Store Register (immediate)
202 str_immediate,
203 /// Store Register (register)
204 str_register,
205 /// Pseudo-instruction: Store byte to stack
206 strb_stack,
207 /// Store Register Byte (immediate)
208 strb_immediate,
209 /// Store Register Byte (register)
210 strb_register,
211 /// Pseudo-instruction: Store halfword to stack
212 strh_stack,
213 /// Store Register Halfword (immediate)
214 strh_immediate,
215 /// Store Register Halfword (register)
216 strh_register,
217 /// Subtract (immediate)
218 sub_immediate,
219 /// Subtract, update condition flags (immediate)
220 subs_immediate,
221 /// Subtract (shifted register)
222 sub_shifted_register,
223 /// Subtract, update condition flags (shifted register)
224 subs_shifted_register,
225 /// Subtract (extended register)
226 sub_extended_register,
227 /// Subtract, update condition flags (extended register)
228 subs_extended_register,
229 /// Supervisor Call
230 svc,
231 /// Test bits (immediate)
232 tst_immediate,
233 /// Unsigned bitfield extract
234 ubfx,
235 /// Unsigned divide
236 udiv,
237 /// Unsigned multiply high
238 umulh,
239 /// Unsigned multiply long
240 umull,
241 /// Unsigned extend byte
242 uxtb,
243 /// Unsigned extend halfword
244 uxth,
245 };
246
247 /// The position of an MIR instruction within the `Mir` instructions array.
248 pub const Index = u32;
249
250 /// All instructions have a 4-byte payload, which is contained within
251 /// this union. `Tag` determines which union field is active, as well as
252 /// how to interpret the data within.
253 pub const Data = union {
254 /// No additional data
255 ///
256 /// Used by e.g. nop
257 nop: void,
258 /// Another instruction
259 ///
260 /// Used by e.g. b
261 inst: Index,
262 /// Relocation for the linker where:
263 /// * `atom_index` is the index of the source
264 /// * `sym_index` is the index of the target
265 ///
266 /// Used by e.g. call_extern
267 relocation: struct {
268 /// Index of the containing atom.
269 atom_index: u32,
270 /// Index into the linker's string table.
271 sym_index: u32,
272 },
273 /// A 16-bit immediate value.
274 ///
275 /// Used by e.g. svc
276 imm16: u16,
277 /// Index into `extra`. Meaning of what can be found there is context-dependent.
278 payload: u32,
279 /// A register
280 ///
281 /// Used by e.g. blr
282 reg: Register,
283 /// Multiple registers
284 ///
285 /// Used by e.g. pop_regs
286 reg_list: u32,
287 /// Another instruction and a condition
288 ///
289 /// Used by e.g. b_cond
290 inst_cond: struct {
291 inst: Index,
292 cond: bits.Instruction.Condition,
293 },
294 /// A register, an unsigned 16-bit immediate, and an optional shift
295 ///
296 /// Used by e.g. movz
297 r_imm16_sh: struct {
298 rd: Register,
299 imm16: u16,
300 hw: u2 = 0,
301 },
302 /// A register and a condition
303 ///
304 /// Used by e.g. cset
305 r_cond: struct {
306 rd: Register,
307 cond: bits.Instruction.Condition,
308 },
309 /// A register and another instruction
310 ///
311 /// Used by e.g. cbz
312 r_inst: struct {
313 rt: Register,
314 inst: Index,
315 },
316 /// A register, an unsigned 12-bit immediate, and an optional shift
317 ///
318 /// Used by e.g. cmp_immediate
319 r_imm12_sh: struct {
320 rn: Register,
321 imm12: u12,
322 sh: u1 = 0,
323 },
324 /// Two registers
325 ///
326 /// Used by e.g. mov_register
327 rr: struct {
328 rd: Register,
329 rn: Register,
330 },
331 /// Two registers, an unsigned 12-bit immediate, and an optional shift
332 ///
333 /// Used by e.g. sub_immediate
334 rr_imm12_sh: struct {
335 rd: Register,
336 rn: Register,
337 imm12: u12,
338 sh: u1 = 0,
339 },
340 /// Two registers and a shift (shift type and 6-bit amount)
341 ///
342 /// Used by e.g. cmp_shifted_register
343 rr_imm6_shift: struct {
344 rn: Register,
345 rm: Register,
346 imm6: u6,
347 shift: bits.Instruction.AddSubtractShiftedRegisterShift,
348 },
349 /// Two registers with sign-extension (extension type and 3-bit shift amount)
350 ///
351 /// Used by e.g. cmp_extended_register
352 rr_extend_shift: struct {
353 rn: Register,
354 rm: Register,
355 ext_type: bits.Instruction.AddSubtractExtendedRegisterOption,
356 imm3: u3,
357 },
358 /// Two registers and a shift (logical instruction version)
359 /// (shift type and 6-bit amount)
360 ///
361 /// Used by e.g. mvn
362 rr_imm6_logical_shift: struct {
363 rd: Register,
364 rm: Register,
365 imm6: u6,
366 shift: bits.Instruction.LogicalShiftedRegisterShift,
367 },
368 /// Two registers and a lsb (range 0-63) and a width (range
369 /// 1-64)
370 ///
371 /// Used by e.g. ubfx
372 rr_lsb_width: struct {
373 rd: Register,
374 rn: Register,
375 lsb: u6,
376 width: u7,
377 },
378 /// Two registers and a bitmask immediate
379 ///
380 /// Used by e.g. eor_immediate
381 rr_bitmask: struct {
382 rd: Register,
383 rn: Register,
384 imms: u6,
385 immr: u6,
386 n: u1,
387 },
388 /// Two registers and a 6-bit unsigned shift
389 ///
390 /// Used by e.g. lsl_immediate
391 rr_shift: struct {
392 rd: Register,
393 rn: Register,
394 shift: u6,
395 },
396 /// Three registers
397 ///
398 /// Used by e.g. mul
399 rrr: struct {
400 rd: Register,
401 rn: Register,
402 rm: Register,
403 },
404 /// Three registers and a condition
405 ///
406 /// Used by e.g. csel
407 rrr_cond: struct {
408 rd: Register,
409 rn: Register,
410 rm: Register,
411 cond: bits.Instruction.Condition,
412 },
413 /// Three registers and a shift (shift type and 6-bit amount)
414 ///
415 /// Used by e.g. add_shifted_register
416 rrr_imm6_shift: struct {
417 rd: Register,
418 rn: Register,
419 rm: Register,
420 imm6: u6,
421 shift: bits.Instruction.AddSubtractShiftedRegisterShift,
422 },
423 /// Three registers with sign-extension (extension type and 3-bit shift amount)
424 ///
425 /// Used by e.g. add_extended_register
426 rrr_extend_shift: struct {
427 rd: Register,
428 rn: Register,
429 rm: Register,
430 ext_type: bits.Instruction.AddSubtractExtendedRegisterOption,
431 imm3: u3,
432 },
433 /// Three registers and a shift (logical instruction version)
434 /// (shift type and 6-bit amount)
435 ///
436 /// Used by e.g. eor_shifted_register
437 rrr_imm6_logical_shift: struct {
438 rd: Register,
439 rn: Register,
440 rm: Register,
441 imm6: u6,
442 shift: bits.Instruction.LogicalShiftedRegisterShift,
443 },
444 /// Two registers and a LoadStoreOffsetImmediate
445 ///
446 /// Used by e.g. str_immediate
447 load_store_register_immediate: struct {
448 rt: Register,
449 rn: Register,
450 offset: bits.Instruction.LoadStoreOffsetImmediate,
451 },
452 /// Two registers and a LoadStoreOffsetRegister
453 ///
454 /// Used by e.g. str_register
455 load_store_register_register: struct {
456 rt: Register,
457 rn: Register,
458 offset: bits.Instruction.LoadStoreOffsetRegister,
459 },
460 /// A register and a stack offset
461 ///
462 /// Used by e.g. str_stack
463 load_store_stack: struct {
464 rt: Register,
465 offset: u32,
466 },
467 /// Three registers and a LoadStorePairOffset
468 ///
469 /// Used by e.g. stp
470 load_store_register_pair: struct {
471 rt: Register,
472 rt2: Register,
473 rn: Register,
474 offset: bits.Instruction.LoadStorePairOffset,
475 },
476 /// Four registers
477 ///
478 /// Used by e.g. msub
479 rrrr: struct {
480 rd: Register,
481 rn: Register,
482 rm: Register,
483 ra: Register,
484 },
485 /// Debug info: line and column
486 ///
487 /// Used by e.g. dbg_line
488 dbg_line_column: struct {
489 line: u32,
490 column: u32,
491 },
492 };
493
494 // Make sure we don't accidentally make instructions bigger than expected.
495 // Note that in safety builds, Zig is allowed to insert a secret field for safety checks.
496 comptime {
497 if (!std.debug.runtime_safety) {
498 assert(@sizeOf(Data) == 8);
499 }
500 }
501};
502
503pub fn deinit(mir: *Mir, gpa: std.mem.Allocator) void {
504 mir.instructions.deinit(gpa);
505 gpa.free(mir.extra);
506 mir.* = undefined;
507}
508
509pub fn emit(
510 mir: Mir,
511 lf: *link.File,
512 pt: Zcu.PerThread,
513 src_loc: Zcu.LazySrcLoc,
514 func_index: InternPool.Index,
515 code: *std.ArrayListUnmanaged(u8),
516 debug_output: link.File.DebugInfoOutput,
517) codegen.CodeGenError!void {
518 const zcu = pt.zcu;
519 const func = zcu.funcInfo(func_index);
520 const nav = func.owner_nav;
521 const mod = zcu.navFileScope(nav).mod.?;
522 var e: Emit = .{
523 .mir = mir,
524 .bin_file = lf,
525 .debug_output = debug_output,
526 .target = &mod.resolved_target.result,
527 .src_loc = src_loc,
528 .code = code,
529 .prev_di_pc = 0,
530 .prev_di_line = func.lbrace_line,
531 .prev_di_column = func.lbrace_column,
532 .stack_size = mir.max_end_stack,
533 .saved_regs_stack_space = mir.saved_regs_stack_space,
534 };
535 defer e.deinit();
536 e.emitMir() catch |err| switch (err) {
537 error.EmitFail => return zcu.codegenFailMsg(nav, e.err_msg.?),
538 else => |e1| return e1,
539 };
540}
541
542/// Returns the requested data, as well as the new index which is at the start of the
543/// trailers for the object.
544pub fn extraData(mir: Mir, comptime T: type, index: usize) struct { data: T, end: usize } {
545 const fields = std.meta.fields(T);
546 var i: usize = index;
547 var result: T = undefined;
548 inline for (fields) |field| {
549 @field(result, field.name) = switch (field.type) {
550 u32 => mir.extra[i],
551 i32 => @as(i32, @bitCast(mir.extra[i])),
552 else => @compileError("bad field type"),
553 };
554 i += 1;
555 }
556 return .{
557 .data = result,
558 .end = i,
559 };
560}
561
562pub const LoadMemoryPie = struct {
563 register: u32,
564 /// Index of the containing atom.
565 atom_index: u32,
566 /// Index into the linker's symbol table.
567 sym_index: u32,
568};
src/arch/aarch64/abi.zig deleted-165
...@@ -1,165 +0,0 @@
1const std = @import("std");
2const builtin = @import("builtin");
3const bits = @import("bits.zig");
4const Register = bits.Register;
5const RegisterManagerFn = @import("../../register_manager.zig").RegisterManager;
6const Type = @import("../../Type.zig");
7const Zcu = @import("../../Zcu.zig");
8
9pub const Class = union(enum) {
10 memory,
11 byval,
12 integer,
13 double_integer,
14 float_array: u8,
15};
16
17/// For `float_array` the second element will be the amount of floats.
18pub fn classifyType(ty: Type, zcu: *Zcu) Class {
19 std.debug.assert(ty.hasRuntimeBitsIgnoreComptime(zcu));
20
21 var maybe_float_bits: ?u16 = null;
22 switch (ty.zigTypeTag(zcu)) {
23 .@"struct" => {
24 if (ty.containerLayout(zcu) == .@"packed") return .byval;
25 const float_count = countFloats(ty, zcu, &maybe_float_bits);
26 if (float_count <= sret_float_count) return .{ .float_array = float_count };
27
28 const bit_size = ty.bitSize(zcu);
29 if (bit_size > 128) return .memory;
30 if (bit_size > 64) return .double_integer;
31 return .integer;
32 },
33 .@"union" => {
34 if (ty.containerLayout(zcu) == .@"packed") return .byval;
35 const float_count = countFloats(ty, zcu, &maybe_float_bits);
36 if (float_count <= sret_float_count) return .{ .float_array = float_count };
37
38 const bit_size = ty.bitSize(zcu);
39 if (bit_size > 128) return .memory;
40 if (bit_size > 64) return .double_integer;
41 return .integer;
42 },
43 .int, .@"enum", .error_set, .float, .bool => return .byval,
44 .vector => {
45 const bit_size = ty.bitSize(zcu);
46 // TODO is this controlled by a cpu feature?
47 if (bit_size > 128) return .memory;
48 return .byval;
49 },
50 .optional => {
51 std.debug.assert(ty.isPtrLikeOptional(zcu));
52 return .byval;
53 },
54 .pointer => {
55 std.debug.assert(!ty.isSlice(zcu));
56 return .byval;
57 },
58 .error_union,
59 .frame,
60 .@"anyframe",
61 .noreturn,
62 .void,
63 .type,
64 .comptime_float,
65 .comptime_int,
66 .undefined,
67 .null,
68 .@"fn",
69 .@"opaque",
70 .enum_literal,
71 .array,
72 => unreachable,
73 }
74}
75
76const sret_float_count = 4;
77fn countFloats(ty: Type, zcu: *Zcu, maybe_float_bits: *?u16) u8 {
78 const ip = &zcu.intern_pool;
79 const target = zcu.getTarget();
80 const invalid = std.math.maxInt(u8);
81 switch (ty.zigTypeTag(zcu)) {
82 .@"union" => {
83 const union_obj = zcu.typeToUnion(ty).?;
84 var max_count: u8 = 0;
85 for (union_obj.field_types.get(ip)) |field_ty| {
86 const field_count = countFloats(Type.fromInterned(field_ty), zcu, maybe_float_bits);
87 if (field_count == invalid) return invalid;
88 if (field_count > max_count) max_count = field_count;
89 if (max_count > sret_float_count) return invalid;
90 }
91 return max_count;
92 },
93 .@"struct" => {
94 const fields_len = ty.structFieldCount(zcu);
95 var count: u8 = 0;
96 var i: u32 = 0;
97 while (i < fields_len) : (i += 1) {
98 const field_ty = ty.fieldType(i, zcu);
99 const field_count = countFloats(field_ty, zcu, maybe_float_bits);
100 if (field_count == invalid) return invalid;
101 count += field_count;
102 if (count > sret_float_count) return invalid;
103 }
104 return count;
105 },
106 .float => {
107 const float_bits = maybe_float_bits.* orelse {
108 maybe_float_bits.* = ty.floatBits(target);
109 return 1;
110 };
111 if (ty.floatBits(target) == float_bits) return 1;
112 return invalid;
113 },
114 .void => return 0,
115 else => return invalid,
116 }
117}
118
119pub fn getFloatArrayType(ty: Type, zcu: *Zcu) ?Type {
120 const ip = &zcu.intern_pool;
121 switch (ty.zigTypeTag(zcu)) {
122 .@"union" => {
123 const union_obj = zcu.typeToUnion(ty).?;
124 for (union_obj.field_types.get(ip)) |field_ty| {
125 if (getFloatArrayType(Type.fromInterned(field_ty), zcu)) |some| return some;
126 }
127 return null;
128 },
129 .@"struct" => {
130 const fields_len = ty.structFieldCount(zcu);
131 var i: u32 = 0;
132 while (i < fields_len) : (i += 1) {
133 const field_ty = ty.fieldType(i, zcu);
134 if (getFloatArrayType(field_ty, zcu)) |some| return some;
135 }
136 return null;
137 },
138 .float => return ty,
139 else => return null,
140 }
141}
142
143pub const callee_preserved_regs = [_]Register{
144 .x19, .x20, .x21, .x22, .x23,
145 .x24, .x25, .x26, .x27, .x28,
146};
147
148pub const c_abi_int_param_regs = [_]Register{ .x0, .x1, .x2, .x3, .x4, .x5, .x6, .x7 };
149pub const c_abi_int_return_regs = [_]Register{ .x0, .x1, .x2, .x3, .x4, .x5, .x6, .x7 };
150
151const allocatable_registers = callee_preserved_regs;
152pub const RegisterManager = RegisterManagerFn(@import("CodeGen.zig"), Register, &allocatable_registers);
153
154// Register classes
155const RegisterBitSet = RegisterManager.RegisterBitSet;
156pub const RegisterClass = struct {
157 pub const gp: RegisterBitSet = blk: {
158 var set = RegisterBitSet.initEmpty();
159 for (callee_preserved_regs) |reg| {
160 const index = RegisterManager.indexOfRegIntoTracked(reg).?;
161 set.set(index);
162 }
163 break :blk set;
164 };
165};
src/arch/arm/CodeGen.zig deleted-6340
...@@ -1,6340 +0,0 @@
1const std = @import("std");
2const builtin = @import("builtin");
3const mem = std.mem;
4const math = std.math;
5const assert = std.debug.assert;
6const codegen = @import("../../codegen.zig");
7const Air = @import("../../Air.zig");
8const Mir = @import("Mir.zig");
9const Emit = @import("Emit.zig");
10const Type = @import("../../Type.zig");
11const Value = @import("../../Value.zig");
12const link = @import("../../link.zig");
13const Zcu = @import("../../Zcu.zig");
14const InternPool = @import("../../InternPool.zig");
15const Compilation = @import("../../Compilation.zig");
16const ErrorMsg = Zcu.ErrorMsg;
17const Target = std.Target;
18const Allocator = mem.Allocator;
19const trace = @import("../../tracy.zig").trace;
20const leb128 = std.leb;
21const log = std.log.scoped(.codegen);
22const build_options = @import("build_options");
23const Alignment = InternPool.Alignment;
24
25const CodeGenError = codegen.CodeGenError;
26
27const bits = @import("bits.zig");
28const abi = @import("abi.zig");
29const errUnionPayloadOffset = codegen.errUnionPayloadOffset;
30const errUnionErrorOffset = codegen.errUnionErrorOffset;
31const RegisterManager = abi.RegisterManager;
32const RegisterLock = RegisterManager.RegisterLock;
33const Register = bits.Register;
34const Instruction = bits.Instruction;
35const Condition = bits.Condition;
36const callee_preserved_regs = abi.callee_preserved_regs;
37const caller_preserved_regs = abi.caller_preserved_regs;
38const c_abi_int_param_regs = abi.c_abi_int_param_regs;
39const c_abi_int_return_regs = abi.c_abi_int_return_regs;
40const gp = abi.RegisterClass.gp;
41
42const InnerError = CodeGenError || error{OutOfRegisters};
43
44pub fn legalizeFeatures(_: *const std.Target) ?*const Air.Legalize.Features {
45 return null;
46}
47
48gpa: Allocator,
49pt: Zcu.PerThread,
50air: Air,
51liveness: Air.Liveness,
52bin_file: *link.File,
53target: *const std.Target,
54func_index: InternPool.Index,
55err_msg: ?*ErrorMsg,
56args: []MCValue,
57ret_mcv: MCValue,
58fn_type: Type,
59arg_index: u32,
60src_loc: Zcu.LazySrcLoc,
61stack_align: u32,
62
63/// MIR Instructions
64mir_instructions: std.MultiArrayList(Mir.Inst) = .{},
65/// MIR extra data
66mir_extra: std.ArrayListUnmanaged(u32) = .empty,
67
68/// Byte offset within the source file of the ending curly.
69end_di_line: u32,
70end_di_column: u32,
71
72/// The value is an offset into the `Function` `code` from the beginning.
73/// To perform the reloc, write 32-bit signed little-endian integer
74/// which is a relative jump, based on the address following the reloc.
75exitlude_jump_relocs: std.ArrayListUnmanaged(usize) = .empty,
76
77reused_operands: std.StaticBitSet(Air.Liveness.bpi - 1) = undefined,
78
79/// We postpone the creation of debug info for function args and locals
80/// until after all Mir instructions have been generated. Only then we
81/// will know saved_regs_stack_space which is necessary in order to
82/// calculate the right stack offsest with respect to the `.fp` register.
83dbg_info_relocs: std.ArrayListUnmanaged(DbgInfoReloc) = .empty,
84
85/// Whenever there is a runtime branch, we push a Branch onto this stack,
86/// and pop it off when the runtime branch joins. This provides an "overlay"
87/// of the table of mappings from instructions to `MCValue` from within the branch.
88/// This way we can modify the `MCValue` for an instruction in different ways
89/// within different branches. Special consideration is needed when a branch
90/// joins with its parent, to make sure all instructions have the same MCValue
91/// across each runtime branch upon joining.
92branch_stack: *std.ArrayList(Branch),
93
94// Key is the block instruction
95blocks: std.AutoHashMapUnmanaged(Air.Inst.Index, BlockData) = .empty,
96
97register_manager: RegisterManager = .{},
98/// Maps offset to what is stored there.
99stack: std.AutoHashMapUnmanaged(u32, StackAllocation) = .empty,
100/// Tracks the current instruction allocated to the compare flags
101cpsr_flags_inst: ?Air.Inst.Index = null,
102
103/// Offset from the stack base, representing the end of the stack frame.
104max_end_stack: u32 = 0,
105/// Represents the current end stack offset. If there is no existing slot
106/// to place a new stack allocation, it goes here, and then bumps `max_end_stack`.
107next_stack_offset: u32 = 0,
108
109saved_regs_stack_space: u32 = 0,
110
111/// Debug field, used to find bugs in the compiler.
112air_bookkeeping: @TypeOf(air_bookkeeping_init) = air_bookkeeping_init,
113
114const air_bookkeeping_init = if (std.debug.runtime_safety) @as(usize, 0) else {};
115
116const MCValue = union(enum) {
117 /// No runtime bits. `void` types, empty structs, u0, enums with 1
118 /// tag, etc.
119 ///
120 /// TODO Look into deleting this tag and using `dead` instead,
121 /// since every use of MCValue.none should be instead looking at
122 /// the type and noticing it is 0 bits.
123 none,
124 /// Control flow will not allow this value to be observed.
125 unreach,
126 /// No more references to this value remain.
127 dead,
128 /// The value is undefined.
129 undef,
130 /// A pointer-sized integer that fits in a register.
131 ///
132 /// If the type is a pointer, this is the pointer address in
133 /// virtual address space.
134 immediate: u32,
135 /// The value is in a target-specific register.
136 register: Register,
137 /// The value is a tuple { wrapped: u32, overflow: u1 } where
138 /// wrapped is stored in the register and the overflow bit is
139 /// stored in the C flag of the CPSR.
140 ///
141 /// This MCValue is only generated by a add_with_overflow or
142 /// sub_with_overflow instruction operating on u32.
143 register_c_flag: Register,
144 /// The value is a tuple { wrapped: i32, overflow: u1 } where
145 /// wrapped is stored in the register and the overflow bit is
146 /// stored in the V flag of the CPSR.
147 ///
148 /// This MCValue is only generated by a add_with_overflow or
149 /// sub_with_overflow instruction operating on i32.
150 register_v_flag: Register,
151 /// The value is in memory at a hard-coded address.
152 ///
153 /// If the type is a pointer, it means the pointer address is at
154 /// this memory location.
155 memory: u64,
156 /// The value is one of the stack variables.
157 ///
158 /// If the type is a pointer, it means the pointer address is in
159 /// the stack at this offset.
160 stack_offset: u32,
161 /// The value is a pointer to one of the stack variables (payload
162 /// is stack offset).
163 ptr_stack_offset: u32,
164 /// The value resides in the N, Z, C, V flags of the Current
165 /// Program Status Register (CPSR). The value is 1 (if the type is
166 /// u1) or true (if the type in bool) iff the specified condition
167 /// is true.
168 cpsr_flags: Condition,
169 /// The value is a function argument passed via the stack.
170 stack_argument_offset: u32,
171};
172
173const Branch = struct {
174 inst_table: std.AutoArrayHashMapUnmanaged(Air.Inst.Index, MCValue) = .empty,
175
176 fn deinit(self: *Branch, gpa: Allocator) void {
177 self.inst_table.deinit(gpa);
178 self.* = undefined;
179 }
180};
181
182const StackAllocation = struct {
183 inst: Air.Inst.Index,
184 /// TODO do we need size? should be determined by inst.ty.abiSize()
185 size: u32,
186};
187
188const BlockData = struct {
189 relocs: std.ArrayListUnmanaged(Mir.Inst.Index),
190 /// The first break instruction encounters `null` here and chooses a
191 /// machine code value for the block result, populating this field.
192 /// Following break instructions encounter that value and use it for
193 /// the location to store their block results.
194 mcv: MCValue,
195};
196
197const BigTomb = struct {
198 function: *Self,
199 inst: Air.Inst.Index,
200 lbt: Air.Liveness.BigTomb,
201
202 fn feed(bt: *BigTomb, op_ref: Air.Inst.Ref) void {
203 const dies = bt.lbt.feed();
204 const op_index = op_ref.toIndex() orelse return;
205 if (!dies) return;
206 bt.function.processDeath(op_index);
207 }
208
209 fn finishAir(bt: *BigTomb, result: MCValue) void {
210 const is_used = !bt.function.liveness.isUnused(bt.inst);
211 if (is_used) {
212 log.debug("%{d} => {}", .{ bt.inst, result });
213 const branch = &bt.function.branch_stack.items[bt.function.branch_stack.items.len - 1];
214 branch.inst_table.putAssumeCapacityNoClobber(bt.inst, result);
215
216 switch (result) {
217 .register => |reg| {
218 // In some cases (such as bitcast), an operand
219 // may be the same MCValue as the result. If
220 // that operand died and was a register, it
221 // was freed by processDeath. We have to
222 // "re-allocate" the register.
223 if (bt.function.register_manager.isRegFree(reg)) {
224 bt.function.register_manager.getRegAssumeFree(reg, bt.inst);
225 }
226 },
227 .register_c_flag,
228 .register_v_flag,
229 => |reg| {
230 if (bt.function.register_manager.isRegFree(reg)) {
231 bt.function.register_manager.getRegAssumeFree(reg, bt.inst);
232 }
233 bt.function.cpsr_flags_inst = bt.inst;
234 },
235 .cpsr_flags => {
236 bt.function.cpsr_flags_inst = bt.inst;
237 },
238 else => {},
239 }
240 }
241 bt.function.finishAirBookkeeping();
242 }
243};
244
245const DbgInfoReloc = struct {
246 tag: Air.Inst.Tag,
247 ty: Type,
248 name: [:0]const u8,
249 mcv: MCValue,
250
251 fn genDbgInfo(reloc: DbgInfoReloc, function: Self) !void {
252 switch (reloc.tag) {
253 .arg,
254 .dbg_arg_inline,
255 => try reloc.genArgDbgInfo(function),
256
257 .dbg_var_ptr,
258 .dbg_var_val,
259 => try reloc.genVarDbgInfo(function),
260
261 else => unreachable,
262 }
263 }
264
265 fn genArgDbgInfo(reloc: DbgInfoReloc, function: Self) !void {
266 // TODO: Add a pseudo-instruction or something to defer this work until Emit.
267 // We aren't allowed to interact with linker state here.
268 if (true) return;
269 switch (function.debug_output) {
270 .dwarf => |dw| {
271 const loc: link.File.Dwarf.Loc = switch (reloc.mcv) {
272 .register => |reg| .{ .reg = reg.dwarfNum() },
273 .stack_offset,
274 .stack_argument_offset,
275 => blk: {
276 const adjusted_stack_offset = switch (reloc.mcv) {
277 .stack_offset => |offset| -@as(i32, @intCast(offset)),
278 .stack_argument_offset => |offset| @as(i32, @intCast(function.saved_regs_stack_space + offset)),
279 else => unreachable,
280 };
281 break :blk .{ .plus = .{
282 &.{ .reg = 11 },
283 &.{ .consts = adjusted_stack_offset },
284 } };
285 },
286 else => unreachable, // not a possible argument
287 };
288
289 try dw.genLocalDebugInfo(.local_arg, reloc.name, reloc.ty, loc);
290 },
291 .plan9 => {},
292 .none => {},
293 }
294 }
295
296 fn genVarDbgInfo(reloc: DbgInfoReloc, function: Self) !void {
297 // TODO: Add a pseudo-instruction or something to defer this work until Emit.
298 // We aren't allowed to interact with linker state here.
299 if (true) return;
300 switch (function.debug_output) {
301 .dwarf => |dw| {
302 const loc: link.File.Dwarf.Loc = switch (reloc.mcv) {
303 .register => |reg| .{ .reg = reg.dwarfNum() },
304 .ptr_stack_offset,
305 .stack_offset,
306 .stack_argument_offset,
307 => |offset| blk: {
308 const adjusted_offset = switch (reloc.mcv) {
309 .ptr_stack_offset,
310 .stack_offset,
311 => -@as(i32, @intCast(offset)),
312 .stack_argument_offset => @as(i32, @intCast(function.saved_regs_stack_space + offset)),
313 else => unreachable,
314 };
315 break :blk .{ .plus = .{
316 &.{ .reg = 11 },
317 &.{ .consts = adjusted_offset },
318 } };
319 },
320 .memory => |address| .{ .constu = address },
321 .immediate => |x| .{ .constu = x },
322 .none => .empty,
323 else => blk: {
324 log.debug("TODO generate debug info for {}", .{reloc.mcv});
325 break :blk .empty;
326 },
327 };
328 try dw.genLocalDebugInfo(.local_var, reloc.name, reloc.ty, loc);
329 },
330 .plan9 => {},
331 .none => {},
332 }
333 }
334};
335
336const Self = @This();
337
338pub fn generate(
339 lf: *link.File,
340 pt: Zcu.PerThread,
341 src_loc: Zcu.LazySrcLoc,
342 func_index: InternPool.Index,
343 air: *const Air,
344 liveness: *const Air.Liveness,
345) CodeGenError!Mir {
346 const zcu = pt.zcu;
347 const gpa = zcu.gpa;
348 const func = zcu.funcInfo(func_index);
349 const func_ty = Type.fromInterned(func.ty);
350 const file_scope = zcu.navFileScope(func.owner_nav);
351 const target = &file_scope.mod.?.resolved_target.result;
352
353 var branch_stack = std.ArrayList(Branch).init(gpa);
354 defer {
355 assert(branch_stack.items.len == 1);
356 branch_stack.items[0].deinit(gpa);
357 branch_stack.deinit();
358 }
359 try branch_stack.append(.{});
360
361 var function: Self = .{
362 .gpa = gpa,
363 .pt = pt,
364 .air = air.*,
365 .liveness = liveness.*,
366 .target = target,
367 .bin_file = lf,
368 .func_index = func_index,
369 .err_msg = null,
370 .args = undefined, // populated after `resolveCallingConventionValues`
371 .ret_mcv = undefined, // populated after `resolveCallingConventionValues`
372 .fn_type = func_ty,
373 .arg_index = 0,
374 .branch_stack = &branch_stack,
375 .src_loc = src_loc,
376 .stack_align = undefined,
377 .end_di_line = func.rbrace_line,
378 .end_di_column = func.rbrace_column,
379 };
380 defer function.stack.deinit(gpa);
381 defer function.blocks.deinit(gpa);
382 defer function.exitlude_jump_relocs.deinit(gpa);
383 defer function.dbg_info_relocs.deinit(gpa);
384
385 var call_info = function.resolveCallingConventionValues(func_ty) catch |err| switch (err) {
386 error.CodegenFail => return error.CodegenFail,
387 else => |e| return e,
388 };
389 defer call_info.deinit(&function);
390
391 function.args = call_info.args;
392 function.ret_mcv = call_info.return_value;
393 function.stack_align = call_info.stack_align;
394 function.max_end_stack = call_info.stack_byte_count;
395
396 function.gen() catch |err| switch (err) {
397 error.CodegenFail => return error.CodegenFail,
398 error.OutOfRegisters => return function.fail("ran out of registers (Zig compiler bug)", .{}),
399 else => |e| return e,
400 };
401
402 for (function.dbg_info_relocs.items) |reloc| {
403 reloc.genDbgInfo(function) catch |err|
404 return function.fail("failed to generate debug info: {s}", .{@errorName(err)});
405 }
406
407 var mir: Mir = .{
408 .instructions = function.mir_instructions.toOwnedSlice(),
409 .extra = &.{}, // fallible, so assign after errdefer
410 .max_end_stack = function.max_end_stack,
411 .saved_regs_stack_space = function.saved_regs_stack_space,
412 };
413 errdefer mir.deinit(gpa);
414 mir.extra = try function.mir_extra.toOwnedSlice(gpa);
415 return mir;
416}
417
418fn addInst(self: *Self, inst: Mir.Inst) error{OutOfMemory}!Mir.Inst.Index {
419 const gpa = self.gpa;
420
421 try self.mir_instructions.ensureUnusedCapacity(gpa, 1);
422
423 const result_index: Mir.Inst.Index = @intCast(self.mir_instructions.len);
424 self.mir_instructions.appendAssumeCapacity(inst);
425 return result_index;
426}
427
428fn addNop(self: *Self) error{OutOfMemory}!Mir.Inst.Index {
429 return try self.addInst(.{
430 .tag = .nop,
431 .data = .{ .nop = {} },
432 });
433}
434
435pub fn addExtra(self: *Self, extra: anytype) Allocator.Error!u32 {
436 const fields = std.meta.fields(@TypeOf(extra));
437 try self.mir_extra.ensureUnusedCapacity(self.gpa, fields.len);
438 return self.addExtraAssumeCapacity(extra);
439}
440
441pub fn addExtraAssumeCapacity(self: *Self, extra: anytype) u32 {
442 const fields = std.meta.fields(@TypeOf(extra));
443 const result: u32 = @intCast(self.mir_extra.items.len);
444 inline for (fields) |field| {
445 self.mir_extra.appendAssumeCapacity(switch (field.type) {
446 u32 => @field(extra, field.name),
447 i32 => @bitCast(@field(extra, field.name)),
448 else => @compileError("bad field type"),
449 });
450 }
451 return result;
452}
453
454fn gen(self: *Self) !void {
455 const pt = self.pt;
456 const zcu = pt.zcu;
457 const cc = self.fn_type.fnCallingConvention(zcu);
458 if (cc != .naked) {
459 // push {fp, lr}
460 const push_reloc = try self.addNop();
461
462 // mov fp, sp
463 _ = try self.addInst(.{
464 .tag = .mov,
465 .data = .{ .r_op_mov = .{
466 .rd = .fp,
467 .op = Instruction.Operand.reg(.sp, Instruction.Operand.Shift.none),
468 } },
469 });
470
471 // sub sp, sp, #reloc
472 const sub_reloc = try self.addNop();
473
474 // The sub_sp_scratch_r4 instruction may use r4, so we mark r4
475 // as allocated by this function.
476 const index = RegisterManager.indexOfRegIntoTracked(.r4).?;
477 self.register_manager.allocated_registers.set(index);
478
479 if (self.ret_mcv == .stack_offset) {
480 // The address of where to store the return value is in
481 // r0. As this register might get overwritten along the
482 // way, save the address to the stack.
483 const stack_offset = try self.allocMem(4, .@"4", null);
484
485 try self.genSetStack(Type.usize, stack_offset, MCValue{ .register = .r0 });
486 self.ret_mcv = MCValue{ .stack_offset = stack_offset };
487 }
488
489 for (self.args, 0..) |*arg, arg_index| {
490 // Copy register arguments to the stack
491 switch (arg.*) {
492 .register => |reg| {
493 // The first AIR instructions of the main body are guaranteed
494 // to be the functions arguments
495 const inst = self.air.getMainBody()[arg_index];
496 assert(self.air.instructions.items(.tag)[@intFromEnum(inst)] == .arg);
497
498 const ty = self.typeOfIndex(inst);
499
500 const abi_size: u32 = @intCast(ty.abiSize(zcu));
501 const abi_align = ty.abiAlignment(zcu);
502 const stack_offset = try self.allocMem(abi_size, abi_align, inst);
503 try self.genSetStack(ty, stack_offset, MCValue{ .register = reg });
504
505 arg.* = MCValue{ .stack_offset = stack_offset };
506 },
507 else => {},
508 }
509 }
510
511 _ = try self.addInst(.{
512 .tag = .dbg_prologue_end,
513 .cond = undefined,
514 .data = .{ .nop = {} },
515 });
516
517 try self.genBody(self.air.getMainBody());
518
519 // Backpatch push callee saved regs
520 var saved_regs = Instruction.RegisterList{
521 .r11 = true, // fp
522 .r14 = true, // lr
523 };
524 self.saved_regs_stack_space = 8;
525 inline for (callee_preserved_regs) |reg| {
526 if (self.register_manager.isRegAllocated(reg)) {
527 @field(saved_regs, @tagName(reg)) = true;
528 self.saved_regs_stack_space += 4;
529 }
530 }
531 self.mir_instructions.set(push_reloc, .{
532 .tag = .push,
533 .data = .{ .register_list = saved_regs },
534 });
535
536 // Backpatch stack offset
537 const total_stack_size = self.max_end_stack + self.saved_regs_stack_space;
538 const aligned_total_stack_end = mem.alignForward(u32, total_stack_size, self.stack_align);
539 const stack_size = aligned_total_stack_end - self.saved_regs_stack_space;
540 self.max_end_stack = stack_size;
541 self.mir_instructions.set(sub_reloc, .{
542 .tag = .sub_sp_scratch_r4,
543 .data = .{ .imm32 = stack_size },
544 });
545
546 _ = try self.addInst(.{
547 .tag = .dbg_epilogue_begin,
548 .cond = undefined,
549 .data = .{ .nop = {} },
550 });
551
552 // exitlude jumps
553 if (self.exitlude_jump_relocs.items.len > 0 and
554 self.exitlude_jump_relocs.items[self.exitlude_jump_relocs.items.len - 1] == self.mir_instructions.len - 2)
555 {
556 // If the last Mir instruction (apart from the
557 // dbg_epilogue_begin) is the last exitlude jump
558 // relocation (which would just jump one instruction
559 // further), it can be safely removed
560 self.mir_instructions.orderedRemove(self.exitlude_jump_relocs.pop().?);
561 }
562
563 for (self.exitlude_jump_relocs.items) |jmp_reloc| {
564 self.mir_instructions.set(jmp_reloc, .{
565 .tag = .b,
566 .data = .{ .inst = @intCast(self.mir_instructions.len) },
567 });
568 }
569
570 // Epilogue: pop callee saved registers (swap lr with pc in saved_regs)
571 saved_regs.r14 = false; // lr
572 saved_regs.r15 = true; // pc
573
574 // mov sp, fp
575 _ = try self.addInst(.{
576 .tag = .mov,
577 .data = .{ .r_op_mov = .{
578 .rd = .sp,
579 .op = Instruction.Operand.reg(.fp, Instruction.Operand.Shift.none),
580 } },
581 });
582
583 // pop {fp, pc}
584 _ = try self.addInst(.{
585 .tag = .pop,
586 .data = .{ .register_list = saved_regs },
587 });
588 } else {
589 _ = try self.addInst(.{
590 .tag = .dbg_prologue_end,
591 .cond = undefined,
592 .data = .{ .nop = {} },
593 });
594
595 try self.genBody(self.air.getMainBody());
596
597 _ = try self.addInst(.{
598 .tag = .dbg_epilogue_begin,
599 .cond = undefined,
600 .data = .{ .nop = {} },
601 });
602 }
603
604 // Drop them off at the rbrace.
605 _ = try self.addInst(.{
606 .tag = .dbg_line,
607 .cond = undefined,
608 .data = .{ .dbg_line_column = .{
609 .line = self.end_di_line,
610 .column = self.end_di_column,
611 } },
612 });
613}
614
615fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
616 const pt = self.pt;
617 const zcu = pt.zcu;
618 const ip = &zcu.intern_pool;
619 const air_tags = self.air.instructions.items(.tag);
620
621 for (body) |inst| {
622 // TODO: remove now-redundant isUnused calls from AIR handler functions
623 if (self.liveness.isUnused(inst) and !self.air.mustLower(inst, ip))
624 continue;
625
626 const old_air_bookkeeping = self.air_bookkeeping;
627 try self.ensureProcessDeathCapacity(Air.Liveness.bpi);
628
629 self.reused_operands = @TypeOf(self.reused_operands).initEmpty();
630 switch (air_tags[@intFromEnum(inst)]) {
631 // zig fmt: off
632 .add, => try self.airBinOp(inst, .add),
633 .add_wrap => try self.airBinOp(inst, .add_wrap),
634 .sub, => try self.airBinOp(inst, .sub),
635 .sub_wrap => try self.airBinOp(inst, .sub_wrap),
636 .mul => try self.airBinOp(inst, .mul),
637 .mul_wrap => try self.airBinOp(inst, .mul_wrap),
638 .shl => try self.airBinOp(inst, .shl),
639 .shl_exact => try self.airBinOp(inst, .shl_exact),
640 .bool_and => try self.airBinOp(inst, .bool_and),
641 .bool_or => try self.airBinOp(inst, .bool_or),
642 .bit_and => try self.airBinOp(inst, .bit_and),
643 .bit_or => try self.airBinOp(inst, .bit_or),
644 .xor => try self.airBinOp(inst, .xor),
645 .shr => try self.airBinOp(inst, .shr),
646 .shr_exact => try self.airBinOp(inst, .shr_exact),
647 .div_float => try self.airBinOp(inst, .div_float),
648 .div_trunc => try self.airBinOp(inst, .div_trunc),
649 .div_floor => try self.airBinOp(inst, .div_floor),
650 .div_exact => try self.airBinOp(inst, .div_exact),
651 .rem => try self.airBinOp(inst, .rem),
652 .mod => try self.airBinOp(inst, .mod),
653
654 .ptr_add => try self.airPtrArithmetic(inst, .ptr_add),
655 .ptr_sub => try self.airPtrArithmetic(inst, .ptr_sub),
656
657 .min => try self.airMinMax(inst),
658 .max => try self.airMinMax(inst),
659
660 .add_sat => try self.airAddSat(inst),
661 .sub_sat => try self.airSubSat(inst),
662 .mul_sat => try self.airMulSat(inst),
663 .shl_sat => try self.airShlSat(inst),
664 .slice => try self.airSlice(inst),
665
666 .sqrt,
667 .sin,
668 .cos,
669 .tan,
670 .exp,
671 .exp2,
672 .log,
673 .log2,
674 .log10,
675 .floor,
676 .ceil,
677 .round,
678 .trunc_float,
679 .neg,
680 => try self.airUnaryMath(inst),
681
682 .add_with_overflow => try self.airOverflow(inst),
683 .sub_with_overflow => try self.airOverflow(inst),
684 .mul_with_overflow => try self.airMulWithOverflow(inst),
685 .shl_with_overflow => try self.airShlWithOverflow(inst),
686
687 .cmp_lt => try self.airCmp(inst, .lt),
688 .cmp_lte => try self.airCmp(inst, .lte),
689 .cmp_eq => try self.airCmp(inst, .eq),
690 .cmp_gte => try self.airCmp(inst, .gte),
691 .cmp_gt => try self.airCmp(inst, .gt),
692 .cmp_neq => try self.airCmp(inst, .neq),
693
694 .cmp_vector => try self.airCmpVector(inst),
695 .cmp_lt_errors_len => try self.airCmpLtErrorsLen(inst),
696
697 .alloc => try self.airAlloc(inst),
698 .ret_ptr => try self.airRetPtr(inst),
699 .arg => try self.airArg(inst),
700 .assembly => try self.airAsm(inst),
701 .bitcast => try self.airBitCast(inst),
702 .block => try self.airBlock(inst),
703 .br => try self.airBr(inst),
704 .repeat => return self.fail("TODO implement `repeat`", .{}),
705 .switch_dispatch => return self.fail("TODO implement `switch_dispatch`", .{}),
706 .trap => try self.airTrap(),
707 .breakpoint => try self.airBreakpoint(),
708 .ret_addr => try self.airRetAddr(inst),
709 .frame_addr => try self.airFrameAddress(inst),
710 .cond_br => try self.airCondBr(inst),
711 .fptrunc => try self.airFptrunc(inst),
712 .fpext => try self.airFpext(inst),
713 .intcast => try self.airIntCast(inst),
714 .trunc => try self.airTrunc(inst),
715 .is_non_null => try self.airIsNonNull(inst),
716 .is_non_null_ptr => try self.airIsNonNullPtr(inst),
717 .is_null => try self.airIsNull(inst),
718 .is_null_ptr => try self.airIsNullPtr(inst),
719 .is_non_err => try self.airIsNonErr(inst),
720 .is_non_err_ptr => try self.airIsNonErrPtr(inst),
721 .is_err => try self.airIsErr(inst),
722 .is_err_ptr => try self.airIsErrPtr(inst),
723 .load => try self.airLoad(inst),
724 .loop => try self.airLoop(inst),
725 .not => try self.airNot(inst),
726 .ret => try self.airRet(inst),
727 .ret_safe => try self.airRet(inst), // TODO
728 .ret_load => try self.airRetLoad(inst),
729 .store => try self.airStore(inst, false),
730 .store_safe => try self.airStore(inst, true),
731 .struct_field_ptr=> try self.airStructFieldPtr(inst),
732 .struct_field_val=> try self.airStructFieldVal(inst),
733 .array_to_slice => try self.airArrayToSlice(inst),
734 .float_from_int => try self.airFloatFromInt(inst),
735 .int_from_float => try self.airIntFromFloat(inst),
736 .cmpxchg_strong => try self.airCmpxchg(inst),
737 .cmpxchg_weak => try self.airCmpxchg(inst),
738 .atomic_rmw => try self.airAtomicRmw(inst),
739 .atomic_load => try self.airAtomicLoad(inst),
740 .memcpy => try self.airMemcpy(inst),
741 .memmove => try self.airMemmove(inst),
742 .memset => try self.airMemset(inst, false),
743 .memset_safe => try self.airMemset(inst, true),
744 .set_union_tag => try self.airSetUnionTag(inst),
745 .get_union_tag => try self.airGetUnionTag(inst),
746 .clz => try self.airClz(inst),
747 .ctz => try self.airCtz(inst),
748 .popcount => try self.airPopcount(inst),
749 .abs => try self.airAbs(inst),
750 .byte_swap => try self.airByteSwap(inst),
751 .bit_reverse => try self.airBitReverse(inst),
752 .tag_name => try self.airTagName(inst),
753 .error_name => try self.airErrorName(inst),
754 .splat => try self.airSplat(inst),
755 .select => try self.airSelect(inst),
756 .shuffle_one => try self.airShuffleOne(inst),
757 .shuffle_two => try self.airShuffleTwo(inst),
758 .reduce => try self.airReduce(inst),
759 .aggregate_init => try self.airAggregateInit(inst),
760 .union_init => try self.airUnionInit(inst),
761 .prefetch => try self.airPrefetch(inst),
762 .mul_add => try self.airMulAdd(inst),
763 .addrspace_cast => return self.fail("TODO implement addrspace_cast", .{}),
764
765 .@"try" => try self.airTry(inst),
766 .try_cold => try self.airTry(inst),
767 .try_ptr => try self.airTryPtr(inst),
768 .try_ptr_cold => try self.airTryPtr(inst),
769
770 .dbg_stmt => try self.airDbgStmt(inst),
771 .dbg_empty_stmt => self.finishAirBookkeeping(),
772 .dbg_inline_block => try self.airDbgInlineBlock(inst),
773 .dbg_var_ptr,
774 .dbg_var_val,
775 .dbg_arg_inline,
776 => try self.airDbgVar(inst),
777
778 .call => try self.airCall(inst, .auto),
779 .call_always_tail => try self.airCall(inst, .always_tail),
780 .call_never_tail => try self.airCall(inst, .never_tail),
781 .call_never_inline => try self.airCall(inst, .never_inline),
782
783 .atomic_store_unordered => try self.airAtomicStore(inst, .unordered),
784 .atomic_store_monotonic => try self.airAtomicStore(inst, .monotonic),
785 .atomic_store_release => try self.airAtomicStore(inst, .release),
786 .atomic_store_seq_cst => try self.airAtomicStore(inst, .seq_cst),
787
788 .struct_field_ptr_index_0 => try self.airStructFieldPtrIndex(inst, 0),
789 .struct_field_ptr_index_1 => try self.airStructFieldPtrIndex(inst, 1),
790 .struct_field_ptr_index_2 => try self.airStructFieldPtrIndex(inst, 2),
791 .struct_field_ptr_index_3 => try self.airStructFieldPtrIndex(inst, 3),
792
793 .field_parent_ptr => try self.airFieldParentPtr(inst),
794
795 .switch_br => try self.airSwitch(inst),
796 .loop_switch_br => return self.fail("TODO implement `loop_switch_br`", .{}),
797 .slice_ptr => try self.airSlicePtr(inst),
798 .slice_len => try self.airSliceLen(inst),
799
800 .ptr_slice_len_ptr => try self.airPtrSliceLenPtr(inst),
801 .ptr_slice_ptr_ptr => try self.airPtrSlicePtrPtr(inst),
802
803 .array_elem_val => try self.airArrayElemVal(inst),
804 .slice_elem_val => try self.airSliceElemVal(inst),
805 .slice_elem_ptr => try self.airSliceElemPtr(inst),
806 .ptr_elem_val => try self.airPtrElemVal(inst),
807 .ptr_elem_ptr => try self.airPtrElemPtr(inst),
808
809 .inferred_alloc, .inferred_alloc_comptime => unreachable,
810 .unreach => self.finishAirBookkeeping(),
811
812 .optional_payload => try self.airOptionalPayload(inst),
813 .optional_payload_ptr => try self.airOptionalPayloadPtr(inst),
814 .optional_payload_ptr_set => try self.airOptionalPayloadPtrSet(inst),
815 .unwrap_errunion_err => try self.airUnwrapErrErr(inst),
816 .unwrap_errunion_payload => try self.airUnwrapErrPayload(inst),
817 .unwrap_errunion_err_ptr => try self.airUnwrapErrErrPtr(inst),
818 .unwrap_errunion_payload_ptr=> try self.airUnwrapErrPayloadPtr(inst),
819 .errunion_payload_ptr_set => try self.airErrUnionPayloadPtrSet(inst),
820 .err_return_trace => try self.airErrReturnTrace(inst),
821 .set_err_return_trace => try self.airSetErrReturnTrace(inst),
822 .save_err_return_trace_index=> try self.airSaveErrReturnTraceIndex(inst),
823
824 .wrap_optional => try self.airWrapOptional(inst),
825 .wrap_errunion_payload => try self.airWrapErrUnionPayload(inst),
826 .wrap_errunion_err => try self.airWrapErrUnionErr(inst),
827
828 .add_optimized,
829 .sub_optimized,
830 .mul_optimized,
831 .div_float_optimized,
832 .div_trunc_optimized,
833 .div_floor_optimized,
834 .div_exact_optimized,
835 .rem_optimized,
836 .mod_optimized,
837 .neg_optimized,
838 .cmp_lt_optimized,
839 .cmp_lte_optimized,
840 .cmp_eq_optimized,
841 .cmp_gte_optimized,
842 .cmp_gt_optimized,
843 .cmp_neq_optimized,
844 .cmp_vector_optimized,
845 .reduce_optimized,
846 .int_from_float_optimized,
847 => return self.fail("TODO implement optimized float mode", .{}),
848
849 .add_safe,
850 .sub_safe,
851 .mul_safe,
852 .intcast_safe,
853 .int_from_float_safe,
854 .int_from_float_optimized_safe,
855 => return self.fail("TODO implement safety_checked_instructions", .{}),
856
857 .is_named_enum_value => return self.fail("TODO implement is_named_enum_value", .{}),
858 .error_set_has_value => return self.fail("TODO implement error_set_has_value", .{}),
859 .vector_store_elem => return self.fail("TODO implement vector_store_elem", .{}),
860 .runtime_nav_ptr => return self.fail("TODO implement runtime_nav_ptr", .{}),
861
862 .c_va_arg => return self.fail("TODO implement c_va_arg", .{}),
863 .c_va_copy => return self.fail("TODO implement c_va_copy", .{}),
864 .c_va_end => return self.fail("TODO implement c_va_end", .{}),
865 .c_va_start => return self.fail("TODO implement c_va_start", .{}),
866
867 .wasm_memory_size => unreachable,
868 .wasm_memory_grow => unreachable,
869
870 .work_item_id => unreachable,
871 .work_group_size => unreachable,
872 .work_group_id => unreachable,
873 // zig fmt: on
874 }
875
876 assert(!self.register_manager.lockedRegsExist());
877
878 if (std.debug.runtime_safety) {
879 if (self.air_bookkeeping < old_air_bookkeeping + 1) {
880 std.debug.panic("in codegen.zig, handling of AIR instruction %{d} ('{}') did not do proper bookkeeping. Look for a missing call to finishAir.", .{ inst, air_tags[@intFromEnum(inst)] });
881 }
882 }
883 }
884}
885
886/// Asserts there is already capacity to insert into top branch inst_table.
887fn processDeath(self: *Self, inst: Air.Inst.Index) void {
888 // When editing this function, note that the logic must synchronize with `reuseOperand`.
889 const prev_value = self.getResolvedInstValue(inst);
890 const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
891 branch.inst_table.putAssumeCapacity(inst, .dead);
892 switch (prev_value) {
893 .register => |reg| {
894 self.register_manager.freeReg(reg);
895 },
896 .register_c_flag,
897 .register_v_flag,
898 => |reg| {
899 self.register_manager.freeReg(reg);
900 self.cpsr_flags_inst = null;
901 },
902 .cpsr_flags => {
903 self.cpsr_flags_inst = null;
904 },
905 else => {}, // TODO process stack allocation death
906 }
907}
908
909/// Called when there are no operands, and the instruction is always unreferenced.
910fn finishAirBookkeeping(self: *Self) void {
911 if (std.debug.runtime_safety) {
912 self.air_bookkeeping += 1;
913 }
914}
915
916fn finishAir(self: *Self, inst: Air.Inst.Index, result: MCValue, operands: [Air.Liveness.bpi - 1]Air.Inst.Ref) void {
917 const tomb_bits = self.liveness.getTombBits(inst);
918 for (0.., operands) |op_index, op| {
919 if (tomb_bits & @as(Air.Liveness.Bpi, 1) << @intCast(op_index) == 0) continue;
920 if (self.reused_operands.isSet(op_index)) continue;
921 self.processDeath(op.toIndexAllowNone() orelse continue);
922 }
923 if (tomb_bits & 1 << (Air.Liveness.bpi - 1) == 0) {
924 log.debug("%{d} => {}", .{ inst, result });
925 const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
926 branch.inst_table.putAssumeCapacityNoClobber(inst, result);
927
928 switch (result) {
929 .register => |reg| {
930 // In some cases (such as bitcast), an operand
931 // may be the same MCValue as the result. If
932 // that operand died and was a register, it
933 // was freed by processDeath. We have to
934 // "re-allocate" the register.
935 if (self.register_manager.isRegFree(reg)) {
936 self.register_manager.getRegAssumeFree(reg, inst);
937 }
938 },
939 .register_c_flag,
940 .register_v_flag,
941 => |reg| {
942 if (self.register_manager.isRegFree(reg)) {
943 self.register_manager.getRegAssumeFree(reg, inst);
944 }
945 self.cpsr_flags_inst = inst;
946 },
947 .cpsr_flags => {
948 self.cpsr_flags_inst = inst;
949 },
950 else => {},
951 }
952 }
953 self.finishAirBookkeeping();
954}
955
956fn ensureProcessDeathCapacity(self: *Self, additional_count: usize) !void {
957 const table = &self.branch_stack.items[self.branch_stack.items.len - 1].inst_table;
958 try table.ensureUnusedCapacity(self.gpa, additional_count);
959}
960
961fn allocMem(
962 self: *Self,
963 abi_size: u32,
964 abi_align: Alignment,
965 maybe_inst: ?Air.Inst.Index,
966) !u32 {
967 assert(abi_size > 0);
968 assert(abi_align != .none);
969
970 // TODO find a free slot instead of always appending
971 const offset: u32 = @intCast(abi_align.forward(self.next_stack_offset) + abi_size);
972 self.next_stack_offset = offset;
973 self.max_end_stack = @max(self.max_end_stack, self.next_stack_offset);
974
975 if (maybe_inst) |inst| {
976 try self.stack.putNoClobber(self.gpa, offset, .{
977 .inst = inst,
978 .size = abi_size,
979 });
980 }
981
982 return offset;
983}
984
985/// Use a pointer instruction as the basis for allocating stack memory.
986fn allocMemPtr(self: *Self, inst: Air.Inst.Index) !u32 {
987 const pt = self.pt;
988 const zcu = pt.zcu;
989 const elem_ty = self.typeOfIndex(inst).childType(zcu);
990
991 if (!elem_ty.hasRuntimeBits(zcu)) {
992 // As this stack item will never be dereferenced at runtime,
993 // return the stack offset 0. Stack offset 0 will be where all
994 // zero-sized stack allocations live as non-zero-sized
995 // allocations will always have an offset > 0.
996 return 0;
997 }
998
999 const abi_size = math.cast(u32, elem_ty.abiSize(zcu)) orelse {
1000 return self.fail("type '{}' too big to fit into stack frame", .{elem_ty.fmt(pt)});
1001 };
1002 // TODO swap this for inst.ty.ptrAlign
1003 const abi_align = elem_ty.abiAlignment(zcu);
1004
1005 return self.allocMem(abi_size, abi_align, inst);
1006}
1007
1008fn allocRegOrMem(self: *Self, elem_ty: Type, reg_ok: bool, maybe_inst: ?Air.Inst.Index) !MCValue {
1009 const pt = self.pt;
1010 const abi_size = math.cast(u32, elem_ty.abiSize(pt.zcu)) orelse {
1011 return self.fail("type '{}' too big to fit into stack frame", .{elem_ty.fmt(pt)});
1012 };
1013 const abi_align = elem_ty.abiAlignment(pt.zcu);
1014
1015 if (reg_ok) {
1016 // Make sure the type can fit in a register before we try to allocate one.
1017 const ptr_bits = self.target.ptrBitWidth();
1018 const ptr_bytes: u64 = @divExact(ptr_bits, 8);
1019 if (abi_size <= ptr_bytes) {
1020 if (self.register_manager.tryAllocReg(maybe_inst, gp)) |reg| {
1021 return MCValue{ .register = reg };
1022 }
1023 }
1024 }
1025
1026 const stack_offset = try self.allocMem(abi_size, abi_align, maybe_inst);
1027 return MCValue{ .stack_offset = stack_offset };
1028}
1029
1030pub fn spillInstruction(self: *Self, reg: Register, inst: Air.Inst.Index) !void {
1031 const stack_mcv = try self.allocRegOrMem(self.typeOfIndex(inst), false, inst);
1032 log.debug("spilling {} (%{d}) to stack mcv {any}", .{ reg, inst, stack_mcv });
1033
1034 const reg_mcv = self.getResolvedInstValue(inst);
1035 switch (reg_mcv) {
1036 .register,
1037 .register_c_flag,
1038 .register_v_flag,
1039 => |r| assert(r == reg),
1040 else => unreachable, // not a register
1041 }
1042
1043 const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
1044 try branch.inst_table.put(self.gpa, inst, stack_mcv);
1045 try self.genSetStack(self.typeOfIndex(inst), stack_mcv.stack_offset, reg_mcv);
1046}
1047
1048/// Save the current instruction stored in the compare flags if
1049/// occupied
1050fn spillCompareFlagsIfOccupied(self: *Self) !void {
1051 if (self.cpsr_flags_inst) |inst_to_save| {
1052 const ty = self.typeOfIndex(inst_to_save);
1053 const mcv = self.getResolvedInstValue(inst_to_save);
1054 const new_mcv = switch (mcv) {
1055 .cpsr_flags => try self.allocRegOrMem(ty, true, inst_to_save),
1056 .register_c_flag,
1057 .register_v_flag,
1058 => try self.allocRegOrMem(ty, false, inst_to_save),
1059 else => unreachable, // mcv doesn't occupy the compare flags
1060 };
1061
1062 try self.setRegOrMem(self.typeOfIndex(inst_to_save), new_mcv, mcv);
1063 log.debug("spilling {d} to mcv {any}", .{ inst_to_save, new_mcv });
1064
1065 const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
1066 try branch.inst_table.put(self.gpa, inst_to_save, new_mcv);
1067
1068 self.cpsr_flags_inst = null;
1069
1070 // TODO consolidate with register manager and spillInstruction
1071 // this call should really belong in the register manager!
1072 switch (mcv) {
1073 .register_c_flag,
1074 .register_v_flag,
1075 => |reg| self.register_manager.freeReg(reg),
1076 else => {},
1077 }
1078 }
1079}
1080
1081/// Copies a value to a register without tracking the register. The register is not considered
1082/// allocated. A second call to `copyToTmpRegister` may return the same register.
1083/// This can have a side effect of spilling instructions to the stack to free up a register.
1084fn copyToTmpRegister(self: *Self, ty: Type, mcv: MCValue) !Register {
1085 const reg = try self.register_manager.allocReg(null, gp);
1086 try self.genSetReg(ty, reg, mcv);
1087 return reg;
1088}
1089
1090fn airAlloc(self: *Self, inst: Air.Inst.Index) !void {
1091 const stack_offset = try self.allocMemPtr(inst);
1092 return self.finishAir(inst, .{ .ptr_stack_offset = stack_offset }, .{ .none, .none, .none });
1093}
1094
1095fn airRetPtr(self: *Self, inst: Air.Inst.Index) !void {
1096 const pt = self.pt;
1097 const zcu = pt.zcu;
1098 const result: MCValue = switch (self.ret_mcv) {
1099 .none, .register => .{ .ptr_stack_offset = try self.allocMemPtr(inst) },
1100 .stack_offset => blk: {
1101 // self.ret_mcv is an address to where this function
1102 // should store its result into
1103 const ret_ty = self.fn_type.fnReturnType(zcu);
1104 const ptr_ty = try pt.singleMutPtrType(ret_ty);
1105
1106 // addr_reg will contain the address of where to store the
1107 // result into
1108 const addr_reg = try self.copyToTmpRegister(ptr_ty, self.ret_mcv);
1109 break :blk .{ .register = addr_reg };
1110 },
1111 else => unreachable, // invalid return result
1112 };
1113
1114 return self.finishAir(inst, result, .{ .none, .none, .none });
1115}
1116
1117fn airFptrunc(self: *Self, inst: Air.Inst.Index) !void {
1118 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
1119 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement airFptrunc for {}", .{self.target.cpu.arch});
1120 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1121}
1122
1123fn airFpext(self: *Self, inst: Air.Inst.Index) !void {
1124 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
1125 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement airFpext for {}", .{self.target.cpu.arch});
1126 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1127}
1128
1129fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
1130 const pt = self.pt;
1131 const zcu = pt.zcu;
1132 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
1133 if (self.liveness.isUnused(inst))
1134 return self.finishAir(inst, .dead, .{ ty_op.operand, .none, .none });
1135
1136 const operand = try self.resolveInst(ty_op.operand);
1137 const operand_ty = self.typeOf(ty_op.operand);
1138 const dest_ty = self.typeOfIndex(inst);
1139
1140 const operand_abi_size = operand_ty.abiSize(zcu);
1141 const dest_abi_size = dest_ty.abiSize(zcu);
1142 const info_a = operand_ty.intInfo(zcu);
1143 const info_b = dest_ty.intInfo(zcu);
1144
1145 const dst_mcv: MCValue = blk: {
1146 if (info_a.bits == info_b.bits) {
1147 break :blk operand;
1148 }
1149 if (operand_abi_size > 4 or dest_abi_size > 4) {
1150 return self.fail("TODO implement intCast for abi sizes larger than 4", .{});
1151 }
1152
1153 const operand_lock: ?RegisterLock = switch (operand) {
1154 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
1155 else => null,
1156 };
1157 defer if (operand_lock) |lock| self.register_manager.unlockReg(lock);
1158
1159 const reg = try self.register_manager.allocReg(inst, gp);
1160 try self.genSetReg(dest_ty, reg, operand);
1161 break :blk MCValue{ .register = reg };
1162 };
1163
1164 return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none });
1165}
1166
1167fn truncRegister(
1168 self: *Self,
1169 operand_reg: Register,
1170 dest_reg: Register,
1171 int_signedness: std.builtin.Signedness,
1172 int_bits: u16,
1173) !void {
1174 // TODO check if sxtb/uxtb/sxth/uxth are more efficient
1175 _ = try self.addInst(.{
1176 .tag = switch (int_signedness) {
1177 .signed => .sbfx,
1178 .unsigned => .ubfx,
1179 },
1180 .data = .{ .rr_lsb_width = .{
1181 .rd = dest_reg,
1182 .rn = operand_reg,
1183 .lsb = 0,
1184 .width = @intCast(int_bits),
1185 } },
1186 });
1187}
1188
1189/// Asserts that both operand_ty and dest_ty are integer types
1190fn trunc(
1191 self: *Self,
1192 maybe_inst: ?Air.Inst.Index,
1193 operand_bind: ReadArg.Bind,
1194 operand_ty: Type,
1195 dest_ty: Type,
1196) !MCValue {
1197 const pt = self.pt;
1198 const zcu = pt.zcu;
1199 const info_a = operand_ty.intInfo(zcu);
1200 const info_b = dest_ty.intInfo(zcu);
1201
1202 if (info_b.bits <= 32) {
1203 if (info_a.bits > 32) {
1204 return self.fail("TODO load least significant word into register", .{});
1205 }
1206
1207 var operand_reg: Register = undefined;
1208 var dest_reg: Register = undefined;
1209
1210 const read_args = [_]ReadArg{
1211 .{ .ty = operand_ty, .bind = operand_bind, .class = gp, .reg = &operand_reg },
1212 };
1213 const write_args = [_]WriteArg{
1214 .{ .ty = dest_ty, .bind = .none, .class = gp, .reg = &dest_reg },
1215 };
1216 try self.allocRegs(
1217 &read_args,
1218 &write_args,
1219 if (maybe_inst) |inst| .{
1220 .corresponding_inst = inst,
1221 .operand_mapping = &.{0},
1222 } else null,
1223 );
1224
1225 switch (info_b.bits) {
1226 32 => {
1227 try self.genSetReg(operand_ty, dest_reg, .{ .register = operand_reg });
1228 },
1229 else => {
1230 try self.truncRegister(operand_reg, dest_reg, info_b.signedness, info_b.bits);
1231 },
1232 }
1233
1234 return MCValue{ .register = dest_reg };
1235 } else {
1236 return self.fail("TODO: truncate to ints > 32 bits", .{});
1237 }
1238}
1239
1240fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {
1241 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
1242 const operand_bind: ReadArg.Bind = .{ .inst = ty_op.operand };
1243 const operand_ty = self.typeOf(ty_op.operand);
1244 const dest_ty = self.typeOfIndex(inst);
1245
1246 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else blk: {
1247 break :blk try self.trunc(inst, operand_bind, operand_ty, dest_ty);
1248 };
1249
1250 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1251}
1252
1253fn airNot(self: *Self, inst: Air.Inst.Index) !void {
1254 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
1255 const pt = self.pt;
1256 const zcu = pt.zcu;
1257 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
1258 const operand_bind: ReadArg.Bind = .{ .inst = ty_op.operand };
1259 const operand_ty = self.typeOf(ty_op.operand);
1260 switch (try operand_bind.resolveToMcv(self)) {
1261 .dead => unreachable,
1262 .unreach => unreachable,
1263 .cpsr_flags => |cond| break :result MCValue{ .cpsr_flags = cond.negate() },
1264 else => {
1265 switch (operand_ty.zigTypeTag(zcu)) {
1266 .bool => {
1267 var op_reg: Register = undefined;
1268 var dest_reg: Register = undefined;
1269
1270 const read_args = [_]ReadArg{
1271 .{ .ty = operand_ty, .bind = operand_bind, .class = gp, .reg = &op_reg },
1272 };
1273 const write_args = [_]WriteArg{
1274 .{ .ty = operand_ty, .bind = .none, .class = gp, .reg = &dest_reg },
1275 };
1276 try self.allocRegs(
1277 &read_args,
1278 &write_args,
1279 ReuseMetadata{
1280 .corresponding_inst = inst,
1281 .operand_mapping = &.{0},
1282 },
1283 );
1284
1285 _ = try self.addInst(.{
1286 .tag = .eor,
1287 .data = .{ .rr_op = .{
1288 .rd = dest_reg,
1289 .rn = op_reg,
1290 .op = Instruction.Operand.fromU32(1).?,
1291 } },
1292 });
1293
1294 break :result MCValue{ .register = dest_reg };
1295 },
1296 .vector => return self.fail("TODO bitwise not for vectors", .{}),
1297 .int => {
1298 const int_info = operand_ty.intInfo(zcu);
1299 if (int_info.bits <= 32) {
1300 var op_reg: Register = undefined;
1301 var dest_reg: Register = undefined;
1302
1303 const read_args = [_]ReadArg{
1304 .{ .ty = operand_ty, .bind = operand_bind, .class = gp, .reg = &op_reg },
1305 };
1306 const write_args = [_]WriteArg{
1307 .{ .ty = operand_ty, .bind = .none, .class = gp, .reg = &dest_reg },
1308 };
1309 try self.allocRegs(
1310 &read_args,
1311 &write_args,
1312 ReuseMetadata{
1313 .corresponding_inst = inst,
1314 .operand_mapping = &.{0},
1315 },
1316 );
1317
1318 _ = try self.addInst(.{
1319 .tag = .mvn,
1320 .data = .{ .r_op_mov = .{
1321 .rd = dest_reg,
1322 .op = Instruction.Operand.reg(op_reg, Instruction.Operand.Shift.none),
1323 } },
1324 });
1325
1326 if (int_info.bits < 32) {
1327 try self.truncRegister(dest_reg, dest_reg, int_info.signedness, int_info.bits);
1328 }
1329
1330 break :result MCValue{ .register = dest_reg };
1331 } else {
1332 return self.fail("TODO ARM not on integers > u32/i32", .{});
1333 }
1334 },
1335 else => unreachable,
1336 }
1337 },
1338 }
1339 };
1340 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1341}
1342
1343fn minMax(
1344 self: *Self,
1345 tag: Air.Inst.Tag,
1346 lhs_bind: ReadArg.Bind,
1347 rhs_bind: ReadArg.Bind,
1348 lhs_ty: Type,
1349 rhs_ty: Type,
1350 maybe_inst: ?Air.Inst.Index,
1351) !MCValue {
1352 const pt = self.pt;
1353 const zcu = pt.zcu;
1354 switch (lhs_ty.zigTypeTag(zcu)) {
1355 .float => return self.fail("TODO ARM min/max on floats", .{}),
1356 .vector => return self.fail("TODO ARM min/max on vectors", .{}),
1357 .int => {
1358 assert(lhs_ty.eql(rhs_ty, zcu));
1359 const int_info = lhs_ty.intInfo(zcu);
1360 if (int_info.bits <= 32) {
1361 var lhs_reg: Register = undefined;
1362 var rhs_reg: Register = undefined;
1363 var dest_reg: Register = undefined;
1364
1365 const read_args = [_]ReadArg{
1366 .{ .ty = lhs_ty, .bind = lhs_bind, .class = gp, .reg = &lhs_reg },
1367 .{ .ty = rhs_ty, .bind = rhs_bind, .class = gp, .reg = &rhs_reg },
1368 };
1369 const write_args = [_]WriteArg{
1370 .{ .ty = lhs_ty, .bind = .none, .class = gp, .reg = &dest_reg },
1371 };
1372 try self.allocRegs(
1373 &read_args,
1374 &write_args,
1375 if (maybe_inst) |inst| .{
1376 .corresponding_inst = inst,
1377 .operand_mapping = &.{ 0, 1 },
1378 } else null,
1379 );
1380
1381 // lhs == reg should have been checked by airMinMax
1382 //
1383 // By guaranteeing lhs != rhs, we guarantee (dst !=
1384 // lhs) or (dst != rhs), which is a property we use to
1385 // omit generating one instruction when we reuse a
1386 // register.
1387 assert(lhs_reg != rhs_reg); // see note above
1388
1389 _ = try self.addInst(.{
1390 .tag = .cmp,
1391 .data = .{ .r_op_cmp = .{
1392 .rn = lhs_reg,
1393 .op = Instruction.Operand.reg(rhs_reg, Instruction.Operand.Shift.none),
1394 } },
1395 });
1396
1397 const cond_choose_lhs: Condition = switch (tag) {
1398 .max => switch (int_info.signedness) {
1399 .signed => Condition.gt,
1400 .unsigned => Condition.hi,
1401 },
1402 .min => switch (int_info.signedness) {
1403 .signed => Condition.lt,
1404 .unsigned => Condition.cc,
1405 },
1406 else => unreachable,
1407 };
1408 const cond_choose_rhs = cond_choose_lhs.negate();
1409
1410 if (dest_reg != lhs_reg) {
1411 _ = try self.addInst(.{
1412 .tag = .mov,
1413 .cond = cond_choose_lhs,
1414 .data = .{ .r_op_mov = .{
1415 .rd = dest_reg,
1416 .op = Instruction.Operand.reg(lhs_reg, Instruction.Operand.Shift.none),
1417 } },
1418 });
1419 }
1420 if (dest_reg != rhs_reg) {
1421 _ = try self.addInst(.{
1422 .tag = .mov,
1423 .cond = cond_choose_rhs,
1424 .data = .{ .r_op_mov = .{
1425 .rd = dest_reg,
1426 .op = Instruction.Operand.reg(rhs_reg, Instruction.Operand.Shift.none),
1427 } },
1428 });
1429 }
1430
1431 return MCValue{ .register = dest_reg };
1432 } else {
1433 return self.fail("TODO ARM min/max on integers > u32/i32", .{});
1434 }
1435 },
1436 else => unreachable,
1437 }
1438}
1439
1440fn airMinMax(self: *Self, inst: Air.Inst.Index) !void {
1441 const tag = self.air.instructions.items(.tag)[@intFromEnum(inst)];
1442 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
1443 const lhs_ty = self.typeOf(bin_op.lhs);
1444 const rhs_ty = self.typeOf(bin_op.rhs);
1445
1446 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
1447 const lhs_bind: ReadArg.Bind = .{ .inst = bin_op.lhs };
1448 const rhs_bind: ReadArg.Bind = .{ .inst = bin_op.rhs };
1449
1450 const lhs = try self.resolveInst(bin_op.lhs);
1451 if (bin_op.lhs == bin_op.rhs) break :result lhs;
1452
1453 break :result try self.minMax(tag, lhs_bind, rhs_bind, lhs_ty, rhs_ty, inst);
1454 };
1455 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1456}
1457
1458fn airSlice(self: *Self, inst: Air.Inst.Index) !void {
1459 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
1460 const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;
1461 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
1462 const ptr = try self.resolveInst(bin_op.lhs);
1463 const ptr_ty = self.typeOf(bin_op.lhs);
1464 const len = try self.resolveInst(bin_op.rhs);
1465 const len_ty = self.typeOf(bin_op.rhs);
1466
1467 const stack_offset = try self.allocMem(8, .@"4", inst);
1468 try self.genSetStack(ptr_ty, stack_offset, ptr);
1469 try self.genSetStack(len_ty, stack_offset - 4, len);
1470 break :result MCValue{ .stack_offset = stack_offset };
1471 };
1472 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1473}
1474
1475fn airBinOp(self: *Self, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void {
1476 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
1477 const lhs_ty = self.typeOf(bin_op.lhs);
1478 const rhs_ty = self.typeOf(bin_op.rhs);
1479
1480 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
1481 const lhs_bind: ReadArg.Bind = .{ .inst = bin_op.lhs };
1482 const rhs_bind: ReadArg.Bind = .{ .inst = bin_op.rhs };
1483
1484 break :result switch (tag) {
1485 .add => try self.addSub(tag, lhs_bind, rhs_bind, lhs_ty, rhs_ty, inst),
1486 .sub => try self.addSub(tag, lhs_bind, rhs_bind, lhs_ty, rhs_ty, inst),
1487
1488 .mul => try self.mul(lhs_bind, rhs_bind, lhs_ty, rhs_ty, inst),
1489
1490 .div_float => try self.divFloat(lhs_bind, rhs_bind, lhs_ty, rhs_ty, inst),
1491
1492 .div_trunc => try self.divTrunc(lhs_bind, rhs_bind, lhs_ty, rhs_ty, inst),
1493
1494 .div_floor => try self.divFloor(lhs_bind, rhs_bind, lhs_ty, rhs_ty, inst),
1495
1496 .div_exact => try self.divExact(lhs_bind, rhs_bind, lhs_ty, rhs_ty, inst),
1497
1498 .rem => try self.rem(lhs_bind, rhs_bind, lhs_ty, rhs_ty, inst),
1499
1500 .mod => try self.modulo(lhs_bind, rhs_bind, lhs_ty, rhs_ty, inst),
1501
1502 .add_wrap => try self.wrappingArithmetic(tag, lhs_bind, rhs_bind, lhs_ty, rhs_ty, inst),
1503 .sub_wrap => try self.wrappingArithmetic(tag, lhs_bind, rhs_bind, lhs_ty, rhs_ty, inst),
1504 .mul_wrap => try self.wrappingArithmetic(tag, lhs_bind, rhs_bind, lhs_ty, rhs_ty, inst),
1505
1506 .bit_and => try self.bitwise(tag, lhs_bind, rhs_bind, lhs_ty, rhs_ty, inst),
1507 .bit_or => try self.bitwise(tag, lhs_bind, rhs_bind, lhs_ty, rhs_ty, inst),
1508 .xor => try self.bitwise(tag, lhs_bind, rhs_bind, lhs_ty, rhs_ty, inst),
1509
1510 .shl_exact => try self.shiftExact(tag, lhs_bind, rhs_bind, lhs_ty, rhs_ty, inst),
1511 .shr_exact => try self.shiftExact(tag, lhs_bind, rhs_bind, lhs_ty, rhs_ty, inst),
1512
1513 .shl => try self.shiftNormal(tag, lhs_bind, rhs_bind, lhs_ty, rhs_ty, inst),
1514 .shr => try self.shiftNormal(tag, lhs_bind, rhs_bind, lhs_ty, rhs_ty, inst),
1515
1516 .bool_and => try self.booleanOp(tag, lhs_bind, rhs_bind, lhs_ty, rhs_ty, inst),
1517 .bool_or => try self.booleanOp(tag, lhs_bind, rhs_bind, lhs_ty, rhs_ty, inst),
1518
1519 else => unreachable,
1520 };
1521 };
1522 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1523}
1524
1525fn airPtrArithmetic(self: *Self, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void {
1526 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
1527 const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;
1528 const lhs_ty = self.typeOf(bin_op.lhs);
1529 const rhs_ty = self.typeOf(bin_op.rhs);
1530
1531 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
1532 const lhs_bind: ReadArg.Bind = .{ .inst = bin_op.lhs };
1533 const rhs_bind: ReadArg.Bind = .{ .inst = bin_op.rhs };
1534
1535 break :result try self.ptrArithmetic(tag, lhs_bind, rhs_bind, lhs_ty, rhs_ty, inst);
1536 };
1537 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1538}
1539
1540fn airAddSat(self: *Self, inst: Air.Inst.Index) !void {
1541 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
1542 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement add_sat for {}", .{self.target.cpu.arch});
1543 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1544}
1545
1546fn airSubSat(self: *Self, inst: Air.Inst.Index) !void {
1547 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
1548 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement sub_sat for {}", .{self.target.cpu.arch});
1549 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1550}
1551
1552fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {
1553 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
1554 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement mul_sat for {}", .{self.target.cpu.arch});
1555 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1556}
1557
1558fn airOverflow(self: *Self, inst: Air.Inst.Index) !void {
1559 const tag = self.air.instructions.items(.tag)[@intFromEnum(inst)];
1560 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
1561 const extra = self.air.extraData(Air.Bin, ty_pl.payload).data;
1562 const pt = self.pt;
1563 const zcu = pt.zcu;
1564 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
1565 const lhs_bind: ReadArg.Bind = .{ .inst = extra.lhs };
1566 const rhs_bind: ReadArg.Bind = .{ .inst = extra.rhs };
1567 const lhs_ty = self.typeOf(extra.lhs);
1568 const rhs_ty = self.typeOf(extra.rhs);
1569
1570 const tuple_ty = self.typeOfIndex(inst);
1571 const tuple_size: u32 = @intCast(tuple_ty.abiSize(zcu));
1572 const tuple_align = tuple_ty.abiAlignment(zcu);
1573 const overflow_bit_offset: u32 = @intCast(tuple_ty.structFieldOffset(1, zcu));
1574
1575 switch (lhs_ty.zigTypeTag(zcu)) {
1576 .vector => return self.fail("TODO implement add_with_overflow/sub_with_overflow for vectors", .{}),
1577 .int => {
1578 assert(lhs_ty.eql(rhs_ty, zcu));
1579 const int_info = lhs_ty.intInfo(zcu);
1580 if (int_info.bits < 32) {
1581 const stack_offset = try self.allocMem(tuple_size, tuple_align, inst);
1582
1583 try self.spillCompareFlagsIfOccupied();
1584
1585 const base_tag: Air.Inst.Tag = switch (tag) {
1586 .add_with_overflow => .add,
1587 .sub_with_overflow => .sub,
1588 else => unreachable,
1589 };
1590 const dest = try self.addSub(base_tag, lhs_bind, rhs_bind, lhs_ty, rhs_ty, null);
1591 const dest_reg = dest.register;
1592 const dest_reg_lock = self.register_manager.lockRegAssumeUnused(dest_reg);
1593 defer self.register_manager.unlockReg(dest_reg_lock);
1594
1595 const truncated_reg = try self.register_manager.allocReg(null, gp);
1596 const truncated_reg_lock = self.register_manager.lockRegAssumeUnused(truncated_reg);
1597 defer self.register_manager.unlockReg(truncated_reg_lock);
1598
1599 // sbfx/ubfx truncated, dest, #0, #bits
1600 try self.truncRegister(dest_reg, truncated_reg, int_info.signedness, int_info.bits);
1601
1602 // cmp dest, truncated
1603 _ = try self.addInst(.{
1604 .tag = .cmp,
1605 .data = .{ .r_op_cmp = .{
1606 .rn = dest_reg,
1607 .op = Instruction.Operand.reg(truncated_reg, Instruction.Operand.Shift.none),
1608 } },
1609 });
1610
1611 try self.genSetStack(lhs_ty, stack_offset, .{ .register = truncated_reg });
1612 try self.genSetStack(Type.u1, stack_offset - overflow_bit_offset, .{ .cpsr_flags = .ne });
1613
1614 break :result MCValue{ .stack_offset = stack_offset };
1615 } else if (int_info.bits == 32) {
1616 const lhs_immediate = try lhs_bind.resolveToImmediate(self);
1617 const rhs_immediate = try rhs_bind.resolveToImmediate(self);
1618
1619 // Only say yes if the operation is
1620 // commutative, i.e. we can swap both of the
1621 // operands
1622 const lhs_immediate_ok = switch (tag) {
1623 .add_with_overflow => if (lhs_immediate) |imm| Instruction.Operand.fromU32(imm) != null else false,
1624 .sub_with_overflow => false,
1625 else => unreachable,
1626 };
1627 const rhs_immediate_ok = switch (tag) {
1628 .add_with_overflow,
1629 .sub_with_overflow,
1630 => if (rhs_immediate) |imm| Instruction.Operand.fromU32(imm) != null else false,
1631 else => unreachable,
1632 };
1633
1634 const mir_tag: Mir.Inst.Tag = switch (tag) {
1635 .add_with_overflow => .adds,
1636 .sub_with_overflow => .subs,
1637 else => unreachable,
1638 };
1639
1640 try self.spillCompareFlagsIfOccupied();
1641 self.cpsr_flags_inst = inst;
1642
1643 const dest = blk: {
1644 if (rhs_immediate_ok) {
1645 break :blk try self.binOpImmediate(mir_tag, lhs_bind, rhs_immediate.?, lhs_ty, false, null);
1646 } else if (lhs_immediate_ok) {
1647 // swap lhs and rhs
1648 break :blk try self.binOpImmediate(mir_tag, rhs_bind, lhs_immediate.?, rhs_ty, true, null);
1649 } else {
1650 break :blk try self.binOpRegister(mir_tag, lhs_bind, rhs_bind, lhs_ty, rhs_ty, null);
1651 }
1652 };
1653
1654 if (tag == .sub_with_overflow) {
1655 break :result MCValue{ .register_v_flag = dest.register };
1656 }
1657
1658 switch (int_info.signedness) {
1659 .unsigned => break :result MCValue{ .register_c_flag = dest.register },
1660 .signed => break :result MCValue{ .register_v_flag = dest.register },
1661 }
1662 } else {
1663 return self.fail("TODO ARM overflow operations on integers > u32/i32", .{});
1664 }
1665 },
1666 else => unreachable,
1667 }
1668 };
1669 return self.finishAir(inst, result, .{ extra.lhs, extra.rhs, .none });
1670}
1671
1672fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
1673 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
1674 const extra = self.air.extraData(Air.Bin, ty_pl.payload).data;
1675 if (self.liveness.isUnused(inst)) return self.finishAir(inst, .dead, .{ extra.lhs, extra.rhs, .none });
1676 const pt = self.pt;
1677 const zcu = pt.zcu;
1678 const result: MCValue = result: {
1679 const lhs_bind: ReadArg.Bind = .{ .inst = extra.lhs };
1680 const rhs_bind: ReadArg.Bind = .{ .inst = extra.rhs };
1681 const lhs_ty = self.typeOf(extra.lhs);
1682 const rhs_ty = self.typeOf(extra.rhs);
1683
1684 const tuple_ty = self.typeOfIndex(inst);
1685 const tuple_size: u32 = @intCast(tuple_ty.abiSize(zcu));
1686 const tuple_align = tuple_ty.abiAlignment(zcu);
1687 const overflow_bit_offset: u32 = @intCast(tuple_ty.structFieldOffset(1, zcu));
1688
1689 switch (lhs_ty.zigTypeTag(zcu)) {
1690 .vector => return self.fail("TODO implement mul_with_overflow for vectors", .{}),
1691 .int => {
1692 assert(lhs_ty.eql(rhs_ty, zcu));
1693 const int_info = lhs_ty.intInfo(zcu);
1694 if (int_info.bits <= 16) {
1695 const stack_offset = try self.allocMem(tuple_size, tuple_align, inst);
1696
1697 try self.spillCompareFlagsIfOccupied();
1698
1699 const base_tag: Mir.Inst.Tag = switch (int_info.signedness) {
1700 .signed => .smulbb,
1701 .unsigned => .mul,
1702 };
1703
1704 const dest = try self.binOpRegister(base_tag, lhs_bind, rhs_bind, lhs_ty, rhs_ty, null);
1705 const dest_reg = dest.register;
1706 const dest_reg_lock = self.register_manager.lockRegAssumeUnused(dest_reg);
1707 defer self.register_manager.unlockReg(dest_reg_lock);
1708
1709 const truncated_reg = try self.register_manager.allocReg(null, gp);
1710 const truncated_reg_lock = self.register_manager.lockRegAssumeUnused(truncated_reg);
1711 defer self.register_manager.unlockReg(truncated_reg_lock);
1712
1713 // sbfx/ubfx truncated, dest, #0, #bits
1714 try self.truncRegister(dest_reg, truncated_reg, int_info.signedness, int_info.bits);
1715
1716 // cmp dest, truncated
1717 _ = try self.addInst(.{
1718 .tag = .cmp,
1719 .data = .{ .r_op_cmp = .{
1720 .rn = dest_reg,
1721 .op = Instruction.Operand.reg(truncated_reg, Instruction.Operand.Shift.none),
1722 } },
1723 });
1724
1725 try self.genSetStack(lhs_ty, stack_offset, .{ .register = truncated_reg });
1726 try self.genSetStack(Type.u1, stack_offset - overflow_bit_offset, .{ .cpsr_flags = .ne });
1727
1728 break :result MCValue{ .stack_offset = stack_offset };
1729 } else if (int_info.bits <= 32) {
1730 const stack_offset = try self.allocMem(tuple_size, tuple_align, inst);
1731
1732 try self.spillCompareFlagsIfOccupied();
1733
1734 const base_tag: Mir.Inst.Tag = switch (int_info.signedness) {
1735 .signed => .smull,
1736 .unsigned => .umull,
1737 };
1738
1739 var lhs_reg: Register = undefined;
1740 var rhs_reg: Register = undefined;
1741 var rdhi: Register = undefined;
1742 var rdlo: Register = undefined;
1743 var truncated_reg: Register = undefined;
1744
1745 const read_args = [_]ReadArg{
1746 .{ .ty = lhs_ty, .bind = lhs_bind, .class = gp, .reg = &lhs_reg },
1747 .{ .ty = rhs_ty, .bind = rhs_bind, .class = gp, .reg = &rhs_reg },
1748 };
1749 const write_args = [_]WriteArg{
1750 .{ .ty = lhs_ty, .bind = .none, .class = gp, .reg = &rdhi },
1751 .{ .ty = lhs_ty, .bind = .none, .class = gp, .reg = &rdlo },
1752 .{ .ty = lhs_ty, .bind = .none, .class = gp, .reg = &truncated_reg },
1753 };
1754 try self.allocRegs(
1755 &read_args,
1756 &write_args,
1757 null,
1758 );
1759
1760 _ = try self.addInst(.{
1761 .tag = base_tag,
1762 .data = .{ .rrrr = .{
1763 .rdlo = rdlo,
1764 .rdhi = rdhi,
1765 .rn = lhs_reg,
1766 .rm = rhs_reg,
1767 } },
1768 });
1769
1770 // sbfx/ubfx truncated, rdlo, #0, #bits
1771 try self.truncRegister(rdlo, truncated_reg, int_info.signedness, int_info.bits);
1772
1773 // str truncated, [...]
1774 try self.genSetStack(lhs_ty, stack_offset, .{ .register = truncated_reg });
1775
1776 // cmp truncated, rdlo
1777 _ = try self.addInst(.{
1778 .tag = .cmp,
1779 .data = .{ .r_op_cmp = .{
1780 .rn = truncated_reg,
1781 .op = Instruction.Operand.reg(rdlo, Instruction.Operand.Shift.none),
1782 } },
1783 });
1784
1785 // mov rdlo, #0
1786 _ = try self.addInst(.{
1787 .tag = .mov,
1788 .data = .{ .r_op_mov = .{
1789 .rd = rdlo,
1790 .op = Instruction.Operand.fromU32(0).?,
1791 } },
1792 });
1793
1794 // movne rdlo, #1
1795 _ = try self.addInst(.{
1796 .tag = .mov,
1797 .cond = .ne,
1798 .data = .{ .r_op_mov = .{
1799 .rd = rdlo,
1800 .op = Instruction.Operand.fromU32(1).?,
1801 } },
1802 });
1803
1804 // cmp rdhi, #0
1805 _ = try self.addInst(.{
1806 .tag = .cmp,
1807 .data = .{ .r_op_cmp = .{
1808 .rn = rdhi,
1809 .op = Instruction.Operand.fromU32(0).?,
1810 } },
1811 });
1812
1813 // movne rdlo, #1
1814 _ = try self.addInst(.{
1815 .tag = .mov,
1816 .cond = .ne,
1817 .data = .{ .r_op_mov = .{
1818 .rd = rdlo,
1819 .op = Instruction.Operand.fromU32(1).?,
1820 } },
1821 });
1822
1823 // strb rdlo, [...]
1824 try self.genSetStack(Type.u1, stack_offset - overflow_bit_offset, .{ .register = rdlo });
1825
1826 break :result MCValue{ .stack_offset = stack_offset };
1827 } else {
1828 return self.fail("TODO ARM overflow operations on integers > u32/i32", .{});
1829 }
1830 },
1831 else => unreachable,
1832 }
1833 };
1834 return self.finishAir(inst, result, .{ extra.lhs, extra.rhs, .none });
1835}
1836
1837fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
1838 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
1839 const extra = self.air.extraData(Air.Bin, ty_pl.payload).data;
1840 if (self.liveness.isUnused(inst)) return self.finishAir(inst, .dead, .{ extra.lhs, extra.rhs, .none });
1841 const pt = self.pt;
1842 const zcu = pt.zcu;
1843 const result: MCValue = result: {
1844 const lhs_ty = self.typeOf(extra.lhs);
1845 const rhs_ty = self.typeOf(extra.rhs);
1846
1847 const tuple_ty = self.typeOfIndex(inst);
1848 const tuple_size: u32 = @intCast(tuple_ty.abiSize(zcu));
1849 const tuple_align = tuple_ty.abiAlignment(zcu);
1850 const overflow_bit_offset: u32 = @intCast(tuple_ty.structFieldOffset(1, zcu));
1851
1852 switch (lhs_ty.zigTypeTag(zcu)) {
1853 .vector => if (!rhs_ty.isVector(zcu))
1854 return self.fail("TODO implement vector shl_with_overflow with scalar rhs", .{})
1855 else
1856 return self.fail("TODO implement shl_with_overflow for vectors", .{}),
1857 .int => {
1858 const int_info = lhs_ty.intInfo(zcu);
1859 if (int_info.bits <= 32) {
1860 const stack_offset = try self.allocMem(tuple_size, tuple_align, inst);
1861
1862 try self.spillCompareFlagsIfOccupied();
1863
1864 const shr_mir_tag: Mir.Inst.Tag = switch (int_info.signedness) {
1865 .signed => Mir.Inst.Tag.asr,
1866 .unsigned => Mir.Inst.Tag.lsr,
1867 };
1868
1869 var lhs_reg: Register = undefined;
1870 var rhs_reg: Register = undefined;
1871 var dest_reg: Register = undefined;
1872 var reconstructed_reg: Register = undefined;
1873
1874 const rhs_mcv = try self.resolveInst(extra.rhs);
1875 const rhs_immediate_ok = rhs_mcv == .immediate and Instruction.Operand.fromU32(rhs_mcv.immediate) != null;
1876
1877 const lhs_bind: ReadArg.Bind = .{ .inst = extra.lhs };
1878 const rhs_bind: ReadArg.Bind = .{ .inst = extra.rhs };
1879
1880 if (rhs_immediate_ok) {
1881 const read_args = [_]ReadArg{
1882 .{ .ty = lhs_ty, .bind = lhs_bind, .class = gp, .reg = &lhs_reg },
1883 };
1884 const write_args = [_]WriteArg{
1885 .{ .ty = lhs_ty, .bind = .none, .class = gp, .reg = &dest_reg },
1886 .{ .ty = lhs_ty, .bind = .none, .class = gp, .reg = &reconstructed_reg },
1887 };
1888 try self.allocRegs(
1889 &read_args,
1890 &write_args,
1891 null,
1892 );
1893
1894 // lsl dest, lhs, rhs
1895 _ = try self.addInst(.{
1896 .tag = .lsl,
1897 .data = .{ .rr_shift = .{
1898 .rd = dest_reg,
1899 .rm = lhs_reg,
1900 .shift_amount = Instruction.ShiftAmount.imm(@intCast(rhs_mcv.immediate)),
1901 } },
1902 });
1903
1904 try self.truncRegister(dest_reg, dest_reg, int_info.signedness, int_info.bits);
1905
1906 // asr/lsr reconstructed, dest, rhs
1907 _ = try self.addInst(.{
1908 .tag = shr_mir_tag,
1909 .data = .{ .rr_shift = .{
1910 .rd = reconstructed_reg,
1911 .rm = dest_reg,
1912 .shift_amount = Instruction.ShiftAmount.imm(@intCast(rhs_mcv.immediate)),
1913 } },
1914 });
1915 } else {
1916 const read_args = [_]ReadArg{
1917 .{ .ty = lhs_ty, .bind = lhs_bind, .class = gp, .reg = &lhs_reg },
1918 .{ .ty = rhs_ty, .bind = rhs_bind, .class = gp, .reg = &rhs_reg },
1919 };
1920 const write_args = [_]WriteArg{
1921 .{ .ty = lhs_ty, .bind = .none, .class = gp, .reg = &dest_reg },
1922 .{ .ty = lhs_ty, .bind = .none, .class = gp, .reg = &reconstructed_reg },
1923 };
1924 try self.allocRegs(
1925 &read_args,
1926 &write_args,
1927 null,
1928 );
1929
1930 // lsl dest, lhs, rhs
1931 _ = try self.addInst(.{
1932 .tag = .lsl,
1933 .data = .{ .rr_shift = .{
1934 .rd = dest_reg,
1935 .rm = lhs_reg,
1936 .shift_amount = Instruction.ShiftAmount.reg(rhs_reg),
1937 } },
1938 });
1939
1940 try self.truncRegister(dest_reg, dest_reg, int_info.signedness, int_info.bits);
1941
1942 // asr/lsr reconstructed, dest, rhs
1943 _ = try self.addInst(.{
1944 .tag = shr_mir_tag,
1945 .data = .{ .rr_shift = .{
1946 .rd = reconstructed_reg,
1947 .rm = dest_reg,
1948 .shift_amount = Instruction.ShiftAmount.reg(rhs_reg),
1949 } },
1950 });
1951 }
1952
1953 // cmp lhs, reconstructed
1954 _ = try self.addInst(.{
1955 .tag = .cmp,
1956 .data = .{ .r_op_cmp = .{
1957 .rn = lhs_reg,
1958 .op = Instruction.Operand.reg(reconstructed_reg, Instruction.Operand.Shift.none),
1959 } },
1960 });
1961
1962 try self.genSetStack(lhs_ty, stack_offset, .{ .register = dest_reg });
1963 try self.genSetStack(Type.u1, stack_offset - overflow_bit_offset, .{ .cpsr_flags = .ne });
1964
1965 break :result MCValue{ .stack_offset = stack_offset };
1966 } else {
1967 return self.fail("TODO ARM overflow operations on integers > u32/i32", .{});
1968 }
1969 },
1970 else => unreachable,
1971 }
1972 };
1973 return self.finishAir(inst, result, .{ extra.lhs, extra.rhs, .none });
1974}
1975
1976fn airShlSat(self: *Self, inst: Air.Inst.Index) !void {
1977 const zcu = self.pt.zcu;
1978 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
1979 const result: MCValue = if (self.liveness.isUnused(inst))
1980 .dead
1981 else if (self.typeOf(bin_op.lhs).isVector(zcu) and !self.typeOf(bin_op.rhs).isVector(zcu))
1982 return self.fail("TODO implement vector shl_sat with scalar rhs for {}", .{self.target.cpu.arch})
1983 else
1984 return self.fail("TODO implement shl_sat for {}", .{self.target.cpu.arch});
1985 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1986}
1987
1988fn airOptionalPayload(self: *Self, inst: Air.Inst.Index) !void {
1989 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
1990 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement .optional_payload for {}", .{self.target.cpu.arch});
1991 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1992}
1993
1994fn airOptionalPayloadPtr(self: *Self, inst: Air.Inst.Index) !void {
1995 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
1996 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement .optional_payload_ptr for {}", .{self.target.cpu.arch});
1997 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1998}
1999
2000fn airOptionalPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {
2001 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
2002 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement .optional_payload_ptr_set for {}", .{self.target.cpu.arch});
2003 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
2004}
2005
2006fn airWrapOptional(self: *Self, inst: Air.Inst.Index) !void {
2007 const pt = self.pt;
2008 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
2009 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
2010 const optional_ty = self.typeOfIndex(inst);
2011 const abi_size: u32 = @intCast(optional_ty.abiSize(pt.zcu));
2012
2013 // Optional with a zero-bit payload type is just a boolean true
2014 if (abi_size == 1) {
2015 break :result MCValue{ .immediate = 1 };
2016 } else {
2017 return self.fail("TODO implement wrap optional for {}", .{self.target.cpu.arch});
2018 }
2019 };
2020 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
2021}
2022
2023/// Given an error union, returns the error
2024fn errUnionErr(
2025 self: *Self,
2026 error_union_bind: ReadArg.Bind,
2027 error_union_ty: Type,
2028 maybe_inst: ?Air.Inst.Index,
2029) !MCValue {
2030 const pt = self.pt;
2031 const zcu = pt.zcu;
2032 const err_ty = error_union_ty.errorUnionSet(zcu);
2033 const payload_ty = error_union_ty.errorUnionPayload(zcu);
2034 if (err_ty.errorSetIsEmpty(zcu)) {
2035 return MCValue{ .immediate = 0 };
2036 }
2037 if (!payload_ty.hasRuntimeBitsIgnoreComptime(zcu)) {
2038 return try error_union_bind.resolveToMcv(self);
2039 }
2040
2041 const err_offset: u32 = @intCast(errUnionErrorOffset(payload_ty, zcu));
2042 switch (try error_union_bind.resolveToMcv(self)) {
2043 .register => {
2044 var operand_reg: Register = undefined;
2045 var dest_reg: Register = undefined;
2046
2047 const read_args = [_]ReadArg{
2048 .{ .ty = error_union_ty, .bind = error_union_bind, .class = gp, .reg = &operand_reg },
2049 };
2050 const write_args = [_]WriteArg{
2051 .{ .ty = err_ty, .bind = .none, .class = gp, .reg = &dest_reg },
2052 };
2053 try self.allocRegs(
2054 &read_args,
2055 &write_args,
2056 if (maybe_inst) |inst| .{
2057 .corresponding_inst = inst,
2058 .operand_mapping = &.{0},
2059 } else null,
2060 );
2061
2062 const err_bit_offset = err_offset * 8;
2063 const err_bit_size: u32 = @intCast(err_ty.abiSize(zcu) * 8);
2064
2065 _ = try self.addInst(.{
2066 .tag = .ubfx, // errors are unsigned integers
2067 .data = .{ .rr_lsb_width = .{
2068 .rd = dest_reg,
2069 .rn = operand_reg,
2070 .lsb = @intCast(err_bit_offset),
2071 .width = @intCast(err_bit_size),
2072 } },
2073 });
2074
2075 return MCValue{ .register = dest_reg };
2076 },
2077 .stack_argument_offset => |off| {
2078 return MCValue{ .stack_argument_offset = off + err_offset };
2079 },
2080 .stack_offset => |off| {
2081 return MCValue{ .stack_offset = off - err_offset };
2082 },
2083 .memory => |addr| {
2084 return MCValue{ .memory = addr + err_offset };
2085 },
2086 else => unreachable, // invalid MCValue for an error union
2087 }
2088}
2089
2090fn airUnwrapErrErr(self: *Self, inst: Air.Inst.Index) !void {
2091 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
2092 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
2093 const error_union_bind: ReadArg.Bind = .{ .inst = ty_op.operand };
2094 const error_union_ty = self.typeOf(ty_op.operand);
2095
2096 break :result try self.errUnionErr(error_union_bind, error_union_ty, inst);
2097 };
2098 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
2099}
2100
2101/// Given an error union, returns the payload
2102fn errUnionPayload(
2103 self: *Self,
2104 error_union_bind: ReadArg.Bind,
2105 error_union_ty: Type,
2106 maybe_inst: ?Air.Inst.Index,
2107) !MCValue {
2108 const pt = self.pt;
2109 const zcu = pt.zcu;
2110 const err_ty = error_union_ty.errorUnionSet(zcu);
2111 const payload_ty = error_union_ty.errorUnionPayload(zcu);
2112 if (err_ty.errorSetIsEmpty(zcu)) {
2113 return try error_union_bind.resolveToMcv(self);
2114 }
2115 if (!payload_ty.hasRuntimeBitsIgnoreComptime(zcu)) {
2116 return MCValue.none;
2117 }
2118
2119 const payload_offset: u32 = @intCast(errUnionPayloadOffset(payload_ty, zcu));
2120 switch (try error_union_bind.resolveToMcv(self)) {
2121 .register => {
2122 var operand_reg: Register = undefined;
2123 var dest_reg: Register = undefined;
2124
2125 const read_args = [_]ReadArg{
2126 .{ .ty = error_union_ty, .bind = error_union_bind, .class = gp, .reg = &operand_reg },
2127 };
2128 const write_args = [_]WriteArg{
2129 .{ .ty = err_ty, .bind = .none, .class = gp, .reg = &dest_reg },
2130 };
2131 try self.allocRegs(
2132 &read_args,
2133 &write_args,
2134 if (maybe_inst) |inst| .{
2135 .corresponding_inst = inst,
2136 .operand_mapping = &.{0},
2137 } else null,
2138 );
2139
2140 const payload_bit_offset = payload_offset * 8;
2141 const payload_bit_size: u32 = @intCast(payload_ty.abiSize(zcu) * 8);
2142
2143 _ = try self.addInst(.{
2144 .tag = if (payload_ty.isSignedInt(zcu)) Mir.Inst.Tag.sbfx else .ubfx,
2145 .data = .{ .rr_lsb_width = .{
2146 .rd = dest_reg,
2147 .rn = operand_reg,
2148 .lsb = @intCast(payload_bit_offset),
2149 .width = @intCast(payload_bit_size),
2150 } },
2151 });
2152
2153 return MCValue{ .register = dest_reg };
2154 },
2155 .stack_argument_offset => |off| {
2156 return MCValue{ .stack_argument_offset = off + payload_offset };
2157 },
2158 .stack_offset => |off| {
2159 return MCValue{ .stack_offset = off - payload_offset };
2160 },
2161 .memory => |addr| {
2162 return MCValue{ .memory = addr + payload_offset };
2163 },
2164 else => unreachable, // invalid MCValue for an error union
2165 }
2166}
2167
2168fn airUnwrapErrPayload(self: *Self, inst: Air.Inst.Index) !void {
2169 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
2170 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
2171 const error_union_bind: ReadArg.Bind = .{ .inst = ty_op.operand };
2172 const error_union_ty = self.typeOf(ty_op.operand);
2173
2174 break :result try self.errUnionPayload(error_union_bind, error_union_ty, inst);
2175 };
2176 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
2177}
2178
2179// *(E!T) -> E
2180fn airUnwrapErrErrPtr(self: *Self, inst: Air.Inst.Index) !void {
2181 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
2182 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement unwrap error union error ptr for {}", .{self.target.cpu.arch});
2183 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
2184}
2185
2186// *(E!T) -> *T
2187fn airUnwrapErrPayloadPtr(self: *Self, inst: Air.Inst.Index) !void {
2188 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
2189 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement unwrap error union payload ptr for {}", .{self.target.cpu.arch});
2190 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
2191}
2192
2193fn airErrUnionPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {
2194 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
2195 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement .errunion_payload_ptr_set for {}", .{self.target.cpu.arch});
2196 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
2197}
2198
2199fn airErrReturnTrace(self: *Self, inst: Air.Inst.Index) !void {
2200 const result: MCValue = if (self.liveness.isUnused(inst))
2201 .dead
2202 else
2203 return self.fail("TODO implement airErrReturnTrace for {}", .{self.target.cpu.arch});
2204 return self.finishAir(inst, result, .{ .none, .none, .none });
2205}
2206
2207fn airSetErrReturnTrace(self: *Self, inst: Air.Inst.Index) !void {
2208 _ = inst;
2209 return self.fail("TODO implement airSetErrReturnTrace for {}", .{self.target.cpu.arch});
2210}
2211
2212fn airSaveErrReturnTraceIndex(self: *Self, inst: Air.Inst.Index) !void {
2213 _ = inst;
2214 return self.fail("TODO implement airSaveErrReturnTraceIndex for {}", .{self.target.cpu.arch});
2215}
2216
2217/// T to E!T
2218fn airWrapErrUnionPayload(self: *Self, inst: Air.Inst.Index) !void {
2219 const pt = self.pt;
2220 const zcu = pt.zcu;
2221 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
2222 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
2223 const error_union_ty = ty_op.ty.toType();
2224 const error_ty = error_union_ty.errorUnionSet(zcu);
2225 const payload_ty = error_union_ty.errorUnionPayload(zcu);
2226 const operand = try self.resolveInst(ty_op.operand);
2227 if (!payload_ty.hasRuntimeBitsIgnoreComptime(zcu)) break :result operand;
2228
2229 const abi_size: u32 = @intCast(error_union_ty.abiSize(zcu));
2230 const abi_align = error_union_ty.abiAlignment(zcu);
2231 const stack_offset: u32 = @intCast(try self.allocMem(abi_size, abi_align, inst));
2232 const payload_off = errUnionPayloadOffset(payload_ty, zcu);
2233 const err_off = errUnionErrorOffset(payload_ty, zcu);
2234 try self.genSetStack(payload_ty, stack_offset - @as(u32, @intCast(payload_off)), operand);
2235 try self.genSetStack(error_ty, stack_offset - @as(u32, @intCast(err_off)), .{ .immediate = 0 });
2236
2237 break :result MCValue{ .stack_offset = stack_offset };
2238 };
2239 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
2240}
2241
2242/// E to E!T
2243fn airWrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {
2244 const pt = self.pt;
2245 const zcu = pt.zcu;
2246 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
2247 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
2248 const error_union_ty = ty_op.ty.toType();
2249 const error_ty = error_union_ty.errorUnionSet(zcu);
2250 const payload_ty = error_union_ty.errorUnionPayload(zcu);
2251 const operand = try self.resolveInst(ty_op.operand);
2252 if (!payload_ty.hasRuntimeBitsIgnoreComptime(zcu)) break :result operand;
2253
2254 const abi_size: u32 = @intCast(error_union_ty.abiSize(zcu));
2255 const abi_align = error_union_ty.abiAlignment(zcu);
2256 const stack_offset: u32 = @intCast(try self.allocMem(abi_size, abi_align, inst));
2257 const payload_off = errUnionPayloadOffset(payload_ty, zcu);
2258 const err_off = errUnionErrorOffset(payload_ty, zcu);
2259 try self.genSetStack(error_ty, stack_offset - @as(u32, @intCast(err_off)), operand);
2260 try self.genSetStack(payload_ty, stack_offset - @as(u32, @intCast(payload_off)), .undef);
2261
2262 break :result MCValue{ .stack_offset = stack_offset };
2263 };
2264 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
2265}
2266
2267/// Given a slice, returns the length
2268fn slicePtr(mcv: MCValue) MCValue {
2269 switch (mcv) {
2270 .register => unreachable, // a slice doesn't fit in one register
2271 .stack_argument_offset => |off| {
2272 return MCValue{ .stack_argument_offset = off };
2273 },
2274 .stack_offset => |off| {
2275 return MCValue{ .stack_offset = off };
2276 },
2277 .memory => |addr| {
2278 return MCValue{ .memory = addr };
2279 },
2280 else => unreachable, // invalid MCValue for a slice
2281 }
2282}
2283
2284fn airSlicePtr(self: *Self, inst: Air.Inst.Index) !void {
2285 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
2286 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
2287 const mcv = try self.resolveInst(ty_op.operand);
2288 break :result slicePtr(mcv);
2289 };
2290 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
2291}
2292
2293fn airSliceLen(self: *Self, inst: Air.Inst.Index) !void {
2294 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
2295 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
2296 const mcv = try self.resolveInst(ty_op.operand);
2297 switch (mcv) {
2298 .register => unreachable, // a slice doesn't fit in one register
2299 .stack_argument_offset => |off| {
2300 break :result MCValue{ .stack_argument_offset = off + 4 };
2301 },
2302 .stack_offset => |off| {
2303 break :result MCValue{ .stack_offset = off - 4 };
2304 },
2305 .memory => |addr| {
2306 break :result MCValue{ .memory = addr + 4 };
2307 },
2308 else => unreachable, // invalid MCValue for a slice
2309 }
2310 };
2311 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
2312}
2313
2314fn airPtrSliceLenPtr(self: *Self, inst: Air.Inst.Index) !void {
2315 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
2316 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
2317 const mcv = try self.resolveInst(ty_op.operand);
2318 switch (mcv) {
2319 .dead, .unreach => unreachable,
2320 .ptr_stack_offset => |off| {
2321 break :result MCValue{ .ptr_stack_offset = off - 4 };
2322 },
2323 else => {
2324 const lhs_bind: ReadArg.Bind = .{ .mcv = mcv };
2325 const rhs_bind: ReadArg.Bind = .{ .mcv = .{ .immediate = 4 } };
2326
2327 break :result try self.addSub(.add, lhs_bind, rhs_bind, Type.usize, Type.usize, null);
2328 },
2329 }
2330 };
2331 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
2332}
2333
2334fn airPtrSlicePtrPtr(self: *Self, inst: Air.Inst.Index) !void {
2335 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
2336 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
2337 const mcv = try self.resolveInst(ty_op.operand);
2338 switch (mcv) {
2339 .dead, .unreach => unreachable,
2340 .ptr_stack_offset => |off| {
2341 break :result MCValue{ .ptr_stack_offset = off };
2342 },
2343 else => {
2344 if (self.reuseOperand(inst, ty_op.operand, 0, mcv)) {
2345 break :result mcv;
2346 } else {
2347 break :result MCValue{ .register = try self.copyToTmpRegister(Type.usize, mcv) };
2348 }
2349 },
2350 }
2351 };
2352 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
2353}
2354
2355fn ptrElemVal(
2356 self: *Self,
2357 ptr_bind: ReadArg.Bind,
2358 index_bind: ReadArg.Bind,
2359 ptr_ty: Type,
2360 maybe_inst: ?Air.Inst.Index,
2361) !MCValue {
2362 const pt = self.pt;
2363 const zcu = pt.zcu;
2364 const elem_ty = ptr_ty.childType(zcu);
2365 const elem_size: u32 = @intCast(elem_ty.abiSize(zcu));
2366
2367 switch (elem_size) {
2368 1, 4 => {
2369 var base_reg: Register = undefined;
2370 var index_reg: Register = undefined;
2371 var dest_reg: Register = undefined;
2372
2373 const read_args = [_]ReadArg{
2374 .{ .ty = ptr_ty, .bind = ptr_bind, .class = gp, .reg = &base_reg },
2375 .{ .ty = Type.usize, .bind = index_bind, .class = gp, .reg = &index_reg },
2376 };
2377 const write_args = [_]WriteArg{
2378 .{ .ty = elem_ty, .bind = .none, .class = gp, .reg = &dest_reg },
2379 };
2380 try self.allocRegs(
2381 &read_args,
2382 &write_args,
2383 if (maybe_inst) |inst| .{
2384 .corresponding_inst = inst,
2385 .operand_mapping = &.{ 0, 1 },
2386 } else null,
2387 );
2388
2389 const tag: Mir.Inst.Tag = switch (elem_size) {
2390 1 => .ldrb,
2391 4 => .ldr,
2392 else => unreachable,
2393 };
2394 const shift: u5 = switch (elem_size) {
2395 1 => 0,
2396 4 => 2,
2397 else => unreachable,
2398 };
2399
2400 _ = try self.addInst(.{
2401 .tag = tag,
2402 .data = .{ .rr_offset = .{
2403 .rt = dest_reg,
2404 .rn = base_reg,
2405 .offset = .{ .offset = Instruction.Offset.reg(index_reg, .{ .lsl = shift }) },
2406 } },
2407 });
2408
2409 return MCValue{ .register = dest_reg };
2410 },
2411 else => {
2412 const addr = try self.ptrArithmetic(.ptr_add, ptr_bind, index_bind, ptr_ty, Type.usize, null);
2413
2414 const dest = try self.allocRegOrMem(elem_ty, true, maybe_inst);
2415 try self.load(dest, addr, ptr_ty);
2416 return dest;
2417 },
2418 }
2419}
2420
2421fn airSliceElemVal(self: *Self, inst: Air.Inst.Index) !void {
2422 const pt = self.pt;
2423 const zcu = pt.zcu;
2424 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
2425 const slice_ty = self.typeOf(bin_op.lhs);
2426 const result: MCValue = if (!slice_ty.isVolatilePtr(zcu) and self.liveness.isUnused(inst)) .dead else result: {
2427 const ptr_ty = slice_ty.slicePtrFieldType(zcu);
2428
2429 const slice_mcv = try self.resolveInst(bin_op.lhs);
2430 const base_mcv = slicePtr(slice_mcv);
2431
2432 const base_bind: ReadArg.Bind = .{ .mcv = base_mcv };
2433 const index_bind: ReadArg.Bind = .{ .inst = bin_op.rhs };
2434
2435 break :result try self.ptrElemVal(base_bind, index_bind, ptr_ty, inst);
2436 };
2437 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
2438}
2439
2440fn airSliceElemPtr(self: *Self, inst: Air.Inst.Index) !void {
2441 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
2442 const extra = self.air.extraData(Air.Bin, ty_pl.payload).data;
2443 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
2444 const slice_mcv = try self.resolveInst(extra.lhs);
2445 const base_mcv = slicePtr(slice_mcv);
2446
2447 const base_bind: ReadArg.Bind = .{ .mcv = base_mcv };
2448 const index_bind: ReadArg.Bind = .{ .inst = extra.rhs };
2449
2450 const slice_ty = self.typeOf(extra.lhs);
2451 const index_ty = self.typeOf(extra.rhs);
2452
2453 const addr = try self.ptrArithmetic(.ptr_add, base_bind, index_bind, slice_ty, index_ty, null);
2454 break :result addr;
2455 };
2456 return self.finishAir(inst, result, .{ extra.lhs, extra.rhs, .none });
2457}
2458
2459fn arrayElemVal(
2460 self: *Self,
2461 array_bind: ReadArg.Bind,
2462 index_bind: ReadArg.Bind,
2463 array_ty: Type,
2464 maybe_inst: ?Air.Inst.Index,
2465) InnerError!MCValue {
2466 const pt = self.pt;
2467 const zcu = pt.zcu;
2468 const elem_ty = array_ty.childType(zcu);
2469
2470 const mcv = try array_bind.resolveToMcv(self);
2471 switch (mcv) {
2472 .stack_offset,
2473 .memory,
2474 .stack_argument_offset,
2475 => {
2476 const ptr_to_mcv = switch (mcv) {
2477 .stack_offset => |off| MCValue{ .ptr_stack_offset = off },
2478 .memory => |addr| MCValue{ .immediate = @intCast(addr) },
2479 .stack_argument_offset => |off| blk: {
2480 const reg = try self.register_manager.allocReg(null, gp);
2481
2482 _ = try self.addInst(.{
2483 .tag = .ldr_ptr_stack_argument,
2484 .data = .{ .r_stack_offset = .{
2485 .rt = reg,
2486 .stack_offset = off,
2487 } },
2488 });
2489
2490 break :blk MCValue{ .register = reg };
2491 },
2492 else => unreachable,
2493 };
2494 const ptr_to_mcv_lock: ?RegisterLock = switch (ptr_to_mcv) {
2495 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
2496 else => null,
2497 };
2498 defer if (ptr_to_mcv_lock) |lock| self.register_manager.unlockReg(lock);
2499
2500 const base_bind: ReadArg.Bind = .{ .mcv = ptr_to_mcv };
2501
2502 const ptr_ty = try pt.singleMutPtrType(elem_ty);
2503
2504 return try self.ptrElemVal(base_bind, index_bind, ptr_ty, maybe_inst);
2505 },
2506 else => return self.fail("TODO implement array_elem_val for {}", .{mcv}),
2507 }
2508}
2509
2510fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {
2511 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
2512 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
2513 const array_bind: ReadArg.Bind = .{ .inst = bin_op.lhs };
2514 const index_bind: ReadArg.Bind = .{ .inst = bin_op.rhs };
2515 const array_ty = self.typeOf(bin_op.lhs);
2516
2517 break :result try self.arrayElemVal(array_bind, index_bind, array_ty, inst);
2518 };
2519 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
2520}
2521
2522fn airPtrElemVal(self: *Self, inst: Air.Inst.Index) !void {
2523 const pt = self.pt;
2524 const zcu = pt.zcu;
2525 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
2526 const ptr_ty = self.typeOf(bin_op.lhs);
2527 const result: MCValue = if (!ptr_ty.isVolatilePtr(zcu) and self.liveness.isUnused(inst)) .dead else result: {
2528 const base_bind: ReadArg.Bind = .{ .inst = bin_op.lhs };
2529 const index_bind: ReadArg.Bind = .{ .inst = bin_op.rhs };
2530
2531 break :result try self.ptrElemVal(base_bind, index_bind, ptr_ty, inst);
2532 };
2533 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
2534}
2535
2536fn airPtrElemPtr(self: *Self, inst: Air.Inst.Index) !void {
2537 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
2538 const extra = self.air.extraData(Air.Bin, ty_pl.payload).data;
2539 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
2540 const ptr_bind: ReadArg.Bind = .{ .inst = extra.lhs };
2541 const index_bind: ReadArg.Bind = .{ .inst = extra.rhs };
2542
2543 const ptr_ty = self.typeOf(extra.lhs);
2544 const index_ty = self.typeOf(extra.rhs);
2545
2546 const addr = try self.ptrArithmetic(.ptr_add, ptr_bind, index_bind, ptr_ty, index_ty, null);
2547 break :result addr;
2548 };
2549 return self.finishAir(inst, result, .{ extra.lhs, extra.rhs, .none });
2550}
2551
2552fn airSetUnionTag(self: *Self, inst: Air.Inst.Index) !void {
2553 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
2554 _ = bin_op;
2555 return self.fail("TODO implement airSetUnionTag for {}", .{self.target.cpu.arch});
2556 // return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
2557}
2558
2559fn airGetUnionTag(self: *Self, inst: Air.Inst.Index) !void {
2560 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
2561 _ = ty_op;
2562 return self.fail("TODO implement airGetUnionTag for {}", .{self.target.cpu.arch});
2563 // return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
2564}
2565
2566fn airClz(self: *Self, inst: Air.Inst.Index) !void {
2567 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
2568 _ = ty_op;
2569 return self.fail("TODO implement airClz for {}", .{self.target.cpu.arch});
2570 // return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
2571}
2572
2573fn airCtz(self: *Self, inst: Air.Inst.Index) !void {
2574 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
2575 _ = ty_op;
2576 return self.fail("TODO implement airCtz for {}", .{self.target.cpu.arch});
2577 // return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
2578}
2579
2580fn airPopcount(self: *Self, inst: Air.Inst.Index) !void {
2581 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
2582 _ = ty_op;
2583 return self.fail("TODO implement airPopcount for {}", .{self.target.cpu.arch});
2584 // return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
2585}
2586
2587fn airAbs(self: *Self, inst: Air.Inst.Index) !void {
2588 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
2589 _ = ty_op;
2590 return self.fail("TODO implement airAbs for {}", .{self.target.cpu.arch});
2591 // return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
2592}
2593
2594fn airByteSwap(self: *Self, inst: Air.Inst.Index) !void {
2595 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
2596 _ = ty_op;
2597 return self.fail("TODO implement airByteSwap for {}", .{self.target.cpu.arch});
2598 // return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
2599}
2600
2601fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void {
2602 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
2603 _ = ty_op;
2604 return self.fail("TODO implement airBitReverse for {}", .{self.target.cpu.arch});
2605 // return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
2606}
2607
2608fn airUnaryMath(self: *Self, inst: Air.Inst.Index) !void {
2609 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
2610 const result: MCValue = if (self.liveness.isUnused(inst))
2611 .dead
2612 else
2613 return self.fail("TODO implement airUnaryMath for {}", .{self.target.cpu.arch});
2614 return self.finishAir(inst, result, .{ un_op, .none, .none });
2615}
2616
2617fn reuseOperand(
2618 self: *Self,
2619 inst: Air.Inst.Index,
2620 operand: Air.Inst.Ref,
2621 op_index: Air.Liveness.OperandInt,
2622 mcv: MCValue,
2623) bool {
2624 if (!self.liveness.operandDies(inst, op_index))
2625 return false;
2626
2627 switch (mcv) {
2628 .register => |reg| {
2629 // We assert that this register is allocatable by asking
2630 // for its index
2631 const index = RegisterManager.indexOfRegIntoTracked(reg).?; // see note above
2632 if (!self.register_manager.isRegFree(reg)) {
2633 self.register_manager.registers[index] = inst;
2634 }
2635
2636 log.debug("%{d} => {} (reused)", .{ inst, reg });
2637 },
2638 .stack_offset => |off| {
2639 log.debug("%{d} => stack offset {d} (reused)", .{ inst, off });
2640 },
2641 .cpsr_flags => {
2642 log.debug("%{d} => cpsr_flags (reused)", .{inst});
2643 },
2644 else => return false,
2645 }
2646
2647 // Prevent the operand deaths processing code from deallocating it.
2648 self.reused_operands.set(op_index);
2649
2650 // That makes us responsible for doing the rest of the stuff that processDeath would have done.
2651 const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
2652 branch.inst_table.putAssumeCapacity(operand.toIndex().?, .dead);
2653
2654 return true;
2655}
2656
2657fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!void {
2658 const pt = self.pt;
2659 const zcu = pt.zcu;
2660 const elem_ty = ptr_ty.childType(zcu);
2661 const elem_size: u32 = @intCast(elem_ty.abiSize(zcu));
2662
2663 switch (ptr) {
2664 .none => unreachable,
2665 .undef => unreachable,
2666 .unreach => unreachable,
2667 .dead => unreachable,
2668 .cpsr_flags,
2669 .register_c_flag,
2670 .register_v_flag,
2671 => unreachable, // cannot hold an address
2672 .immediate => |imm| {
2673 try self.setRegOrMem(elem_ty, dst_mcv, .{ .memory = imm });
2674 },
2675 .ptr_stack_offset => |off| {
2676 try self.setRegOrMem(elem_ty, dst_mcv, .{ .stack_offset = off });
2677 },
2678 .register => |reg| {
2679 const reg_lock = self.register_manager.lockReg(reg);
2680 defer if (reg_lock) |reg_locked| self.register_manager.unlockReg(reg_locked);
2681
2682 switch (dst_mcv) {
2683 .register => |dst_reg| {
2684 try self.genLdrRegister(dst_reg, reg, elem_ty);
2685 },
2686 .stack_offset => |off| {
2687 if (elem_size <= 4) {
2688 const tmp_reg = try self.register_manager.allocReg(null, gp);
2689 const tmp_reg_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
2690 defer self.register_manager.unlockReg(tmp_reg_lock);
2691
2692 try self.load(.{ .register = tmp_reg }, ptr, ptr_ty);
2693 try self.genSetStack(elem_ty, off, MCValue{ .register = tmp_reg });
2694 } else {
2695 // TODO optimize the register allocation
2696 const regs = try self.register_manager.allocRegs(4, .{ null, null, null, null }, gp);
2697 const regs_locks = self.register_manager.lockRegsAssumeUnused(4, regs);
2698 defer for (regs_locks) |reg_locked| {
2699 self.register_manager.unlockReg(reg_locked);
2700 };
2701
2702 const src_reg = reg;
2703 const dst_reg = regs[0];
2704 const len_reg = regs[1];
2705 const count_reg = regs[2];
2706 const tmp_reg = regs[3];
2707
2708 // sub dst_reg, fp, #off
2709 try self.genSetReg(ptr_ty, dst_reg, .{ .ptr_stack_offset = off });
2710
2711 // mov len, #elem_size
2712 try self.genSetReg(Type.usize, len_reg, .{ .immediate = elem_size });
2713
2714 // memcpy(src, dst, len)
2715 try self.genInlineMemcpy(src_reg, dst_reg, len_reg, count_reg, tmp_reg);
2716 }
2717 },
2718 else => unreachable, // attempting to load into non-register or non-stack MCValue
2719 }
2720 },
2721 .memory,
2722 .stack_offset,
2723 .stack_argument_offset,
2724 => {
2725 const reg = try self.register_manager.allocReg(null, gp);
2726 const reg_lock = self.register_manager.lockRegAssumeUnused(reg);
2727 defer self.register_manager.unlockReg(reg_lock);
2728
2729 try self.genSetReg(ptr_ty, reg, ptr);
2730 try self.load(dst_mcv, .{ .register = reg }, ptr_ty);
2731 },
2732 }
2733}
2734
2735fn airLoad(self: *Self, inst: Air.Inst.Index) !void {
2736 const pt = self.pt;
2737 const zcu = pt.zcu;
2738 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
2739 const elem_ty = self.typeOfIndex(inst);
2740 const result: MCValue = result: {
2741 if (!elem_ty.hasRuntimeBits(zcu))
2742 break :result MCValue.none;
2743
2744 const ptr = try self.resolveInst(ty_op.operand);
2745 const is_volatile = self.typeOf(ty_op.operand).isVolatilePtr(zcu);
2746 if (self.liveness.isUnused(inst) and !is_volatile)
2747 break :result MCValue.dead;
2748
2749 const dest_mcv: MCValue = blk: {
2750 const ptr_fits_dest = elem_ty.abiSize(zcu) <= 4;
2751 if (ptr_fits_dest and self.reuseOperand(inst, ty_op.operand, 0, ptr)) {
2752 // The MCValue that holds the pointer can be re-used as the value.
2753 break :blk ptr;
2754 } else {
2755 break :blk try self.allocRegOrMem(elem_ty, true, inst);
2756 }
2757 };
2758 try self.load(dest_mcv, ptr, self.typeOf(ty_op.operand));
2759
2760 break :result dest_mcv;
2761 };
2762 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
2763}
2764
2765fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type) InnerError!void {
2766 const pt = self.pt;
2767 const elem_size: u32 = @intCast(value_ty.abiSize(pt.zcu));
2768
2769 switch (ptr) {
2770 .none => unreachable,
2771 .undef => unreachable,
2772 .unreach => unreachable,
2773 .dead => unreachable,
2774 .cpsr_flags,
2775 .register_c_flag,
2776 .register_v_flag,
2777 => unreachable, // cannot hold an address
2778 .immediate => |imm| {
2779 try self.setRegOrMem(value_ty, .{ .memory = imm }, value);
2780 },
2781 .ptr_stack_offset => |off| {
2782 try self.genSetStack(value_ty, off, value);
2783 },
2784 .register => |addr_reg| {
2785 const addr_reg_lock = self.register_manager.lockReg(addr_reg);
2786 defer if (addr_reg_lock) |reg| self.register_manager.unlockReg(reg);
2787
2788 switch (value) {
2789 .dead => unreachable,
2790 .undef => {
2791 try self.genSetReg(value_ty, addr_reg, value);
2792 },
2793 .register => |value_reg| {
2794 try self.genStrRegister(value_reg, addr_reg, value_ty);
2795 },
2796 else => {
2797 if (elem_size <= 4) {
2798 const tmp_reg = try self.register_manager.allocReg(null, gp);
2799 const tmp_reg_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
2800 defer self.register_manager.unlockReg(tmp_reg_lock);
2801
2802 try self.genSetReg(value_ty, tmp_reg, value);
2803 try self.store(ptr, .{ .register = tmp_reg }, ptr_ty, value_ty);
2804 } else {
2805 const regs = try self.register_manager.allocRegs(4, .{ null, null, null, null }, gp);
2806 const regs_locks = self.register_manager.lockRegsAssumeUnused(4, regs);
2807 defer for (regs_locks) |reg| {
2808 self.register_manager.unlockReg(reg);
2809 };
2810
2811 const src_reg = regs[0];
2812 const dst_reg = addr_reg;
2813 const len_reg = regs[1];
2814 const count_reg = regs[2];
2815 const tmp_reg = regs[3];
2816
2817 switch (value) {
2818 .stack_offset => |off| {
2819 // sub src_reg, fp, #off
2820 try self.genSetReg(ptr_ty, src_reg, .{ .ptr_stack_offset = off });
2821 },
2822 .memory => |addr| try self.genSetReg(ptr_ty, src_reg, .{ .immediate = @intCast(addr) }),
2823 .stack_argument_offset => |off| {
2824 _ = try self.addInst(.{
2825 .tag = .ldr_ptr_stack_argument,
2826 .data = .{ .r_stack_offset = .{
2827 .rt = src_reg,
2828 .stack_offset = off,
2829 } },
2830 });
2831 },
2832 else => return self.fail("TODO store {} to register", .{value}),
2833 }
2834
2835 // mov len, #elem_size
2836 try self.genSetReg(Type.usize, len_reg, .{ .immediate = elem_size });
2837
2838 // memcpy(src, dst, len)
2839 try self.genInlineMemcpy(src_reg, dst_reg, len_reg, count_reg, tmp_reg);
2840 }
2841 },
2842 }
2843 },
2844 .memory,
2845 .stack_offset,
2846 .stack_argument_offset,
2847 => {
2848 const addr_reg = try self.copyToTmpRegister(ptr_ty, ptr);
2849 try self.store(.{ .register = addr_reg }, value, ptr_ty, value_ty);
2850 },
2851 }
2852}
2853
2854fn airStore(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
2855 if (safety) {
2856 // TODO if the value is undef, write 0xaa bytes to dest
2857 } else {
2858 // TODO if the value is undef, don't lower this instruction
2859 }
2860 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
2861 const ptr = try self.resolveInst(bin_op.lhs);
2862 const value = try self.resolveInst(bin_op.rhs);
2863 const ptr_ty = self.typeOf(bin_op.lhs);
2864 const value_ty = self.typeOf(bin_op.rhs);
2865
2866 try self.store(ptr, value, ptr_ty, value_ty);
2867
2868 return self.finishAir(inst, .dead, .{ bin_op.lhs, bin_op.rhs, .none });
2869}
2870
2871fn airStructFieldPtr(self: *Self, inst: Air.Inst.Index) !void {
2872 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
2873 const extra = self.air.extraData(Air.StructField, ty_pl.payload).data;
2874 const result = try self.structFieldPtr(inst, extra.struct_operand, extra.field_index);
2875 return self.finishAir(inst, result, .{ extra.struct_operand, .none, .none });
2876}
2877
2878fn airStructFieldPtrIndex(self: *Self, inst: Air.Inst.Index, index: u8) !void {
2879 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
2880 const result = try self.structFieldPtr(inst, ty_op.operand, index);
2881 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
2882}
2883
2884fn structFieldPtr(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, index: u32) !MCValue {
2885 return if (self.liveness.isUnused(inst)) .dead else result: {
2886 const pt = self.pt;
2887 const zcu = pt.zcu;
2888 const mcv = try self.resolveInst(operand);
2889 const ptr_ty = self.typeOf(operand);
2890 const struct_ty = ptr_ty.childType(zcu);
2891 const struct_field_offset: u32 = @intCast(struct_ty.structFieldOffset(index, zcu));
2892 switch (mcv) {
2893 .ptr_stack_offset => |off| {
2894 break :result MCValue{ .ptr_stack_offset = off - struct_field_offset };
2895 },
2896 else => {
2897 const lhs_bind: ReadArg.Bind = .{ .mcv = mcv };
2898 const rhs_bind: ReadArg.Bind = .{ .mcv = .{ .immediate = struct_field_offset } };
2899
2900 break :result try self.addSub(.add, lhs_bind, rhs_bind, Type.usize, Type.usize, null);
2901 },
2902 }
2903 };
2904}
2905
2906fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
2907 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
2908 const extra = self.air.extraData(Air.StructField, ty_pl.payload).data;
2909 const operand = extra.struct_operand;
2910 const index = extra.field_index;
2911 const pt = self.pt;
2912 const zcu = pt.zcu;
2913 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
2914 const mcv = try self.resolveInst(operand);
2915 const struct_ty = self.typeOf(operand);
2916 const struct_field_offset: u32 = @intCast(struct_ty.structFieldOffset(index, zcu));
2917 const struct_field_ty = struct_ty.fieldType(index, zcu);
2918
2919 switch (mcv) {
2920 .dead, .unreach => unreachable,
2921 .stack_argument_offset => |off| {
2922 break :result MCValue{ .stack_argument_offset = off + struct_field_offset };
2923 },
2924 .stack_offset => |off| {
2925 break :result MCValue{ .stack_offset = off - struct_field_offset };
2926 },
2927 .memory => |addr| {
2928 break :result MCValue{ .memory = addr + struct_field_offset };
2929 },
2930 .register_c_flag,
2931 .register_v_flag,
2932 => |reg| {
2933 const reg_lock = self.register_manager.lockRegAssumeUnused(reg);
2934 defer self.register_manager.unlockReg(reg_lock);
2935
2936 const field: MCValue = switch (index) {
2937 // get wrapped value: return register
2938 0 => MCValue{ .register = reg },
2939
2940 // get overflow bit: return C or V flag
2941 1 => MCValue{ .cpsr_flags = switch (mcv) {
2942 .register_c_flag => .cs,
2943 .register_v_flag => .vs,
2944 else => unreachable,
2945 } },
2946
2947 else => unreachable,
2948 };
2949
2950 if (self.reuseOperand(inst, operand, 0, field)) {
2951 break :result field;
2952 } else {
2953 // Copy to new register
2954 const dest_reg = try self.register_manager.allocReg(null, gp);
2955 try self.genSetReg(struct_field_ty, dest_reg, field);
2956
2957 break :result MCValue{ .register = dest_reg };
2958 }
2959 },
2960 .register => {
2961 var operand_reg: Register = undefined;
2962 var dest_reg: Register = undefined;
2963
2964 const read_args = [_]ReadArg{
2965 .{ .ty = struct_ty, .bind = .{ .mcv = mcv }, .class = gp, .reg = &operand_reg },
2966 };
2967 const write_args = [_]WriteArg{
2968 .{ .ty = struct_field_ty, .bind = .none, .class = gp, .reg = &dest_reg },
2969 };
2970 try self.allocRegs(
2971 &read_args,
2972 &write_args,
2973 ReuseMetadata{
2974 .corresponding_inst = inst,
2975 .operand_mapping = &.{0},
2976 },
2977 );
2978
2979 const field_bit_offset = struct_field_offset * 8;
2980 const field_bit_size: u32 = @intCast(struct_field_ty.abiSize(zcu) * 8);
2981
2982 _ = try self.addInst(.{
2983 .tag = if (struct_field_ty.isSignedInt(zcu)) Mir.Inst.Tag.sbfx else .ubfx,
2984 .data = .{ .rr_lsb_width = .{
2985 .rd = dest_reg,
2986 .rn = operand_reg,
2987 .lsb = @intCast(field_bit_offset),
2988 .width = @intCast(field_bit_size),
2989 } },
2990 });
2991
2992 break :result MCValue{ .register = dest_reg };
2993 },
2994 else => return self.fail("TODO implement codegen struct_field_val for {}", .{mcv}),
2995 }
2996 };
2997
2998 return self.finishAir(inst, result, .{ extra.struct_operand, .none, .none });
2999}
3000
3001fn airFieldParentPtr(self: *Self, inst: Air.Inst.Index) !void {
3002 const pt = self.pt;
3003 const zcu = pt.zcu;
3004 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
3005 const extra = self.air.extraData(Air.FieldParentPtr, ty_pl.payload).data;
3006 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
3007 const field_ptr = try self.resolveInst(extra.field_ptr);
3008 const struct_ty = ty_pl.ty.toType().childType(zcu);
3009
3010 if (struct_ty.zigTypeTag(zcu) == .@"union") {
3011 return self.fail("TODO implement @fieldParentPtr codegen for unions", .{});
3012 }
3013
3014 const struct_field_offset: u32 = @intCast(struct_ty.structFieldOffset(extra.field_index, zcu));
3015 switch (field_ptr) {
3016 .ptr_stack_offset => |off| {
3017 break :result MCValue{ .ptr_stack_offset = off + struct_field_offset };
3018 },
3019 else => {
3020 const lhs_bind: ReadArg.Bind = .{ .mcv = field_ptr };
3021 const rhs_bind: ReadArg.Bind = .{ .mcv = .{ .immediate = struct_field_offset } };
3022
3023 break :result try self.addSub(.sub, lhs_bind, rhs_bind, Type.usize, Type.usize, null);
3024 },
3025 }
3026 };
3027 return self.finishAir(inst, result, .{ extra.field_ptr, .none, .none });
3028}
3029
3030/// An argument to a Mir instruction which is read (and possibly also
3031/// written to) by the respective instruction
3032const ReadArg = struct {
3033 ty: Type,
3034 bind: Bind,
3035 class: RegisterManager.RegisterBitSet,
3036 reg: *Register,
3037
3038 const Bind = union(enum) {
3039 inst: Air.Inst.Ref,
3040 mcv: MCValue,
3041
3042 fn resolveToMcv(bind: Bind, function: *Self) InnerError!MCValue {
3043 return switch (bind) {
3044 .inst => |inst| try function.resolveInst(inst),
3045 .mcv => |mcv| mcv,
3046 };
3047 }
3048
3049 fn resolveToImmediate(bind: Bind, function: *Self) InnerError!?u32 {
3050 switch (bind) {
3051 .inst => |inst| {
3052 // TODO resolve independently of inst_table
3053 const mcv = try function.resolveInst(inst);
3054 switch (mcv) {
3055 .immediate => |imm| return imm,
3056 else => return null,
3057 }
3058 },
3059 .mcv => |mcv| {
3060 switch (mcv) {
3061 .immediate => |imm| return imm,
3062 else => return null,
3063 }
3064 },
3065 }
3066 }
3067 };
3068};
3069
3070/// An argument to a Mir instruction which is written to (but not read
3071/// from) by the respective instruction
3072const WriteArg = struct {
3073 ty: Type,
3074 bind: Bind,
3075 class: RegisterManager.RegisterBitSet,
3076 reg: *Register,
3077
3078 const Bind = union(enum) {
3079 reg: Register,
3080 none: void,
3081 };
3082};
3083
3084/// Holds all data necessary for enabling the potential reuse of
3085/// operand registers as destinations
3086const ReuseMetadata = struct {
3087 corresponding_inst: Air.Inst.Index,
3088
3089 /// Maps every element index of read_args to the corresponding
3090 /// index in the Air instruction
3091 ///
3092 /// When the order of read_args corresponds exactly to the order
3093 /// of the inputs of the Air instruction, this would be e.g.
3094 /// &.{ 0, 1 }. However, when the order is not the same or some
3095 /// inputs to the Air instruction are omitted (e.g. when they can
3096 /// be represented as immediates to the Mir instruction),
3097 /// operand_mapping should reflect that fact.
3098 operand_mapping: []const Air.Liveness.OperandInt,
3099};
3100
3101/// Allocate a set of registers for use as arguments for a Mir
3102/// instruction
3103///
3104/// If the Mir instruction these registers are allocated for
3105/// corresponds exactly to a single Air instruction, populate
3106/// reuse_metadata in order to enable potential reuse of an operand as
3107/// the destination (provided that that operand dies in this
3108/// instruction).
3109///
3110/// Reusing an operand register as destination is the only time two
3111/// arguments may share the same register. In all other cases,
3112/// allocRegs guarantees that a register will never be allocated to
3113/// more than one argument.
3114///
3115/// Furthermore, allocReg guarantees that all arguments which are
3116/// already bound to registers before calling allocRegs will not
3117/// change their register binding. This is done by locking these
3118/// registers.
3119fn allocRegs(
3120 self: *Self,
3121 read_args: []const ReadArg,
3122 write_args: []const WriteArg,
3123 reuse_metadata: ?ReuseMetadata,
3124) InnerError!void {
3125 // Air instructions have exactly one output
3126 assert(!(reuse_metadata != null and write_args.len != 1)); // see note above
3127
3128 // The operand mapping is a 1:1 mapping of read args to their
3129 // corresponding operand index in the Air instruction
3130 assert(!(reuse_metadata != null and reuse_metadata.?.operand_mapping.len != read_args.len)); // see note above
3131
3132 const locks = try self.gpa.alloc(?RegisterLock, read_args.len + write_args.len);
3133 defer self.gpa.free(locks);
3134 const read_locks = locks[0..read_args.len];
3135 const write_locks = locks[read_args.len..];
3136
3137 @memset(locks, null);
3138 defer for (locks) |lock| {
3139 if (lock) |locked_reg| self.register_manager.unlockReg(locked_reg);
3140 };
3141
3142 // When we reuse a read_arg as a destination, the corresponding
3143 // MCValue of the read_arg will be set to .dead. In that case, we
3144 // skip allocating this read_arg.
3145 var reused_read_arg: ?usize = null;
3146
3147 // Lock all args which are already allocated to registers
3148 for (read_args, 0..) |arg, i| {
3149 const mcv = try arg.bind.resolveToMcv(self);
3150 if (mcv == .register) {
3151 read_locks[i] = self.register_manager.lockReg(mcv.register);
3152 }
3153 }
3154
3155 for (write_args, 0..) |arg, i| {
3156 if (arg.bind == .reg) {
3157 write_locks[i] = self.register_manager.lockReg(arg.bind.reg);
3158 }
3159 }
3160
3161 // Allocate registers for all args which aren't allocated to
3162 // registers yet
3163 for (read_args, 0..) |arg, i| {
3164 const mcv = try arg.bind.resolveToMcv(self);
3165 if (mcv == .register) {
3166 arg.reg.* = mcv.register;
3167 } else {
3168 const track_inst: ?Air.Inst.Index = switch (arg.bind) {
3169 .inst => |inst| inst.toIndex().?,
3170 else => null,
3171 };
3172 arg.reg.* = try self.register_manager.allocReg(track_inst, arg.class);
3173 read_locks[i] = self.register_manager.lockReg(arg.reg.*);
3174 }
3175 }
3176
3177 if (reuse_metadata != null) {
3178 const inst = reuse_metadata.?.corresponding_inst;
3179 const operand_mapping = reuse_metadata.?.operand_mapping;
3180 const arg = write_args[0];
3181 if (arg.bind == .reg) {
3182 arg.reg.* = arg.bind.reg;
3183 } else {
3184 reuse_operand: for (read_args, 0..) |read_arg, i| {
3185 if (read_arg.bind == .inst) {
3186 const operand = read_arg.bind.inst;
3187 const mcv = try self.resolveInst(operand);
3188 if (mcv == .register and
3189 std.meta.eql(arg.class, read_arg.class) and
3190 self.reuseOperand(inst, operand, operand_mapping[i], mcv))
3191 {
3192 arg.reg.* = mcv.register;
3193 write_locks[0] = null;
3194 reused_read_arg = i;
3195 break :reuse_operand;
3196 }
3197 }
3198 } else {
3199 arg.reg.* = try self.register_manager.allocReg(inst, arg.class);
3200 write_locks[0] = self.register_manager.lockReg(arg.reg.*);
3201 }
3202 }
3203 } else {
3204 for (write_args, 0..) |arg, i| {
3205 if (arg.bind == .reg) {
3206 arg.reg.* = arg.bind.reg;
3207 } else {
3208 arg.reg.* = try self.register_manager.allocReg(null, arg.class);
3209 write_locks[i] = self.register_manager.lockReg(arg.reg.*);
3210 }
3211 }
3212 }
3213
3214 // For all read_args which need to be moved from non-register to
3215 // register, perform the move
3216 for (read_args, 0..) |arg, i| {
3217 if (reused_read_arg) |j| {
3218 // Check whether this read_arg was reused
3219 if (i == j) continue;
3220 }
3221
3222 const mcv = try arg.bind.resolveToMcv(self);
3223 if (mcv != .register) {
3224 if (arg.bind == .inst) {
3225 const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
3226 const inst = arg.bind.inst.toIndex().?;
3227
3228 // Overwrite the MCValue associated with this inst
3229 branch.inst_table.putAssumeCapacity(inst, .{ .register = arg.reg.* });
3230
3231 // If the previous MCValue occupied some space we track, we
3232 // need to make sure it is marked as free now.
3233 switch (mcv) {
3234 .cpsr_flags => {
3235 assert(self.cpsr_flags_inst.? == inst);
3236 self.cpsr_flags_inst = null;
3237 },
3238 .register => |prev_reg| {
3239 assert(!self.register_manager.isRegFree(prev_reg));
3240 self.register_manager.freeReg(prev_reg);
3241 },
3242 else => {},
3243 }
3244 }
3245
3246 try self.genSetReg(arg.ty, arg.reg.*, mcv);
3247 }
3248 }
3249}
3250
3251/// Wrapper around allocRegs and addInst tailored for specific Mir
3252/// instructions which are binary operations acting on two registers
3253///
3254/// Returns the destination register
3255fn binOpRegister(
3256 self: *Self,
3257 mir_tag: Mir.Inst.Tag,
3258 lhs_bind: ReadArg.Bind,
3259 rhs_bind: ReadArg.Bind,
3260 lhs_ty: Type,
3261 rhs_ty: Type,
3262 maybe_inst: ?Air.Inst.Index,
3263) !MCValue {
3264 var lhs_reg: Register = undefined;
3265 var rhs_reg: Register = undefined;
3266 var dest_reg: Register = undefined;
3267
3268 const read_args = [_]ReadArg{
3269 .{ .ty = lhs_ty, .bind = lhs_bind, .class = gp, .reg = &lhs_reg },
3270 .{ .ty = rhs_ty, .bind = rhs_bind, .class = gp, .reg = &rhs_reg },
3271 };
3272 const write_args = [_]WriteArg{
3273 .{ .ty = lhs_ty, .bind = .none, .class = gp, .reg = &dest_reg },
3274 };
3275 try self.allocRegs(
3276 &read_args,
3277 &write_args,
3278 if (maybe_inst) |inst| .{
3279 .corresponding_inst = inst,
3280 .operand_mapping = &.{ 0, 1 },
3281 } else null,
3282 );
3283
3284 const mir_data: Mir.Inst.Data = switch (mir_tag) {
3285 .add,
3286 .adds,
3287 .sub,
3288 .subs,
3289 .@"and",
3290 .orr,
3291 .eor,
3292 => .{ .rr_op = .{
3293 .rd = dest_reg,
3294 .rn = lhs_reg,
3295 .op = Instruction.Operand.reg(rhs_reg, Instruction.Operand.Shift.none),
3296 } },
3297 .lsl,
3298 .asr,
3299 .lsr,
3300 => .{ .rr_shift = .{
3301 .rd = dest_reg,
3302 .rm = lhs_reg,
3303 .shift_amount = Instruction.ShiftAmount.reg(rhs_reg),
3304 } },
3305 .mul,
3306 .smulbb,
3307 => .{ .rrr = .{
3308 .rd = dest_reg,
3309 .rn = lhs_reg,
3310 .rm = rhs_reg,
3311 } },
3312 else => unreachable,
3313 };
3314
3315 _ = try self.addInst(.{
3316 .tag = mir_tag,
3317 .data = mir_data,
3318 });
3319
3320 return MCValue{ .register = dest_reg };
3321}
3322
3323/// Wrapper around allocRegs and addInst tailored for specific Mir
3324/// instructions which are binary operations acting on a register and
3325/// an immediate
3326///
3327/// Returns the destination register
3328fn binOpImmediate(
3329 self: *Self,
3330 mir_tag: Mir.Inst.Tag,
3331 lhs_bind: ReadArg.Bind,
3332 rhs_immediate: u32,
3333 lhs_ty: Type,
3334 lhs_and_rhs_swapped: bool,
3335 maybe_inst: ?Air.Inst.Index,
3336) !MCValue {
3337 var lhs_reg: Register = undefined;
3338 var dest_reg: Register = undefined;
3339
3340 const read_args = [_]ReadArg{
3341 .{ .ty = lhs_ty, .bind = lhs_bind, .class = gp, .reg = &lhs_reg },
3342 };
3343 const write_args = [_]WriteArg{
3344 .{ .ty = lhs_ty, .bind = .none, .class = gp, .reg = &dest_reg },
3345 };
3346 const operand_mapping: []const Air.Liveness.OperandInt = if (lhs_and_rhs_swapped) &.{1} else &.{0};
3347 try self.allocRegs(
3348 &read_args,
3349 &write_args,
3350 if (maybe_inst) |inst| .{
3351 .corresponding_inst = inst,
3352 .operand_mapping = operand_mapping,
3353 } else null,
3354 );
3355
3356 const mir_data: Mir.Inst.Data = switch (mir_tag) {
3357 .add,
3358 .adds,
3359 .sub,
3360 .subs,
3361 .@"and",
3362 .orr,
3363 .eor,
3364 => .{ .rr_op = .{
3365 .rd = dest_reg,
3366 .rn = lhs_reg,
3367 .op = Instruction.Operand.fromU32(rhs_immediate).?,
3368 } },
3369 .lsl,
3370 .asr,
3371 .lsr,
3372 => .{ .rr_shift = .{
3373 .rd = dest_reg,
3374 .rm = lhs_reg,
3375 .shift_amount = Instruction.ShiftAmount.imm(@intCast(rhs_immediate)),
3376 } },
3377 else => unreachable,
3378 };
3379
3380 _ = try self.addInst(.{
3381 .tag = mir_tag,
3382 .data = mir_data,
3383 });
3384
3385 return MCValue{ .register = dest_reg };
3386}
3387
3388fn addSub(
3389 self: *Self,
3390 tag: Air.Inst.Tag,
3391 lhs_bind: ReadArg.Bind,
3392 rhs_bind: ReadArg.Bind,
3393 lhs_ty: Type,
3394 rhs_ty: Type,
3395 maybe_inst: ?Air.Inst.Index,
3396) InnerError!MCValue {
3397 const pt = self.pt;
3398 const zcu = pt.zcu;
3399 switch (lhs_ty.zigTypeTag(zcu)) {
3400 .float => return self.fail("TODO ARM binary operations on floats", .{}),
3401 .vector => return self.fail("TODO ARM binary operations on vectors", .{}),
3402 .int => {
3403 assert(lhs_ty.eql(rhs_ty, zcu));
3404 const int_info = lhs_ty.intInfo(zcu);
3405 if (int_info.bits <= 32) {
3406 const lhs_immediate = try lhs_bind.resolveToImmediate(self);
3407 const rhs_immediate = try rhs_bind.resolveToImmediate(self);
3408
3409 // Only say yes if the operation is
3410 // commutative, i.e. we can swap both of the
3411 // operands
3412 const lhs_immediate_ok = switch (tag) {
3413 .add => if (lhs_immediate) |imm| Instruction.Operand.fromU32(imm) != null else false,
3414 .sub => false,
3415 else => unreachable,
3416 };
3417 const rhs_immediate_ok = switch (tag) {
3418 .add,
3419 .sub,
3420 => if (rhs_immediate) |imm| Instruction.Operand.fromU32(imm) != null else false,
3421 else => unreachable,
3422 };
3423
3424 const mir_tag: Mir.Inst.Tag = switch (tag) {
3425 .add => .add,
3426 .sub => .sub,
3427 else => unreachable,
3428 };
3429
3430 if (rhs_immediate_ok) {
3431 return try self.binOpImmediate(mir_tag, lhs_bind, rhs_immediate.?, lhs_ty, false, maybe_inst);
3432 } else if (lhs_immediate_ok) {
3433 // swap lhs and rhs
3434 return try self.binOpImmediate(mir_tag, rhs_bind, lhs_immediate.?, rhs_ty, true, maybe_inst);
3435 } else {
3436 return try self.binOpRegister(mir_tag, lhs_bind, rhs_bind, lhs_ty, rhs_ty, maybe_inst);
3437 }
3438 } else {
3439 return self.fail("TODO ARM binary operations on integers > u32/i32", .{});
3440 }
3441 },
3442 else => unreachable,
3443 }
3444}
3445
3446fn mul(
3447 self: *Self,
3448 lhs_bind: ReadArg.Bind,
3449 rhs_bind: ReadArg.Bind,
3450 lhs_ty: Type,
3451 rhs_ty: Type,
3452 maybe_inst: ?Air.Inst.Index,
3453) InnerError!MCValue {
3454 const pt = self.pt;
3455 const zcu = pt.zcu;
3456 switch (lhs_ty.zigTypeTag(zcu)) {
3457 .float => return self.fail("TODO ARM binary operations on floats", .{}),
3458 .vector => return self.fail("TODO ARM binary operations on vectors", .{}),
3459 .int => {
3460 assert(lhs_ty.eql(rhs_ty, zcu));
3461 const int_info = lhs_ty.intInfo(zcu);
3462 if (int_info.bits <= 32) {
3463 // TODO add optimisations for multiplication
3464 // with immediates, for example a * 2 can be
3465 // lowered to a << 1
3466 return try self.binOpRegister(.mul, lhs_bind, rhs_bind, lhs_ty, rhs_ty, maybe_inst);
3467 } else {
3468 return self.fail("TODO ARM binary operations on integers > u32/i32", .{});
3469 }
3470 },
3471 else => unreachable,
3472 }
3473}
3474
3475fn divFloat(
3476 self: *Self,
3477 lhs_bind: ReadArg.Bind,
3478 rhs_bind: ReadArg.Bind,
3479 lhs_ty: Type,
3480 rhs_ty: Type,
3481 maybe_inst: ?Air.Inst.Index,
3482) InnerError!MCValue {
3483 _ = lhs_bind;
3484 _ = rhs_bind;
3485 _ = rhs_ty;
3486 _ = maybe_inst;
3487
3488 const pt = self.pt;
3489 const zcu = pt.zcu;
3490 switch (lhs_ty.zigTypeTag(zcu)) {
3491 .float => return self.fail("TODO ARM binary operations on floats", .{}),
3492 .vector => return self.fail("TODO ARM binary operations on vectors", .{}),
3493 else => unreachable,
3494 }
3495}
3496
3497fn divTrunc(
3498 self: *Self,
3499 lhs_bind: ReadArg.Bind,
3500 rhs_bind: ReadArg.Bind,
3501 lhs_ty: Type,
3502 rhs_ty: Type,
3503 maybe_inst: ?Air.Inst.Index,
3504) InnerError!MCValue {
3505 const pt = self.pt;
3506 const zcu = pt.zcu;
3507 switch (lhs_ty.zigTypeTag(zcu)) {
3508 .float => return self.fail("TODO ARM binary operations on floats", .{}),
3509 .vector => return self.fail("TODO ARM binary operations on vectors", .{}),
3510 .int => {
3511 assert(lhs_ty.eql(rhs_ty, zcu));
3512 const int_info = lhs_ty.intInfo(zcu);
3513 if (int_info.bits <= 32) {
3514 switch (int_info.signedness) {
3515 .signed => {
3516 return self.fail("TODO ARM signed integer division", .{});
3517 },
3518 .unsigned => {
3519 const rhs_immediate = try rhs_bind.resolveToImmediate(self);
3520
3521 if (rhs_immediate) |imm| {
3522 if (std.math.isPowerOfTwo(imm)) {
3523 const shift = std.math.log2_int(u32, imm);
3524 return try self.binOpImmediate(.lsr, lhs_bind, shift, lhs_ty, false, maybe_inst);
3525 } else {
3526 return self.fail("TODO ARM integer division by constants", .{});
3527 }
3528 } else {
3529 return self.fail("TODO ARM integer division", .{});
3530 }
3531 },
3532 }
3533 } else {
3534 return self.fail("TODO ARM integer division for integers > u32/i32", .{});
3535 }
3536 },
3537 else => unreachable,
3538 }
3539}
3540
3541fn divFloor(
3542 self: *Self,
3543 lhs_bind: ReadArg.Bind,
3544 rhs_bind: ReadArg.Bind,
3545 lhs_ty: Type,
3546 rhs_ty: Type,
3547 maybe_inst: ?Air.Inst.Index,
3548) InnerError!MCValue {
3549 const pt = self.pt;
3550 const zcu = pt.zcu;
3551 switch (lhs_ty.zigTypeTag(zcu)) {
3552 .float => return self.fail("TODO ARM binary operations on floats", .{}),
3553 .vector => return self.fail("TODO ARM binary operations on vectors", .{}),
3554 .int => {
3555 assert(lhs_ty.eql(rhs_ty, zcu));
3556 const int_info = lhs_ty.intInfo(zcu);
3557 if (int_info.bits <= 32) {
3558 switch (int_info.signedness) {
3559 .signed => {
3560 return self.fail("TODO ARM signed integer division", .{});
3561 },
3562 .unsigned => {
3563 const rhs_immediate = try rhs_bind.resolveToImmediate(self);
3564
3565 if (rhs_immediate) |imm| {
3566 if (std.math.isPowerOfTwo(imm)) {
3567 const shift = std.math.log2_int(u32, imm);
3568 return try self.binOpImmediate(.lsr, lhs_bind, shift, lhs_ty, false, maybe_inst);
3569 } else {
3570 return self.fail("TODO ARM integer division by constants", .{});
3571 }
3572 } else {
3573 return self.fail("TODO ARM integer division", .{});
3574 }
3575 },
3576 }
3577 } else {
3578 return self.fail("TODO ARM integer division for integers > u32/i32", .{});
3579 }
3580 },
3581 else => unreachable,
3582 }
3583}
3584
3585fn divExact(
3586 self: *Self,
3587 lhs_bind: ReadArg.Bind,
3588 rhs_bind: ReadArg.Bind,
3589 lhs_ty: Type,
3590 rhs_ty: Type,
3591 maybe_inst: ?Air.Inst.Index,
3592) InnerError!MCValue {
3593 _ = lhs_bind;
3594 _ = rhs_bind;
3595 _ = rhs_ty;
3596 _ = maybe_inst;
3597
3598 const pt = self.pt;
3599 const zcu = pt.zcu;
3600 switch (lhs_ty.zigTypeTag(zcu)) {
3601 .float => return self.fail("TODO ARM binary operations on floats", .{}),
3602 .vector => return self.fail("TODO ARM binary operations on vectors", .{}),
3603 .int => return self.fail("TODO ARM div_exact", .{}),
3604 else => unreachable,
3605 }
3606}
3607
3608fn rem(
3609 self: *Self,
3610 lhs_bind: ReadArg.Bind,
3611 rhs_bind: ReadArg.Bind,
3612 lhs_ty: Type,
3613 rhs_ty: Type,
3614 maybe_inst: ?Air.Inst.Index,
3615) InnerError!MCValue {
3616 const pt = self.pt;
3617 const zcu = pt.zcu;
3618 switch (lhs_ty.zigTypeTag(zcu)) {
3619 .float => return self.fail("TODO ARM binary operations on floats", .{}),
3620 .vector => return self.fail("TODO ARM binary operations on vectors", .{}),
3621 .int => {
3622 assert(lhs_ty.eql(rhs_ty, zcu));
3623 const int_info = lhs_ty.intInfo(zcu);
3624 if (int_info.bits <= 32) {
3625 switch (int_info.signedness) {
3626 .signed => {
3627 return self.fail("TODO ARM signed integer zcu", .{});
3628 },
3629 .unsigned => {
3630 const rhs_immediate = try rhs_bind.resolveToImmediate(self);
3631
3632 if (rhs_immediate) |imm| {
3633 if (std.math.isPowerOfTwo(imm)) {
3634 const log2 = std.math.log2_int(u32, imm);
3635
3636 var lhs_reg: Register = undefined;
3637 var dest_reg: Register = undefined;
3638
3639 const read_args = [_]ReadArg{
3640 .{ .ty = lhs_ty, .bind = lhs_bind, .class = gp, .reg = &lhs_reg },
3641 };
3642 const write_args = [_]WriteArg{
3643 .{ .ty = lhs_ty, .bind = .none, .class = gp, .reg = &dest_reg },
3644 };
3645 try self.allocRegs(
3646 &read_args,
3647 &write_args,
3648 if (maybe_inst) |inst| .{
3649 .corresponding_inst = inst,
3650 .operand_mapping = &.{0},
3651 } else null,
3652 );
3653
3654 try self.truncRegister(lhs_reg, dest_reg, int_info.signedness, log2);
3655
3656 return MCValue{ .register = dest_reg };
3657 } else {
3658 return self.fail("TODO ARM integer zcu by constants", .{});
3659 }
3660 } else {
3661 return self.fail("TODO ARM integer zcu", .{});
3662 }
3663 },
3664 }
3665 } else {
3666 return self.fail("TODO ARM integer division for integers > u32/i32", .{});
3667 }
3668 },
3669 else => unreachable,
3670 }
3671}
3672
3673fn modulo(
3674 self: *Self,
3675 lhs_bind: ReadArg.Bind,
3676 rhs_bind: ReadArg.Bind,
3677 lhs_ty: Type,
3678 rhs_ty: Type,
3679 maybe_inst: ?Air.Inst.Index,
3680) InnerError!MCValue {
3681 _ = lhs_bind;
3682 _ = rhs_bind;
3683 _ = rhs_ty;
3684 _ = maybe_inst;
3685
3686 const pt = self.pt;
3687 const zcu = pt.zcu;
3688 switch (lhs_ty.zigTypeTag(zcu)) {
3689 .float => return self.fail("TODO ARM binary operations on floats", .{}),
3690 .vector => return self.fail("TODO ARM binary operations on vectors", .{}),
3691 .int => return self.fail("TODO ARM zcu", .{}),
3692 else => unreachable,
3693 }
3694}
3695
3696fn wrappingArithmetic(
3697 self: *Self,
3698 tag: Air.Inst.Tag,
3699 lhs_bind: ReadArg.Bind,
3700 rhs_bind: ReadArg.Bind,
3701 lhs_ty: Type,
3702 rhs_ty: Type,
3703 maybe_inst: ?Air.Inst.Index,
3704) InnerError!MCValue {
3705 const pt = self.pt;
3706 const zcu = pt.zcu;
3707 switch (lhs_ty.zigTypeTag(zcu)) {
3708 .vector => return self.fail("TODO ARM binary operations on vectors", .{}),
3709 .int => {
3710 const int_info = lhs_ty.intInfo(zcu);
3711 if (int_info.bits <= 32) {
3712 // Generate an add/sub/mul
3713 const result: MCValue = switch (tag) {
3714 .add_wrap => try self.addSub(.add, lhs_bind, rhs_bind, lhs_ty, rhs_ty, maybe_inst),
3715 .sub_wrap => try self.addSub(.sub, lhs_bind, rhs_bind, lhs_ty, rhs_ty, maybe_inst),
3716 .mul_wrap => try self.mul(lhs_bind, rhs_bind, lhs_ty, rhs_ty, maybe_inst),
3717 else => unreachable,
3718 };
3719
3720 // Truncate if necessary
3721 const result_reg = result.register;
3722 if (int_info.bits < 32) {
3723 try self.truncRegister(result_reg, result_reg, int_info.signedness, int_info.bits);
3724 }
3725
3726 return result;
3727 } else {
3728 return self.fail("TODO ARM binary operations on integers > u32/i32", .{});
3729 }
3730 },
3731 else => unreachable,
3732 }
3733}
3734
3735fn bitwise(
3736 self: *Self,
3737 tag: Air.Inst.Tag,
3738 lhs_bind: ReadArg.Bind,
3739 rhs_bind: ReadArg.Bind,
3740 lhs_ty: Type,
3741 rhs_ty: Type,
3742 maybe_inst: ?Air.Inst.Index,
3743) InnerError!MCValue {
3744 const pt = self.pt;
3745 const zcu = pt.zcu;
3746 switch (lhs_ty.zigTypeTag(zcu)) {
3747 .vector => return self.fail("TODO ARM binary operations on vectors", .{}),
3748 .int => {
3749 assert(lhs_ty.eql(rhs_ty, zcu));
3750 const int_info = lhs_ty.intInfo(zcu);
3751 if (int_info.bits <= 32) {
3752 const lhs_immediate = try lhs_bind.resolveToImmediate(self);
3753 const rhs_immediate = try rhs_bind.resolveToImmediate(self);
3754
3755 const lhs_immediate_ok = if (lhs_immediate) |imm| Instruction.Operand.fromU32(imm) != null else false;
3756 const rhs_immediate_ok = if (rhs_immediate) |imm| Instruction.Operand.fromU32(imm) != null else false;
3757
3758 const mir_tag: Mir.Inst.Tag = switch (tag) {
3759 .bit_and => .@"and",
3760 .bit_or => .orr,
3761 .xor => .eor,
3762 else => unreachable,
3763 };
3764
3765 if (rhs_immediate_ok) {
3766 return try self.binOpImmediate(mir_tag, lhs_bind, rhs_immediate.?, lhs_ty, false, maybe_inst);
3767 } else if (lhs_immediate_ok) {
3768 // swap lhs and rhs
3769 return try self.binOpImmediate(mir_tag, rhs_bind, lhs_immediate.?, rhs_ty, true, maybe_inst);
3770 } else {
3771 return try self.binOpRegister(mir_tag, lhs_bind, rhs_bind, lhs_ty, rhs_ty, maybe_inst);
3772 }
3773 } else {
3774 return self.fail("TODO ARM binary operations on integers > u32/i32", .{});
3775 }
3776 },
3777 else => unreachable,
3778 }
3779}
3780
3781fn shiftExact(
3782 self: *Self,
3783 tag: Air.Inst.Tag,
3784 lhs_bind: ReadArg.Bind,
3785 rhs_bind: ReadArg.Bind,
3786 lhs_ty: Type,
3787 rhs_ty: Type,
3788 maybe_inst: ?Air.Inst.Index,
3789) InnerError!MCValue {
3790 const pt = self.pt;
3791 const zcu = pt.zcu;
3792 switch (lhs_ty.zigTypeTag(zcu)) {
3793 .vector => if (!rhs_ty.isVector(zcu))
3794 return self.fail("TODO ARM vector shift with scalar rhs", .{})
3795 else
3796 return self.fail("TODO ARM binary operations on vectors", .{}),
3797 .int => {
3798 const int_info = lhs_ty.intInfo(zcu);
3799 if (int_info.bits <= 32) {
3800 const rhs_immediate = try rhs_bind.resolveToImmediate(self);
3801
3802 const mir_tag: Mir.Inst.Tag = switch (tag) {
3803 .shl_exact => .lsl,
3804 .shr_exact => switch (lhs_ty.intInfo(zcu).signedness) {
3805 .signed => Mir.Inst.Tag.asr,
3806 .unsigned => Mir.Inst.Tag.lsr,
3807 },
3808 else => unreachable,
3809 };
3810
3811 if (rhs_immediate) |imm| {
3812 return try self.binOpImmediate(mir_tag, lhs_bind, imm, lhs_ty, false, maybe_inst);
3813 } else {
3814 return try self.binOpRegister(mir_tag, lhs_bind, rhs_bind, lhs_ty, rhs_ty, maybe_inst);
3815 }
3816 } else {
3817 return self.fail("TODO ARM binary operations on integers > u32/i32", .{});
3818 }
3819 },
3820 else => unreachable,
3821 }
3822}
3823
3824fn shiftNormal(
3825 self: *Self,
3826 tag: Air.Inst.Tag,
3827 lhs_bind: ReadArg.Bind,
3828 rhs_bind: ReadArg.Bind,
3829 lhs_ty: Type,
3830 rhs_ty: Type,
3831 maybe_inst: ?Air.Inst.Index,
3832) InnerError!MCValue {
3833 const pt = self.pt;
3834 const zcu = pt.zcu;
3835 switch (lhs_ty.zigTypeTag(zcu)) {
3836 .vector => if (!rhs_ty.isVector(zcu))
3837 return self.fail("TODO ARM vector shift with scalar rhs", .{})
3838 else
3839 return self.fail("TODO ARM binary operations on vectors", .{}),
3840 .int => {
3841 const int_info = lhs_ty.intInfo(zcu);
3842 if (int_info.bits <= 32) {
3843 // Generate a shl_exact/shr_exact
3844 const result: MCValue = switch (tag) {
3845 .shl => try self.shiftExact(.shl_exact, lhs_bind, rhs_bind, lhs_ty, rhs_ty, maybe_inst),
3846 .shr => try self.shiftExact(.shr_exact, lhs_bind, rhs_bind, lhs_ty, rhs_ty, maybe_inst),
3847 else => unreachable,
3848 };
3849
3850 // Truncate if necessary
3851 switch (tag) {
3852 .shr => return result,
3853 .shl => {
3854 const result_reg = result.register;
3855 if (int_info.bits < 32) {
3856 try self.truncRegister(result_reg, result_reg, int_info.signedness, int_info.bits);
3857 }
3858
3859 return result;
3860 },
3861 else => unreachable,
3862 }
3863 } else {
3864 return self.fail("TODO ARM binary operations on integers > u32/i32", .{});
3865 }
3866 },
3867 else => unreachable,
3868 }
3869}
3870
3871fn booleanOp(
3872 self: *Self,
3873 tag: Air.Inst.Tag,
3874 lhs_bind: ReadArg.Bind,
3875 rhs_bind: ReadArg.Bind,
3876 lhs_ty: Type,
3877 rhs_ty: Type,
3878 maybe_inst: ?Air.Inst.Index,
3879) InnerError!MCValue {
3880 const pt = self.pt;
3881 const zcu = pt.zcu;
3882 switch (lhs_ty.zigTypeTag(zcu)) {
3883 .bool => {
3884 const lhs_immediate = try lhs_bind.resolveToImmediate(self);
3885 const rhs_immediate = try rhs_bind.resolveToImmediate(self);
3886
3887 const mir_tag: Mir.Inst.Tag = switch (tag) {
3888 .bool_and => .@"and",
3889 .bool_or => .orr,
3890 else => unreachable,
3891 };
3892
3893 if (rhs_immediate) |imm| {
3894 return try self.binOpImmediate(mir_tag, lhs_bind, imm, lhs_ty, false, maybe_inst);
3895 } else if (lhs_immediate) |imm| {
3896 // swap lhs and rhs
3897 return try self.binOpImmediate(mir_tag, rhs_bind, imm, rhs_ty, true, maybe_inst);
3898 } else {
3899 return try self.binOpRegister(mir_tag, lhs_bind, rhs_bind, lhs_ty, rhs_ty, maybe_inst);
3900 }
3901 },
3902 else => unreachable,
3903 }
3904}
3905
3906fn ptrArithmetic(
3907 self: *Self,
3908 tag: Air.Inst.Tag,
3909 lhs_bind: ReadArg.Bind,
3910 rhs_bind: ReadArg.Bind,
3911 lhs_ty: Type,
3912 rhs_ty: Type,
3913 maybe_inst: ?Air.Inst.Index,
3914) InnerError!MCValue {
3915 const pt = self.pt;
3916 const zcu = pt.zcu;
3917 switch (lhs_ty.zigTypeTag(zcu)) {
3918 .pointer => {
3919 assert(rhs_ty.eql(Type.usize, zcu));
3920
3921 const ptr_ty = lhs_ty;
3922 const elem_ty = switch (ptr_ty.ptrSize(zcu)) {
3923 .one => ptr_ty.childType(zcu).childType(zcu), // ptr to array, so get array element type
3924 else => ptr_ty.childType(zcu),
3925 };
3926 const elem_size: u32 = @intCast(elem_ty.abiSize(zcu));
3927
3928 const base_tag: Air.Inst.Tag = switch (tag) {
3929 .ptr_add => .add,
3930 .ptr_sub => .sub,
3931 else => unreachable,
3932 };
3933
3934 if (elem_size == 1) {
3935 return try self.addSub(base_tag, lhs_bind, rhs_bind, Type.usize, Type.usize, maybe_inst);
3936 } else {
3937 // convert the offset into a byte offset by
3938 // multiplying it with elem_size
3939 const imm_bind = ReadArg.Bind{ .mcv = .{ .immediate = elem_size } };
3940
3941 const offset = try self.mul(rhs_bind, imm_bind, Type.usize, Type.usize, null);
3942 const offset_bind = ReadArg.Bind{ .mcv = offset };
3943
3944 const addr = try self.addSub(base_tag, lhs_bind, offset_bind, Type.usize, Type.usize, null);
3945 return addr;
3946 }
3947 },
3948 else => unreachable,
3949 }
3950}
3951
3952fn genLdrRegister(self: *Self, dest_reg: Register, addr_reg: Register, ty: Type) !void {
3953 const pt = self.pt;
3954 const zcu = pt.zcu;
3955 const abi_size = ty.abiSize(zcu);
3956
3957 const tag: Mir.Inst.Tag = switch (abi_size) {
3958 1 => if (ty.isSignedInt(zcu)) Mir.Inst.Tag.ldrsb else .ldrb,
3959 2 => if (ty.isSignedInt(zcu)) Mir.Inst.Tag.ldrsh else .ldrh,
3960 3, 4 => .ldr,
3961 else => unreachable,
3962 };
3963
3964 const rr_offset: Mir.Inst.Data = .{ .rr_offset = .{
3965 .rt = dest_reg,
3966 .rn = addr_reg,
3967 .offset = .{ .offset = Instruction.Offset.none },
3968 } };
3969 const rr_extra_offset: Mir.Inst.Data = .{ .rr_extra_offset = .{
3970 .rt = dest_reg,
3971 .rn = addr_reg,
3972 .offset = .{ .offset = Instruction.ExtraLoadStoreOffset.none },
3973 } };
3974
3975 const data: Mir.Inst.Data = switch (abi_size) {
3976 1 => if (ty.isSignedInt(zcu)) rr_extra_offset else rr_offset,
3977 2 => rr_extra_offset,
3978 3, 4 => rr_offset,
3979 else => unreachable,
3980 };
3981
3982 _ = try self.addInst(.{
3983 .tag = tag,
3984 .data = data,
3985 });
3986}
3987
3988fn genStrRegister(self: *Self, source_reg: Register, addr_reg: Register, ty: Type) !void {
3989 const pt = self.pt;
3990 const abi_size = ty.abiSize(pt.zcu);
3991
3992 const tag: Mir.Inst.Tag = switch (abi_size) {
3993 1 => .strb,
3994 2 => .strh,
3995 4 => .str,
3996 3 => return self.fail("TODO: genStrRegister for abi_size={}", .{abi_size}),
3997 else => unreachable,
3998 };
3999
4000 const rr_offset: Mir.Inst.Data = .{ .rr_offset = .{
4001 .rt = source_reg,
4002 .rn = addr_reg,
4003 .offset = .{ .offset = Instruction.Offset.none },
4004 } };
4005 const rr_extra_offset: Mir.Inst.Data = .{ .rr_extra_offset = .{
4006 .rt = source_reg,
4007 .rn = addr_reg,
4008 .offset = .{ .offset = Instruction.ExtraLoadStoreOffset.none },
4009 } };
4010
4011 const data: Mir.Inst.Data = switch (abi_size) {
4012 1, 4 => rr_offset,
4013 2 => rr_extra_offset,
4014 else => unreachable,
4015 };
4016
4017 _ = try self.addInst(.{
4018 .tag = tag,
4019 .data = data,
4020 });
4021}
4022
4023fn genInlineMemcpy(
4024 self: *Self,
4025 src: Register,
4026 dst: Register,
4027 len: Register,
4028 count: Register,
4029 tmp: Register,
4030) !void {
4031 // mov count, #0
4032 _ = try self.addInst(.{
4033 .tag = .mov,
4034 .data = .{ .r_op_mov = .{
4035 .rd = count,
4036 .op = Instruction.Operand.imm(0, 0),
4037 } },
4038 });
4039
4040 // loop:
4041 // cmp count, len
4042 _ = try self.addInst(.{
4043 .tag = .cmp,
4044 .data = .{ .r_op_cmp = .{
4045 .rn = count,
4046 .op = Instruction.Operand.reg(len, Instruction.Operand.Shift.none),
4047 } },
4048 });
4049
4050 // bge end
4051 _ = try self.addInst(.{
4052 .tag = .b,
4053 .cond = .ge,
4054 .data = .{ .inst = @intCast(self.mir_instructions.len + 5) },
4055 });
4056
4057 // ldrb tmp, [src, count]
4058 _ = try self.addInst(.{
4059 .tag = .ldrb,
4060 .data = .{ .rr_offset = .{
4061 .rt = tmp,
4062 .rn = src,
4063 .offset = .{ .offset = Instruction.Offset.reg(count, .none) },
4064 } },
4065 });
4066
4067 // strb tmp, [src, count]
4068 _ = try self.addInst(.{
4069 .tag = .strb,
4070 .data = .{ .rr_offset = .{
4071 .rt = tmp,
4072 .rn = dst,
4073 .offset = .{ .offset = Instruction.Offset.reg(count, .none) },
4074 } },
4075 });
4076
4077 // add count, count, #1
4078 _ = try self.addInst(.{
4079 .tag = .add,
4080 .data = .{ .rr_op = .{
4081 .rd = count,
4082 .rn = count,
4083 .op = Instruction.Operand.imm(1, 0),
4084 } },
4085 });
4086
4087 // b loop
4088 _ = try self.addInst(.{
4089 .tag = .b,
4090 .data = .{ .inst = @intCast(self.mir_instructions.len - 5) },
4091 });
4092
4093 // end:
4094}
4095
4096fn genInlineMemset(
4097 self: *Self,
4098 dst: MCValue,
4099 val: MCValue,
4100 len: MCValue,
4101) !void {
4102 const dst_reg = switch (dst) {
4103 .register => |r| r,
4104 else => try self.copyToTmpRegister(Type.manyptr_u8, dst),
4105 };
4106 const dst_reg_lock = self.register_manager.lockReg(dst_reg);
4107 defer if (dst_reg_lock) |lock| self.register_manager.unlockReg(lock);
4108
4109 const val_reg = switch (val) {
4110 .register => |r| r,
4111 else => try self.copyToTmpRegister(Type.u8, val),
4112 };
4113 const val_reg_lock = self.register_manager.lockReg(val_reg);
4114 defer if (val_reg_lock) |lock| self.register_manager.unlockReg(lock);
4115
4116 const len_reg = switch (len) {
4117 .register => |r| r,
4118 else => try self.copyToTmpRegister(Type.usize, len),
4119 };
4120 const len_reg_lock = self.register_manager.lockReg(len_reg);
4121 defer if (len_reg_lock) |lock| self.register_manager.unlockReg(lock);
4122
4123 const count_reg = try self.register_manager.allocReg(null, gp);
4124
4125 try self.genInlineMemsetCode(dst_reg, val_reg, len_reg, count_reg);
4126}
4127
4128fn genInlineMemsetCode(
4129 self: *Self,
4130 dst: Register,
4131 val: Register,
4132 len: Register,
4133 count: Register,
4134) !void {
4135 // mov count, #0
4136 _ = try self.addInst(.{
4137 .tag = .mov,
4138 .data = .{ .r_op_mov = .{
4139 .rd = count,
4140 .op = Instruction.Operand.imm(0, 0),
4141 } },
4142 });
4143
4144 // loop:
4145 // cmp count, len
4146 _ = try self.addInst(.{
4147 .tag = .cmp,
4148 .data = .{ .r_op_cmp = .{
4149 .rn = count,
4150 .op = Instruction.Operand.reg(len, Instruction.Operand.Shift.none),
4151 } },
4152 });
4153
4154 // bge end
4155 _ = try self.addInst(.{
4156 .tag = .b,
4157 .cond = .ge,
4158 .data = .{ .inst = @intCast(self.mir_instructions.len + 4) },
4159 });
4160
4161 // strb val, [src, count]
4162 _ = try self.addInst(.{
4163 .tag = .strb,
4164 .data = .{ .rr_offset = .{
4165 .rt = val,
4166 .rn = dst,
4167 .offset = .{ .offset = Instruction.Offset.reg(count, .none) },
4168 } },
4169 });
4170
4171 // add count, count, #1
4172 _ = try self.addInst(.{
4173 .tag = .add,
4174 .data = .{ .rr_op = .{
4175 .rd = count,
4176 .rn = count,
4177 .op = Instruction.Operand.imm(1, 0),
4178 } },
4179 });
4180
4181 // b loop
4182 _ = try self.addInst(.{
4183 .tag = .b,
4184 .data = .{ .inst = @intCast(self.mir_instructions.len - 4) },
4185 });
4186
4187 // end:
4188}
4189
4190fn airArg(self: *Self, inst: Air.Inst.Index) !void {
4191 // skip zero-bit arguments as they don't have a corresponding arg instruction
4192 var arg_index = self.arg_index;
4193 while (self.args[arg_index] == .none) arg_index += 1;
4194 self.arg_index = arg_index + 1;
4195
4196 const zcu = self.pt.zcu;
4197 const func_zir = zcu.funcInfo(self.func_index).zir_body_inst.resolveFull(&zcu.intern_pool).?;
4198 const file = zcu.fileByIndex(func_zir.file);
4199 if (!file.mod.?.strip) {
4200 const tag = self.air.instructions.items(.tag)[@intFromEnum(inst)];
4201 const arg = self.air.instructions.items(.data)[@intFromEnum(inst)].arg;
4202 const ty = self.typeOfIndex(inst);
4203 const zir = &file.zir.?;
4204 const name = zir.nullTerminatedString(zir.getParamName(zir.getParamBody(func_zir.inst)[arg.zir_param_index]).?);
4205 try self.dbg_info_relocs.append(self.gpa, .{
4206 .tag = tag,
4207 .ty = ty,
4208 .name = name,
4209 .mcv = self.args[arg_index],
4210 });
4211 }
4212
4213 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else self.args[arg_index];
4214 return self.finishAir(inst, result, .{ .none, .none, .none });
4215}
4216
4217fn airTrap(self: *Self) !void {
4218 _ = try self.addInst(.{
4219 .tag = .undefined_instruction,
4220 .data = .{ .nop = {} },
4221 });
4222 return self.finishAirBookkeeping();
4223}
4224
4225fn airBreakpoint(self: *Self) !void {
4226 _ = try self.addInst(.{
4227 .tag = .bkpt,
4228 .data = .{ .imm16 = 0 },
4229 });
4230 return self.finishAirBookkeeping();
4231}
4232
4233fn airRetAddr(self: *Self, inst: Air.Inst.Index) !void {
4234 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement airRetAddr for arm", .{});
4235 return self.finishAir(inst, result, .{ .none, .none, .none });
4236}
4237
4238fn airFrameAddress(self: *Self, inst: Air.Inst.Index) !void {
4239 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement airFrameAddress for arm", .{});
4240 return self.finishAir(inst, result, .{ .none, .none, .none });
4241}
4242
4243fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier) !void {
4244 if (modifier == .always_tail) return self.fail("TODO implement tail calls for arm", .{});
4245 const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
4246 const callee = pl_op.operand;
4247 const extra = self.air.extraData(Air.Call, pl_op.payload);
4248 const args: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[extra.end..][0..extra.data.args_len]);
4249 const ty = self.typeOf(callee);
4250 const pt = self.pt;
4251 const zcu = pt.zcu;
4252 const ip = &zcu.intern_pool;
4253
4254 const fn_ty = switch (ty.zigTypeTag(zcu)) {
4255 .@"fn" => ty,
4256 .pointer => ty.childType(zcu),
4257 else => unreachable,
4258 };
4259
4260 var info = try self.resolveCallingConventionValues(fn_ty);
4261 defer info.deinit(self);
4262
4263 // According to the Procedure Call Standard for the ARM
4264 // Architecture, compare flags are not preserved across
4265 // calls. Therefore, if some value is currently stored there, we
4266 // need to save it.
4267 try self.spillCompareFlagsIfOccupied();
4268
4269 // Save caller-saved registers, but crucially *after* we save the
4270 // compare flags as saving compare flags may require a new
4271 // caller-saved register
4272 for (caller_preserved_regs) |reg| {
4273 try self.register_manager.getReg(reg, null);
4274 }
4275
4276 // If returning by reference, r0 will contain the address of where
4277 // to put the result into. In that case, make sure that r0 remains
4278 // untouched by the parameter passing code
4279 const r0_lock: ?RegisterLock = if (info.return_value == .stack_offset) blk: {
4280 log.debug("airCall: return by reference", .{});
4281 const ret_ty = fn_ty.fnReturnType(zcu);
4282 const ret_abi_size: u32 = @intCast(ret_ty.abiSize(zcu));
4283 const ret_abi_align = ret_ty.abiAlignment(zcu);
4284 const stack_offset = try self.allocMem(ret_abi_size, ret_abi_align, inst);
4285
4286 const ptr_ty = try pt.singleMutPtrType(ret_ty);
4287 try self.register_manager.getReg(.r0, null);
4288 try self.genSetReg(ptr_ty, .r0, .{ .ptr_stack_offset = stack_offset });
4289
4290 info.return_value = .{ .stack_offset = stack_offset };
4291
4292 break :blk self.register_manager.lockRegAssumeUnused(.r0);
4293 } else null;
4294 defer if (r0_lock) |reg| self.register_manager.unlockReg(reg);
4295
4296 // Make space for the arguments passed via the stack
4297 self.max_end_stack += info.stack_byte_count;
4298
4299 for (info.args, 0..) |mc_arg, arg_i| {
4300 const arg = args[arg_i];
4301 const arg_ty = self.typeOf(arg);
4302 const arg_mcv = try self.resolveInst(args[arg_i]);
4303
4304 switch (mc_arg) {
4305 .none => continue,
4306 .register => |reg| {
4307 try self.register_manager.getReg(reg, null);
4308 try self.genSetReg(arg_ty, reg, arg_mcv);
4309 },
4310 .stack_offset => unreachable,
4311 .stack_argument_offset => |offset| try self.genSetStackArgument(
4312 arg_ty,
4313 offset,
4314 arg_mcv,
4315 ),
4316 else => unreachable,
4317 }
4318 }
4319
4320 // Due to incremental compilation, how function calls are generated depends
4321 // on linking.
4322 if (try self.air.value(callee, pt)) |func_value| switch (ip.indexToKey(func_value.toIntern())) {
4323 .func => {
4324 return self.fail("TODO implement calling functions", .{});
4325 },
4326 .@"extern" => {
4327 return self.fail("TODO implement calling extern functions", .{});
4328 },
4329 else => {
4330 return self.fail("TODO implement calling bitcasted functions", .{});
4331 },
4332 } else {
4333 assert(ty.zigTypeTag(zcu) == .pointer);
4334 const mcv = try self.resolveInst(callee);
4335
4336 try self.genSetReg(Type.usize, .lr, mcv);
4337 }
4338
4339 // TODO: add Instruction.supportedOn
4340 // function for ARM
4341 if (self.target.cpu.has(.arm, .has_v5t)) {
4342 _ = try self.addInst(.{
4343 .tag = .blx,
4344 .data = .{ .reg = .lr },
4345 });
4346 } else {
4347 return self.fail("TODO fix blx emulation for ARM <v5", .{});
4348 // _ = try self.addInst(.{
4349 // .tag = .mov,
4350 // .data = .{ .rr_op = .{
4351 // .rd = .lr,
4352 // .rn = .r0,
4353 // .op = Instruction.Operand.reg(.pc, Instruction.Operand.Shift.none),
4354 // } },
4355 // });
4356 // _ = try self.addInst(.{
4357 // .tag = .bx,
4358 // .data = .{ .reg = .lr },
4359 // });
4360 }
4361
4362 const result: MCValue = result: {
4363 switch (info.return_value) {
4364 .register => |reg| {
4365 if (RegisterManager.indexOfRegIntoTracked(reg) == null) {
4366 // Save function return value into a tracked register
4367 log.debug("airCall: copying {} as it is not tracked", .{reg});
4368 const new_reg = try self.copyToTmpRegister(fn_ty.fnReturnType(zcu), info.return_value);
4369 break :result MCValue{ .register = new_reg };
4370 }
4371 },
4372 else => {},
4373 }
4374 break :result info.return_value;
4375 };
4376
4377 if (args.len <= Air.Liveness.bpi - 2) {
4378 var buf = [1]Air.Inst.Ref{.none} ** (Air.Liveness.bpi - 1);
4379 buf[0] = callee;
4380 @memcpy(buf[1..][0..args.len], args);
4381 return self.finishAir(inst, result, buf);
4382 }
4383 var bt = try self.iterateBigTomb(inst, 1 + args.len);
4384 bt.feed(callee);
4385 for (args) |arg| {
4386 bt.feed(arg);
4387 }
4388 return bt.finishAir(result);
4389}
4390
4391fn airRet(self: *Self, inst: Air.Inst.Index) !void {
4392 const pt = self.pt;
4393 const zcu = pt.zcu;
4394 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
4395 const operand = try self.resolveInst(un_op);
4396 const ret_ty = self.fn_type.fnReturnType(zcu);
4397
4398 switch (self.ret_mcv) {
4399 .none => {},
4400 .immediate => {
4401 assert(ret_ty.isError(zcu));
4402 },
4403 .register => |reg| {
4404 // Return result by value
4405 try self.genSetReg(ret_ty, reg, operand);
4406 },
4407 .stack_offset => {
4408 // Return result by reference
4409 //
4410 // self.ret_mcv is an address to where this function
4411 // should store its result into
4412 const ptr_ty = try pt.singleMutPtrType(ret_ty);
4413 try self.store(self.ret_mcv, operand, ptr_ty, ret_ty);
4414 },
4415 else => unreachable, // invalid return result
4416 }
4417
4418 // Just add space for an instruction, patch this later
4419 try self.exitlude_jump_relocs.append(self.gpa, try self.addNop());
4420
4421 return self.finishAir(inst, .dead, .{ un_op, .none, .none });
4422}
4423
4424fn airRetLoad(self: *Self, inst: Air.Inst.Index) !void {
4425 const pt = self.pt;
4426 const zcu = pt.zcu;
4427 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
4428 const ptr = try self.resolveInst(un_op);
4429 const ptr_ty = self.typeOf(un_op);
4430 const ret_ty = self.fn_type.fnReturnType(zcu);
4431
4432 switch (self.ret_mcv) {
4433 .none => {},
4434 .register => {
4435 // Return result by value
4436 try self.load(self.ret_mcv, ptr, ptr_ty);
4437 },
4438 .stack_offset => {
4439 // Return result by reference
4440 //
4441 // self.ret_mcv is an address to where this function
4442 // should store its result into
4443 //
4444 // If the operand is a ret_ptr instruction, we are done
4445 // here. Else we need to load the result from the location
4446 // pointed to by the operand and store it to the result
4447 // location.
4448 const op_inst = un_op.toIndex().?;
4449 if (self.air.instructions.items(.tag)[@intFromEnum(op_inst)] != .ret_ptr) {
4450 const abi_size: u32 = @intCast(ret_ty.abiSize(zcu));
4451 const abi_align = ret_ty.abiAlignment(zcu);
4452
4453 const offset = try self.allocMem(abi_size, abi_align, null);
4454
4455 const tmp_mcv = MCValue{ .stack_offset = offset };
4456 try self.load(tmp_mcv, ptr, ptr_ty);
4457 try self.store(self.ret_mcv, tmp_mcv, ptr_ty, ret_ty);
4458 }
4459 },
4460 else => unreachable, // invalid return result
4461 }
4462
4463 // Just add space for an instruction, patch this later
4464 try self.exitlude_jump_relocs.append(self.gpa, try self.addNop());
4465
4466 return self.finishAir(inst, .dead, .{ un_op, .none, .none });
4467}
4468
4469fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
4470 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
4471 const lhs_ty = self.typeOf(bin_op.lhs);
4472
4473 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else blk: {
4474 break :blk try self.cmp(.{ .inst = bin_op.lhs }, .{ .inst = bin_op.rhs }, lhs_ty, op);
4475 };
4476
4477 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
4478}
4479
4480fn cmp(
4481 self: *Self,
4482 lhs: ReadArg.Bind,
4483 rhs: ReadArg.Bind,
4484 lhs_ty: Type,
4485 op: math.CompareOperator,
4486) !MCValue {
4487 const pt = self.pt;
4488 const zcu = pt.zcu;
4489 const int_ty = switch (lhs_ty.zigTypeTag(zcu)) {
4490 .optional => blk: {
4491 const payload_ty = lhs_ty.optionalChild(zcu);
4492 if (!payload_ty.hasRuntimeBitsIgnoreComptime(zcu)) {
4493 break :blk Type.u1;
4494 } else if (lhs_ty.isPtrLikeOptional(zcu)) {
4495 break :blk Type.usize;
4496 } else {
4497 return self.fail("TODO ARM cmp non-pointer optionals", .{});
4498 }
4499 },
4500 .float => return self.fail("TODO ARM cmp floats", .{}),
4501 .@"enum" => lhs_ty.intTagType(zcu),
4502 .int => lhs_ty,
4503 .bool => Type.u1,
4504 .pointer => Type.usize,
4505 .error_set => Type.u16,
4506 else => unreachable,
4507 };
4508
4509 const int_info = int_ty.intInfo(zcu);
4510 if (int_info.bits <= 32) {
4511 try self.spillCompareFlagsIfOccupied();
4512
4513 var lhs_reg: Register = undefined;
4514 var rhs_reg: Register = undefined;
4515
4516 const rhs_immediate = try rhs.resolveToImmediate(self);
4517 const rhs_immediate_ok = if (rhs_immediate) |imm| Instruction.Operand.fromU32(imm) != null else false;
4518
4519 if (rhs_immediate_ok) {
4520 const read_args = [_]ReadArg{
4521 .{ .ty = int_ty, .bind = lhs, .class = gp, .reg = &lhs_reg },
4522 };
4523 try self.allocRegs(
4524 &read_args,
4525 &.{},
4526 null, // we won't be able to reuse a register as there are no write_regs
4527 );
4528
4529 _ = try self.addInst(.{
4530 .tag = .cmp,
4531 .data = .{ .r_op_cmp = .{
4532 .rn = lhs_reg,
4533 .op = Instruction.Operand.fromU32(rhs_immediate.?).?,
4534 } },
4535 });
4536 } else {
4537 const read_args = [_]ReadArg{
4538 .{ .ty = int_ty, .bind = lhs, .class = gp, .reg = &lhs_reg },
4539 .{ .ty = int_ty, .bind = rhs, .class = gp, .reg = &rhs_reg },
4540 };
4541 try self.allocRegs(
4542 &read_args,
4543 &.{},
4544 null, // we won't be able to reuse a register as there are no write_regs
4545 );
4546
4547 _ = try self.addInst(.{
4548 .tag = .cmp,
4549 .data = .{ .r_op_cmp = .{
4550 .rn = lhs_reg,
4551 .op = Instruction.Operand.reg(rhs_reg, Instruction.Operand.Shift.none),
4552 } },
4553 });
4554 }
4555
4556 return switch (int_info.signedness) {
4557 .signed => MCValue{ .cpsr_flags = Condition.fromCompareOperatorSigned(op) },
4558 .unsigned => MCValue{ .cpsr_flags = Condition.fromCompareOperatorUnsigned(op) },
4559 };
4560 } else {
4561 return self.fail("TODO ARM cmp for ints > 32 bits", .{});
4562 }
4563}
4564
4565fn airCmpVector(self: *Self, inst: Air.Inst.Index) !void {
4566 _ = inst;
4567 return self.fail("TODO implement airCmpVector for {}", .{self.target.cpu.arch});
4568}
4569
4570fn airCmpLtErrorsLen(self: *Self, inst: Air.Inst.Index) !void {
4571 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
4572 const operand = try self.resolveInst(un_op);
4573 _ = operand;
4574 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement airCmpLtErrorsLen for {}", .{self.target.cpu.arch});
4575 return self.finishAir(inst, result, .{ un_op, .none, .none });
4576}
4577
4578fn airDbgStmt(self: *Self, inst: Air.Inst.Index) !void {
4579 const dbg_stmt = self.air.instructions.items(.data)[@intFromEnum(inst)].dbg_stmt;
4580
4581 _ = try self.addInst(.{
4582 .tag = .dbg_line,
4583 .cond = undefined,
4584 .data = .{ .dbg_line_column = .{
4585 .line = dbg_stmt.line,
4586 .column = dbg_stmt.column,
4587 } },
4588 });
4589
4590 return self.finishAirBookkeeping();
4591}
4592
4593fn airDbgInlineBlock(self: *Self, inst: Air.Inst.Index) !void {
4594 const pt = self.pt;
4595 const zcu = pt.zcu;
4596 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
4597 const extra = self.air.extraData(Air.DbgInlineBlock, ty_pl.payload);
4598 const func = zcu.funcInfo(extra.data.func);
4599 // TODO emit debug info for function change
4600 _ = func;
4601 try self.lowerBlock(inst, @ptrCast(self.air.extra.items[extra.end..][0..extra.data.body_len]));
4602}
4603
4604fn airDbgVar(self: *Self, inst: Air.Inst.Index) !void {
4605 const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
4606 const operand = pl_op.operand;
4607 const tag = self.air.instructions.items(.tag)[@intFromEnum(inst)];
4608 const ty = self.typeOf(operand);
4609 const mcv = try self.resolveInst(operand);
4610 const name: Air.NullTerminatedString = @enumFromInt(pl_op.payload);
4611
4612 log.debug("airDbgVar: %{d}: {}, {}", .{ inst, ty.fmtDebug(), mcv });
4613
4614 try self.dbg_info_relocs.append(self.gpa, .{
4615 .tag = tag,
4616 .ty = ty,
4617 .name = name.toSlice(self.air),
4618 .mcv = mcv,
4619 });
4620
4621 return self.finishAir(inst, .dead, .{ operand, .none, .none });
4622}
4623
4624/// Given a boolean condition, emit a jump that is taken when that
4625/// condition is false.
4626fn condBr(self: *Self, condition: MCValue) !Mir.Inst.Index {
4627 const condition_code: Condition = switch (condition) {
4628 .cpsr_flags => |cond| cond.negate(),
4629 else => blk: {
4630 const reg = switch (condition) {
4631 .register => |r| r,
4632 else => try self.copyToTmpRegister(Type.bool, condition),
4633 };
4634
4635 try self.spillCompareFlagsIfOccupied();
4636
4637 // cmp reg, 1
4638 // bne ...
4639 _ = try self.addInst(.{
4640 .tag = .cmp,
4641 .data = .{ .r_op_cmp = .{
4642 .rn = reg,
4643 .op = Instruction.Operand.imm(1, 0),
4644 } },
4645 });
4646
4647 break :blk .ne;
4648 },
4649 };
4650
4651 return try self.addInst(.{
4652 .tag = .b,
4653 .cond = condition_code,
4654 .data = .{ .inst = undefined }, // populated later through performReloc
4655 });
4656}
4657
4658fn airCondBr(self: *Self, inst: Air.Inst.Index) !void {
4659 const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
4660 const cond_inst = try self.resolveInst(pl_op.operand);
4661 const extra = self.air.extraData(Air.CondBr, pl_op.payload);
4662 const then_body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[extra.end..][0..extra.data.then_body_len]);
4663 const else_body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[extra.end + then_body.len ..][0..extra.data.else_body_len]);
4664 const liveness_condbr = self.liveness.getCondBr(inst);
4665
4666 const reloc: Mir.Inst.Index = try self.condBr(cond_inst);
4667
4668 // If the condition dies here in this condbr instruction, process
4669 // that death now instead of later as this has an effect on
4670 // whether it needs to be spilled in the branches
4671 if (self.liveness.operandDies(inst, 0)) {
4672 if (pl_op.operand.toIndex()) |op_index| {
4673 self.processDeath(op_index);
4674 }
4675 }
4676
4677 // Capture the state of register and stack allocation state so that we can revert to it.
4678 const parent_next_stack_offset = self.next_stack_offset;
4679 const parent_free_registers = self.register_manager.free_registers;
4680 var parent_stack = try self.stack.clone(self.gpa);
4681 defer parent_stack.deinit(self.gpa);
4682 const parent_registers = self.register_manager.registers;
4683 const parent_cpsr_flags_inst = self.cpsr_flags_inst;
4684
4685 try self.branch_stack.append(.{});
4686 errdefer {
4687 _ = self.branch_stack.pop().?;
4688 }
4689
4690 try self.ensureProcessDeathCapacity(liveness_condbr.then_deaths.len);
4691 for (liveness_condbr.then_deaths) |operand| {
4692 self.processDeath(operand);
4693 }
4694 try self.genBody(then_body);
4695
4696 // Revert to the previous register and stack allocation state.
4697
4698 var saved_then_branch = self.branch_stack.pop().?;
4699 defer saved_then_branch.deinit(self.gpa);
4700
4701 self.register_manager.registers = parent_registers;
4702 self.cpsr_flags_inst = parent_cpsr_flags_inst;
4703
4704 self.stack.deinit(self.gpa);
4705 self.stack = parent_stack;
4706 parent_stack = .{};
4707
4708 self.next_stack_offset = parent_next_stack_offset;
4709 self.register_manager.free_registers = parent_free_registers;
4710
4711 try self.performReloc(reloc);
4712 const else_branch = self.branch_stack.addOneAssumeCapacity();
4713 else_branch.* = .{};
4714
4715 try self.ensureProcessDeathCapacity(liveness_condbr.else_deaths.len);
4716 for (liveness_condbr.else_deaths) |operand| {
4717 self.processDeath(operand);
4718 }
4719 try self.genBody(else_body);
4720
4721 // At this point, each branch will possibly have conflicting values for where
4722 // each instruction is stored. They agree, however, on which instructions are alive/dead.
4723 // We use the first ("then") branch as canonical, and here emit
4724 // instructions into the second ("else") branch to make it conform.
4725 // We continue respect the data structure semantic guarantees of the else_branch so
4726 // that we can use all the code emitting abstractions. This is why at the bottom we
4727 // assert that parent_branch.free_registers equals the saved_then_branch.free_registers
4728 // rather than assigning it.
4729 const parent_branch = &self.branch_stack.items[self.branch_stack.items.len - 2];
4730 try parent_branch.inst_table.ensureUnusedCapacity(self.gpa, else_branch.inst_table.count());
4731
4732 const else_slice = else_branch.inst_table.entries.slice();
4733 const else_keys = else_slice.items(.key);
4734 const else_values = else_slice.items(.value);
4735 for (else_keys, 0..) |else_key, else_idx| {
4736 const else_value = else_values[else_idx];
4737 const canon_mcv = if (saved_then_branch.inst_table.fetchSwapRemove(else_key)) |then_entry| blk: {
4738 // The instruction's MCValue is overridden in both branches.
4739 parent_branch.inst_table.putAssumeCapacity(else_key, then_entry.value);
4740 if (else_value == .dead) {
4741 assert(then_entry.value == .dead);
4742 continue;
4743 }
4744 break :blk then_entry.value;
4745 } else blk: {
4746 if (else_value == .dead)
4747 continue;
4748 // The instruction is only overridden in the else branch.
4749 var i: usize = self.branch_stack.items.len - 1;
4750 while (true) {
4751 i -= 1; // If this overflows, the question is: why wasn't the instruction marked dead?
4752 if (self.branch_stack.items[i].inst_table.get(else_key)) |mcv| {
4753 assert(mcv != .dead);
4754 break :blk mcv;
4755 }
4756 }
4757 };
4758 log.debug("consolidating else_entry {d} {}=>{}", .{ else_key, else_value, canon_mcv });
4759 // TODO make sure the destination stack offset / register does not already have something
4760 // going on there.
4761 try self.setRegOrMem(self.typeOfIndex(else_key), canon_mcv, else_value);
4762 // TODO track the new register / stack allocation
4763 }
4764 try parent_branch.inst_table.ensureUnusedCapacity(self.gpa, saved_then_branch.inst_table.count());
4765 const then_slice = saved_then_branch.inst_table.entries.slice();
4766 const then_keys = then_slice.items(.key);
4767 const then_values = then_slice.items(.value);
4768 for (then_keys, 0..) |then_key, then_idx| {
4769 const then_value = then_values[then_idx];
4770 // We already deleted the items from this table that matched the else_branch.
4771 // So these are all instructions that are only overridden in the then branch.
4772 parent_branch.inst_table.putAssumeCapacity(then_key, then_value);
4773 if (then_value == .dead)
4774 continue;
4775 const parent_mcv = blk: {
4776 var i: usize = self.branch_stack.items.len - 1;
4777 while (true) {
4778 i -= 1;
4779 if (self.branch_stack.items[i].inst_table.get(then_key)) |mcv| {
4780 assert(mcv != .dead);
4781 break :blk mcv;
4782 }
4783 }
4784 };
4785 log.debug("consolidating then_entry {d} {}=>{}", .{ then_key, parent_mcv, then_value });
4786 // TODO make sure the destination stack offset / register does not already have something
4787 // going on there.
4788 try self.setRegOrMem(self.typeOfIndex(then_key), parent_mcv, then_value);
4789 // TODO track the new register / stack allocation
4790 }
4791
4792 {
4793 var item = self.branch_stack.pop().?;
4794 item.deinit(self.gpa);
4795 }
4796
4797 // We already took care of pl_op.operand earlier, so we're going
4798 // to pass .none here
4799 return self.finishAir(inst, .unreach, .{ .none, .none, .none });
4800}
4801
4802fn isNull(
4803 self: *Self,
4804 operand_bind: ReadArg.Bind,
4805 operand_ty: Type,
4806) !MCValue {
4807 const pt = self.pt;
4808 const zcu = pt.zcu;
4809 if (operand_ty.isPtrLikeOptional(zcu)) {
4810 assert(operand_ty.abiSize(zcu) == 4);
4811
4812 const imm_bind: ReadArg.Bind = .{ .mcv = .{ .immediate = 0 } };
4813 return self.cmp(operand_bind, imm_bind, Type.usize, .eq);
4814 } else {
4815 return self.fail("TODO implement non-pointer optionals", .{});
4816 }
4817}
4818
4819fn isNonNull(
4820 self: *Self,
4821 operand_bind: ReadArg.Bind,
4822 operand_ty: Type,
4823) !MCValue {
4824 const is_null_result = try self.isNull(operand_bind, operand_ty);
4825 assert(is_null_result.cpsr_flags == .eq);
4826
4827 return MCValue{ .cpsr_flags = .ne };
4828}
4829
4830fn airIsNull(self: *Self, inst: Air.Inst.Index) !void {
4831 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
4832 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
4833 const operand_bind: ReadArg.Bind = .{ .inst = un_op };
4834 const operand_ty = self.typeOf(un_op);
4835
4836 break :result try self.isNull(operand_bind, operand_ty);
4837 };
4838 return self.finishAir(inst, result, .{ un_op, .none, .none });
4839}
4840
4841fn airIsNullPtr(self: *Self, inst: Air.Inst.Index) !void {
4842 const pt = self.pt;
4843 const zcu = pt.zcu;
4844 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
4845 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
4846 const operand_ptr = try self.resolveInst(un_op);
4847 const ptr_ty = self.typeOf(un_op);
4848 const elem_ty = ptr_ty.childType(zcu);
4849
4850 const operand = try self.allocRegOrMem(elem_ty, true, null);
4851 try self.load(operand, operand_ptr, ptr_ty);
4852
4853 break :result try self.isNull(.{ .mcv = operand }, elem_ty);
4854 };
4855 return self.finishAir(inst, result, .{ un_op, .none, .none });
4856}
4857
4858fn airIsNonNull(self: *Self, inst: Air.Inst.Index) !void {
4859 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
4860 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
4861 const operand_bind: ReadArg.Bind = .{ .inst = un_op };
4862 const operand_ty = self.typeOf(un_op);
4863
4864 break :result try self.isNonNull(operand_bind, operand_ty);
4865 };
4866 return self.finishAir(inst, result, .{ un_op, .none, .none });
4867}
4868
4869fn airIsNonNullPtr(self: *Self, inst: Air.Inst.Index) !void {
4870 const pt = self.pt;
4871 const zcu = pt.zcu;
4872 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
4873 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
4874 const operand_ptr = try self.resolveInst(un_op);
4875 const ptr_ty = self.typeOf(un_op);
4876 const elem_ty = ptr_ty.childType(zcu);
4877
4878 const operand = try self.allocRegOrMem(elem_ty, true, null);
4879 try self.load(operand, operand_ptr, ptr_ty);
4880
4881 break :result try self.isNonNull(.{ .mcv = operand }, elem_ty);
4882 };
4883 return self.finishAir(inst, result, .{ un_op, .none, .none });
4884}
4885
4886fn isErr(
4887 self: *Self,
4888 error_union_bind: ReadArg.Bind,
4889 error_union_ty: Type,
4890) !MCValue {
4891 const pt = self.pt;
4892 const zcu = pt.zcu;
4893 const error_type = error_union_ty.errorUnionSet(zcu);
4894
4895 if (error_type.errorSetIsEmpty(zcu)) {
4896 return MCValue{ .immediate = 0 }; // always false
4897 }
4898
4899 const error_mcv = try self.errUnionErr(error_union_bind, error_union_ty, null);
4900 return try self.cmp(.{ .mcv = error_mcv }, .{ .mcv = .{ .immediate = 0 } }, error_type, .gt);
4901}
4902
4903fn isNonErr(
4904 self: *Self,
4905 error_union_bind: ReadArg.Bind,
4906 error_union_ty: Type,
4907) !MCValue {
4908 const is_err_result = try self.isErr(error_union_bind, error_union_ty);
4909 switch (is_err_result) {
4910 .cpsr_flags => |cond| {
4911 assert(cond == .hi);
4912 return MCValue{ .cpsr_flags = cond.negate() };
4913 },
4914 .immediate => |imm| {
4915 assert(imm == 0);
4916 return MCValue{ .immediate = 1 };
4917 },
4918 else => unreachable,
4919 }
4920}
4921
4922fn airIsErr(self: *Self, inst: Air.Inst.Index) !void {
4923 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
4924 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
4925 const error_union_bind: ReadArg.Bind = .{ .inst = un_op };
4926 const error_union_ty = self.typeOf(un_op);
4927
4928 break :result try self.isErr(error_union_bind, error_union_ty);
4929 };
4930 return self.finishAir(inst, result, .{ un_op, .none, .none });
4931}
4932
4933fn airIsErrPtr(self: *Self, inst: Air.Inst.Index) !void {
4934 const pt = self.pt;
4935 const zcu = pt.zcu;
4936 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
4937 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
4938 const operand_ptr = try self.resolveInst(un_op);
4939 const ptr_ty = self.typeOf(un_op);
4940 const elem_ty = ptr_ty.childType(zcu);
4941
4942 const operand = try self.allocRegOrMem(elem_ty, true, null);
4943 try self.load(operand, operand_ptr, ptr_ty);
4944
4945 break :result try self.isErr(.{ .mcv = operand }, elem_ty);
4946 };
4947 return self.finishAir(inst, result, .{ un_op, .none, .none });
4948}
4949
4950fn airIsNonErr(self: *Self, inst: Air.Inst.Index) !void {
4951 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
4952 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
4953 const error_union_bind: ReadArg.Bind = .{ .inst = un_op };
4954 const error_union_ty = self.typeOf(un_op);
4955
4956 break :result try self.isNonErr(error_union_bind, error_union_ty);
4957 };
4958 return self.finishAir(inst, result, .{ un_op, .none, .none });
4959}
4960
4961fn airIsNonErrPtr(self: *Self, inst: Air.Inst.Index) !void {
4962 const pt = self.pt;
4963 const zcu = pt.zcu;
4964 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
4965 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
4966 const operand_ptr = try self.resolveInst(un_op);
4967 const ptr_ty = self.typeOf(un_op);
4968 const elem_ty = ptr_ty.childType(zcu);
4969
4970 const operand = try self.allocRegOrMem(elem_ty, true, null);
4971 try self.load(operand, operand_ptr, ptr_ty);
4972
4973 break :result try self.isNonErr(.{ .mcv = operand }, elem_ty);
4974 };
4975 return self.finishAir(inst, result, .{ un_op, .none, .none });
4976}
4977
4978fn airLoop(self: *Self, inst: Air.Inst.Index) !void {
4979 // A loop is a setup to be able to jump back to the beginning.
4980 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
4981 const loop = self.air.extraData(Air.Block, ty_pl.payload);
4982 const body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[loop.end..][0..loop.data.body_len]);
4983 const start_index: Mir.Inst.Index = @intCast(self.mir_instructions.len);
4984
4985 try self.genBody(body);
4986 try self.jump(start_index);
4987
4988 return self.finishAirBookkeeping();
4989}
4990
4991/// Send control flow to `inst`.
4992fn jump(self: *Self, inst: Mir.Inst.Index) !void {
4993 _ = try self.addInst(.{
4994 .tag = .b,
4995 .data = .{ .inst = inst },
4996 });
4997}
4998
4999fn airBlock(self: *Self, inst: Air.Inst.Index) !void {
5000 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
5001 const extra = self.air.extraData(Air.Block, ty_pl.payload);
5002 try self.lowerBlock(inst, @ptrCast(self.air.extra.items[extra.end..][0..extra.data.body_len]));
5003}
5004
5005fn lowerBlock(self: *Self, inst: Air.Inst.Index, body: []const Air.Inst.Index) !void {
5006 try self.blocks.putNoClobber(self.gpa, inst, .{
5007 // A block is a setup to be able to jump to the end.
5008 .relocs = .{},
5009 // It also acts as a receptacle for break operands.
5010 // Here we use `MCValue.none` to represent a null value so that the first
5011 // break instruction will choose a MCValue for the block result and overwrite
5012 // this field. Following break instructions will use that MCValue to put their
5013 // block results.
5014 .mcv = MCValue{ .none = {} },
5015 });
5016 defer self.blocks.getPtr(inst).?.relocs.deinit(self.gpa);
5017
5018 // TODO emit debug info lexical block
5019 try self.genBody(body);
5020
5021 // relocations for `br` instructions
5022 const relocs = &self.blocks.getPtr(inst).?.relocs;
5023 if (relocs.items.len > 0 and relocs.items[relocs.items.len - 1] == self.mir_instructions.len - 1) {
5024 // If the last Mir instruction is the last relocation (which
5025 // would just jump one instruction further), it can be safely
5026 // removed
5027 self.mir_instructions.orderedRemove(relocs.pop().?);
5028 }
5029 for (relocs.items) |reloc| {
5030 try self.performReloc(reloc);
5031 }
5032
5033 const result = self.blocks.getPtr(inst).?.mcv;
5034 return self.finishAir(inst, result, .{ .none, .none, .none });
5035}
5036
5037fn airSwitch(self: *Self, inst: Air.Inst.Index) !void {
5038 const switch_br = self.air.unwrapSwitch(inst);
5039 const condition_ty = self.typeOf(switch_br.operand);
5040 const liveness = try self.liveness.getSwitchBr(
5041 self.gpa,
5042 inst,
5043 switch_br.cases_len + 1,
5044 );
5045 defer self.gpa.free(liveness.deaths);
5046
5047 var it = switch_br.iterateCases();
5048 while (it.next()) |case| {
5049 if (case.ranges.len > 0) return self.fail("TODO: switch with ranges", .{});
5050 // For every item, we compare it to condition and branch into
5051 // the prong if they are equal. After we compared to all
5052 // items, we branch into the next prong (or if no other prongs
5053 // exist out of the switch statement).
5054 //
5055 // cmp condition, item1
5056 // beq prong
5057 // cmp condition, item2
5058 // beq prong
5059 // cmp condition, item3
5060 // beq prong
5061 // b out
5062 // prong: ...
5063 // ...
5064 // out: ...
5065 const branch_into_prong_relocs = try self.gpa.alloc(u32, case.items.len);
5066 defer self.gpa.free(branch_into_prong_relocs);
5067
5068 for (case.items, 0..) |item, idx| {
5069 const cmp_result = try self.cmp(.{ .inst = switch_br.operand }, .{ .inst = item }, condition_ty, .neq);
5070 branch_into_prong_relocs[idx] = try self.condBr(cmp_result);
5071 }
5072
5073 const branch_away_from_prong_reloc = try self.addInst(.{
5074 .tag = .b,
5075 .data = .{ .inst = undefined }, // populated later through performReloc
5076 });
5077
5078 for (branch_into_prong_relocs) |reloc| {
5079 try self.performReloc(reloc);
5080 }
5081
5082 // Capture the state of register and stack allocation state so that we can revert to it.
5083 const parent_next_stack_offset = self.next_stack_offset;
5084 const parent_free_registers = self.register_manager.free_registers;
5085 const parent_cpsr_flags_inst = self.cpsr_flags_inst;
5086 var parent_stack = try self.stack.clone(self.gpa);
5087 defer parent_stack.deinit(self.gpa);
5088 const parent_registers = self.register_manager.registers;
5089
5090 try self.branch_stack.append(.{});
5091 errdefer {
5092 _ = self.branch_stack.pop().?;
5093 }
5094
5095 try self.ensureProcessDeathCapacity(liveness.deaths[case.idx].len);
5096 for (liveness.deaths[case.idx]) |operand| {
5097 self.processDeath(operand);
5098 }
5099 try self.genBody(case.body);
5100
5101 // Revert to the previous register and stack allocation state.
5102 var saved_case_branch = self.branch_stack.pop().?;
5103 defer saved_case_branch.deinit(self.gpa);
5104
5105 self.register_manager.registers = parent_registers;
5106 self.cpsr_flags_inst = parent_cpsr_flags_inst;
5107 self.stack.deinit(self.gpa);
5108 self.stack = parent_stack;
5109 parent_stack = .{};
5110
5111 self.next_stack_offset = parent_next_stack_offset;
5112 self.register_manager.free_registers = parent_free_registers;
5113
5114 try self.performReloc(branch_away_from_prong_reloc);
5115 }
5116
5117 if (switch_br.else_body_len > 0) {
5118 const else_body = it.elseBody();
5119
5120 // Capture the state of register and stack allocation state so that we can revert to it.
5121 const parent_next_stack_offset = self.next_stack_offset;
5122 const parent_free_registers = self.register_manager.free_registers;
5123 const parent_cpsr_flags_inst = self.cpsr_flags_inst;
5124 var parent_stack = try self.stack.clone(self.gpa);
5125 defer parent_stack.deinit(self.gpa);
5126 const parent_registers = self.register_manager.registers;
5127
5128 try self.branch_stack.append(.{});
5129 errdefer {
5130 _ = self.branch_stack.pop().?;
5131 }
5132
5133 const else_deaths = liveness.deaths.len - 1;
5134 try self.ensureProcessDeathCapacity(liveness.deaths[else_deaths].len);
5135 for (liveness.deaths[else_deaths]) |operand| {
5136 self.processDeath(operand);
5137 }
5138 try self.genBody(else_body);
5139
5140 // Revert to the previous register and stack allocation state.
5141 var saved_case_branch = self.branch_stack.pop().?;
5142 defer saved_case_branch.deinit(self.gpa);
5143
5144 self.register_manager.registers = parent_registers;
5145 self.cpsr_flags_inst = parent_cpsr_flags_inst;
5146 self.stack.deinit(self.gpa);
5147 self.stack = parent_stack;
5148 parent_stack = .{};
5149
5150 self.next_stack_offset = parent_next_stack_offset;
5151 self.register_manager.free_registers = parent_free_registers;
5152
5153 // TODO consolidate returned MCValues between prongs and else branch like we do
5154 // in airCondBr.
5155 }
5156
5157 return self.finishAir(inst, .unreach, .{ switch_br.operand, .none, .none });
5158}
5159
5160fn performReloc(self: *Self, inst: Mir.Inst.Index) !void {
5161 const tag = self.mir_instructions.items(.tag)[inst];
5162 switch (tag) {
5163 .b => self.mir_instructions.items(.data)[inst].inst = @intCast(self.mir_instructions.len),
5164 else => unreachable,
5165 }
5166}
5167
5168fn airBr(self: *Self, inst: Air.Inst.Index) !void {
5169 const branch = self.air.instructions.items(.data)[@intFromEnum(inst)].br;
5170 try self.br(branch.block_inst, branch.operand);
5171 return self.finishAir(inst, .dead, .{ branch.operand, .none, .none });
5172}
5173
5174fn br(self: *Self, block: Air.Inst.Index, operand: Air.Inst.Ref) !void {
5175 const zcu = self.pt.zcu;
5176 const block_data = self.blocks.getPtr(block).?;
5177
5178 if (self.typeOf(operand).hasRuntimeBits(zcu)) {
5179 const operand_mcv = try self.resolveInst(operand);
5180 const block_mcv = block_data.mcv;
5181 if (block_mcv == .none) {
5182 block_data.mcv = switch (operand_mcv) {
5183 .none, .dead, .unreach => unreachable,
5184 .register, .stack_offset, .memory => operand_mcv,
5185 .immediate, .stack_argument_offset, .cpsr_flags => blk: {
5186 const new_mcv = try self.allocRegOrMem(self.typeOfIndex(block), true, block);
5187 try self.setRegOrMem(self.typeOfIndex(block), new_mcv, operand_mcv);
5188 break :blk new_mcv;
5189 },
5190 else => return self.fail("TODO implement block_data.mcv = operand_mcv for {}", .{operand_mcv}),
5191 };
5192 } else {
5193 try self.setRegOrMem(self.typeOfIndex(block), block_mcv, operand_mcv);
5194 }
5195 }
5196 return self.brVoid(block);
5197}
5198
5199fn brVoid(self: *Self, block: Air.Inst.Index) !void {
5200 const block_data = self.blocks.getPtr(block).?;
5201
5202 // Emit a jump with a relocation. It will be patched up after the block ends.
5203 try block_data.relocs.append(self.gpa, try self.addInst(.{
5204 .tag = .b,
5205 .data = .{ .inst = undefined }, // populated later through performReloc
5206 }));
5207}
5208
5209fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
5210 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
5211 const extra = self.air.extraData(Air.Asm, ty_pl.payload);
5212 const is_volatile = @as(u1, @truncate(extra.data.flags >> 31)) != 0;
5213 const clobbers_len: u31 = @truncate(extra.data.flags);
5214 var extra_i: usize = extra.end;
5215 const outputs: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[extra_i..][0..extra.data.outputs_len]);
5216 extra_i += outputs.len;
5217 const inputs: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[extra_i..][0..extra.data.inputs_len]);
5218 extra_i += inputs.len;
5219
5220 const dead = !is_volatile and self.liveness.isUnused(inst);
5221 const result: MCValue = if (dead) .dead else result: {
5222 if (outputs.len > 1) {
5223 return self.fail("TODO implement codegen for asm with more than 1 output", .{});
5224 }
5225
5226 const output_constraint: ?[]const u8 = for (outputs) |output| {
5227 if (output != .none) {
5228 return self.fail("TODO implement codegen for non-expr asm", .{});
5229 }
5230 const extra_bytes = std.mem.sliceAsBytes(self.air.extra.items[extra_i..]);
5231 const constraint = std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra.items[extra_i..]), 0);
5232 const name = std.mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0);
5233 // This equation accounts for the fact that even if we have exactly 4 bytes
5234 // for the string, we still use the next u32 for the null terminator.
5235 extra_i += (constraint.len + name.len + (2 + 3)) / 4;
5236
5237 break constraint;
5238 } else null;
5239
5240 for (inputs) |input| {
5241 const input_bytes = std.mem.sliceAsBytes(self.air.extra.items[extra_i..]);
5242 const constraint = std.mem.sliceTo(input_bytes, 0);
5243 const name = std.mem.sliceTo(input_bytes[constraint.len + 1 ..], 0);
5244 // This equation accounts for the fact that even if we have exactly 4 bytes
5245 // for the string, we still use the next u32 for the null terminator.
5246 extra_i += (constraint.len + name.len + (2 + 3)) / 4;
5247
5248 if (constraint.len < 3 or constraint[0] != '{' or constraint[constraint.len - 1] != '}') {
5249 return self.fail("unrecognized asm input constraint: '{s}'", .{constraint});
5250 }
5251 const reg_name = constraint[1 .. constraint.len - 1];
5252 const reg = parseRegName(reg_name) orelse
5253 return self.fail("unrecognized register: '{s}'", .{reg_name});
5254
5255 const arg_mcv = try self.resolveInst(input);
5256 try self.register_manager.getReg(reg, null);
5257 try self.genSetReg(self.typeOf(input), reg, arg_mcv);
5258 }
5259
5260 {
5261 var clobber_i: u32 = 0;
5262 while (clobber_i < clobbers_len) : (clobber_i += 1) {
5263 const clobber = std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra.items[extra_i..]), 0);
5264 // This equation accounts for the fact that even if we have exactly 4 bytes
5265 // for the string, we still use the next u32 for the null terminator.
5266 extra_i += clobber.len / 4 + 1;
5267
5268 // TODO honor these
5269 }
5270 }
5271
5272 const asm_source = std.mem.sliceAsBytes(self.air.extra.items[extra_i..])[0..extra.data.source_len];
5273
5274 if (mem.eql(u8, asm_source, "svc #0")) {
5275 _ = try self.addInst(.{
5276 .tag = .svc,
5277 .data = .{ .imm24 = 0 },
5278 });
5279 } else {
5280 return self.fail("TODO implement support for more arm assembly instructions", .{});
5281 }
5282
5283 if (output_constraint) |output| {
5284 if (output.len < 4 or output[0] != '=' or output[1] != '{' or output[output.len - 1] != '}') {
5285 return self.fail("unrecognized asm output constraint: '{s}'", .{output});
5286 }
5287 const reg_name = output[2 .. output.len - 1];
5288 const reg = parseRegName(reg_name) orelse
5289 return self.fail("unrecognized register: '{s}'", .{reg_name});
5290
5291 break :result MCValue{ .register = reg };
5292 } else {
5293 break :result MCValue{ .none = {} };
5294 }
5295 };
5296
5297 simple: {
5298 var buf = [1]Air.Inst.Ref{.none} ** (Air.Liveness.bpi - 1);
5299 var buf_index: usize = 0;
5300 for (outputs) |output| {
5301 if (output == .none) continue;
5302
5303 if (buf_index >= buf.len) break :simple;
5304 buf[buf_index] = output;
5305 buf_index += 1;
5306 }
5307 if (buf_index + inputs.len > buf.len) break :simple;
5308 @memcpy(buf[buf_index..][0..inputs.len], inputs);
5309 return self.finishAir(inst, result, buf);
5310 }
5311 var bt = try self.iterateBigTomb(inst, outputs.len + inputs.len);
5312 for (outputs) |output| {
5313 if (output == .none) continue;
5314
5315 bt.feed(output);
5316 }
5317 for (inputs) |input| {
5318 bt.feed(input);
5319 }
5320 return bt.finishAir(result);
5321}
5322
5323fn iterateBigTomb(self: *Self, inst: Air.Inst.Index, operand_count: usize) !BigTomb {
5324 try self.ensureProcessDeathCapacity(operand_count + 1);
5325 return BigTomb{
5326 .function = self,
5327 .inst = inst,
5328 .lbt = self.liveness.iterateBigTomb(inst),
5329 };
5330}
5331
5332/// Sets the value without any modifications to register allocation metadata or stack allocation metadata.
5333fn setRegOrMem(self: *Self, ty: Type, loc: MCValue, val: MCValue) !void {
5334 switch (loc) {
5335 .none => return,
5336 .register => |reg| return self.genSetReg(ty, reg, val),
5337 .stack_offset => |off| return self.genSetStack(ty, off, val),
5338 .memory => {
5339 return self.fail("TODO implement setRegOrMem for memory", .{});
5340 },
5341 else => unreachable,
5342 }
5343}
5344
5345fn genSetStack(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerError!void {
5346 const pt = self.pt;
5347 const zcu = pt.zcu;
5348 const abi_size: u32 = @intCast(ty.abiSize(zcu));
5349 switch (mcv) {
5350 .dead => unreachable,
5351 .unreach, .none => return, // Nothing to do.
5352 .undef => {
5353 if (!self.wantSafety())
5354 return; // The already existing value will do just fine.
5355 // TODO Upgrade this to a memset call when we have that available.
5356 switch (abi_size) {
5357 1 => try self.genSetStack(ty, stack_offset, .{ .immediate = 0xaa }),
5358 2 => try self.genSetStack(ty, stack_offset, .{ .immediate = 0xaaaa }),
5359 4 => try self.genSetStack(ty, stack_offset, .{ .immediate = 0xaaaaaaaa }),
5360 else => try self.genInlineMemset(
5361 .{ .ptr_stack_offset = stack_offset },
5362 .{ .immediate = 0xaa },
5363 .{ .immediate = abi_size },
5364 ),
5365 }
5366 },
5367 .cpsr_flags,
5368 .immediate,
5369 .ptr_stack_offset,
5370 => {
5371 const reg = try self.copyToTmpRegister(ty, mcv);
5372 return self.genSetStack(ty, stack_offset, MCValue{ .register = reg });
5373 },
5374 .register => |reg| {
5375 switch (abi_size) {
5376 1, 4 => {
5377 const offset = if (math.cast(u12, stack_offset)) |imm| blk: {
5378 break :blk Instruction.Offset.imm(imm);
5379 } else Instruction.Offset.reg(try self.copyToTmpRegister(Type.u32, MCValue{ .immediate = stack_offset }), .none);
5380
5381 const tag: Mir.Inst.Tag = switch (abi_size) {
5382 1 => .strb,
5383 4 => .str,
5384 else => unreachable,
5385 };
5386
5387 _ = try self.addInst(.{
5388 .tag = tag,
5389 .data = .{ .rr_offset = .{
5390 .rt = reg,
5391 .rn = .fp,
5392 .offset = .{
5393 .offset = offset,
5394 .positive = false,
5395 },
5396 } },
5397 });
5398 },
5399 2 => {
5400 const offset = if (stack_offset <= math.maxInt(u8)) blk: {
5401 break :blk Instruction.ExtraLoadStoreOffset.imm(@intCast(stack_offset));
5402 } else Instruction.ExtraLoadStoreOffset.reg(try self.copyToTmpRegister(Type.u32, MCValue{ .immediate = stack_offset }));
5403
5404 _ = try self.addInst(.{
5405 .tag = .strh,
5406 .data = .{ .rr_extra_offset = .{
5407 .rt = reg,
5408 .rn = .fp,
5409 .offset = .{
5410 .offset = offset,
5411 .positive = false,
5412 },
5413 } },
5414 });
5415 },
5416 else => return self.fail("TODO implement storing other types abi_size={}", .{abi_size}),
5417 }
5418 },
5419 .register_c_flag,
5420 .register_v_flag,
5421 => |reg| {
5422 const reg_lock = self.register_manager.lockReg(reg);
5423 defer if (reg_lock) |locked_reg| self.register_manager.unlockReg(locked_reg);
5424
5425 const wrapped_ty = ty.fieldType(0, zcu);
5426 try self.genSetStack(wrapped_ty, stack_offset, .{ .register = reg });
5427
5428 const overflow_bit_ty = ty.fieldType(1, zcu);
5429 const overflow_bit_offset: u32 = @intCast(ty.structFieldOffset(1, zcu));
5430 const cond_reg = try self.register_manager.allocReg(null, gp);
5431
5432 // C flag: movcs reg, #1
5433 // V flag: movvs reg, #1
5434 _ = try self.addInst(.{
5435 .tag = .mov,
5436 .cond = switch (mcv) {
5437 .register_c_flag => .cs,
5438 .register_v_flag => .vs,
5439 else => unreachable,
5440 },
5441 .data = .{ .r_op_mov = .{
5442 .rd = cond_reg,
5443 .op = Instruction.Operand.fromU32(1).?,
5444 } },
5445 });
5446
5447 try self.genSetStack(overflow_bit_ty, stack_offset - overflow_bit_offset, .{
5448 .register = cond_reg,
5449 });
5450 },
5451 .memory,
5452 .stack_argument_offset,
5453 .stack_offset,
5454 => {
5455 switch (mcv) {
5456 .stack_offset => |off| {
5457 if (stack_offset == off)
5458 return; // Copy stack variable to itself; nothing to do.
5459 },
5460 else => {},
5461 }
5462
5463 if (abi_size <= 4) {
5464 const reg = try self.copyToTmpRegister(ty, mcv);
5465 return self.genSetStack(ty, stack_offset, MCValue{ .register = reg });
5466 } else {
5467 const ptr_ty = try pt.singleMutPtrType(ty);
5468
5469 // TODO call extern memcpy
5470 const regs = try self.register_manager.allocRegs(5, .{ null, null, null, null, null }, gp);
5471 const src_reg = regs[0];
5472 const dst_reg = regs[1];
5473 const len_reg = regs[2];
5474 const count_reg = regs[3];
5475 const tmp_reg = regs[4];
5476
5477 switch (mcv) {
5478 .stack_offset => |off| {
5479 // sub src_reg, fp, #off
5480 try self.genSetReg(ptr_ty, src_reg, .{ .ptr_stack_offset = off });
5481 },
5482 .memory => |addr| try self.genSetReg(ptr_ty, src_reg, .{ .immediate = @intCast(addr) }),
5483 .stack_argument_offset => |off| {
5484 _ = try self.addInst(.{
5485 .tag = .ldr_ptr_stack_argument,
5486 .data = .{ .r_stack_offset = .{
5487 .rt = src_reg,
5488 .stack_offset = off,
5489 } },
5490 });
5491 },
5492 else => unreachable,
5493 }
5494
5495 // sub dst_reg, fp, #stack_offset
5496 try self.genSetReg(ptr_ty, dst_reg, .{ .ptr_stack_offset = stack_offset });
5497
5498 // mov len, #abi_size
5499 try self.genSetReg(Type.usize, len_reg, .{ .immediate = abi_size });
5500
5501 // memcpy(src, dst, len)
5502 try self.genInlineMemcpy(src_reg, dst_reg, len_reg, count_reg, tmp_reg);
5503 }
5504 },
5505 }
5506}
5507
5508fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void {
5509 const pt = self.pt;
5510 const zcu = pt.zcu;
5511 switch (mcv) {
5512 .dead => unreachable,
5513 .unreach, .none => return, // Nothing to do.
5514 .undef => {
5515 if (!self.wantSafety())
5516 return; // The already existing value will do just fine.
5517 // Write the debug undefined value.
5518 return self.genSetReg(ty, reg, .{ .immediate = 0xaaaaaaaa });
5519 },
5520 .ptr_stack_offset => |off| {
5521 // TODO: maybe addressing from sp instead of fp
5522 const op = Instruction.Operand.fromU32(off) orelse
5523 return self.fail("TODO larger stack offsets", .{});
5524
5525 _ = try self.addInst(.{
5526 .tag = .sub,
5527 .data = .{ .rr_op = .{
5528 .rd = reg,
5529 .rn = .fp,
5530 .op = op,
5531 } },
5532 });
5533 },
5534 .cpsr_flags => |condition| {
5535 const zero = Instruction.Operand.imm(0, 0);
5536 const one = Instruction.Operand.imm(1, 0);
5537
5538 // mov reg, 0
5539 _ = try self.addInst(.{
5540 .tag = .mov,
5541 .data = .{ .r_op_mov = .{
5542 .rd = reg,
5543 .op = zero,
5544 } },
5545 });
5546
5547 // moveq reg, 1
5548 _ = try self.addInst(.{
5549 .tag = .mov,
5550 .cond = condition,
5551 .data = .{ .r_op_mov = .{
5552 .rd = reg,
5553 .op = one,
5554 } },
5555 });
5556 },
5557 .immediate => |x| {
5558 if (Instruction.Operand.fromU32(x)) |op| {
5559 _ = try self.addInst(.{
5560 .tag = .mov,
5561 .data = .{ .r_op_mov = .{
5562 .rd = reg,
5563 .op = op,
5564 } },
5565 });
5566 } else if (Instruction.Operand.fromU32(~x)) |op| {
5567 _ = try self.addInst(.{
5568 .tag = .mvn,
5569 .data = .{ .r_op_mov = .{
5570 .rd = reg,
5571 .op = op,
5572 } },
5573 });
5574 } else if (x <= math.maxInt(u16)) {
5575 if (self.target.cpu.has(.arm, .has_v7)) {
5576 _ = try self.addInst(.{
5577 .tag = .movw,
5578 .data = .{ .r_imm16 = .{
5579 .rd = reg,
5580 .imm16 = @intCast(x),
5581 } },
5582 });
5583 } else {
5584 _ = try self.addInst(.{
5585 .tag = .mov,
5586 .data = .{ .r_op_mov = .{
5587 .rd = reg,
5588 .op = Instruction.Operand.imm(@truncate(x), 0),
5589 } },
5590 });
5591 _ = try self.addInst(.{
5592 .tag = .orr,
5593 .data = .{ .rr_op = .{
5594 .rd = reg,
5595 .rn = reg,
5596 .op = Instruction.Operand.imm(@truncate(x >> 8), 12),
5597 } },
5598 });
5599 }
5600 } else {
5601 // TODO write constant to code and load
5602 // relative to pc
5603 if (self.target.cpu.has(.arm, .has_v7)) {
5604 // immediate: 0xaaaabbbb
5605 // movw reg, #0xbbbb
5606 // movt reg, #0xaaaa
5607 _ = try self.addInst(.{
5608 .tag = .movw,
5609 .data = .{ .r_imm16 = .{
5610 .rd = reg,
5611 .imm16 = @truncate(x),
5612 } },
5613 });
5614 _ = try self.addInst(.{
5615 .tag = .movt,
5616 .data = .{ .r_imm16 = .{
5617 .rd = reg,
5618 .imm16 = @truncate(x >> 16),
5619 } },
5620 });
5621 } else {
5622 // immediate: 0xaabbccdd
5623 // mov reg, #0xaa
5624 // orr reg, reg, #0xbb, 24
5625 // orr reg, reg, #0xcc, 16
5626 // orr reg, reg, #0xdd, 8
5627 _ = try self.addInst(.{
5628 .tag = .mov,
5629 .data = .{ .r_op_mov = .{
5630 .rd = reg,
5631 .op = Instruction.Operand.imm(@truncate(x), 0),
5632 } },
5633 });
5634 _ = try self.addInst(.{
5635 .tag = .orr,
5636 .data = .{ .rr_op = .{
5637 .rd = reg,
5638 .rn = reg,
5639 .op = Instruction.Operand.imm(@truncate(x >> 8), 12),
5640 } },
5641 });
5642 _ = try self.addInst(.{
5643 .tag = .orr,
5644 .data = .{ .rr_op = .{
5645 .rd = reg,
5646 .rn = reg,
5647 .op = Instruction.Operand.imm(@truncate(x >> 16), 8),
5648 } },
5649 });
5650 _ = try self.addInst(.{
5651 .tag = .orr,
5652 .data = .{ .rr_op = .{
5653 .rd = reg,
5654 .rn = reg,
5655 .op = Instruction.Operand.imm(@truncate(x >> 24), 4),
5656 } },
5657 });
5658 }
5659 }
5660 },
5661 .register => |src_reg| {
5662 // If the registers are the same, nothing to do.
5663 if (src_reg.id() == reg.id())
5664 return;
5665
5666 // mov reg, src_reg
5667 _ = try self.addInst(.{
5668 .tag = .mov,
5669 .data = .{ .r_op_mov = .{
5670 .rd = reg,
5671 .op = Instruction.Operand.reg(src_reg, Instruction.Operand.Shift.none),
5672 } },
5673 });
5674 },
5675 .register_c_flag => unreachable, // doesn't fit into a register
5676 .register_v_flag => unreachable, // doesn't fit into a register
5677 .memory => |addr| {
5678 // The value is in memory at a hard-coded address.
5679 // If the type is a pointer, it means the pointer address is at this memory location.
5680 try self.genSetReg(ty, reg, .{ .immediate = @intCast(addr) });
5681 try self.genLdrRegister(reg, reg, ty);
5682 },
5683 .stack_offset => |off| {
5684 // TODO: maybe addressing from sp instead of fp
5685 const abi_size: u32 = @intCast(ty.abiSize(zcu));
5686
5687 const tag: Mir.Inst.Tag = switch (abi_size) {
5688 1 => if (ty.isSignedInt(zcu)) Mir.Inst.Tag.ldrsb else .ldrb,
5689 2 => if (ty.isSignedInt(zcu)) Mir.Inst.Tag.ldrsh else .ldrh,
5690 3, 4 => .ldr,
5691 else => unreachable,
5692 };
5693
5694 const extra_offset = switch (abi_size) {
5695 1 => ty.isSignedInt(zcu),
5696 2 => true,
5697 3, 4 => false,
5698 else => unreachable,
5699 };
5700
5701 if (extra_offset) {
5702 const offset = if (off <= math.maxInt(u8)) blk: {
5703 break :blk Instruction.ExtraLoadStoreOffset.imm(@intCast(off));
5704 } else Instruction.ExtraLoadStoreOffset.reg(try self.copyToTmpRegister(Type.usize, MCValue{ .immediate = off }));
5705
5706 _ = try self.addInst(.{
5707 .tag = tag,
5708 .data = .{ .rr_extra_offset = .{
5709 .rt = reg,
5710 .rn = .fp,
5711 .offset = .{
5712 .offset = offset,
5713 .positive = false,
5714 },
5715 } },
5716 });
5717 } else {
5718 const offset = if (off <= math.maxInt(u12)) blk: {
5719 break :blk Instruction.Offset.imm(@intCast(off));
5720 } else Instruction.Offset.reg(try self.copyToTmpRegister(Type.usize, MCValue{ .immediate = off }), .none);
5721
5722 _ = try self.addInst(.{
5723 .tag = tag,
5724 .data = .{ .rr_offset = .{
5725 .rt = reg,
5726 .rn = .fp,
5727 .offset = .{
5728 .offset = offset,
5729 .positive = false,
5730 },
5731 } },
5732 });
5733 }
5734 },
5735 .stack_argument_offset => |off| {
5736 const abi_size = ty.abiSize(zcu);
5737
5738 const tag: Mir.Inst.Tag = switch (abi_size) {
5739 1 => if (ty.isSignedInt(zcu)) Mir.Inst.Tag.ldrsb_stack_argument else .ldrb_stack_argument,
5740 2 => if (ty.isSignedInt(zcu)) Mir.Inst.Tag.ldrsh_stack_argument else .ldrh_stack_argument,
5741 3, 4 => .ldr_stack_argument,
5742 else => unreachable,
5743 };
5744
5745 _ = try self.addInst(.{
5746 .tag = tag,
5747 .data = .{ .r_stack_offset = .{
5748 .rt = reg,
5749 .stack_offset = off,
5750 } },
5751 });
5752 },
5753 }
5754}
5755
5756fn genSetStackArgument(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerError!void {
5757 const pt = self.pt;
5758 const abi_size: u32 = @intCast(ty.abiSize(pt.zcu));
5759 switch (mcv) {
5760 .dead => unreachable,
5761 .none, .unreach => return,
5762 .undef => {
5763 if (!self.wantSafety())
5764 return; // The already existing value will do just fine.
5765 // TODO Upgrade this to a memset call when we have that available.
5766 switch (abi_size) {
5767 1 => try self.genSetStackArgument(ty, stack_offset, .{ .immediate = 0xaa }),
5768 2 => try self.genSetStackArgument(ty, stack_offset, .{ .immediate = 0xaaaa }),
5769 4 => try self.genSetStackArgument(ty, stack_offset, .{ .immediate = 0xaaaaaaaa }),
5770 else => return self.fail("TODO implement memset", .{}),
5771 }
5772 },
5773 .register => |reg| {
5774 switch (abi_size) {
5775 1, 4 => {
5776 const offset = if (math.cast(u12, stack_offset)) |imm| blk: {
5777 break :blk Instruction.Offset.imm(imm);
5778 } else Instruction.Offset.reg(try self.copyToTmpRegister(Type.u32, MCValue{ .immediate = stack_offset }), .none);
5779
5780 const tag: Mir.Inst.Tag = switch (abi_size) {
5781 1 => .strb,
5782 4 => .str,
5783 else => unreachable,
5784 };
5785
5786 _ = try self.addInst(.{
5787 .tag = tag,
5788 .data = .{ .rr_offset = .{
5789 .rt = reg,
5790 .rn = .sp,
5791 .offset = .{ .offset = offset },
5792 } },
5793 });
5794 },
5795 2 => {
5796 const offset = if (stack_offset <= math.maxInt(u8)) blk: {
5797 break :blk Instruction.ExtraLoadStoreOffset.imm(@intCast(stack_offset));
5798 } else Instruction.ExtraLoadStoreOffset.reg(try self.copyToTmpRegister(Type.u32, MCValue{ .immediate = stack_offset }));
5799
5800 _ = try self.addInst(.{
5801 .tag = .strh,
5802 .data = .{ .rr_extra_offset = .{
5803 .rt = reg,
5804 .rn = .sp,
5805 .offset = .{ .offset = offset },
5806 } },
5807 });
5808 },
5809 else => return self.fail("TODO implement storing other types abi_size={}", .{abi_size}),
5810 }
5811 },
5812 .register_c_flag,
5813 .register_v_flag,
5814 => {
5815 return self.fail("TODO implement genSetStack {}", .{mcv});
5816 },
5817 .stack_offset,
5818 .memory,
5819 .stack_argument_offset,
5820 => {
5821 if (abi_size <= 4) {
5822 const reg = try self.copyToTmpRegister(ty, mcv);
5823 return self.genSetStackArgument(ty, stack_offset, MCValue{ .register = reg });
5824 } else {
5825 const ptr_ty = try pt.singleMutPtrType(ty);
5826
5827 // TODO call extern memcpy
5828 const regs = try self.register_manager.allocRegs(5, .{ null, null, null, null, null }, gp);
5829 const src_reg = regs[0];
5830 const dst_reg = regs[1];
5831 const len_reg = regs[2];
5832 const count_reg = regs[3];
5833 const tmp_reg = regs[4];
5834
5835 switch (mcv) {
5836 .stack_offset => |off| {
5837 // sub src_reg, fp, #off
5838 try self.genSetReg(ptr_ty, src_reg, .{ .ptr_stack_offset = off });
5839 },
5840 .memory => |addr| try self.genSetReg(ptr_ty, src_reg, .{ .immediate = @intCast(addr) }),
5841 .stack_argument_offset => |off| {
5842 _ = try self.addInst(.{
5843 .tag = .ldr_ptr_stack_argument,
5844 .data = .{ .r_stack_offset = .{
5845 .rt = src_reg,
5846 .stack_offset = off,
5847 } },
5848 });
5849 },
5850 else => unreachable,
5851 }
5852
5853 // add dst_reg, sp, #stack_offset
5854 const dst_offset_op: Instruction.Operand = if (Instruction.Operand.fromU32(stack_offset)) |x| x else {
5855 return self.fail("TODO load: set reg to stack offset with all possible offsets", .{});
5856 };
5857 _ = try self.addInst(.{
5858 .tag = .add,
5859 .data = .{ .rr_op = .{
5860 .rd = dst_reg,
5861 .rn = .sp,
5862 .op = dst_offset_op,
5863 } },
5864 });
5865
5866 // mov len, #abi_size
5867 try self.genSetReg(Type.usize, len_reg, .{ .immediate = abi_size });
5868
5869 // memcpy(src, dst, len)
5870 try self.genInlineMemcpy(src_reg, dst_reg, len_reg, count_reg, tmp_reg);
5871 }
5872 },
5873 .cpsr_flags,
5874 .immediate,
5875 .ptr_stack_offset,
5876 => {
5877 const reg = try self.copyToTmpRegister(ty, mcv);
5878 return self.genSetStackArgument(ty, stack_offset, MCValue{ .register = reg });
5879 },
5880 }
5881}
5882
5883fn airBitCast(self: *Self, inst: Air.Inst.Index) !void {
5884 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
5885 const result = if (self.liveness.isUnused(inst)) .dead else result: {
5886 const operand = try self.resolveInst(ty_op.operand);
5887 if (self.reuseOperand(inst, ty_op.operand, 0, operand)) break :result operand;
5888
5889 const operand_lock = switch (operand) {
5890 .register,
5891 .register_c_flag,
5892 .register_v_flag,
5893 => |reg| self.register_manager.lockReg(reg),
5894 else => null,
5895 };
5896 defer if (operand_lock) |lock| self.register_manager.unlockReg(lock);
5897
5898 const dest_ty = self.typeOfIndex(inst);
5899 const dest = try self.allocRegOrMem(dest_ty, true, inst);
5900 try self.setRegOrMem(dest_ty, dest, operand);
5901 break :result dest;
5902 };
5903 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
5904}
5905
5906fn airArrayToSlice(self: *Self, inst: Air.Inst.Index) !void {
5907 const pt = self.pt;
5908 const zcu = pt.zcu;
5909 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
5910 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
5911 const ptr_ty = self.typeOf(ty_op.operand);
5912 const ptr = try self.resolveInst(ty_op.operand);
5913 const array_ty = ptr_ty.childType(zcu);
5914 const array_len: u32 = @intCast(array_ty.arrayLen(zcu));
5915
5916 const stack_offset = try self.allocMem(8, .@"8", inst);
5917 try self.genSetStack(ptr_ty, stack_offset, ptr);
5918 try self.genSetStack(Type.usize, stack_offset - 4, .{ .immediate = array_len });
5919 break :result MCValue{ .stack_offset = stack_offset };
5920 };
5921 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
5922}
5923
5924fn airFloatFromInt(self: *Self, inst: Air.Inst.Index) !void {
5925 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
5926 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement airFloatFromInt for {}", .{
5927 self.target.cpu.arch,
5928 });
5929 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
5930}
5931
5932fn airIntFromFloat(self: *Self, inst: Air.Inst.Index) !void {
5933 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
5934 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement airIntFromFloat for {}", .{
5935 self.target.cpu.arch,
5936 });
5937 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
5938}
5939
5940fn airCmpxchg(self: *Self, inst: Air.Inst.Index) !void {
5941 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
5942 const extra = self.air.extraData(Air.Block, ty_pl.payload);
5943 _ = extra;
5944
5945 return self.fail("TODO implement airCmpxchg for {}", .{
5946 self.target.cpu.arch,
5947 });
5948}
5949
5950fn airAtomicRmw(self: *Self, inst: Air.Inst.Index) !void {
5951 _ = inst;
5952 return self.fail("TODO implement airCmpxchg for {}", .{self.target.cpu.arch});
5953}
5954
5955fn airAtomicLoad(self: *Self, inst: Air.Inst.Index) !void {
5956 _ = inst;
5957 return self.fail("TODO implement airAtomicLoad for {}", .{self.target.cpu.arch});
5958}
5959
5960fn airAtomicStore(self: *Self, inst: Air.Inst.Index, order: std.builtin.AtomicOrder) !void {
5961 _ = inst;
5962 _ = order;
5963 return self.fail("TODO implement airAtomicStore for {}", .{self.target.cpu.arch});
5964}
5965
5966fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
5967 if (safety) {
5968 // TODO if the value is undef, write 0xaa bytes to dest
5969 } else {
5970 // TODO if the value is undef, don't lower this instruction
5971 }
5972 _ = inst;
5973 return self.fail("TODO implement airMemset for {}", .{self.target.cpu.arch});
5974}
5975
5976fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {
5977 _ = inst;
5978 return self.fail("TODO implement airMemcpy for {}", .{self.target.cpu.arch});
5979}
5980
5981fn airMemmove(self: *Self, inst: Air.Inst.Index) !void {
5982 _ = inst;
5983 return self.fail("TODO implement airMemmove for {}", .{self.target.cpu.arch});
5984}
5985
5986fn airTagName(self: *Self, inst: Air.Inst.Index) !void {
5987 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
5988 const operand = try self.resolveInst(un_op);
5989 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else {
5990 _ = operand;
5991 return self.fail("TODO implement airTagName for arm", .{});
5992 };
5993 return self.finishAir(inst, result, .{ un_op, .none, .none });
5994}
5995
5996fn airErrorName(self: *Self, inst: Air.Inst.Index) !void {
5997 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
5998 const operand = try self.resolveInst(un_op);
5999 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else {
6000 _ = operand;
6001 return self.fail("TODO implement airErrorName for arm", .{});
6002 };
6003 return self.finishAir(inst, result, .{ un_op, .none, .none });
6004}
6005
6006fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
6007 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
6008 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement airSplat for arm", .{});
6009 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
6010}
6011
6012fn airSelect(self: *Self, inst: Air.Inst.Index) !void {
6013 const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
6014 const extra = self.air.extraData(Air.Bin, pl_op.payload).data;
6015 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement airSelect for arm", .{});
6016 return self.finishAir(inst, result, .{ pl_op.operand, extra.lhs, extra.rhs });
6017}
6018
6019fn airShuffleOne(self: *Self, inst: Air.Inst.Index) !void {
6020 _ = inst;
6021 return self.fail("TODO implement airShuffleOne for arm", .{});
6022}
6023
6024fn airShuffleTwo(self: *Self, inst: Air.Inst.Index) !void {
6025 _ = inst;
6026 return self.fail("TODO implement airShuffleTwo for arm", .{});
6027}
6028
6029fn airReduce(self: *Self, inst: Air.Inst.Index) !void {
6030 const reduce = self.air.instructions.items(.data)[@intFromEnum(inst)].reduce;
6031 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement airReduce for arm", .{});
6032 return self.finishAir(inst, result, .{ reduce.operand, .none, .none });
6033}
6034
6035fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {
6036 const pt = self.pt;
6037 const zcu = pt.zcu;
6038 const vector_ty = self.typeOfIndex(inst);
6039 const len = vector_ty.vectorLen(zcu);
6040 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
6041 const elements: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[ty_pl.payload..][0..len]);
6042 const result: MCValue = res: {
6043 if (self.liveness.isUnused(inst)) break :res MCValue.dead;
6044 return self.fail("TODO implement airAggregateInit for arm", .{});
6045 };
6046
6047 if (elements.len <= Air.Liveness.bpi - 1) {
6048 var buf = [1]Air.Inst.Ref{.none} ** (Air.Liveness.bpi - 1);
6049 @memcpy(buf[0..elements.len], elements);
6050 return self.finishAir(inst, result, buf);
6051 }
6052 var bt = try self.iterateBigTomb(inst, elements.len);
6053 for (elements) |elem| {
6054 bt.feed(elem);
6055 }
6056 return bt.finishAir(result);
6057}
6058
6059fn airUnionInit(self: *Self, inst: Air.Inst.Index) !void {
6060 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
6061 const extra = self.air.extraData(Air.UnionInit, ty_pl.payload).data;
6062 _ = extra;
6063
6064 return self.fail("TODO implement airUnionInit for arm", .{});
6065}
6066
6067fn airPrefetch(self: *Self, inst: Air.Inst.Index) !void {
6068 const prefetch = self.air.instructions.items(.data)[@intFromEnum(inst)].prefetch;
6069 return self.finishAir(inst, MCValue.dead, .{ prefetch.ptr, .none, .none });
6070}
6071
6072fn airMulAdd(self: *Self, inst: Air.Inst.Index) !void {
6073 const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
6074 const extra = self.air.extraData(Air.Bin, pl_op.payload).data;
6075 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else {
6076 return self.fail("TODO implement airMulAdd for arm", .{});
6077 };
6078 return self.finishAir(inst, result, .{ extra.lhs, extra.rhs, pl_op.operand });
6079}
6080
6081fn airTry(self: *Self, inst: Air.Inst.Index) !void {
6082 const pt = self.pt;
6083 const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
6084 const extra = self.air.extraData(Air.Try, pl_op.payload);
6085 const body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[extra.end..][0..extra.data.body_len]);
6086 const result: MCValue = result: {
6087 const error_union_bind: ReadArg.Bind = .{ .inst = pl_op.operand };
6088 const error_union_ty = self.typeOf(pl_op.operand);
6089 const error_union_size: u32 = @intCast(error_union_ty.abiSize(pt.zcu));
6090 const error_union_align = error_union_ty.abiAlignment(pt.zcu);
6091
6092 // The error union will die in the body. However, we need the
6093 // error union after the body in order to extract the payload
6094 // of the error union, so we create a copy of it
6095 const error_union_copy = try self.allocMem(error_union_size, error_union_align, null);
6096 try self.genSetStack(error_union_ty, error_union_copy, try error_union_bind.resolveToMcv(self));
6097
6098 const is_err_result = try self.isErr(error_union_bind, error_union_ty);
6099 const reloc = try self.condBr(is_err_result);
6100
6101 try self.genBody(body);
6102 try self.performReloc(reloc);
6103
6104 break :result try self.errUnionPayload(.{ .mcv = .{ .stack_offset = error_union_copy } }, error_union_ty, null);
6105 };
6106 return self.finishAir(inst, result, .{ pl_op.operand, .none, .none });
6107}
6108
6109fn airTryPtr(self: *Self, inst: Air.Inst.Index) !void {
6110 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
6111 const extra = self.air.extraData(Air.TryPtr, ty_pl.payload);
6112 const body = self.air.extra.items[extra.end..][0..extra.data.body_len];
6113 _ = body;
6114 return self.fail("TODO implement airTryPtr for arm", .{});
6115 // return self.finishAir(inst, result, .{ extra.data.ptr, .none, .none });
6116}
6117
6118fn resolveInst(self: *Self, inst: Air.Inst.Ref) InnerError!MCValue {
6119 const pt = self.pt;
6120 const zcu = pt.zcu;
6121
6122 // If the type has no codegen bits, no need to store it.
6123 const inst_ty = self.typeOf(inst);
6124 if (!inst_ty.hasRuntimeBitsIgnoreComptime(zcu) and !inst_ty.isError(zcu))
6125 return MCValue{ .none = {} };
6126
6127 const inst_index = inst.toIndex() orelse return self.genTypedValue((try self.air.value(inst, pt)).?);
6128
6129 return self.getResolvedInstValue(inst_index);
6130}
6131
6132fn getResolvedInstValue(self: *Self, inst: Air.Inst.Index) MCValue {
6133 // Treat each stack item as a "layer" on top of the previous one.
6134 var i: usize = self.branch_stack.items.len;
6135 while (true) {
6136 i -= 1;
6137 if (self.branch_stack.items[i].inst_table.get(inst)) |mcv| {
6138 assert(mcv != .dead);
6139 return mcv;
6140 }
6141 }
6142}
6143
6144fn genTypedValue(self: *Self, val: Value) InnerError!MCValue {
6145 const pt = self.pt;
6146 const mcv: MCValue = switch (try codegen.genTypedValue(
6147 self.bin_file,
6148 pt,
6149 self.src_loc,
6150 val,
6151 self.target,
6152 )) {
6153 .mcv => |mcv| switch (mcv) {
6154 .none => .none,
6155 .undef => .undef,
6156 .load_got, .load_symbol, .load_direct, .lea_symbol, .lea_direct => unreachable, // TODO
6157 .immediate => |imm| .{ .immediate = @truncate(imm) },
6158 .memory => |addr| .{ .memory = addr },
6159 },
6160 .fail => |msg| {
6161 self.err_msg = msg;
6162 return error.CodegenFail;
6163 },
6164 };
6165 return mcv;
6166}
6167
6168const CallMCValues = struct {
6169 args: []MCValue,
6170 return_value: MCValue,
6171 stack_byte_count: u32,
6172 stack_align: u32,
6173
6174 fn deinit(self: *CallMCValues, func: *Self) void {
6175 func.gpa.free(self.args);
6176 self.* = undefined;
6177 }
6178};
6179
6180/// Caller must call `CallMCValues.deinit`.
6181fn resolveCallingConventionValues(self: *Self, fn_ty: Type) !CallMCValues {
6182 const pt = self.pt;
6183 const zcu = pt.zcu;
6184 const ip = &zcu.intern_pool;
6185 const fn_info = zcu.typeToFunc(fn_ty).?;
6186 const cc = fn_info.cc;
6187 var result: CallMCValues = .{
6188 .args = try self.gpa.alloc(MCValue, fn_info.param_types.len),
6189 // These undefined values must be populated before returning from this function.
6190 .return_value = undefined,
6191 .stack_byte_count = undefined,
6192 .stack_align = undefined,
6193 };
6194 errdefer self.gpa.free(result.args);
6195
6196 const ret_ty = fn_ty.fnReturnType(zcu);
6197
6198 switch (cc) {
6199 .naked => {
6200 assert(result.args.len == 0);
6201 result.return_value = .{ .unreach = {} };
6202 result.stack_byte_count = 0;
6203 result.stack_align = 1;
6204 return result;
6205 },
6206 .arm_aapcs => {
6207 // ARM Procedure Call Standard, Chapter 6.5
6208 var ncrn: usize = 0; // Next Core Register Number
6209 var nsaa: u32 = 0; // Next stacked argument address
6210
6211 if (ret_ty.zigTypeTag(zcu) == .noreturn) {
6212 result.return_value = .{ .unreach = {} };
6213 } else if (!ret_ty.hasRuntimeBitsIgnoreComptime(zcu)) {
6214 result.return_value = .{ .none = {} };
6215 } else {
6216 const ret_ty_size: u32 = @intCast(ret_ty.abiSize(zcu));
6217 // TODO handle cases where multiple registers are used
6218 if (ret_ty_size <= 4) {
6219 result.return_value = .{ .register = c_abi_int_return_regs[0] };
6220 } else {
6221 // The result is returned by reference, not by
6222 // value. This means that r0 will contain the
6223 // address of where this function should write the
6224 // result into.
6225 result.return_value = .{ .stack_offset = 0 };
6226 ncrn = 1;
6227 }
6228 }
6229
6230 for (fn_info.param_types.get(ip), result.args) |ty, *result_arg| {
6231 if (Type.fromInterned(ty).abiAlignment(zcu) == .@"8")
6232 ncrn = std.mem.alignForward(usize, ncrn, 2);
6233
6234 const param_size: u32 = @intCast(Type.fromInterned(ty).abiSize(zcu));
6235 if (std.math.divCeil(u32, param_size, 4) catch unreachable <= 4 - ncrn) {
6236 if (param_size <= 4) {
6237 result_arg.* = .{ .register = c_abi_int_param_regs[ncrn] };
6238 ncrn += 1;
6239 } else {
6240 return self.fail("TODO MCValues with multiple registers", .{});
6241 }
6242 } else if (ncrn < 4 and nsaa == 0) {
6243 return self.fail("TODO MCValues split between registers and stack", .{});
6244 } else {
6245 ncrn = 4;
6246 if (Type.fromInterned(ty).abiAlignment(zcu) == .@"8")
6247 nsaa = std.mem.alignForward(u32, nsaa, 8);
6248
6249 result_arg.* = .{ .stack_argument_offset = nsaa };
6250 nsaa += param_size;
6251 }
6252 }
6253
6254 result.stack_byte_count = nsaa;
6255 result.stack_align = 8;
6256 },
6257 .auto => {
6258 if (ret_ty.zigTypeTag(zcu) == .noreturn) {
6259 result.return_value = .{ .unreach = {} };
6260 } else if (!ret_ty.hasRuntimeBitsIgnoreComptime(zcu) and !ret_ty.isError(zcu)) {
6261 result.return_value = .{ .none = {} };
6262 } else {
6263 const ret_ty_size: u32 = @intCast(ret_ty.abiSize(zcu));
6264 if (ret_ty_size == 0) {
6265 assert(ret_ty.isError(zcu));
6266 result.return_value = .{ .immediate = 0 };
6267 } else if (ret_ty_size <= 4) {
6268 result.return_value = .{ .register = .r0 };
6269 } else {
6270 // The result is returned by reference, not by
6271 // value. This means that r0 will contain the
6272 // address of where this function should write the
6273 // result into.
6274 result.return_value = .{ .stack_offset = 0 };
6275 }
6276 }
6277
6278 var stack_offset: u32 = 0;
6279
6280 for (fn_info.param_types.get(ip), result.args) |ty, *result_arg| {
6281 if (Type.fromInterned(ty).abiSize(zcu) > 0) {
6282 const param_size: u32 = @intCast(Type.fromInterned(ty).abiSize(zcu));
6283 const param_alignment = Type.fromInterned(ty).abiAlignment(zcu);
6284
6285 stack_offset = @intCast(param_alignment.forward(stack_offset));
6286 result_arg.* = .{ .stack_argument_offset = stack_offset };
6287 stack_offset += param_size;
6288 } else {
6289 result_arg.* = .{ .none = {} };
6290 }
6291 }
6292
6293 result.stack_byte_count = stack_offset;
6294 result.stack_align = 8;
6295 },
6296 else => return self.fail("TODO implement function parameters for {} on arm", .{cc}),
6297 }
6298
6299 return result;
6300}
6301
6302/// TODO support scope overrides. Also note this logic is duplicated with `Zcu.wantSafety`.
6303fn wantSafety(self: *Self) bool {
6304 return switch (self.bin_file.comp.root_mod.optimize_mode) {
6305 .Debug => true,
6306 .ReleaseSafe => true,
6307 .ReleaseFast => false,
6308 .ReleaseSmall => false,
6309 };
6310}
6311
6312fn fail(self: *Self, comptime format: []const u8, args: anytype) error{ OutOfMemory, CodegenFail } {
6313 @branchHint(.cold);
6314 const zcu = self.pt.zcu;
6315 const func = zcu.funcInfo(self.func_index);
6316 const msg = try ErrorMsg.create(zcu.gpa, self.src_loc, format, args);
6317 return zcu.codegenFailMsg(func.owner_nav, msg);
6318}
6319
6320fn failMsg(self: *Self, msg: *ErrorMsg) error{ OutOfMemory, CodegenFail } {
6321 @branchHint(.cold);
6322 const zcu = self.pt.zcu;
6323 const func = zcu.funcInfo(self.func_index);
6324 return zcu.codegenFailMsg(func.owner_nav, msg);
6325}
6326
6327fn parseRegName(name: []const u8) ?Register {
6328 if (@hasDecl(Register, "parseRegName")) {
6329 return Register.parseRegName(name);
6330 }
6331 return std.meta.stringToEnum(Register, name);
6332}
6333
6334fn typeOf(self: *Self, inst: Air.Inst.Ref) Type {
6335 return self.air.typeOf(inst, &self.pt.zcu.intern_pool);
6336}
6337
6338fn typeOfIndex(self: *Self, inst: Air.Inst.Index) Type {
6339 return self.air.typeOfIndex(inst, &self.pt.zcu.intern_pool);
6340}
src/arch/arm/Emit.zig deleted-712
...@@ -1,712 +0,0 @@
1//! This file contains the functionality for lowering AArch32 MIR into
2//! machine code
3
4const Emit = @This();
5const builtin = @import("builtin");
6const std = @import("std");
7const math = std.math;
8const Mir = @import("Mir.zig");
9const bits = @import("bits.zig");
10const link = @import("../../link.zig");
11const Zcu = @import("../../Zcu.zig");
12const Type = @import("../../Type.zig");
13const ErrorMsg = Zcu.ErrorMsg;
14const Target = std.Target;
15const assert = std.debug.assert;
16const Instruction = bits.Instruction;
17const Register = bits.Register;
18const log = std.log.scoped(.aarch32_emit);
19const CodeGen = @import("CodeGen.zig");
20
21mir: Mir,
22bin_file: *link.File,
23debug_output: link.File.DebugInfoOutput,
24target: *const std.Target,
25err_msg: ?*ErrorMsg = null,
26src_loc: Zcu.LazySrcLoc,
27code: *std.ArrayListUnmanaged(u8),
28
29prev_di_line: u32,
30prev_di_column: u32,
31/// Relative to the beginning of `code`.
32prev_di_pc: usize,
33
34/// The amount of stack space consumed by the saved callee-saved
35/// registers in bytes
36saved_regs_stack_space: u32,
37
38/// The final stack frame size of the function (already aligned to the
39/// respective stack alignment). Does not include prologue stack space.
40stack_size: u32,
41
42/// The branch type of every branch
43branch_types: std.AutoHashMapUnmanaged(Mir.Inst.Index, BranchType) = .empty,
44/// For every forward branch, maps the target instruction to a list of
45/// branches which branch to this target instruction
46branch_forward_origins: std.AutoHashMapUnmanaged(Mir.Inst.Index, std.ArrayListUnmanaged(Mir.Inst.Index)) = .empty,
47/// For backward branches: stores the code offset of the target
48/// instruction
49///
50/// For forward branches: stores the code offset of the branch
51/// instruction
52code_offset_mapping: std.AutoHashMapUnmanaged(Mir.Inst.Index, usize) = .empty,
53
54const InnerError = error{
55 OutOfMemory,
56 EmitFail,
57};
58
59const BranchType = enum {
60 b,
61
62 fn default(tag: Mir.Inst.Tag) BranchType {
63 return switch (tag) {
64 .b => .b,
65 else => unreachable,
66 };
67 }
68};
69
70pub fn emitMir(
71 emit: *Emit,
72) !void {
73 const mir_tags = emit.mir.instructions.items(.tag);
74
75 // Find smallest lowerings for branch instructions
76 try emit.lowerBranches();
77
78 // Emit machine code
79 for (mir_tags, 0..) |tag, index| {
80 const inst = @as(u32, @intCast(index));
81 switch (tag) {
82 .add => try emit.mirDataProcessing(inst),
83 .adds => try emit.mirDataProcessing(inst),
84 .@"and" => try emit.mirDataProcessing(inst),
85 .cmp => try emit.mirDataProcessing(inst),
86 .eor => try emit.mirDataProcessing(inst),
87 .mov => try emit.mirDataProcessing(inst),
88 .mvn => try emit.mirDataProcessing(inst),
89 .orr => try emit.mirDataProcessing(inst),
90 .rsb => try emit.mirDataProcessing(inst),
91 .sub => try emit.mirDataProcessing(inst),
92 .subs => try emit.mirDataProcessing(inst),
93
94 .sub_sp_scratch_r4 => try emit.mirSubStackPointer(inst),
95
96 .asr => try emit.mirShift(inst),
97 .lsl => try emit.mirShift(inst),
98 .lsr => try emit.mirShift(inst),
99
100 .b => try emit.mirBranch(inst),
101
102 .undefined_instruction => try emit.mirUndefinedInstruction(),
103 .bkpt => try emit.mirExceptionGeneration(inst),
104
105 .blx => try emit.mirBranchExchange(inst),
106 .bx => try emit.mirBranchExchange(inst),
107
108 .dbg_line => try emit.mirDbgLine(inst),
109
110 .dbg_prologue_end => try emit.mirDebugPrologueEnd(),
111
112 .dbg_epilogue_begin => try emit.mirDebugEpilogueBegin(),
113
114 .ldr => try emit.mirLoadStore(inst),
115 .ldrb => try emit.mirLoadStore(inst),
116 .str => try emit.mirLoadStore(inst),
117 .strb => try emit.mirLoadStore(inst),
118
119 .ldr_ptr_stack_argument => try emit.mirLoadStackArgument(inst),
120 .ldr_stack_argument => try emit.mirLoadStackArgument(inst),
121 .ldrb_stack_argument => try emit.mirLoadStackArgument(inst),
122 .ldrh_stack_argument => try emit.mirLoadStackArgument(inst),
123 .ldrsb_stack_argument => try emit.mirLoadStackArgument(inst),
124 .ldrsh_stack_argument => try emit.mirLoadStackArgument(inst),
125
126 .ldrh => try emit.mirLoadStoreExtra(inst),
127 .ldrsb => try emit.mirLoadStoreExtra(inst),
128 .ldrsh => try emit.mirLoadStoreExtra(inst),
129 .strh => try emit.mirLoadStoreExtra(inst),
130
131 .movw => try emit.mirSpecialMove(inst),
132 .movt => try emit.mirSpecialMove(inst),
133
134 .mul => try emit.mirMultiply(inst),
135 .smulbb => try emit.mirMultiply(inst),
136
137 .smull => try emit.mirMultiplyLong(inst),
138 .umull => try emit.mirMultiplyLong(inst),
139
140 .nop => try emit.mirNop(),
141
142 .pop => try emit.mirBlockDataTransfer(inst),
143 .push => try emit.mirBlockDataTransfer(inst),
144
145 .svc => try emit.mirSupervisorCall(inst),
146
147 .sbfx => try emit.mirBitFieldExtract(inst),
148 .ubfx => try emit.mirBitFieldExtract(inst),
149 }
150 }
151}
152
153pub fn deinit(emit: *Emit) void {
154 const comp = emit.bin_file.comp;
155 const gpa = comp.gpa;
156
157 var iter = emit.branch_forward_origins.valueIterator();
158 while (iter.next()) |origin_list| {
159 origin_list.deinit(gpa);
160 }
161
162 emit.branch_types.deinit(gpa);
163 emit.branch_forward_origins.deinit(gpa);
164 emit.code_offset_mapping.deinit(gpa);
165 emit.* = undefined;
166}
167
168fn optimalBranchType(emit: *Emit, tag: Mir.Inst.Tag, offset: i64) !BranchType {
169 assert(std.mem.isAlignedGeneric(i64, offset, 4)); // misaligned offset
170
171 switch (tag) {
172 .b => {
173 if (std.math.cast(i24, @divExact(offset, 4))) |_| {
174 return BranchType.b;
175 } else {
176 return emit.fail("TODO support larger branches", .{});
177 }
178 },
179 else => unreachable,
180 }
181}
182
183fn instructionSize(emit: *Emit, inst: Mir.Inst.Index) usize {
184 const tag = emit.mir.instructions.items(.tag)[inst];
185
186 if (isBranch(tag)) {
187 switch (emit.branch_types.get(inst).?) {
188 .b => return 4,
189 }
190 }
191
192 switch (tag) {
193 .dbg_line,
194 .dbg_epilogue_begin,
195 .dbg_prologue_end,
196 => return 0,
197
198 .sub_sp_scratch_r4 => {
199 const imm32 = emit.mir.instructions.items(.data)[inst].imm32;
200
201 if (imm32 == 0) {
202 return 0 * 4;
203 } else if (Instruction.Operand.fromU32(imm32) != null) {
204 // sub
205 return 1 * 4;
206 } else if (emit.target.cpu.has(.arm, .has_v7)) {
207 // movw; movt; sub
208 return 3 * 4;
209 } else {
210 // mov; orr; orr; orr; sub
211 return 5 * 4;
212 }
213 },
214
215 else => return 4,
216 }
217}
218
219fn isBranch(tag: Mir.Inst.Tag) bool {
220 return switch (tag) {
221 .b => true,
222 else => false,
223 };
224}
225
226fn branchTarget(emit: *Emit, inst: Mir.Inst.Index) Mir.Inst.Index {
227 const tag = emit.mir.instructions.items(.tag)[inst];
228
229 switch (tag) {
230 .b => return emit.mir.instructions.items(.data)[inst].inst,
231 else => unreachable,
232 }
233}
234
235fn lowerBranches(emit: *Emit) !void {
236 const comp = emit.bin_file.comp;
237 const gpa = comp.gpa;
238 const mir_tags = emit.mir.instructions.items(.tag);
239
240 // First pass: Note down all branches and their target
241 // instructions, i.e. populate branch_types,
242 // branch_forward_origins, and code_offset_mapping
243 //
244 // TODO optimization opportunity: do this in codegen while
245 // generating MIR
246 for (mir_tags, 0..) |tag, index| {
247 const inst = @as(u32, @intCast(index));
248 if (isBranch(tag)) {
249 const target_inst = emit.branchTarget(inst);
250
251 // Remember this branch instruction
252 try emit.branch_types.put(gpa, inst, BranchType.default(tag));
253
254 // Forward branches require some extra stuff: We only
255 // know their offset once we arrive at the target
256 // instruction. Therefore, we need to be able to
257 // access the branch instruction when we visit the
258 // target instruction in order to manipulate its type
259 // etc.
260 if (target_inst > inst) {
261 // Remember the branch instruction index
262 try emit.code_offset_mapping.put(gpa, inst, 0);
263
264 if (emit.branch_forward_origins.getPtr(target_inst)) |origin_list| {
265 try origin_list.append(gpa, inst);
266 } else {
267 var origin_list: std.ArrayListUnmanaged(Mir.Inst.Index) = .empty;
268 try origin_list.append(gpa, inst);
269 try emit.branch_forward_origins.put(gpa, target_inst, origin_list);
270 }
271 }
272
273 // Remember the target instruction index so that we
274 // update the real code offset in all future passes
275 //
276 // putNoClobber may not be used as the put operation
277 // may clobber the entry when multiple branches branch
278 // to the same target instruction
279 try emit.code_offset_mapping.put(gpa, target_inst, 0);
280 }
281 }
282
283 // Further passes: Until all branches are lowered, interate
284 // through all instructions and calculate new offsets and
285 // potentially new branch types
286 var all_branches_lowered = false;
287 while (!all_branches_lowered) {
288 all_branches_lowered = true;
289 var current_code_offset: usize = 0;
290
291 for (mir_tags, 0..) |tag, index| {
292 const inst = @as(u32, @intCast(index));
293
294 // If this instruction contained in the code offset
295 // mapping (when it is a target of a branch or if it is a
296 // forward branch), update the code offset
297 if (emit.code_offset_mapping.getPtr(inst)) |offset| {
298 offset.* = current_code_offset;
299 }
300
301 // If this instruction is a backward branch, calculate the
302 // offset, which may potentially update the branch type
303 if (isBranch(tag)) {
304 const target_inst = emit.branchTarget(inst);
305 if (target_inst < inst) {
306 const target_offset = emit.code_offset_mapping.get(target_inst).?;
307 const offset = @as(i64, @intCast(target_offset)) - @as(i64, @intCast(current_code_offset + 8));
308 const branch_type = emit.branch_types.getPtr(inst).?;
309 const optimal_branch_type = try emit.optimalBranchType(tag, offset);
310 if (branch_type.* != optimal_branch_type) {
311 branch_type.* = optimal_branch_type;
312 all_branches_lowered = false;
313 }
314
315 log.debug("lowerBranches: branch {} has offset {}", .{ inst, offset });
316 }
317 }
318
319 // If this instruction is the target of one or more
320 // forward branches, calculate the offset, which may
321 // potentially update the branch type
322 if (emit.branch_forward_origins.get(inst)) |origin_list| {
323 for (origin_list.items) |forward_branch_inst| {
324 const branch_tag = emit.mir.instructions.items(.tag)[forward_branch_inst];
325 const forward_branch_inst_offset = emit.code_offset_mapping.get(forward_branch_inst).?;
326 const offset = @as(i64, @intCast(current_code_offset)) - @as(i64, @intCast(forward_branch_inst_offset + 8));
327 const branch_type = emit.branch_types.getPtr(forward_branch_inst).?;
328 const optimal_branch_type = try emit.optimalBranchType(branch_tag, offset);
329 if (branch_type.* != optimal_branch_type) {
330 branch_type.* = optimal_branch_type;
331 all_branches_lowered = false;
332 }
333
334 log.debug("lowerBranches: branch {} has offset {}", .{ forward_branch_inst, offset });
335 }
336 }
337
338 // Increment code offset
339 current_code_offset += emit.instructionSize(inst);
340 }
341 }
342}
343
344fn writeInstruction(emit: *Emit, instruction: Instruction) !void {
345 const comp = emit.bin_file.comp;
346 const gpa = comp.gpa;
347 const endian = emit.target.cpu.arch.endian();
348 std.mem.writeInt(u32, try emit.code.addManyAsArray(gpa, 4), instruction.toU32(), endian);
349}
350
351fn fail(emit: *Emit, comptime format: []const u8, args: anytype) InnerError {
352 @branchHint(.cold);
353 assert(emit.err_msg == null);
354 const comp = emit.bin_file.comp;
355 const gpa = comp.gpa;
356 emit.err_msg = try ErrorMsg.create(gpa, emit.src_loc, format, args);
357 return error.EmitFail;
358}
359
360fn dbgAdvancePCAndLine(self: *Emit, line: u32, column: u32) !void {
361 const delta_line = @as(i32, @intCast(line)) - @as(i32, @intCast(self.prev_di_line));
362 const delta_pc: usize = self.code.items.len - self.prev_di_pc;
363 switch (self.debug_output) {
364 .dwarf => |dw| {
365 try dw.advancePCAndLine(delta_line, delta_pc);
366 self.prev_di_line = line;
367 self.prev_di_column = column;
368 self.prev_di_pc = self.code.items.len;
369 },
370 .plan9 => |dbg_out| {
371 if (delta_pc <= 0) return; // only do this when the pc changes
372
373 // increasing the line number
374 try link.File.Plan9.changeLine(&dbg_out.dbg_line, delta_line);
375 // increasing the pc
376 const d_pc_p9 = @as(i64, @intCast(delta_pc)) - dbg_out.pc_quanta;
377 if (d_pc_p9 > 0) {
378 // minus one because if its the last one, we want to leave space to change the line which is one pc quanta
379 try dbg_out.dbg_line.append(@as(u8, @intCast(@divExact(d_pc_p9, dbg_out.pc_quanta) + 128)) - dbg_out.pc_quanta);
380 if (dbg_out.pcop_change_index) |pci|
381 dbg_out.dbg_line.items[pci] += 1;
382 dbg_out.pcop_change_index = @as(u32, @intCast(dbg_out.dbg_line.items.len - 1));
383 } else if (d_pc_p9 == 0) {
384 // we don't need to do anything, because adding the pc quanta does it for us
385 } else unreachable;
386 if (dbg_out.start_line == null)
387 dbg_out.start_line = self.prev_di_line;
388 dbg_out.end_line = line;
389 // only do this if the pc changed
390 self.prev_di_line = line;
391 self.prev_di_column = column;
392 self.prev_di_pc = self.code.items.len;
393 },
394 .none => {},
395 }
396}
397
398fn mirDataProcessing(emit: *Emit, inst: Mir.Inst.Index) !void {
399 const tag = emit.mir.instructions.items(.tag)[inst];
400 const cond = emit.mir.instructions.items(.cond)[inst];
401
402 switch (tag) {
403 .add,
404 .adds,
405 .@"and",
406 .eor,
407 .orr,
408 .rsb,
409 .sub,
410 .subs,
411 => {
412 const rr_op = emit.mir.instructions.items(.data)[inst].rr_op;
413 switch (tag) {
414 .add => try emit.writeInstruction(Instruction.add(cond, rr_op.rd, rr_op.rn, rr_op.op)),
415 .adds => try emit.writeInstruction(Instruction.adds(cond, rr_op.rd, rr_op.rn, rr_op.op)),
416 .@"and" => try emit.writeInstruction(Instruction.@"and"(cond, rr_op.rd, rr_op.rn, rr_op.op)),
417 .eor => try emit.writeInstruction(Instruction.eor(cond, rr_op.rd, rr_op.rn, rr_op.op)),
418 .orr => try emit.writeInstruction(Instruction.orr(cond, rr_op.rd, rr_op.rn, rr_op.op)),
419 .rsb => try emit.writeInstruction(Instruction.rsb(cond, rr_op.rd, rr_op.rn, rr_op.op)),
420 .sub => try emit.writeInstruction(Instruction.sub(cond, rr_op.rd, rr_op.rn, rr_op.op)),
421 .subs => try emit.writeInstruction(Instruction.subs(cond, rr_op.rd, rr_op.rn, rr_op.op)),
422 else => unreachable,
423 }
424 },
425 .cmp => {
426 const r_op_cmp = emit.mir.instructions.items(.data)[inst].r_op_cmp;
427 try emit.writeInstruction(Instruction.cmp(cond, r_op_cmp.rn, r_op_cmp.op));
428 },
429 .mov,
430 .mvn,
431 => {
432 const r_op_mov = emit.mir.instructions.items(.data)[inst].r_op_mov;
433 switch (tag) {
434 .mov => try emit.writeInstruction(Instruction.mov(cond, r_op_mov.rd, r_op_mov.op)),
435 .mvn => try emit.writeInstruction(Instruction.mvn(cond, r_op_mov.rd, r_op_mov.op)),
436 else => unreachable,
437 }
438 },
439 else => unreachable,
440 }
441}
442
443fn mirSubStackPointer(emit: *Emit, inst: Mir.Inst.Index) !void {
444 const tag = emit.mir.instructions.items(.tag)[inst];
445 const cond = emit.mir.instructions.items(.cond)[inst];
446 const imm32 = emit.mir.instructions.items(.data)[inst].imm32;
447
448 switch (tag) {
449 .sub_sp_scratch_r4 => {
450 if (imm32 == 0) return;
451
452 const operand = Instruction.Operand.fromU32(imm32) orelse blk: {
453 const scratch: Register = .r4;
454
455 if (emit.target.cpu.has(.arm, .has_v7)) {
456 try emit.writeInstruction(Instruction.movw(cond, scratch, @as(u16, @truncate(imm32))));
457 try emit.writeInstruction(Instruction.movt(cond, scratch, @as(u16, @truncate(imm32 >> 16))));
458 } else {
459 try emit.writeInstruction(Instruction.mov(cond, scratch, Instruction.Operand.imm(@as(u8, @truncate(imm32)), 0)));
460 try emit.writeInstruction(Instruction.orr(cond, scratch, scratch, Instruction.Operand.imm(@as(u8, @truncate(imm32 >> 8)), 12)));
461 try emit.writeInstruction(Instruction.orr(cond, scratch, scratch, Instruction.Operand.imm(@as(u8, @truncate(imm32 >> 16)), 8)));
462 try emit.writeInstruction(Instruction.orr(cond, scratch, scratch, Instruction.Operand.imm(@as(u8, @truncate(imm32 >> 24)), 4)));
463 }
464
465 break :blk Instruction.Operand.reg(scratch, Instruction.Operand.Shift.none);
466 };
467
468 try emit.writeInstruction(Instruction.sub(cond, .sp, .sp, operand));
469 },
470 else => unreachable,
471 }
472}
473
474fn mirShift(emit: *Emit, inst: Mir.Inst.Index) !void {
475 const tag = emit.mir.instructions.items(.tag)[inst];
476 const cond = emit.mir.instructions.items(.cond)[inst];
477 const rr_shift = emit.mir.instructions.items(.data)[inst].rr_shift;
478
479 switch (tag) {
480 .asr => try emit.writeInstruction(Instruction.asr(cond, rr_shift.rd, rr_shift.rm, rr_shift.shift_amount)),
481 .lsl => try emit.writeInstruction(Instruction.lsl(cond, rr_shift.rd, rr_shift.rm, rr_shift.shift_amount)),
482 .lsr => try emit.writeInstruction(Instruction.lsr(cond, rr_shift.rd, rr_shift.rm, rr_shift.shift_amount)),
483 else => unreachable,
484 }
485}
486
487fn mirBranch(emit: *Emit, inst: Mir.Inst.Index) !void {
488 const tag = emit.mir.instructions.items(.tag)[inst];
489 const cond = emit.mir.instructions.items(.cond)[inst];
490 const target_inst = emit.mir.instructions.items(.data)[inst].inst;
491
492 const offset = @as(i64, @intCast(emit.code_offset_mapping.get(target_inst).?)) - @as(i64, @intCast(emit.code.items.len + 8));
493 const branch_type = emit.branch_types.get(inst).?;
494
495 switch (branch_type) {
496 .b => switch (tag) {
497 .b => try emit.writeInstruction(Instruction.b(cond, @as(i26, @intCast(offset)))),
498 else => unreachable,
499 },
500 }
501}
502
503fn mirUndefinedInstruction(emit: *Emit) !void {
504 try emit.writeInstruction(Instruction.undefinedInstruction());
505}
506
507fn mirExceptionGeneration(emit: *Emit, inst: Mir.Inst.Index) !void {
508 const tag = emit.mir.instructions.items(.tag)[inst];
509 const imm16 = emit.mir.instructions.items(.data)[inst].imm16;
510
511 switch (tag) {
512 .bkpt => try emit.writeInstruction(Instruction.bkpt(imm16)),
513 else => unreachable,
514 }
515}
516
517fn mirBranchExchange(emit: *Emit, inst: Mir.Inst.Index) !void {
518 const tag = emit.mir.instructions.items(.tag)[inst];
519 const cond = emit.mir.instructions.items(.cond)[inst];
520 const reg = emit.mir.instructions.items(.data)[inst].reg;
521
522 switch (tag) {
523 .blx => try emit.writeInstruction(Instruction.blx(cond, reg)),
524 .bx => try emit.writeInstruction(Instruction.bx(cond, reg)),
525 else => unreachable,
526 }
527}
528
529fn mirDbgLine(emit: *Emit, inst: Mir.Inst.Index) !void {
530 const tag = emit.mir.instructions.items(.tag)[inst];
531 const dbg_line_column = emit.mir.instructions.items(.data)[inst].dbg_line_column;
532
533 switch (tag) {
534 .dbg_line => try emit.dbgAdvancePCAndLine(dbg_line_column.line, dbg_line_column.column),
535 else => unreachable,
536 }
537}
538
539fn mirDebugPrologueEnd(emit: *Emit) !void {
540 switch (emit.debug_output) {
541 .dwarf => |dw| {
542 try dw.setPrologueEnd();
543 try emit.dbgAdvancePCAndLine(emit.prev_di_line, emit.prev_di_column);
544 },
545 .plan9 => {},
546 .none => {},
547 }
548}
549
550fn mirDebugEpilogueBegin(emit: *Emit) !void {
551 switch (emit.debug_output) {
552 .dwarf => |dw| {
553 try dw.setEpilogueBegin();
554 try emit.dbgAdvancePCAndLine(emit.prev_di_line, emit.prev_di_column);
555 },
556 .plan9 => {},
557 .none => {},
558 }
559}
560
561fn mirLoadStore(emit: *Emit, inst: Mir.Inst.Index) !void {
562 const tag = emit.mir.instructions.items(.tag)[inst];
563 const cond = emit.mir.instructions.items(.cond)[inst];
564 const rr_offset = emit.mir.instructions.items(.data)[inst].rr_offset;
565
566 switch (tag) {
567 .ldr => try emit.writeInstruction(Instruction.ldr(cond, rr_offset.rt, rr_offset.rn, rr_offset.offset)),
568 .ldrb => try emit.writeInstruction(Instruction.ldrb(cond, rr_offset.rt, rr_offset.rn, rr_offset.offset)),
569 .str => try emit.writeInstruction(Instruction.str(cond, rr_offset.rt, rr_offset.rn, rr_offset.offset)),
570 .strb => try emit.writeInstruction(Instruction.strb(cond, rr_offset.rt, rr_offset.rn, rr_offset.offset)),
571 else => unreachable,
572 }
573}
574
575fn mirLoadStackArgument(emit: *Emit, inst: Mir.Inst.Index) !void {
576 const tag = emit.mir.instructions.items(.tag)[inst];
577 const cond = emit.mir.instructions.items(.cond)[inst];
578 const r_stack_offset = emit.mir.instructions.items(.data)[inst].r_stack_offset;
579 const rt = r_stack_offset.rt;
580
581 const raw_offset = emit.stack_size + emit.saved_regs_stack_space + r_stack_offset.stack_offset;
582 switch (tag) {
583 .ldr_ptr_stack_argument => {
584 const operand = Instruction.Operand.fromU32(raw_offset) orelse
585 return emit.fail("TODO mirLoadStack larger offsets", .{});
586
587 try emit.writeInstruction(Instruction.add(cond, rt, .sp, operand));
588 },
589 .ldr_stack_argument,
590 .ldrb_stack_argument,
591 => {
592 const offset = if (raw_offset <= math.maxInt(u12)) blk: {
593 break :blk Instruction.Offset.imm(@as(u12, @intCast(raw_offset)));
594 } else return emit.fail("TODO mirLoadStack larger offsets", .{});
595
596 switch (tag) {
597 .ldr_stack_argument => try emit.writeInstruction(Instruction.ldr(cond, rt, .sp, .{ .offset = offset })),
598 .ldrb_stack_argument => try emit.writeInstruction(Instruction.ldrb(cond, rt, .sp, .{ .offset = offset })),
599 else => unreachable,
600 }
601 },
602 .ldrh_stack_argument,
603 .ldrsb_stack_argument,
604 .ldrsh_stack_argument,
605 => {
606 const offset = if (raw_offset <= math.maxInt(u8)) blk: {
607 break :blk Instruction.ExtraLoadStoreOffset.imm(@as(u8, @intCast(raw_offset)));
608 } else return emit.fail("TODO mirLoadStack larger offsets", .{});
609
610 switch (tag) {
611 .ldrh_stack_argument => try emit.writeInstruction(Instruction.ldrh(cond, rt, .sp, .{ .offset = offset })),
612 .ldrsb_stack_argument => try emit.writeInstruction(Instruction.ldrsb(cond, rt, .sp, .{ .offset = offset })),
613 .ldrsh_stack_argument => try emit.writeInstruction(Instruction.ldrsh(cond, rt, .sp, .{ .offset = offset })),
614 else => unreachable,
615 }
616 },
617 else => unreachable,
618 }
619}
620
621fn mirLoadStoreExtra(emit: *Emit, inst: Mir.Inst.Index) !void {
622 const tag = emit.mir.instructions.items(.tag)[inst];
623 const cond = emit.mir.instructions.items(.cond)[inst];
624 const rr_extra_offset = emit.mir.instructions.items(.data)[inst].rr_extra_offset;
625
626 switch (tag) {
627 .ldrh => try emit.writeInstruction(Instruction.ldrh(cond, rr_extra_offset.rt, rr_extra_offset.rn, rr_extra_offset.offset)),
628 .ldrsb => try emit.writeInstruction(Instruction.ldrsb(cond, rr_extra_offset.rt, rr_extra_offset.rn, rr_extra_offset.offset)),
629 .ldrsh => try emit.writeInstruction(Instruction.ldrsh(cond, rr_extra_offset.rt, rr_extra_offset.rn, rr_extra_offset.offset)),
630 .strh => try emit.writeInstruction(Instruction.strh(cond, rr_extra_offset.rt, rr_extra_offset.rn, rr_extra_offset.offset)),
631 else => unreachable,
632 }
633}
634
635fn mirSpecialMove(emit: *Emit, inst: Mir.Inst.Index) !void {
636 const tag = emit.mir.instructions.items(.tag)[inst];
637 const cond = emit.mir.instructions.items(.cond)[inst];
638 const r_imm16 = emit.mir.instructions.items(.data)[inst].r_imm16;
639
640 switch (tag) {
641 .movw => try emit.writeInstruction(Instruction.movw(cond, r_imm16.rd, r_imm16.imm16)),
642 .movt => try emit.writeInstruction(Instruction.movt(cond, r_imm16.rd, r_imm16.imm16)),
643 else => unreachable,
644 }
645}
646
647fn mirMultiply(emit: *Emit, inst: Mir.Inst.Index) !void {
648 const tag = emit.mir.instructions.items(.tag)[inst];
649 const cond = emit.mir.instructions.items(.cond)[inst];
650 const rrr = emit.mir.instructions.items(.data)[inst].rrr;
651
652 switch (tag) {
653 .mul => try emit.writeInstruction(Instruction.mul(cond, rrr.rd, rrr.rn, rrr.rm)),
654 .smulbb => try emit.writeInstruction(Instruction.smulbb(cond, rrr.rd, rrr.rn, rrr.rm)),
655 else => unreachable,
656 }
657}
658
659fn mirMultiplyLong(emit: *Emit, inst: Mir.Inst.Index) !void {
660 const tag = emit.mir.instructions.items(.tag)[inst];
661 const cond = emit.mir.instructions.items(.cond)[inst];
662 const rrrr = emit.mir.instructions.items(.data)[inst].rrrr;
663
664 switch (tag) {
665 .smull => try emit.writeInstruction(Instruction.smull(cond, rrrr.rdlo, rrrr.rdhi, rrrr.rn, rrrr.rm)),
666 .umull => try emit.writeInstruction(Instruction.umull(cond, rrrr.rdlo, rrrr.rdhi, rrrr.rn, rrrr.rm)),
667 else => unreachable,
668 }
669}
670
671fn mirNop(emit: *Emit) !void {
672 try emit.writeInstruction(Instruction.nop());
673}
674
675fn mirBlockDataTransfer(emit: *Emit, inst: Mir.Inst.Index) !void {
676 const tag = emit.mir.instructions.items(.tag)[inst];
677 const cond = emit.mir.instructions.items(.cond)[inst];
678 const register_list = emit.mir.instructions.items(.data)[inst].register_list;
679
680 switch (tag) {
681 .pop => try emit.writeInstruction(Instruction.ldm(cond, .sp, true, register_list)),
682 .push => try emit.writeInstruction(Instruction.stmdb(cond, .sp, true, register_list)),
683 else => unreachable,
684 }
685}
686
687fn mirSupervisorCall(emit: *Emit, inst: Mir.Inst.Index) !void {
688 const tag = emit.mir.instructions.items(.tag)[inst];
689 const cond = emit.mir.instructions.items(.cond)[inst];
690 const imm24 = emit.mir.instructions.items(.data)[inst].imm24;
691
692 switch (tag) {
693 .svc => try emit.writeInstruction(Instruction.svc(cond, imm24)),
694 else => unreachable,
695 }
696}
697
698fn mirBitFieldExtract(emit: *Emit, inst: Mir.Inst.Index) !void {
699 const tag = emit.mir.instructions.items(.tag)[inst];
700 const cond = emit.mir.instructions.items(.cond)[inst];
701 const rr_lsb_width = emit.mir.instructions.items(.data)[inst].rr_lsb_width;
702 const rd = rr_lsb_width.rd;
703 const rn = rr_lsb_width.rn;
704 const lsb = rr_lsb_width.lsb;
705 const width = rr_lsb_width.width;
706
707 switch (tag) {
708 .sbfx => try emit.writeInstruction(Instruction.sbfx(cond, rd, rn, lsb, width)),
709 .ubfx => try emit.writeInstruction(Instruction.ubfx(cond, rd, rn, lsb, width)),
710 else => unreachable,
711 }
712}
src/arch/arm/Mir.zig deleted-340
...@@ -1,340 +0,0 @@
1//! Machine Intermediate Representation.
2//! This data is produced by ARM Codegen or ARM assembly parsing
3//! These instructions have a 1:1 correspondence with machine code instructions
4//! for the target. MIR can be lowered to source-annotated textual assembly code
5//! instructions, or it can be lowered to machine code.
6//! The main purpose of MIR is to postpone the assignment of offsets until Isel,
7//! so that, for example, the smaller encodings of jump instructions can be used.
8
9const Mir = @This();
10const std = @import("std");
11const builtin = @import("builtin");
12const assert = std.debug.assert;
13
14const bits = @import("bits.zig");
15const Register = bits.Register;
16const InternPool = @import("../../InternPool.zig");
17const Emit = @import("Emit.zig");
18const codegen = @import("../../codegen.zig");
19const link = @import("../../link.zig");
20const Zcu = @import("../../Zcu.zig");
21
22max_end_stack: u32,
23saved_regs_stack_space: u32,
24
25instructions: std.MultiArrayList(Inst).Slice,
26/// The meaning of this data is determined by `Inst.Tag` value.
27extra: []const u32,
28
29pub const Inst = struct {
30 tag: Tag,
31 cond: bits.Condition = .al,
32 /// The meaning of this depends on `tag`.
33 data: Data,
34
35 pub const Tag = enum(u16) {
36 /// Add
37 add,
38 /// Add, update condition flags
39 adds,
40 /// Bitwise AND
41 @"and",
42 /// Arithmetic Shift Right
43 asr,
44 /// Branch
45 b,
46 /// Undefined instruction
47 undefined_instruction,
48 /// Breakpoint
49 bkpt,
50 /// Branch with Link and Exchange
51 blx,
52 /// Branch and Exchange
53 bx,
54 /// Compare
55 cmp,
56 /// Pseudo-instruction: End of prologue
57 dbg_prologue_end,
58 /// Pseudo-instruction: Beginning of epilogue
59 dbg_epilogue_begin,
60 /// Pseudo-instruction: Update debug line
61 dbg_line,
62 /// Bitwise Exclusive OR
63 eor,
64 /// Load Register
65 ldr,
66 /// Pseudo-instruction: Load pointer to stack argument offset
67 ldr_ptr_stack_argument,
68 /// Load Register
69 ldr_stack_argument,
70 /// Load Register Byte
71 ldrb,
72 /// Load Register Byte
73 ldrb_stack_argument,
74 /// Load Register Halfword
75 ldrh,
76 /// Load Register Halfword
77 ldrh_stack_argument,
78 /// Load Register Signed Byte
79 ldrsb,
80 /// Load Register Signed Byte
81 ldrsb_stack_argument,
82 /// Load Register Signed Halfword
83 ldrsh,
84 /// Load Register Signed Halfword
85 ldrsh_stack_argument,
86 /// Logical Shift Left
87 lsl,
88 /// Logical Shift Right
89 lsr,
90 /// Move
91 mov,
92 /// Move
93 movw,
94 /// Move Top
95 movt,
96 /// Multiply
97 mul,
98 /// Bitwise NOT
99 mvn,
100 /// No Operation
101 nop,
102 /// Bitwise OR
103 orr,
104 /// Pop multiple registers from Stack
105 pop,
106 /// Push multiple registers to Stack
107 push,
108 /// Reverse Subtract
109 rsb,
110 /// Signed Bit Field Extract
111 sbfx,
112 /// Signed Multiply (halfwords), bottom half, bottom half
113 smulbb,
114 /// Signed Multiply Long
115 smull,
116 /// Store Register
117 str,
118 /// Store Register Byte
119 strb,
120 /// Store Register Halfword
121 strh,
122 /// Subtract
123 sub,
124 /// Pseudo-instruction: Subtract 32-bit immediate from stack
125 ///
126 /// r4 can be used by Emit as a scratch register for loading
127 /// the immediate
128 sub_sp_scratch_r4,
129 /// Subtract, update condition flags
130 subs,
131 /// Supervisor Call
132 svc,
133 /// Unsigned Bit Field Extract
134 ubfx,
135 /// Unsigned Multiply Long
136 umull,
137 };
138
139 /// The position of an MIR instruction within the `Mir` instructions array.
140 pub const Index = u32;
141
142 /// All instructions have a 8-byte payload, which is contained within
143 /// this union. `Tag` determines which union field is active, as well as
144 /// how to interpret the data within.
145 pub const Data = union {
146 /// No additional data
147 ///
148 /// Used by e.g. nop
149 nop: void,
150 /// Another instruction
151 ///
152 /// Used by e.g. b
153 inst: Index,
154 /// A 16-bit immediate value.
155 ///
156 /// Used by e.g. bkpt
157 imm16: u16,
158 /// A 24-bit immediate value.
159 ///
160 /// Used by e.g. svc
161 imm24: u24,
162 /// A 32-bit immediate value.
163 ///
164 /// Used by e.g. sub_sp_scratch_r0
165 imm32: u32,
166 /// Index into `extra`. Meaning of what can be found there is context-dependent.
167 ///
168 /// Used by e.g. load_memory
169 payload: u32,
170 /// A register
171 ///
172 /// Used by e.g. blx
173 reg: Register,
174 /// A register and a stack offset
175 ///
176 /// Used by e.g. ldr_stack_argument
177 r_stack_offset: struct {
178 rt: Register,
179 stack_offset: u32,
180 },
181 /// A register and a 16-bit unsigned immediate
182 ///
183 /// Used by e.g. movw
184 r_imm16: struct {
185 rd: Register,
186 imm16: u16,
187 },
188 /// A register and an operand
189 ///
190 /// Used by mov and mvn
191 r_op_mov: struct {
192 rd: Register,
193 op: bits.Instruction.Operand,
194 },
195 /// A register and an operand
196 ///
197 /// Used by cmp
198 r_op_cmp: struct {
199 rn: Register,
200 op: bits.Instruction.Operand,
201 },
202 /// Two registers and a shift amount
203 ///
204 /// Used by e.g. lsl
205 rr_shift: struct {
206 rd: Register,
207 rm: Register,
208 shift_amount: bits.Instruction.ShiftAmount,
209 },
210 /// Two registers and an operand
211 ///
212 /// Used by e.g. sub
213 rr_op: struct {
214 rd: Register,
215 rn: Register,
216 op: bits.Instruction.Operand,
217 },
218 /// Two registers and an offset
219 ///
220 /// Used by e.g. ldr
221 rr_offset: struct {
222 rt: Register,
223 rn: Register,
224 offset: bits.Instruction.OffsetArgs,
225 },
226 /// Two registers and an extra load/store offset
227 ///
228 /// Used by e.g. ldrh
229 rr_extra_offset: struct {
230 rt: Register,
231 rn: Register,
232 offset: bits.Instruction.ExtraLoadStoreOffsetArgs,
233 },
234 /// Two registers and a lsb (range 0-31) and a width (range
235 /// 1-32)
236 ///
237 /// Used by e.g. sbfx
238 rr_lsb_width: struct {
239 rd: Register,
240 rn: Register,
241 lsb: u5,
242 width: u6,
243 },
244 /// Three registers
245 ///
246 /// Used by e.g. mul
247 rrr: struct {
248 rd: Register,
249 rn: Register,
250 rm: Register,
251 },
252 /// Four registers
253 ///
254 /// Used by e.g. smull
255 rrrr: struct {
256 rdlo: Register,
257 rdhi: Register,
258 rn: Register,
259 rm: Register,
260 },
261 /// An unordered list of registers
262 ///
263 /// Used by e.g. push
264 register_list: bits.Instruction.RegisterList,
265 /// Debug info: line and column
266 ///
267 /// Used by e.g. dbg_line
268 dbg_line_column: struct {
269 line: u32,
270 column: u32,
271 },
272 };
273
274 // Make sure we don't accidentally make instructions bigger than expected.
275 // Note that in safety builds, Zig is allowed to insert a secret field for safety checks.
276 comptime {
277 if (!std.debug.runtime_safety) {
278 assert(@sizeOf(Data) == 8);
279 }
280 }
281};
282
283pub fn deinit(mir: *Mir, gpa: std.mem.Allocator) void {
284 mir.instructions.deinit(gpa);
285 gpa.free(mir.extra);
286 mir.* = undefined;
287}
288
289pub fn emit(
290 mir: Mir,
291 lf: *link.File,
292 pt: Zcu.PerThread,
293 src_loc: Zcu.LazySrcLoc,
294 func_index: InternPool.Index,
295 code: *std.ArrayListUnmanaged(u8),
296 debug_output: link.File.DebugInfoOutput,
297) codegen.CodeGenError!void {
298 const zcu = pt.zcu;
299 const func = zcu.funcInfo(func_index);
300 const nav = func.owner_nav;
301 const mod = zcu.navFileScope(nav).mod.?;
302 var e: Emit = .{
303 .mir = mir,
304 .bin_file = lf,
305 .debug_output = debug_output,
306 .target = &mod.resolved_target.result,
307 .src_loc = src_loc,
308 .code = code,
309 .prev_di_pc = 0,
310 .prev_di_line = func.lbrace_line,
311 .prev_di_column = func.lbrace_column,
312 .stack_size = mir.max_end_stack,
313 .saved_regs_stack_space = mir.saved_regs_stack_space,
314 };
315 defer e.deinit();
316 e.emitMir() catch |err| switch (err) {
317 error.EmitFail => return zcu.codegenFailMsg(nav, e.err_msg.?),
318 else => |e1| return e1,
319 };
320}
321
322/// Returns the requested data, as well as the new index which is at the start of the
323/// trailers for the object.
324pub fn extraData(mir: Mir, comptime T: type, index: usize) struct { data: T, end: usize } {
325 const fields = std.meta.fields(T);
326 var i: usize = index;
327 var result: T = undefined;
328 inline for (fields) |field| {
329 @field(result, field.name) = switch (field.type) {
330 u32 => mir.extra[i],
331 i32 => @as(i32, @bitCast(mir.extra[i])),
332 else => @compileError("bad field type"),
333 };
334 i += 1;
335 }
336 return .{
337 .data = result,
338 .end = i,
339 };
340}
src/arch/arm/abi.zig deleted-187
...@@ -1,187 +0,0 @@
1const std = @import("std");
2const assert = std.debug.assert;
3const bits = @import("bits.zig");
4const Register = bits.Register;
5const RegisterManagerFn = @import("../../register_manager.zig").RegisterManager;
6const Type = @import("../../Type.zig");
7const Zcu = @import("../../Zcu.zig");
8
9pub const Class = union(enum) {
10 memory,
11 byval,
12 i32_array: u8,
13 i64_array: u8,
14
15 fn arrSize(total_size: u64, arr_size: u64) Class {
16 const count = @as(u8, @intCast(std.mem.alignForward(u64, total_size, arr_size) / arr_size));
17 if (arr_size == 32) {
18 return .{ .i32_array = count };
19 } else {
20 return .{ .i64_array = count };
21 }
22 }
23};
24
25pub const Context = enum { ret, arg };
26
27pub fn classifyType(ty: Type, zcu: *Zcu, ctx: Context) Class {
28 assert(ty.hasRuntimeBitsIgnoreComptime(zcu));
29
30 var maybe_float_bits: ?u16 = null;
31 const max_byval_size = 512;
32 const ip = &zcu.intern_pool;
33 switch (ty.zigTypeTag(zcu)) {
34 .@"struct" => {
35 const bit_size = ty.bitSize(zcu);
36 if (ty.containerLayout(zcu) == .@"packed") {
37 if (bit_size > 64) return .memory;
38 return .byval;
39 }
40 if (bit_size > max_byval_size) return .memory;
41 const float_count = countFloats(ty, zcu, &maybe_float_bits);
42 if (float_count <= byval_float_count) return .byval;
43
44 const fields = ty.structFieldCount(zcu);
45 var i: u32 = 0;
46 while (i < fields) : (i += 1) {
47 const field_ty = ty.fieldType(i, zcu);
48 const field_alignment = ty.fieldAlignment(i, zcu);
49 const field_size = field_ty.bitSize(zcu);
50 if (field_size > 32 or field_alignment.compare(.gt, .@"32")) {
51 return Class.arrSize(bit_size, 64);
52 }
53 }
54 return Class.arrSize(bit_size, 32);
55 },
56 .@"union" => {
57 const bit_size = ty.bitSize(zcu);
58 const union_obj = zcu.typeToUnion(ty).?;
59 if (union_obj.flagsUnordered(ip).layout == .@"packed") {
60 if (bit_size > 64) return .memory;
61 return .byval;
62 }
63 if (bit_size > max_byval_size) return .memory;
64 const float_count = countFloats(ty, zcu, &maybe_float_bits);
65 if (float_count <= byval_float_count) return .byval;
66
67 for (union_obj.field_types.get(ip), 0..) |field_ty, field_index| {
68 if (Type.fromInterned(field_ty).bitSize(zcu) > 32 or
69 ty.fieldAlignment(field_index, zcu).compare(.gt, .@"32"))
70 {
71 return Class.arrSize(bit_size, 64);
72 }
73 }
74 return Class.arrSize(bit_size, 32);
75 },
76 .bool, .float => return .byval,
77 .int => {
78 // TODO this is incorrect for _BitInt(128) but implementing
79 // this correctly makes implementing compiler-rt impossible.
80 // const bit_size = ty.bitSize(zcu);
81 // if (bit_size > 64) return .memory;
82 return .byval;
83 },
84 .@"enum", .error_set => {
85 const bit_size = ty.bitSize(zcu);
86 if (bit_size > 64) return .memory;
87 return .byval;
88 },
89 .vector => {
90 const bit_size = ty.bitSize(zcu);
91 // TODO is this controlled by a cpu feature?
92 if (ctx == .ret and bit_size > 128) return .memory;
93 if (bit_size > 512) return .memory;
94 return .byval;
95 },
96 .optional => {
97 assert(ty.isPtrLikeOptional(zcu));
98 return .byval;
99 },
100 .pointer => {
101 assert(!ty.isSlice(zcu));
102 return .byval;
103 },
104 .error_union,
105 .frame,
106 .@"anyframe",
107 .noreturn,
108 .void,
109 .type,
110 .comptime_float,
111 .comptime_int,
112 .undefined,
113 .null,
114 .@"fn",
115 .@"opaque",
116 .enum_literal,
117 .array,
118 => unreachable,
119 }
120}
121
122const byval_float_count = 4;
123fn countFloats(ty: Type, zcu: *Zcu, maybe_float_bits: *?u16) u32 {
124 const ip = &zcu.intern_pool;
125 const target = zcu.getTarget();
126 const invalid = std.math.maxInt(u32);
127 switch (ty.zigTypeTag(zcu)) {
128 .@"union" => {
129 const union_obj = zcu.typeToUnion(ty).?;
130 var max_count: u32 = 0;
131 for (union_obj.field_types.get(ip)) |field_ty| {
132 const field_count = countFloats(Type.fromInterned(field_ty), zcu, maybe_float_bits);
133 if (field_count == invalid) return invalid;
134 if (field_count > max_count) max_count = field_count;
135 if (max_count > byval_float_count) return invalid;
136 }
137 return max_count;
138 },
139 .@"struct" => {
140 const fields_len = ty.structFieldCount(zcu);
141 var count: u32 = 0;
142 var i: u32 = 0;
143 while (i < fields_len) : (i += 1) {
144 const field_ty = ty.fieldType(i, zcu);
145 const field_count = countFloats(field_ty, zcu, maybe_float_bits);
146 if (field_count == invalid) return invalid;
147 count += field_count;
148 if (count > byval_float_count) return invalid;
149 }
150 return count;
151 },
152 .float => {
153 const float_bits = maybe_float_bits.* orelse {
154 const float_bits = ty.floatBits(target);
155 if (float_bits != 32 and float_bits != 64) return invalid;
156 maybe_float_bits.* = float_bits;
157 return 1;
158 };
159 if (ty.floatBits(target) == float_bits) return 1;
160 return invalid;
161 },
162 .void => return 0,
163 else => return invalid,
164 }
165}
166
167pub const callee_preserved_regs = [_]Register{ .r4, .r5, .r6, .r7, .r8, .r10 };
168pub const caller_preserved_regs = [_]Register{ .r0, .r1, .r2, .r3 };
169
170pub const c_abi_int_param_regs = [_]Register{ .r0, .r1, .r2, .r3 };
171pub const c_abi_int_return_regs = [_]Register{ .r0, .r1 };
172
173const allocatable_registers = callee_preserved_regs ++ caller_preserved_regs;
174pub const RegisterManager = RegisterManagerFn(@import("CodeGen.zig"), Register, &allocatable_registers);
175
176// Register classes
177const RegisterBitSet = RegisterManager.RegisterBitSet;
178pub const RegisterClass = struct {
179 pub const gp: RegisterBitSet = blk: {
180 var set = RegisterBitSet.initEmpty();
181 set.setRangeValue(.{
182 .start = 0,
183 .end = caller_preserved_regs.len + callee_preserved_regs.len,
184 }, true);
185 break :blk set;
186 };
187};
src/arch/arm/bits.zig deleted-1566
...@@ -1,1566 +0,0 @@
1const std = @import("std");
2const assert = std.debug.assert;
3const testing = std.testing;
4
5/// The condition field specifies the flags necessary for an
6/// Instruction to be executed
7pub const Condition = enum(u4) {
8 /// equal
9 eq,
10 /// not equal
11 ne,
12 /// unsigned higher or same
13 cs,
14 /// unsigned lower
15 cc,
16 /// negative
17 mi,
18 /// positive or zero
19 pl,
20 /// overflow
21 vs,
22 /// no overflow
23 vc,
24 /// unsigned higer
25 hi,
26 /// unsigned lower or same
27 ls,
28 /// greater or equal
29 ge,
30 /// less than
31 lt,
32 /// greater than
33 gt,
34 /// less than or equal
35 le,
36 /// always
37 al,
38
39 /// Converts a std.math.CompareOperator into a condition flag,
40 /// i.e. returns the condition that is true iff the result of the
41 /// comparison is true. Assumes signed comparison
42 pub fn fromCompareOperatorSigned(op: std.math.CompareOperator) Condition {
43 return switch (op) {
44 .gte => .ge,
45 .gt => .gt,
46 .neq => .ne,
47 .lt => .lt,
48 .lte => .le,
49 .eq => .eq,
50 };
51 }
52
53 /// Converts a std.math.CompareOperator into a condition flag,
54 /// i.e. returns the condition that is true iff the result of the
55 /// comparison is true. Assumes unsigned comparison
56 pub fn fromCompareOperatorUnsigned(op: std.math.CompareOperator) Condition {
57 return switch (op) {
58 .gte => .cs,
59 .gt => .hi,
60 .neq => .ne,
61 .lt => .cc,
62 .lte => .ls,
63 .eq => .eq,
64 };
65 }
66
67 /// Returns the condition which is true iff the given condition is
68 /// false (if such a condition exists)
69 pub fn negate(cond: Condition) Condition {
70 return switch (cond) {
71 .eq => .ne,
72 .ne => .eq,
73 .cs => .cc,
74 .cc => .cs,
75 .mi => .pl,
76 .pl => .mi,
77 .vs => .vc,
78 .vc => .vs,
79 .hi => .ls,
80 .ls => .hi,
81 .ge => .lt,
82 .lt => .ge,
83 .gt => .le,
84 .le => .gt,
85 .al => unreachable,
86 };
87 }
88};
89
90test "condition from CompareOperator" {
91 try testing.expectEqual(@as(Condition, .eq), Condition.fromCompareOperatorSigned(.eq));
92 try testing.expectEqual(@as(Condition, .eq), Condition.fromCompareOperatorUnsigned(.eq));
93
94 try testing.expectEqual(@as(Condition, .gt), Condition.fromCompareOperatorSigned(.gt));
95 try testing.expectEqual(@as(Condition, .hi), Condition.fromCompareOperatorUnsigned(.gt));
96
97 try testing.expectEqual(@as(Condition, .le), Condition.fromCompareOperatorSigned(.lte));
98 try testing.expectEqual(@as(Condition, .ls), Condition.fromCompareOperatorUnsigned(.lte));
99}
100
101test "negate condition" {
102 try testing.expectEqual(@as(Condition, .eq), Condition.ne.negate());
103 try testing.expectEqual(@as(Condition, .ne), Condition.eq.negate());
104}
105
106/// Represents a register in the ARM instruction set architecture
107pub const Register = enum(u5) {
108 r0,
109 r1,
110 r2,
111 r3,
112 r4,
113 r5,
114 r6,
115 r7,
116 r8,
117 r9,
118 r10,
119 r11,
120 r12,
121 r13,
122 r14,
123 r15,
124
125 /// Argument / result / scratch register 1
126 a1,
127 /// Argument / result / scratch register 2
128 a2,
129 /// Argument / scratch register 3
130 a3,
131 /// Argument / scratch register 4
132 a4,
133 /// Variable-register 1
134 v1,
135 /// Variable-register 2
136 v2,
137 /// Variable-register 3
138 v3,
139 /// Variable-register 4
140 v4,
141 /// Variable-register 5
142 v5,
143 /// Platform register
144 v6,
145 /// Variable-register 7
146 v7,
147 /// Frame pointer or Variable-register 8
148 fp,
149 /// Intra-Procedure-call scratch register
150 ip,
151 /// Stack pointer
152 sp,
153 /// Link register
154 lr,
155 /// Program counter
156 pc,
157
158 /// Returns the unique 4-bit ID of this register which is used in
159 /// the machine code
160 pub fn id(reg: Register) u4 {
161 return @truncate(@intFromEnum(reg));
162 }
163
164 pub fn dwarfNum(reg: Register) u4 {
165 return reg.id();
166 }
167};
168
169test "Register.id" {
170 try testing.expectEqual(@as(u4, 15), Register.r15.id());
171 try testing.expectEqual(@as(u4, 15), Register.pc.id());
172}
173
174/// Program status registers containing flags, mode bits and other
175/// vital information
176pub const Psr = enum {
177 cpsr,
178 spsr,
179};
180
181/// Represents an instruction in the ARM instruction set architecture
182pub const Instruction = union(enum) {
183 data_processing: packed struct {
184 // Note to self: The order of the fields top-to-bottom is
185 // right-to-left in the actual 32-bit int representation
186 op2: u12,
187 rd: u4,
188 rn: u4,
189 s: u1,
190 opcode: u4,
191 i: u1,
192 fixed: u2 = 0b00,
193 cond: u4,
194 },
195 multiply: packed struct {
196 rn: u4,
197 fixed_1: u4 = 0b1001,
198 rm: u4,
199 ra: u4,
200 rd: u4,
201 set_cond: u1,
202 accumulate: u1,
203 fixed_2: u6 = 0b000000,
204 cond: u4,
205 },
206 multiply_long: packed struct {
207 rn: u4,
208 fixed_1: u4 = 0b1001,
209 rm: u4,
210 rdlo: u4,
211 rdhi: u4,
212 set_cond: u1,
213 accumulate: u1,
214 unsigned: u1,
215 fixed_2: u5 = 0b00001,
216 cond: u4,
217 },
218 signed_multiply_halfwords: packed struct {
219 rn: u4,
220 fixed_1: u1 = 0b0,
221 n: u1,
222 m: u1,
223 fixed_2: u1 = 0b1,
224 rm: u4,
225 fixed_3: u4 = 0b0000,
226 rd: u4,
227 fixed_4: u8 = 0b00010110,
228 cond: u4,
229 },
230 integer_saturating_arithmetic: packed struct {
231 rm: u4,
232 fixed_1: u8 = 0b0000_0101,
233 rd: u4,
234 rn: u4,
235 fixed_2: u1 = 0b0,
236 opc: u2,
237 fixed_3: u5 = 0b00010,
238 cond: u4,
239 },
240 bit_field_extract: packed struct {
241 rn: u4,
242 fixed_1: u3 = 0b101,
243 lsb: u5,
244 rd: u4,
245 widthm1: u5,
246 fixed_2: u1 = 0b1,
247 unsigned: u1,
248 fixed_3: u5 = 0b01111,
249 cond: u4,
250 },
251 single_data_transfer: packed struct {
252 offset: u12,
253 rd: u4,
254 rn: u4,
255 load_store: u1,
256 write_back: u1,
257 byte_word: u1,
258 up_down: u1,
259 pre_post: u1,
260 imm: u1,
261 fixed: u2 = 0b01,
262 cond: u4,
263 },
264 extra_load_store: packed struct {
265 imm4l: u4,
266 fixed_1: u1 = 0b1,
267 op2: u2,
268 fixed_2: u1 = 0b1,
269 imm4h: u4,
270 rt: u4,
271 rn: u4,
272 o1: u1,
273 write_back: u1,
274 imm: u1,
275 up_down: u1,
276 pre_index: u1,
277 fixed_3: u3 = 0b000,
278 cond: u4,
279 },
280 block_data_transfer: packed struct {
281 register_list: u16,
282 rn: u4,
283 load_store: u1,
284 write_back: u1,
285 psr_or_user: u1,
286 up_down: u1,
287 pre_post: u1,
288 fixed: u3 = 0b100,
289 cond: u4,
290 },
291 branch: packed struct {
292 offset: u24,
293 link: u1,
294 fixed: u3 = 0b101,
295 cond: u4,
296 },
297 branch_exchange: packed struct {
298 rn: u4,
299 fixed_1: u1 = 0b1,
300 link: u1,
301 fixed_2: u22 = 0b0001_0010_1111_1111_1111_00,
302 cond: u4,
303 },
304 supervisor_call: packed struct {
305 comment: u24,
306 fixed: u4 = 0b1111,
307 cond: u4,
308 },
309 undefined_instruction: packed struct {
310 imm32: u32 = 0xe7ffdefe,
311 },
312 breakpoint: packed struct {
313 imm4: u4,
314 fixed_1: u4 = 0b0111,
315 imm12: u12,
316 fixed_2_and_cond: u12 = 0b1110_0001_0010,
317 },
318
319 /// Represents the possible operations which can be performed by a
320 /// Data Processing instruction
321 const Opcode = enum(u4) {
322 // Rd := Op1 AND Op2
323 @"and",
324 // Rd := Op1 EOR Op2
325 eor,
326 // Rd := Op1 - Op2
327 sub,
328 // Rd := Op2 - Op1
329 rsb,
330 // Rd := Op1 + Op2
331 add,
332 // Rd := Op1 + Op2 + C
333 adc,
334 // Rd := Op1 - Op2 + C - 1
335 sbc,
336 // Rd := Op2 - Op1 + C - 1
337 rsc,
338 // set condition codes on Op1 AND Op2
339 tst,
340 // set condition codes on Op1 EOR Op2
341 teq,
342 // set condition codes on Op1 - Op2
343 cmp,
344 // set condition codes on Op1 + Op2
345 cmn,
346 // Rd := Op1 OR Op2
347 orr,
348 // Rd := Op2
349 mov,
350 // Rd := Op1 AND NOT Op2
351 bic,
352 // Rd := NOT Op2
353 mvn,
354 };
355
356 /// Represents the second operand to a data processing instruction
357 /// which can either be content from a register or an immediate
358 /// value
359 pub const Operand = union(enum) {
360 register: packed struct {
361 rm: u4,
362 shift: u8,
363 },
364 immediate: packed struct {
365 imm: u8,
366 rotate: u4,
367 },
368
369 /// Represents multiple ways a register can be shifted. A
370 /// register can be shifted by a specific immediate value or
371 /// by the contents of another register
372 pub const Shift = union(enum) {
373 immediate: packed struct {
374 fixed: u1 = 0b0,
375 typ: u2,
376 amount: u5,
377 },
378 register: packed struct {
379 fixed_1: u1 = 0b1,
380 typ: u2,
381 fixed_2: u1 = 0b0,
382 rs: u4,
383 },
384
385 pub const Type = enum(u2) {
386 logical_left,
387 logical_right,
388 arithmetic_right,
389 rotate_right,
390 };
391
392 pub const none = Shift{
393 .immediate = .{
394 .amount = 0,
395 .typ = 0,
396 },
397 };
398
399 pub fn toU8(self: Shift) u8 {
400 return switch (self) {
401 .register => |v| @as(u8, @bitCast(v)),
402 .immediate => |v| @as(u8, @bitCast(v)),
403 };
404 }
405
406 pub fn reg(rs: Register, typ: Type) Shift {
407 return Shift{
408 .register = .{
409 .rs = rs.id(),
410 .typ = @intFromEnum(typ),
411 },
412 };
413 }
414
415 pub fn imm(amount: u5, typ: Type) Shift {
416 return Shift{
417 .immediate = .{
418 .amount = amount,
419 .typ = @intFromEnum(typ),
420 },
421 };
422 }
423 };
424
425 pub fn toU12(self: Operand) u12 {
426 return switch (self) {
427 .register => |v| @as(u12, @bitCast(v)),
428 .immediate => |v| @as(u12, @bitCast(v)),
429 };
430 }
431
432 pub fn reg(rm: Register, shift: Shift) Operand {
433 return Operand{
434 .register = .{
435 .rm = rm.id(),
436 .shift = shift.toU8(),
437 },
438 };
439 }
440
441 pub fn imm(immediate: u8, rotate: u4) Operand {
442 return Operand{
443 .immediate = .{
444 .imm = immediate,
445 .rotate = rotate,
446 },
447 };
448 }
449
450 /// Tries to convert an unsigned 32 bit integer into an
451 /// immediate operand using rotation. Returns null when there
452 /// is no conversion
453 pub fn fromU32(x: u32) ?Operand {
454 const masks = comptime blk: {
455 const base_mask: u32 = std.math.maxInt(u8);
456 var result = [_]u32{0} ** 16;
457 for (&result, 0..) |*mask, i| mask.* = std.math.rotr(u32, base_mask, 2 * i);
458 break :blk result;
459 };
460
461 return for (masks, 0..) |mask, i| {
462 if (x & mask == x) {
463 break Operand{
464 .immediate = .{
465 .imm = @as(u8, @intCast(std.math.rotl(u32, x, 2 * i))),
466 .rotate = @as(u4, @intCast(i)),
467 },
468 };
469 }
470 } else null;
471 }
472 };
473
474 pub const AddressingMode = enum {
475 /// [<Rn>, <offset>]
476 ///
477 /// Address = Rn + offset
478 offset,
479 /// [<Rn>, <offset>]!
480 ///
481 /// Address = Rn + offset
482 /// Rn = Rn + offset
483 pre_index,
484 /// [<Rn>], <offset>
485 ///
486 /// Address = Rn
487 /// Rn = Rn + offset
488 post_index,
489 };
490
491 /// Represents the offset operand of a load or store
492 /// instruction. Data can be loaded from memory with either an
493 /// immediate offset or an offset that is stored in some register.
494 pub const Offset = union(enum) {
495 immediate: u12,
496 register: packed struct {
497 rm: u4,
498 fixed: u1 = 0b0,
499 stype: u2,
500 imm5: u5,
501 },
502
503 pub const Shift = union(enum) {
504 /// No shift
505 none,
506 /// Logical shift left
507 lsl: u5,
508 /// Logical shift right
509 lsr: u5,
510 /// Arithmetic shift right
511 asr: u5,
512 /// Rotate right
513 ror: u5,
514 /// Rotate right one bit, with extend
515 rrx,
516 };
517
518 pub const none = Offset{
519 .immediate = 0,
520 };
521
522 pub fn toU12(self: Offset) u12 {
523 return switch (self) {
524 .register => |v| @as(u12, @bitCast(v)),
525 .immediate => |v| v,
526 };
527 }
528
529 pub fn reg(rm: Register, shift: Shift) Offset {
530 return Offset{
531 .register = .{
532 .rm = rm.id(),
533 .stype = switch (shift) {
534 .none => 0b00,
535 .lsl => 0b00,
536 .lsr => 0b01,
537 .asr => 0b10,
538 .ror => 0b11,
539 .rrx => 0b11,
540 },
541 .imm5 = switch (shift) {
542 .none => 0,
543 .lsl => |n| n,
544 .lsr => |n| n,
545 .asr => |n| n,
546 .ror => |n| n,
547 .rrx => 0,
548 },
549 },
550 };
551 }
552
553 pub fn imm(immediate: u12) Offset {
554 return Offset{
555 .immediate = immediate,
556 };
557 }
558 };
559
560 /// Represents the offset operand of an extra load or store
561 /// instruction.
562 pub const ExtraLoadStoreOffset = union(enum) {
563 immediate: u8,
564 register: u4,
565
566 pub const none = ExtraLoadStoreOffset{
567 .immediate = 0,
568 };
569
570 pub fn reg(register: Register) ExtraLoadStoreOffset {
571 return ExtraLoadStoreOffset{
572 .register = register.id(),
573 };
574 }
575
576 pub fn imm(immediate: u8) ExtraLoadStoreOffset {
577 return ExtraLoadStoreOffset{
578 .immediate = immediate,
579 };
580 }
581 };
582
583 /// Represents the register list operand to a block data transfer
584 /// instruction
585 pub const RegisterList = packed struct {
586 r0: bool = false,
587 r1: bool = false,
588 r2: bool = false,
589 r3: bool = false,
590 r4: bool = false,
591 r5: bool = false,
592 r6: bool = false,
593 r7: bool = false,
594 r8: bool = false,
595 r9: bool = false,
596 r10: bool = false,
597 r11: bool = false,
598 r12: bool = false,
599 r13: bool = false,
600 r14: bool = false,
601 r15: bool = false,
602 };
603
604 pub fn toU32(self: Instruction) u32 {
605 return switch (self) {
606 .data_processing => |v| @as(u32, @bitCast(v)),
607 .multiply => |v| @as(u32, @bitCast(v)),
608 .multiply_long => |v| @as(u32, @bitCast(v)),
609 .signed_multiply_halfwords => |v| @as(u32, @bitCast(v)),
610 .integer_saturating_arithmetic => |v| @as(u32, @bitCast(v)),
611 .bit_field_extract => |v| @as(u32, @bitCast(v)),
612 .single_data_transfer => |v| @as(u32, @bitCast(v)),
613 .extra_load_store => |v| @as(u32, @bitCast(v)),
614 .block_data_transfer => |v| @as(u32, @bitCast(v)),
615 .branch => |v| @as(u32, @bitCast(v)),
616 .branch_exchange => |v| @as(u32, @bitCast(v)),
617 .supervisor_call => |v| @as(u32, @bitCast(v)),
618 .undefined_instruction => |v| v.imm32,
619 .breakpoint => |v| @as(u32, @intCast(v.imm4)) | (@as(u32, @intCast(v.fixed_1)) << 4) | (@as(u32, @intCast(v.imm12)) << 8) | (@as(u32, @intCast(v.fixed_2_and_cond)) << 20),
620 };
621 }
622
623 // Helper functions for the "real" functions below
624
625 fn dataProcessing(
626 cond: Condition,
627 opcode: Opcode,
628 s: u1,
629 rd: Register,
630 rn: Register,
631 op2: Operand,
632 ) Instruction {
633 return Instruction{
634 .data_processing = .{
635 .cond = @intFromEnum(cond),
636 .i = @intFromBool(op2 == .immediate),
637 .opcode = @intFromEnum(opcode),
638 .s = s,
639 .rn = rn.id(),
640 .rd = rd.id(),
641 .op2 = op2.toU12(),
642 },
643 };
644 }
645
646 fn specialMov(
647 cond: Condition,
648 rd: Register,
649 imm: u16,
650 top: bool,
651 ) Instruction {
652 return Instruction{
653 .data_processing = .{
654 .cond = @intFromEnum(cond),
655 .i = 1,
656 .opcode = if (top) 0b1010 else 0b1000,
657 .s = 0,
658 .rn = @as(u4, @truncate(imm >> 12)),
659 .rd = rd.id(),
660 .op2 = @as(u12, @truncate(imm)),
661 },
662 };
663 }
664
665 fn initMultiply(
666 cond: Condition,
667 set_cond: u1,
668 rd: Register,
669 rn: Register,
670 rm: Register,
671 ra: ?Register,
672 ) Instruction {
673 return Instruction{
674 .multiply = .{
675 .cond = @intFromEnum(cond),
676 .accumulate = @intFromBool(ra != null),
677 .set_cond = set_cond,
678 .rd = rd.id(),
679 .rn = rn.id(),
680 .ra = if (ra) |reg| reg.id() else 0b0000,
681 .rm = rm.id(),
682 },
683 };
684 }
685
686 fn multiplyLong(
687 cond: Condition,
688 signed: u1,
689 accumulate: u1,
690 set_cond: u1,
691 rdhi: Register,
692 rdlo: Register,
693 rm: Register,
694 rn: Register,
695 ) Instruction {
696 return Instruction{
697 .multiply_long = .{
698 .cond = @intFromEnum(cond),
699 .unsigned = signed,
700 .accumulate = accumulate,
701 .set_cond = set_cond,
702 .rdlo = rdlo.id(),
703 .rdhi = rdhi.id(),
704 .rn = rn.id(),
705 .rm = rm.id(),
706 },
707 };
708 }
709
710 fn signedMultiplyHalfwords(
711 n: u1,
712 m: u1,
713 cond: Condition,
714 rd: Register,
715 rn: Register,
716 rm: Register,
717 ) Instruction {
718 return Instruction{
719 .signed_multiply_halfwords = .{
720 .rn = rn.id(),
721 .n = n,
722 .m = m,
723 .rm = rm.id(),
724 .rd = rd.id(),
725 .cond = @intFromEnum(cond),
726 },
727 };
728 }
729
730 fn integerSaturationArithmetic(
731 cond: Condition,
732 rd: Register,
733 rm: Register,
734 rn: Register,
735 opc: u2,
736 ) Instruction {
737 return Instruction{
738 .integer_saturating_arithmetic = .{
739 .rm = rm.id(),
740 .rd = rd.id(),
741 .rn = rn.id(),
742 .opc = opc,
743 .cond = @intFromEnum(cond),
744 },
745 };
746 }
747
748 fn bitFieldExtract(
749 unsigned: u1,
750 cond: Condition,
751 rd: Register,
752 rn: Register,
753 lsb: u5,
754 width: u6,
755 ) Instruction {
756 assert(width > 0 and width <= 32);
757 return Instruction{
758 .bit_field_extract = .{
759 .rn = rn.id(),
760 .lsb = lsb,
761 .rd = rd.id(),
762 .widthm1 = @as(u5, @intCast(width - 1)),
763 .unsigned = unsigned,
764 .cond = @intFromEnum(cond),
765 },
766 };
767 }
768
769 fn singleDataTransfer(
770 cond: Condition,
771 rd: Register,
772 rn: Register,
773 offset: Offset,
774 mode: AddressingMode,
775 positive: bool,
776 byte_word: u1,
777 load_store: u1,
778 ) Instruction {
779 return Instruction{
780 .single_data_transfer = .{
781 .cond = @intFromEnum(cond),
782 .rn = rn.id(),
783 .rd = rd.id(),
784 .offset = offset.toU12(),
785 .load_store = load_store,
786 .write_back = switch (mode) {
787 .offset => 0b0,
788 .pre_index, .post_index => 0b1,
789 },
790 .byte_word = byte_word,
791 .up_down = @intFromBool(positive),
792 .pre_post = switch (mode) {
793 .offset, .pre_index => 0b1,
794 .post_index => 0b0,
795 },
796 .imm = @intFromBool(offset != .immediate),
797 },
798 };
799 }
800
801 fn extraLoadStore(
802 cond: Condition,
803 mode: AddressingMode,
804 positive: bool,
805 o1: u1,
806 op2: u2,
807 rn: Register,
808 rt: Register,
809 offset: ExtraLoadStoreOffset,
810 ) Instruction {
811 const imm4l: u4 = switch (offset) {
812 .immediate => |imm| @as(u4, @truncate(imm)),
813 .register => |reg| reg,
814 };
815 const imm4h: u4 = switch (offset) {
816 .immediate => |imm| @as(u4, @truncate(imm >> 4)),
817 .register => 0b0000,
818 };
819
820 return Instruction{
821 .extra_load_store = .{
822 .imm4l = imm4l,
823 .op2 = op2,
824 .imm4h = imm4h,
825 .rt = rt.id(),
826 .rn = rn.id(),
827 .o1 = o1,
828 .write_back = switch (mode) {
829 .offset => 0b0,
830 .pre_index, .post_index => 0b1,
831 },
832 .imm = @intFromBool(offset == .immediate),
833 .up_down = @intFromBool(positive),
834 .pre_index = switch (mode) {
835 .offset, .pre_index => 0b1,
836 .post_index => 0b0,
837 },
838 .cond = @intFromEnum(cond),
839 },
840 };
841 }
842
843 fn blockDataTransfer(
844 cond: Condition,
845 rn: Register,
846 reg_list: RegisterList,
847 pre_post: u1,
848 up_down: u1,
849 psr_or_user: u1,
850 write_back: bool,
851 load_store: u1,
852 ) Instruction {
853 return Instruction{
854 .block_data_transfer = .{
855 .register_list = @as(u16, @bitCast(reg_list)),
856 .rn = rn.id(),
857 .load_store = load_store,
858 .write_back = @intFromBool(write_back),
859 .psr_or_user = psr_or_user,
860 .up_down = up_down,
861 .pre_post = pre_post,
862 .cond = @intFromEnum(cond),
863 },
864 };
865 }
866
867 fn initBranch(cond: Condition, offset: i26, link: u1) Instruction {
868 return Instruction{
869 .branch = .{
870 .cond = @intFromEnum(cond),
871 .link = link,
872 .offset = @as(u24, @bitCast(@as(i24, @intCast(offset >> 2)))),
873 },
874 };
875 }
876
877 fn branchExchange(cond: Condition, rn: Register, link: u1) Instruction {
878 return Instruction{
879 .branch_exchange = .{
880 .cond = @intFromEnum(cond),
881 .link = link,
882 .rn = rn.id(),
883 },
884 };
885 }
886
887 fn supervisorCall(cond: Condition, comment: u24) Instruction {
888 return Instruction{
889 .supervisor_call = .{
890 .cond = @intFromEnum(cond),
891 .comment = comment,
892 },
893 };
894 }
895
896 // This instruction has no official mnemonic equivalent so it is public as-is.
897 pub fn undefinedInstruction() Instruction {
898 return Instruction{
899 .undefined_instruction = .{},
900 };
901 }
902
903 fn initBreakpoint(imm: u16) Instruction {
904 return Instruction{
905 .breakpoint = .{
906 .imm12 = @as(u12, @truncate(imm >> 4)),
907 .imm4 = @as(u4, @truncate(imm)),
908 },
909 };
910 }
911
912 // Public functions replicating assembler syntax as closely as
913 // possible
914
915 // Data processing
916
917 pub fn @"and"(cond: Condition, rd: Register, rn: Register, op2: Operand) Instruction {
918 return dataProcessing(cond, .@"and", 0, rd, rn, op2);
919 }
920
921 pub fn ands(cond: Condition, rd: Register, rn: Register, op2: Operand) Instruction {
922 return dataProcessing(cond, .@"and", 1, rd, rn, op2);
923 }
924
925 pub fn eor(cond: Condition, rd: Register, rn: Register, op2: Operand) Instruction {
926 return dataProcessing(cond, .eor, 0, rd, rn, op2);
927 }
928
929 pub fn eors(cond: Condition, rd: Register, rn: Register, op2: Operand) Instruction {
930 return dataProcessing(cond, .eor, 1, rd, rn, op2);
931 }
932
933 pub fn sub(cond: Condition, rd: Register, rn: Register, op2: Operand) Instruction {
934 return dataProcessing(cond, .sub, 0, rd, rn, op2);
935 }
936
937 pub fn subs(cond: Condition, rd: Register, rn: Register, op2: Operand) Instruction {
938 return dataProcessing(cond, .sub, 1, rd, rn, op2);
939 }
940
941 pub fn rsb(cond: Condition, rd: Register, rn: Register, op2: Operand) Instruction {
942 return dataProcessing(cond, .rsb, 0, rd, rn, op2);
943 }
944
945 pub fn rsbs(cond: Condition, rd: Register, rn: Register, op2: Operand) Instruction {
946 return dataProcessing(cond, .rsb, 1, rd, rn, op2);
947 }
948
949 pub fn add(cond: Condition, rd: Register, rn: Register, op2: Operand) Instruction {
950 return dataProcessing(cond, .add, 0, rd, rn, op2);
951 }
952
953 pub fn adds(cond: Condition, rd: Register, rn: Register, op2: Operand) Instruction {
954 return dataProcessing(cond, .add, 1, rd, rn, op2);
955 }
956
957 pub fn adc(cond: Condition, rd: Register, rn: Register, op2: Operand) Instruction {
958 return dataProcessing(cond, .adc, 0, rd, rn, op2);
959 }
960
961 pub fn adcs(cond: Condition, rd: Register, rn: Register, op2: Operand) Instruction {
962 return dataProcessing(cond, .adc, 1, rd, rn, op2);
963 }
964
965 pub fn sbc(cond: Condition, rd: Register, rn: Register, op2: Operand) Instruction {
966 return dataProcessing(cond, .sbc, 0, rd, rn, op2);
967 }
968
969 pub fn sbcs(cond: Condition, rd: Register, rn: Register, op2: Operand) Instruction {
970 return dataProcessing(cond, .sbc, 1, rd, rn, op2);
971 }
972
973 pub fn rsc(cond: Condition, rd: Register, rn: Register, op2: Operand) Instruction {
974 return dataProcessing(cond, .rsc, 0, rd, rn, op2);
975 }
976
977 pub fn rscs(cond: Condition, rd: Register, rn: Register, op2: Operand) Instruction {
978 return dataProcessing(cond, .rsc, 1, rd, rn, op2);
979 }
980
981 pub fn tst(cond: Condition, rn: Register, op2: Operand) Instruction {
982 return dataProcessing(cond, .tst, 1, .r0, rn, op2);
983 }
984
985 pub fn teq(cond: Condition, rn: Register, op2: Operand) Instruction {
986 return dataProcessing(cond, .teq, 1, .r0, rn, op2);
987 }
988
989 pub fn cmp(cond: Condition, rn: Register, op2: Operand) Instruction {
990 return dataProcessing(cond, .cmp, 1, .r0, rn, op2);
991 }
992
993 pub fn cmn(cond: Condition, rn: Register, op2: Operand) Instruction {
994 return dataProcessing(cond, .cmn, 1, .r0, rn, op2);
995 }
996
997 pub fn orr(cond: Condition, rd: Register, rn: Register, op2: Operand) Instruction {
998 return dataProcessing(cond, .orr, 0, rd, rn, op2);
999 }
1000
1001 pub fn orrs(cond: Condition, rd: Register, rn: Register, op2: Operand) Instruction {
1002 return dataProcessing(cond, .orr, 1, rd, rn, op2);
1003 }
1004
1005 pub fn mov(cond: Condition, rd: Register, op2: Operand) Instruction {
1006 return dataProcessing(cond, .mov, 0, rd, .r0, op2);
1007 }
1008
1009 pub fn movs(cond: Condition, rd: Register, op2: Operand) Instruction {
1010 return dataProcessing(cond, .mov, 1, rd, .r0, op2);
1011 }
1012
1013 pub fn bic(cond: Condition, rd: Register, rn: Register, op2: Operand) Instruction {
1014 return dataProcessing(cond, .bic, 0, rd, rn, op2);
1015 }
1016
1017 pub fn bics(cond: Condition, rd: Register, rn: Register, op2: Operand) Instruction {
1018 return dataProcessing(cond, .bic, 1, rd, rn, op2);
1019 }
1020
1021 pub fn mvn(cond: Condition, rd: Register, op2: Operand) Instruction {
1022 return dataProcessing(cond, .mvn, 0, rd, .r0, op2);
1023 }
1024
1025 pub fn mvns(cond: Condition, rd: Register, op2: Operand) Instruction {
1026 return dataProcessing(cond, .mvn, 1, rd, .r0, op2);
1027 }
1028
1029 // Integer Saturating Arithmetic
1030
1031 pub fn qadd(cond: Condition, rd: Register, rm: Register, rn: Register) Instruction {
1032 return integerSaturationArithmetic(cond, rd, rm, rn, 0b00);
1033 }
1034
1035 pub fn qsub(cond: Condition, rd: Register, rm: Register, rn: Register) Instruction {
1036 return integerSaturationArithmetic(cond, rd, rm, rn, 0b01);
1037 }
1038
1039 pub fn qdadd(cond: Condition, rd: Register, rm: Register, rn: Register) Instruction {
1040 return integerSaturationArithmetic(cond, rd, rm, rn, 0b10);
1041 }
1042
1043 pub fn qdsub(cond: Condition, rd: Register, rm: Register, rn: Register) Instruction {
1044 return integerSaturationArithmetic(cond, rd, rm, rn, 0b11);
1045 }
1046
1047 // movw and movt
1048
1049 pub fn movw(cond: Condition, rd: Register, imm: u16) Instruction {
1050 return specialMov(cond, rd, imm, false);
1051 }
1052
1053 pub fn movt(cond: Condition, rd: Register, imm: u16) Instruction {
1054 return specialMov(cond, rd, imm, true);
1055 }
1056
1057 // PSR transfer
1058
1059 pub fn mrs(cond: Condition, rd: Register, psr: Psr) Instruction {
1060 return Instruction{
1061 .data_processing = .{
1062 .cond = @intFromEnum(cond),
1063 .i = 0,
1064 .opcode = if (psr == .spsr) 0b1010 else 0b1000,
1065 .s = 0,
1066 .rn = 0b1111,
1067 .rd = rd.id(),
1068 .op2 = 0b0000_0000_0000,
1069 },
1070 };
1071 }
1072
1073 pub fn msr(cond: Condition, psr: Psr, op: Operand) Instruction {
1074 return Instruction{
1075 .data_processing = .{
1076 .cond = @intFromEnum(cond),
1077 .i = 0,
1078 .opcode = if (psr == .spsr) 0b1011 else 0b1001,
1079 .s = 0,
1080 .rn = 0b1111,
1081 .rd = 0b1111,
1082 .op2 = op.toU12(),
1083 },
1084 };
1085 }
1086
1087 // Multiply
1088
1089 pub fn mul(cond: Condition, rd: Register, rn: Register, rm: Register) Instruction {
1090 return initMultiply(cond, 0, rd, rn, rm, null);
1091 }
1092
1093 pub fn muls(cond: Condition, rd: Register, rn: Register, rm: Register) Instruction {
1094 return initMultiply(cond, 1, rd, rn, rm, null);
1095 }
1096
1097 pub fn mla(cond: Condition, rd: Register, rn: Register, rm: Register, ra: Register) Instruction {
1098 return initMultiply(cond, 0, rd, rn, rm, ra);
1099 }
1100
1101 pub fn mlas(cond: Condition, rd: Register, rn: Register, rm: Register, ra: Register) Instruction {
1102 return initMultiply(cond, 1, rd, rn, rm, ra);
1103 }
1104
1105 // Multiply long
1106
1107 pub fn umull(cond: Condition, rdlo: Register, rdhi: Register, rn: Register, rm: Register) Instruction {
1108 return multiplyLong(cond, 0, 0, 0, rdhi, rdlo, rm, rn);
1109 }
1110
1111 pub fn umulls(cond: Condition, rdlo: Register, rdhi: Register, rn: Register, rm: Register) Instruction {
1112 return multiplyLong(cond, 0, 0, 1, rdhi, rdlo, rm, rn);
1113 }
1114
1115 pub fn umlal(cond: Condition, rdlo: Register, rdhi: Register, rn: Register, rm: Register) Instruction {
1116 return multiplyLong(cond, 0, 1, 0, rdhi, rdlo, rm, rn);
1117 }
1118
1119 pub fn umlals(cond: Condition, rdlo: Register, rdhi: Register, rn: Register, rm: Register) Instruction {
1120 return multiplyLong(cond, 0, 1, 1, rdhi, rdlo, rm, rn);
1121 }
1122
1123 pub fn smull(cond: Condition, rdlo: Register, rdhi: Register, rn: Register, rm: Register) Instruction {
1124 return multiplyLong(cond, 1, 0, 0, rdhi, rdlo, rm, rn);
1125 }
1126
1127 pub fn smulls(cond: Condition, rdlo: Register, rdhi: Register, rn: Register, rm: Register) Instruction {
1128 return multiplyLong(cond, 1, 0, 1, rdhi, rdlo, rm, rn);
1129 }
1130
1131 pub fn smlal(cond: Condition, rdlo: Register, rdhi: Register, rn: Register, rm: Register) Instruction {
1132 return multiplyLong(cond, 1, 1, 0, rdhi, rdlo, rm, rn);
1133 }
1134
1135 pub fn smlals(cond: Condition, rdlo: Register, rdhi: Register, rn: Register, rm: Register) Instruction {
1136 return multiplyLong(cond, 1, 1, 1, rdhi, rdlo, rm, rn);
1137 }
1138
1139 // Signed Multiply (halfwords)
1140
1141 pub fn smulbb(cond: Condition, rd: Register, rn: Register, rm: Register) Instruction {
1142 return signedMultiplyHalfwords(0, 0, cond, rd, rn, rm);
1143 }
1144
1145 pub fn smulbt(cond: Condition, rd: Register, rn: Register, rm: Register) Instruction {
1146 return signedMultiplyHalfwords(0, 1, cond, rd, rn, rm);
1147 }
1148
1149 pub fn smultb(cond: Condition, rd: Register, rn: Register, rm: Register) Instruction {
1150 return signedMultiplyHalfwords(1, 0, cond, rd, rn, rm);
1151 }
1152
1153 pub fn smultt(cond: Condition, rd: Register, rn: Register, rm: Register) Instruction {
1154 return signedMultiplyHalfwords(1, 1, cond, rd, rn, rm);
1155 }
1156
1157 // Bit field extract
1158
1159 pub fn ubfx(cond: Condition, rd: Register, rn: Register, lsb: u5, width: u6) Instruction {
1160 return bitFieldExtract(0b1, cond, rd, rn, lsb, width);
1161 }
1162
1163 pub fn sbfx(cond: Condition, rd: Register, rn: Register, lsb: u5, width: u6) Instruction {
1164 return bitFieldExtract(0b0, cond, rd, rn, lsb, width);
1165 }
1166
1167 // Single data transfer
1168
1169 pub const OffsetArgs = struct {
1170 mode: AddressingMode = .offset,
1171 positive: bool = true,
1172 offset: Offset,
1173 };
1174
1175 pub fn ldr(cond: Condition, rd: Register, rn: Register, args: OffsetArgs) Instruction {
1176 return singleDataTransfer(cond, rd, rn, args.offset, args.mode, args.positive, 0, 1);
1177 }
1178
1179 pub fn ldrb(cond: Condition, rd: Register, rn: Register, args: OffsetArgs) Instruction {
1180 return singleDataTransfer(cond, rd, rn, args.offset, args.mode, args.positive, 1, 1);
1181 }
1182
1183 pub fn str(cond: Condition, rd: Register, rn: Register, args: OffsetArgs) Instruction {
1184 return singleDataTransfer(cond, rd, rn, args.offset, args.mode, args.positive, 0, 0);
1185 }
1186
1187 pub fn strb(cond: Condition, rd: Register, rn: Register, args: OffsetArgs) Instruction {
1188 return singleDataTransfer(cond, rd, rn, args.offset, args.mode, args.positive, 1, 0);
1189 }
1190
1191 // Extra load/store
1192
1193 pub const ExtraLoadStoreOffsetArgs = struct {
1194 mode: AddressingMode = .offset,
1195 positive: bool = true,
1196 offset: ExtraLoadStoreOffset,
1197 };
1198
1199 pub fn strh(cond: Condition, rt: Register, rn: Register, args: ExtraLoadStoreOffsetArgs) Instruction {
1200 return extraLoadStore(cond, args.mode, args.positive, 0b0, 0b01, rn, rt, args.offset);
1201 }
1202
1203 pub fn ldrh(cond: Condition, rt: Register, rn: Register, args: ExtraLoadStoreOffsetArgs) Instruction {
1204 return extraLoadStore(cond, args.mode, args.positive, 0b1, 0b01, rn, rt, args.offset);
1205 }
1206
1207 pub fn ldrsh(cond: Condition, rt: Register, rn: Register, args: ExtraLoadStoreOffsetArgs) Instruction {
1208 return extraLoadStore(cond, args.mode, args.positive, 0b1, 0b11, rn, rt, args.offset);
1209 }
1210
1211 pub fn ldrsb(cond: Condition, rt: Register, rn: Register, args: ExtraLoadStoreOffsetArgs) Instruction {
1212 return extraLoadStore(cond, args.mode, args.positive, 0b1, 0b10, rn, rt, args.offset);
1213 }
1214
1215 // Block data transfer
1216
1217 pub fn ldmda(cond: Condition, rn: Register, write_back: bool, reg_list: RegisterList) Instruction {
1218 return blockDataTransfer(cond, rn, reg_list, 0, 0, 0, write_back, 1);
1219 }
1220
1221 pub fn ldmdb(cond: Condition, rn: Register, write_back: bool, reg_list: RegisterList) Instruction {
1222 return blockDataTransfer(cond, rn, reg_list, 1, 0, 0, write_back, 1);
1223 }
1224
1225 pub fn ldmib(cond: Condition, rn: Register, write_back: bool, reg_list: RegisterList) Instruction {
1226 return blockDataTransfer(cond, rn, reg_list, 1, 1, 0, write_back, 1);
1227 }
1228
1229 pub fn ldmia(cond: Condition, rn: Register, write_back: bool, reg_list: RegisterList) Instruction {
1230 return blockDataTransfer(cond, rn, reg_list, 0, 1, 0, write_back, 1);
1231 }
1232
1233 pub const ldmfa = ldmda;
1234 pub const ldmea = ldmdb;
1235 pub const ldmed = ldmib;
1236 pub const ldmfd = ldmia;
1237 pub const ldm = ldmia;
1238
1239 pub fn stmda(cond: Condition, rn: Register, write_back: bool, reg_list: RegisterList) Instruction {
1240 return blockDataTransfer(cond, rn, reg_list, 0, 0, 0, write_back, 0);
1241 }
1242
1243 pub fn stmdb(cond: Condition, rn: Register, write_back: bool, reg_list: RegisterList) Instruction {
1244 return blockDataTransfer(cond, rn, reg_list, 1, 0, 0, write_back, 0);
1245 }
1246
1247 pub fn stmib(cond: Condition, rn: Register, write_back: bool, reg_list: RegisterList) Instruction {
1248 return blockDataTransfer(cond, rn, reg_list, 1, 1, 0, write_back, 0);
1249 }
1250
1251 pub fn stmia(cond: Condition, rn: Register, write_back: bool, reg_list: RegisterList) Instruction {
1252 return blockDataTransfer(cond, rn, reg_list, 0, 1, 0, write_back, 0);
1253 }
1254
1255 pub const stmed = stmda;
1256 pub const stmfd = stmdb;
1257 pub const stmfa = stmib;
1258 pub const stmea = stmia;
1259 pub const stm = stmia;
1260
1261 // Branch
1262
1263 pub fn b(cond: Condition, offset: i26) Instruction {
1264 return initBranch(cond, offset, 0);
1265 }
1266
1267 pub fn bl(cond: Condition, offset: i26) Instruction {
1268 return initBranch(cond, offset, 1);
1269 }
1270
1271 // Branch and exchange
1272
1273 pub fn bx(cond: Condition, rn: Register) Instruction {
1274 return branchExchange(cond, rn, 0);
1275 }
1276
1277 pub fn blx(cond: Condition, rn: Register) Instruction {
1278 return branchExchange(cond, rn, 1);
1279 }
1280
1281 // Supervisor Call
1282
1283 pub const swi = svc;
1284
1285 pub fn svc(cond: Condition, comment: u24) Instruction {
1286 return supervisorCall(cond, comment);
1287 }
1288
1289 // Breakpoint
1290
1291 pub fn bkpt(imm: u16) Instruction {
1292 return initBreakpoint(imm);
1293 }
1294
1295 // Aliases
1296
1297 pub fn nop() Instruction {
1298 return mov(.al, .r0, Instruction.Operand.reg(.r0, Instruction.Operand.Shift.none));
1299 }
1300
1301 pub fn pop(cond: Condition, args: anytype) Instruction {
1302 if (@typeInfo(@TypeOf(args)) != .@"struct") {
1303 @compileError("Expected tuple or struct argument, found " ++ @typeName(@TypeOf(args)));
1304 }
1305
1306 if (args.len < 1) {
1307 @compileError("Expected at least one register");
1308 } else if (args.len == 1) {
1309 const reg = args[0];
1310 return ldr(cond, reg, .sp, .{
1311 .mode = .post_index,
1312 .positive = true,
1313 .offset = Offset.imm(4),
1314 });
1315 } else {
1316 var register_list: u16 = 0;
1317 inline for (args) |arg| {
1318 const reg = @as(Register, arg);
1319 register_list |= @as(u16, 1) << reg.id();
1320 }
1321 return ldm(cond, .sp, true, @as(RegisterList, @bitCast(register_list)));
1322 }
1323 }
1324
1325 pub fn push(cond: Condition, args: anytype) Instruction {
1326 if (@typeInfo(@TypeOf(args)) != .@"struct") {
1327 @compileError("Expected tuple or struct argument, found " ++ @typeName(@TypeOf(args)));
1328 }
1329
1330 if (args.len < 1) {
1331 @compileError("Expected at least one register");
1332 } else if (args.len == 1) {
1333 const reg = args[0];
1334 return str(cond, reg, .sp, .{
1335 .mode = .pre_index,
1336 .positive = false,
1337 .offset = Offset.imm(4),
1338 });
1339 } else {
1340 var register_list: u16 = 0;
1341 inline for (args) |arg| {
1342 const reg = @as(Register, arg);
1343 register_list |= @as(u16, 1) << reg.id();
1344 }
1345 return stmdb(cond, .sp, true, @as(RegisterList, @bitCast(register_list)));
1346 }
1347 }
1348
1349 pub const ShiftAmount = union(enum) {
1350 immediate: u5,
1351 register: Register,
1352
1353 pub fn imm(immediate: u5) ShiftAmount {
1354 return .{
1355 .immediate = immediate,
1356 };
1357 }
1358
1359 pub fn reg(register: Register) ShiftAmount {
1360 return .{
1361 .register = register,
1362 };
1363 }
1364 };
1365
1366 pub fn lsl(cond: Condition, rd: Register, rm: Register, shift: ShiftAmount) Instruction {
1367 return switch (shift) {
1368 .immediate => |imm| mov(cond, rd, Operand.reg(rm, Operand.Shift.imm(imm, .logical_left))),
1369 .register => |reg| mov(cond, rd, Operand.reg(rm, Operand.Shift.reg(reg, .logical_left))),
1370 };
1371 }
1372
1373 pub fn lsr(cond: Condition, rd: Register, rm: Register, shift: ShiftAmount) Instruction {
1374 return switch (shift) {
1375 .immediate => |imm| mov(cond, rd, Operand.reg(rm, Operand.Shift.imm(imm, .logical_right))),
1376 .register => |reg| mov(cond, rd, Operand.reg(rm, Operand.Shift.reg(reg, .logical_right))),
1377 };
1378 }
1379
1380 pub fn asr(cond: Condition, rd: Register, rm: Register, shift: ShiftAmount) Instruction {
1381 return switch (shift) {
1382 .immediate => |imm| mov(cond, rd, Operand.reg(rm, Operand.Shift.imm(imm, .arithmetic_right))),
1383 .register => |reg| mov(cond, rd, Operand.reg(rm, Operand.Shift.reg(reg, .arithmetic_right))),
1384 };
1385 }
1386
1387 pub fn ror(cond: Condition, rd: Register, rm: Register, shift: ShiftAmount) Instruction {
1388 return switch (shift) {
1389 .immediate => |imm| mov(cond, rd, Operand.reg(rm, Operand.Shift.imm(imm, .rotate_right))),
1390 .register => |reg| mov(cond, rd, Operand.reg(rm, Operand.Shift.reg(reg, .rotate_right))),
1391 };
1392 }
1393
1394 pub fn lsls(cond: Condition, rd: Register, rm: Register, shift: ShiftAmount) Instruction {
1395 return switch (shift) {
1396 .immediate => |imm| movs(cond, rd, Operand.reg(rm, Operand.Shift.imm(imm, .logical_left))),
1397 .register => |reg| movs(cond, rd, Operand.reg(rm, Operand.Shift.reg(reg, .logical_left))),
1398 };
1399 }
1400
1401 pub fn lsrs(cond: Condition, rd: Register, rm: Register, shift: ShiftAmount) Instruction {
1402 return switch (shift) {
1403 .immediate => |imm| movs(cond, rd, Operand.reg(rm, Operand.Shift.imm(imm, .logical_right))),
1404 .register => |reg| movs(cond, rd, Operand.reg(rm, Operand.Shift.reg(reg, .logical_right))),
1405 };
1406 }
1407
1408 pub fn asrs(cond: Condition, rd: Register, rm: Register, shift: ShiftAmount) Instruction {
1409 return switch (shift) {
1410 .immediate => |imm| movs(cond, rd, Operand.reg(rm, Operand.Shift.imm(imm, .arithmetic_right))),
1411 .register => |reg| movs(cond, rd, Operand.reg(rm, Operand.Shift.reg(reg, .arithmetic_right))),
1412 };
1413 }
1414
1415 pub fn rors(cond: Condition, rd: Register, rm: Register, shift: ShiftAmount) Instruction {
1416 return switch (shift) {
1417 .immediate => |imm| movs(cond, rd, Operand.reg(rm, Operand.Shift.imm(imm, .rotate_right))),
1418 .register => |reg| movs(cond, rd, Operand.reg(rm, Operand.Shift.reg(reg, .rotate_right))),
1419 };
1420 }
1421};
1422
1423test "serialize instructions" {
1424 const Testcase = struct {
1425 inst: Instruction,
1426 expected: u32,
1427 };
1428
1429 const testcases = [_]Testcase{
1430 .{ // add r0, r0, r0
1431 .inst = Instruction.add(.al, .r0, .r0, Instruction.Operand.reg(.r0, Instruction.Operand.Shift.none)),
1432 .expected = 0b1110_00_0_0100_0_0000_0000_00000000_0000,
1433 },
1434 .{ // mov r4, r2
1435 .inst = Instruction.mov(.al, .r4, Instruction.Operand.reg(.r2, Instruction.Operand.Shift.none)),
1436 .expected = 0b1110_00_0_1101_0_0000_0100_00000000_0010,
1437 },
1438 .{ // mov r0, #42
1439 .inst = Instruction.mov(.al, .r0, Instruction.Operand.imm(42, 0)),
1440 .expected = 0b1110_00_1_1101_0_0000_0000_0000_00101010,
1441 },
1442 .{ // mrs r5, cpsr
1443 .inst = Instruction.mrs(.al, .r5, .cpsr),
1444 .expected = 0b1110_00010_0_001111_0101_000000000000,
1445 },
1446 .{ // mul r0, r1, r2
1447 .inst = Instruction.mul(.al, .r0, .r1, .r2),
1448 .expected = 0b1110_000000_0_0_0000_0000_0010_1001_0001,
1449 },
1450 .{ // umlal r0, r1, r5, r6
1451 .inst = Instruction.umlal(.al, .r0, .r1, .r5, .r6),
1452 .expected = 0b1110_00001_0_1_0_0001_0000_0110_1001_0101,
1453 },
1454 .{ // ldr r0, [r2, #42]
1455 .inst = Instruction.ldr(.al, .r0, .r2, .{
1456 .offset = Instruction.Offset.imm(42),
1457 }),
1458 .expected = 0b1110_01_0_1_1_0_0_1_0010_0000_000000101010,
1459 },
1460 .{ // str r0, [r3]
1461 .inst = Instruction.str(.al, .r0, .r3, .{
1462 .offset = Instruction.Offset.none,
1463 }),
1464 .expected = 0b1110_01_0_1_1_0_0_0_0011_0000_000000000000,
1465 },
1466 .{ // strh r1, [r5]
1467 .inst = Instruction.strh(.al, .r1, .r5, .{
1468 .offset = Instruction.ExtraLoadStoreOffset.none,
1469 }),
1470 .expected = 0b1110_000_1_1_1_0_0_0101_0001_0000_1011_0000,
1471 },
1472 .{ // b #12
1473 .inst = Instruction.b(.al, 12),
1474 .expected = 0b1110_101_0_0000_0000_0000_0000_0000_0011,
1475 },
1476 .{ // bl #-4
1477 .inst = Instruction.bl(.al, -4),
1478 .expected = 0b1110_101_1_1111_1111_1111_1111_1111_1111,
1479 },
1480 .{ // bx lr
1481 .inst = Instruction.bx(.al, .lr),
1482 .expected = 0b1110_0001_0010_1111_1111_1111_0001_1110,
1483 },
1484 .{ // svc #0
1485 .inst = Instruction.svc(.al, 0),
1486 .expected = 0b1110_1111_0000_0000_0000_0000_0000_0000,
1487 },
1488 .{ // bkpt #42
1489 .inst = Instruction.bkpt(42),
1490 .expected = 0b1110_0001_0010_000000000010_0111_1010,
1491 },
1492 .{ // stmdb r9, {r0}
1493 .inst = Instruction.stmdb(.al, .r9, false, .{ .r0 = true }),
1494 .expected = 0b1110_100_1_0_0_0_0_1001_0000000000000001,
1495 },
1496 .{ // ldmea r4!, {r2, r5}
1497 .inst = Instruction.ldmea(.al, .r4, true, .{ .r2 = true, .r5 = true }),
1498 .expected = 0b1110_100_1_0_0_1_1_0100_0000000000100100,
1499 },
1500 .{ // qadd r0, r7, r8
1501 .inst = Instruction.qadd(.al, .r0, .r7, .r8),
1502 .expected = 0b1110_00010_00_0_1000_0000_0000_0101_0111,
1503 },
1504 .{ // smulbt r0, r0, r0
1505 .inst = Instruction.smulbt(.al, .r0, .r0, .r0),
1506 .expected = 0b1110_00010110_0000_0000_0000_1_1_0_0_0000,
1507 },
1508 };
1509
1510 for (testcases) |case| {
1511 const actual = case.inst.toU32();
1512 try testing.expectEqual(case.expected, actual);
1513 }
1514}
1515
1516test "aliases" {
1517 const Testcase = struct {
1518 expected: Instruction,
1519 actual: Instruction,
1520 };
1521
1522 const testcases = [_]Testcase{
1523 .{ // pop { r6 }
1524 .actual = Instruction.pop(.al, .{.r6}),
1525 .expected = Instruction.ldr(.al, .r6, .sp, .{
1526 .mode = .post_index,
1527 .positive = true,
1528 .offset = Instruction.Offset.imm(4),
1529 }),
1530 },
1531 .{ // pop { r1, r5 }
1532 .actual = Instruction.pop(.al, .{ .r1, .r5 }),
1533 .expected = Instruction.ldm(.al, .sp, true, .{ .r1 = true, .r5 = true }),
1534 },
1535 .{ // push { r3 }
1536 .actual = Instruction.push(.al, .{.r3}),
1537 .expected = Instruction.str(.al, .r3, .sp, .{
1538 .mode = .pre_index,
1539 .positive = false,
1540 .offset = Instruction.Offset.imm(4),
1541 }),
1542 },
1543 .{ // push { r0, r2 }
1544 .actual = Instruction.push(.al, .{ .r0, .r2 }),
1545 .expected = Instruction.stmdb(.al, .sp, true, .{ .r0 = true, .r2 = true }),
1546 },
1547 .{ // lsl r4, r5, #5
1548 .actual = Instruction.lsl(.al, .r4, .r5, Instruction.ShiftAmount.imm(5)),
1549 .expected = Instruction.mov(.al, .r4, Instruction.Operand.reg(
1550 .r5,
1551 Instruction.Operand.Shift.imm(5, .logical_left),
1552 )),
1553 },
1554 .{ // asrs r1, r1, r3
1555 .actual = Instruction.asrs(.al, .r1, .r1, Instruction.ShiftAmount.reg(.r3)),
1556 .expected = Instruction.movs(.al, .r1, Instruction.Operand.reg(
1557 .r1,
1558 Instruction.Operand.Shift.reg(.r3, .arithmetic_right),
1559 )),
1560 },
1561 };
1562
1563 for (testcases) |case| {
1564 try testing.expectEqual(case.expected.toU32(), case.actual.toU32());
1565 }
1566}
src/arch/mips/abi.zig deleted-84
...@@ -1,84 +0,0 @@
1const std = @import("std");
2const Type = @import("../../Type.zig");
3const Zcu = @import("../../Zcu.zig");
4const assert = std.debug.assert;
5
6pub const Class = union(enum) {
7 memory,
8 byval,
9 i32_array: u8,
10};
11
12pub const Context = enum { ret, arg };
13
14pub fn classifyType(ty: Type, zcu: *Zcu, ctx: Context) Class {
15 const target = zcu.getTarget();
16 std.debug.assert(ty.hasRuntimeBitsIgnoreComptime(zcu));
17
18 const max_direct_size = target.ptrBitWidth() * 2;
19 switch (ty.zigTypeTag(zcu)) {
20 .@"struct" => {
21 const bit_size = ty.bitSize(zcu);
22 if (ty.containerLayout(zcu) == .@"packed") {
23 if (bit_size > max_direct_size) return .memory;
24 return .byval;
25 }
26 if (bit_size > max_direct_size) return .memory;
27 // TODO: for bit_size <= 32 using byval is more correct, but that needs inreg argument attribute
28 const count = @as(u8, @intCast(std.mem.alignForward(u64, bit_size, 32) / 32));
29 return .{ .i32_array = count };
30 },
31 .@"union" => {
32 const bit_size = ty.bitSize(zcu);
33 if (ty.containerLayout(zcu) == .@"packed") {
34 if (bit_size > max_direct_size) return .memory;
35 return .byval;
36 }
37 if (bit_size > max_direct_size) return .memory;
38
39 return .byval;
40 },
41 .bool => return .byval,
42 .float => return .byval,
43 .int, .@"enum", .error_set => {
44 return .byval;
45 },
46 .vector => {
47 const elem_type = ty.elemType2(zcu);
48 switch (elem_type.zigTypeTag(zcu)) {
49 .bool, .int => {
50 const bit_size = ty.bitSize(zcu);
51 if (ctx == .ret and bit_size > 128) return .memory;
52 if (bit_size > 512) return .memory;
53 // TODO: byval vector arguments with non power of 2 size need inreg attribute
54 return .byval;
55 },
56 .float => return .memory,
57 else => unreachable,
58 }
59 },
60 .optional => {
61 std.debug.assert(ty.isPtrLikeOptional(zcu));
62 return .byval;
63 },
64 .pointer => {
65 std.debug.assert(!ty.isSlice(zcu));
66 return .byval;
67 },
68 .error_union,
69 .frame,
70 .@"anyframe",
71 .noreturn,
72 .void,
73 .type,
74 .comptime_float,
75 .comptime_int,
76 .undefined,
77 .null,
78 .@"fn",
79 .@"opaque",
80 .enum_literal,
81 .array,
82 => unreachable,
83 }
84}
src/arch/powerpc/CodeGen.zig deleted-51
...@@ -1,51 +0,0 @@
1const builtin = @import("builtin");
2const std = @import("std");
3
4const Air = @import("../../Air.zig");
5const codegen = @import("../../codegen.zig");
6const InternPool = @import("../../InternPool.zig");
7const link = @import("../../link.zig");
8const Zcu = @import("../../Zcu.zig");
9
10const assert = std.debug.assert;
11const log = std.log.scoped(.codegen);
12
13pub fn legalizeFeatures(_: *const std.Target) ?*const Air.Legalize.Features {
14 return null;
15}
16
17pub fn generate(
18 bin_file: *link.File,
19 pt: Zcu.PerThread,
20 src_loc: Zcu.LazySrcLoc,
21 func_index: InternPool.Index,
22 air: *const Air,
23 liveness: *const Air.Liveness,
24) codegen.CodeGenError!noreturn {
25 _ = bin_file;
26 _ = pt;
27 _ = src_loc;
28 _ = func_index;
29 _ = air;
30 _ = liveness;
31
32 unreachable;
33}
34
35pub fn generateLazy(
36 bin_file: *link.File,
37 pt: Zcu.PerThread,
38 src_loc: Zcu.LazySrcLoc,
39 lazy_sym: link.File.LazySymbol,
40 code: *std.ArrayListUnmanaged(u8),
41 debug_output: link.File.DebugInfoOutput,
42) codegen.CodeGenError!void {
43 _ = bin_file;
44 _ = pt;
45 _ = src_loc;
46 _ = lazy_sym;
47 _ = code;
48 _ = debug_output;
49
50 unreachable;
51}
src/arch/wasm/CodeGen.zig+1-1
...@@ -17,7 +17,7 @@ const Compilation = @import("../../Compilation.zig");...@@ -17,7 +17,7 @@ const Compilation = @import("../../Compilation.zig");
17const link = @import("../../link.zig");17const link = @import("../../link.zig");
18const Air = @import("../../Air.zig");18const Air = @import("../../Air.zig");
19const Mir = @import("Mir.zig");19const Mir = @import("Mir.zig");
20const abi = @import("abi.zig");20const abi = @import("../../codegen/wasm/abi.zig");
21const Alignment = InternPool.Alignment;21const Alignment = InternPool.Alignment;
22const errUnionPayloadOffset = codegen.errUnionPayloadOffset;22const errUnionPayloadOffset = codegen.errUnionPayloadOffset;
23const errUnionErrorOffset = codegen.errUnionErrorOffset;23const errUnionErrorOffset = codegen.errUnionErrorOffset;
src/arch/wasm/abi.zig deleted-87
...@@ -1,87 +0,0 @@
1//! Classifies Zig types to follow the C-ABI for Wasm.
2//! The convention for Wasm's C-ABI can be found at the tool-conventions repo:
3//! https://github.com/WebAssembly/tool-conventions/blob/main/BasicCABI.md
4//! When not targeting the C-ABI, Zig is allowed to do derail from this convention.
5//! Note: Above mentioned document is not an official specification, therefore called a convention.
6
7const std = @import("std");
8const Target = std.Target;
9const assert = std.debug.assert;
10
11const Type = @import("../../Type.zig");
12const Zcu = @import("../../Zcu.zig");
13
14/// Defines how to pass a type as part of a function signature,
15/// both for parameters as well as return values.
16pub const Class = union(enum) {
17 direct: Type,
18 indirect,
19};
20
21/// Classifies a given Zig type to determine how they must be passed
22/// or returned as value within a wasm function.
23pub fn classifyType(ty: Type, zcu: *const Zcu) Class {
24 const ip = &zcu.intern_pool;
25 assert(ty.hasRuntimeBitsIgnoreComptime(zcu));
26 switch (ty.zigTypeTag(zcu)) {
27 .int, .@"enum", .error_set => return .{ .direct = ty },
28 .float => return .{ .direct = ty },
29 .bool => return .{ .direct = ty },
30 .vector => return .{ .direct = ty },
31 .array => return .indirect,
32 .optional => {
33 assert(ty.isPtrLikeOptional(zcu));
34 return .{ .direct = ty };
35 },
36 .pointer => {
37 assert(!ty.isSlice(zcu));
38 return .{ .direct = ty };
39 },
40 .@"struct" => {
41 const struct_type = zcu.typeToStruct(ty).?;
42 if (struct_type.layout == .@"packed") {
43 return .{ .direct = ty };
44 }
45 if (struct_type.field_types.len > 1) {
46 // The struct type is non-scalar.
47 return .indirect;
48 }
49 const field_ty = Type.fromInterned(struct_type.field_types.get(ip)[0]);
50 const explicit_align = struct_type.fieldAlign(ip, 0);
51 if (explicit_align != .none) {
52 if (explicit_align.compareStrict(.gt, field_ty.abiAlignment(zcu)))
53 return .indirect;
54 }
55 return classifyType(field_ty, zcu);
56 },
57 .@"union" => {
58 const union_obj = zcu.typeToUnion(ty).?;
59 if (union_obj.flagsUnordered(ip).layout == .@"packed") {
60 return .{ .direct = ty };
61 }
62 const layout = ty.unionGetLayout(zcu);
63 assert(layout.tag_size == 0);
64 if (union_obj.field_types.len > 1) return .indirect;
65 const first_field_ty = Type.fromInterned(union_obj.field_types.get(ip)[0]);
66 return classifyType(first_field_ty, zcu);
67 },
68 .error_union,
69 .frame,
70 .@"anyframe",
71 .noreturn,
72 .void,
73 .type,
74 .comptime_float,
75 .comptime_int,
76 .undefined,
77 .null,
78 .@"fn",
79 .@"opaque",
80 .enum_literal,
81 => unreachable,
82 }
83}
84
85pub fn lowerAsDoubleI64(scalar_ty: Type, zcu: *const Zcu) bool {
86 return scalar_ty.bitSize(zcu) > 64;
87}
src/arch/x86/bits.zig deleted-100
...@@ -1,100 +0,0 @@
1const std = @import("std");
2
3// zig fmt: off
4pub const Register = enum(u8) {
5 // 0 through 7, 32-bit registers. id is int value
6 eax, ecx, edx, ebx, esp, ebp, esi, edi,
7
8 // 8-15, 16-bit registers. id is int value - 8.
9 ax, cx, dx, bx, sp, bp, si, di,
10
11 // 16-23, 8-bit registers. id is int value - 16.
12 al, cl, dl, bl, ah, ch, dh, bh,
13
14 /// Returns the bit-width of the register.
15 pub fn size(self: Register) u7 {
16 return switch (@intFromEnum(self)) {
17 0...7 => 32,
18 8...15 => 16,
19 16...23 => 8,
20 else => unreachable,
21 };
22 }
23
24 /// Returns the register's id. This is used in practically every opcode the
25 /// x86 has. It is embedded in some instructions, such as the `B8 +rd` move
26 /// instruction, and is used in the R/M byte.
27 pub fn id(self: Register) u3 {
28 return @truncate(@intFromEnum(self));
29 }
30
31 /// Convert from any register to its 32 bit alias.
32 pub fn to32(self: Register) Register {
33 return @enumFromInt(@as(u8, self.id()));
34 }
35
36 /// Convert from any register to its 16 bit alias.
37 pub fn to16(self: Register) Register {
38 return @enumFromInt(@as(u8, self.id()) + 8);
39 }
40
41 /// Convert from any register to its 8 bit alias.
42 pub fn to8(self: Register) Register {
43 return @enumFromInt(@as(u8, self.id()) + 16);
44 }
45
46 pub fn dwarfNum(reg: Register) u8 {
47 return @intFromEnum(reg.to32());
48 }
49};
50
51// zig fmt: on
52
53/// TODO this set is actually a set of caller-saved registers.
54pub const callee_preserved_regs = [_]Register{ .eax, .ecx, .edx, .esi, .edi };
55
56// TODO add these to Register enum and corresponding dwarfNum
57// // Return Address register. This is stored in `0(%esp, "")` and is not a physical register.
58// RA = (8, "RA"),
59//
60// ST0 = (11, "st0"),
61// ST1 = (12, "st1"),
62// ST2 = (13, "st2"),
63// ST3 = (14, "st3"),
64// ST4 = (15, "st4"),
65// ST5 = (16, "st5"),
66// ST6 = (17, "st6"),
67// ST7 = (18, "st7"),
68//
69// XMM0 = (21, "xmm0"),
70// XMM1 = (22, "xmm1"),
71// XMM2 = (23, "xmm2"),
72// XMM3 = (24, "xmm3"),
73// XMM4 = (25, "xmm4"),
74// XMM5 = (26, "xmm5"),
75// XMM6 = (27, "xmm6"),
76// XMM7 = (28, "xmm7"),
77//
78// MM0 = (29, "mm0"),
79// MM1 = (30, "mm1"),
80// MM2 = (31, "mm2"),
81// MM3 = (32, "mm3"),
82// MM4 = (33, "mm4"),
83// MM5 = (34, "mm5"),
84// MM6 = (35, "mm6"),
85// MM7 = (36, "mm7"),
86//
87// MXCSR = (39, "mxcsr"),
88//
89// ES = (40, "es"),
90// CS = (41, "cs"),
91// SS = (42, "ss"),
92// DS = (43, "ds"),
93// FS = (44, "fs"),
94// GS = (45, "gs"),
95//
96// TR = (48, "tr"),
97// LDTR = (49, "ldtr"),
98//
99// FS_BASE = (93, "fs.base"),
100// GS_BASE = (94, "gs.base"),
src/codegen.zig+8-27
...@@ -34,7 +34,7 @@ fn devFeatureForBackend(backend: std.builtin.CompilerBackend) dev.Feature {...@@ -34,7 +34,7 @@ fn devFeatureForBackend(backend: std.builtin.CompilerBackend) dev.Feature {
34 .stage2_arm => .arm_backend,34 .stage2_arm => .arm_backend,
35 .stage2_c => .c_backend,35 .stage2_c => .c_backend,
36 .stage2_llvm => .llvm_backend,36 .stage2_llvm => .llvm_backend,
37 .stage2_powerpc => .powerpc_backend,37 .stage2_powerpc => unreachable,
38 .stage2_riscv64 => .riscv64_backend,38 .stage2_riscv64 => .riscv64_backend,
39 .stage2_sparc64 => .sparc64_backend,39 .stage2_sparc64 => .sparc64_backend,
40 .stage2_spirv => .spirv_backend,40 .stage2_spirv => .spirv_backend,
...@@ -48,11 +48,11 @@ fn devFeatureForBackend(backend: std.builtin.CompilerBackend) dev.Feature {...@@ -48,11 +48,11 @@ fn devFeatureForBackend(backend: std.builtin.CompilerBackend) dev.Feature {
48fn importBackend(comptime backend: std.builtin.CompilerBackend) type {48fn importBackend(comptime backend: std.builtin.CompilerBackend) type {
49 return switch (backend) {49 return switch (backend) {
50 .other, .stage1 => unreachable,50 .other, .stage1 => unreachable,
51 .stage2_aarch64 => @import("arch/aarch64/CodeGen.zig"),51 .stage2_aarch64 => unreachable,
52 .stage2_arm => @import("arch/arm/CodeGen.zig"),52 .stage2_arm => unreachable,
53 .stage2_c => @import("codegen/c.zig"),53 .stage2_c => @import("codegen/c.zig"),
54 .stage2_llvm => @import("codegen/llvm.zig"),54 .stage2_llvm => @import("codegen/llvm.zig"),
55 .stage2_powerpc => @import("arch/powerpc/CodeGen.zig"),55 .stage2_powerpc => unreachable,
56 .stage2_riscv64 => @import("arch/riscv64/CodeGen.zig"),56 .stage2_riscv64 => @import("arch/riscv64/CodeGen.zig"),
57 .stage2_sparc64 => @import("arch/sparc64/CodeGen.zig"),57 .stage2_sparc64 => @import("arch/sparc64/CodeGen.zig"),
58 .stage2_spirv => @import("codegen/spirv.zig"),58 .stage2_spirv => @import("codegen/spirv.zig"),
...@@ -70,14 +70,11 @@ pub fn legalizeFeatures(pt: Zcu.PerThread, nav_index: InternPool.Nav.Index) ?*co...@@ -70,14 +70,11 @@ pub fn legalizeFeatures(pt: Zcu.PerThread, nav_index: InternPool.Nav.Index) ?*co
70 inline .stage2_llvm,70 inline .stage2_llvm,
71 .stage2_c,71 .stage2_c,
72 .stage2_wasm,72 .stage2_wasm,
73 .stage2_arm,
74 .stage2_x86_64,73 .stage2_x86_64,
75 .stage2_aarch64,
76 .stage2_x86,74 .stage2_x86,
77 .stage2_riscv64,75 .stage2_riscv64,
78 .stage2_sparc64,76 .stage2_sparc64,
79 .stage2_spirv,77 .stage2_spirv,
80 .stage2_powerpc,
81 => |backend| {78 => |backend| {
82 dev.check(devFeatureForBackend(backend));79 dev.check(devFeatureForBackend(backend));
83 return importBackend(backend).legalizeFeatures(target);80 return importBackend(backend).legalizeFeatures(target);
...@@ -89,9 +86,6 @@ pub fn legalizeFeatures(pt: Zcu.PerThread, nav_index: InternPool.Nav.Index) ?*co...@@ -89,9 +86,6 @@ pub fn legalizeFeatures(pt: Zcu.PerThread, nav_index: InternPool.Nav.Index) ?*co
89/// MIR from codegen to the linker *regardless* of which backend is in use. So, we use this: a86/// MIR from codegen to the linker *regardless* of which backend is in use. So, we use this: a
90/// union of all MIR types. The active tag is known from the backend in use; see `AnyMir.tag`.87/// union of all MIR types. The active tag is known from the backend in use; see `AnyMir.tag`.
91pub const AnyMir = union {88pub const AnyMir = union {
92 aarch64: @import("arch/aarch64/Mir.zig"),
93 arm: @import("arch/arm/Mir.zig"),
94 powerpc: noreturn, //@import("arch/powerpc/Mir.zig"),
95 riscv64: @import("arch/riscv64/Mir.zig"),89 riscv64: @import("arch/riscv64/Mir.zig"),
96 sparc64: @import("arch/sparc64/Mir.zig"),90 sparc64: @import("arch/sparc64/Mir.zig"),
97 x86_64: @import("arch/x86_64/Mir.zig"),91 x86_64: @import("arch/x86_64/Mir.zig"),
...@@ -102,7 +96,6 @@ pub const AnyMir = union {...@@ -102,7 +96,6 @@ pub const AnyMir = union {
102 return switch (backend) {96 return switch (backend) {
103 .stage2_aarch64 => "aarch64",97 .stage2_aarch64 => "aarch64",
104 .stage2_arm => "arm",98 .stage2_arm => "arm",
105 .stage2_powerpc => "powerpc",
106 .stage2_riscv64 => "riscv64",99 .stage2_riscv64 => "riscv64",
107 .stage2_sparc64 => "sparc64",100 .stage2_sparc64 => "sparc64",
108 .stage2_x86_64 => "x86_64",101 .stage2_x86_64 => "x86_64",
...@@ -117,10 +110,7 @@ pub const AnyMir = union {...@@ -117,10 +110,7 @@ pub const AnyMir = union {
117 const backend = target_util.zigBackend(&zcu.root_mod.resolved_target.result, zcu.comp.config.use_llvm);110 const backend = target_util.zigBackend(&zcu.root_mod.resolved_target.result, zcu.comp.config.use_llvm);
118 switch (backend) {111 switch (backend) {
119 else => unreachable,112 else => unreachable,
120 inline .stage2_aarch64,113 inline .stage2_riscv64,
121 .stage2_arm,
122 .stage2_powerpc,
123 .stage2_riscv64,
124 .stage2_sparc64,114 .stage2_sparc64,
125 .stage2_x86_64,115 .stage2_x86_64,
126 .stage2_wasm,116 .stage2_wasm,
...@@ -148,10 +138,7 @@ pub fn generateFunction(...@@ -148,10 +138,7 @@ pub fn generateFunction(
148 const target = &zcu.navFileScope(func.owner_nav).mod.?.resolved_target.result;138 const target = &zcu.navFileScope(func.owner_nav).mod.?.resolved_target.result;
149 switch (target_util.zigBackend(target, false)) {139 switch (target_util.zigBackend(target, false)) {
150 else => unreachable,140 else => unreachable,
151 inline .stage2_aarch64,141 inline .stage2_riscv64,
152 .stage2_arm,
153 .stage2_powerpc,
154 .stage2_riscv64,
155 .stage2_sparc64,142 .stage2_sparc64,
156 .stage2_x86_64,143 .stage2_x86_64,
157 .stage2_wasm,144 .stage2_wasm,
...@@ -186,10 +173,7 @@ pub fn emitFunction(...@@ -186,10 +173,7 @@ pub fn emitFunction(
186 const target = &zcu.navFileScope(func.owner_nav).mod.?.resolved_target.result;173 const target = &zcu.navFileScope(func.owner_nav).mod.?.resolved_target.result;
187 switch (target_util.zigBackend(target, zcu.comp.config.use_llvm)) {174 switch (target_util.zigBackend(target, zcu.comp.config.use_llvm)) {
188 else => unreachable,175 else => unreachable,
189 inline .stage2_aarch64,176 inline .stage2_riscv64,
190 .stage2_arm,
191 .stage2_powerpc,
192 .stage2_riscv64,
193 .stage2_sparc64,177 .stage2_sparc64,
194 .stage2_x86_64,178 .stage2_x86_64,
195 => |backend| {179 => |backend| {
...@@ -215,10 +199,7 @@ pub fn generateLazyFunction(...@@ -215,10 +199,7 @@ pub fn generateLazyFunction(
215 zcu.getTarget();199 zcu.getTarget();
216 switch (target_util.zigBackend(target, zcu.comp.config.use_llvm)) {200 switch (target_util.zigBackend(target, zcu.comp.config.use_llvm)) {
217 else => unreachable,201 else => unreachable,
218 inline .stage2_powerpc,202 inline .stage2_riscv64, .stage2_x86_64 => |backend| {
219 .stage2_riscv64,
220 .stage2_x86_64,
221 => |backend| {
222 dev.check(devFeatureForBackend(backend));203 dev.check(devFeatureForBackend(backend));
223 return importBackend(backend).generateLazy(lf, pt, src_loc, lazy_sym, code, debug_output);204 return importBackend(backend).generateLazy(lf, pt, src_loc, lazy_sym, code, debug_output);
224 },205 },
src/codegen/aarch64/abi.zig created+150
...@@ -0,0 +1,150 @@
1const std = @import("std");
2const builtin = @import("builtin");
3const bits = @import("../../arch/aarch64/bits.zig");
4const Register = bits.Register;
5const Type = @import("../../Type.zig");
6const Zcu = @import("../../Zcu.zig");
7
8pub const Class = union(enum) {
9 memory,
10 byval,
11 integer,
12 double_integer,
13 float_array: u8,
14};
15
16/// For `float_array` the second element will be the amount of floats.
17pub fn classifyType(ty: Type, zcu: *Zcu) Class {
18 std.debug.assert(ty.hasRuntimeBitsIgnoreComptime(zcu));
19
20 var maybe_float_bits: ?u16 = null;
21 switch (ty.zigTypeTag(zcu)) {
22 .@"struct" => {
23 if (ty.containerLayout(zcu) == .@"packed") return .byval;
24 const float_count = countFloats(ty, zcu, &maybe_float_bits);
25 if (float_count <= sret_float_count) return .{ .float_array = float_count };
26
27 const bit_size = ty.bitSize(zcu);
28 if (bit_size > 128) return .memory;
29 if (bit_size > 64) return .double_integer;
30 return .integer;
31 },
32 .@"union" => {
33 if (ty.containerLayout(zcu) == .@"packed") return .byval;
34 const float_count = countFloats(ty, zcu, &maybe_float_bits);
35 if (float_count <= sret_float_count) return .{ .float_array = float_count };
36
37 const bit_size = ty.bitSize(zcu);
38 if (bit_size > 128) return .memory;
39 if (bit_size > 64) return .double_integer;
40 return .integer;
41 },
42 .int, .@"enum", .error_set, .float, .bool => return .byval,
43 .vector => {
44 const bit_size = ty.bitSize(zcu);
45 // TODO is this controlled by a cpu feature?
46 if (bit_size > 128) return .memory;
47 return .byval;
48 },
49 .optional => {
50 std.debug.assert(ty.isPtrLikeOptional(zcu));
51 return .byval;
52 },
53 .pointer => {
54 std.debug.assert(!ty.isSlice(zcu));
55 return .byval;
56 },
57 .error_union,
58 .frame,
59 .@"anyframe",
60 .noreturn,
61 .void,
62 .type,
63 .comptime_float,
64 .comptime_int,
65 .undefined,
66 .null,
67 .@"fn",
68 .@"opaque",
69 .enum_literal,
70 .array,
71 => unreachable,
72 }
73}
74
75const sret_float_count = 4;
76fn countFloats(ty: Type, zcu: *Zcu, maybe_float_bits: *?u16) u8 {
77 const ip = &zcu.intern_pool;
78 const target = zcu.getTarget();
79 const invalid = std.math.maxInt(u8);
80 switch (ty.zigTypeTag(zcu)) {
81 .@"union" => {
82 const union_obj = zcu.typeToUnion(ty).?;
83 var max_count: u8 = 0;
84 for (union_obj.field_types.get(ip)) |field_ty| {
85 const field_count = countFloats(Type.fromInterned(field_ty), zcu, maybe_float_bits);
86 if (field_count == invalid) return invalid;
87 if (field_count > max_count) max_count = field_count;
88 if (max_count > sret_float_count) return invalid;
89 }
90 return max_count;
91 },
92 .@"struct" => {
93 const fields_len = ty.structFieldCount(zcu);
94 var count: u8 = 0;
95 var i: u32 = 0;
96 while (i < fields_len) : (i += 1) {
97 const field_ty = ty.fieldType(i, zcu);
98 const field_count = countFloats(field_ty, zcu, maybe_float_bits);
99 if (field_count == invalid) return invalid;
100 count += field_count;
101 if (count > sret_float_count) return invalid;
102 }
103 return count;
104 },
105 .float => {
106 const float_bits = maybe_float_bits.* orelse {
107 maybe_float_bits.* = ty.floatBits(target);
108 return 1;
109 };
110 if (ty.floatBits(target) == float_bits) return 1;
111 return invalid;
112 },
113 .void => return 0,
114 else => return invalid,
115 }
116}
117
118pub fn getFloatArrayType(ty: Type, zcu: *Zcu) ?Type {
119 const ip = &zcu.intern_pool;
120 switch (ty.zigTypeTag(zcu)) {
121 .@"union" => {
122 const union_obj = zcu.typeToUnion(ty).?;
123 for (union_obj.field_types.get(ip)) |field_ty| {
124 if (getFloatArrayType(Type.fromInterned(field_ty), zcu)) |some| return some;
125 }
126 return null;
127 },
128 .@"struct" => {
129 const fields_len = ty.structFieldCount(zcu);
130 var i: u32 = 0;
131 while (i < fields_len) : (i += 1) {
132 const field_ty = ty.fieldType(i, zcu);
133 if (getFloatArrayType(field_ty, zcu)) |some| return some;
134 }
135 return null;
136 },
137 .float => return ty,
138 else => return null,
139 }
140}
141
142pub const callee_preserved_regs = [_]Register{
143 .x19, .x20, .x21, .x22, .x23,
144 .x24, .x25, .x26, .x27, .x28,
145};
146
147pub const c_abi_int_param_regs = [_]Register{ .x0, .x1, .x2, .x3, .x4, .x5, .x6, .x7 };
148pub const c_abi_int_return_regs = [_]Register{ .x0, .x1, .x2, .x3, .x4, .x5, .x6, .x7 };
149
150const allocatable_registers = callee_preserved_regs;
src/codegen/arm/abi.zig created+163
...@@ -0,0 +1,163 @@
1const std = @import("std");
2const assert = std.debug.assert;
3const RegisterManagerFn = @import("../../register_manager.zig").RegisterManager;
4const Type = @import("../../Type.zig");
5const Zcu = @import("../../Zcu.zig");
6
7pub const Class = union(enum) {
8 memory,
9 byval,
10 i32_array: u8,
11 i64_array: u8,
12
13 fn arrSize(total_size: u64, arr_size: u64) Class {
14 const count = @as(u8, @intCast(std.mem.alignForward(u64, total_size, arr_size) / arr_size));
15 if (arr_size == 32) {
16 return .{ .i32_array = count };
17 } else {
18 return .{ .i64_array = count };
19 }
20 }
21};
22
23pub const Context = enum { ret, arg };
24
25pub fn classifyType(ty: Type, zcu: *Zcu, ctx: Context) Class {
26 assert(ty.hasRuntimeBitsIgnoreComptime(zcu));
27
28 var maybe_float_bits: ?u16 = null;
29 const max_byval_size = 512;
30 const ip = &zcu.intern_pool;
31 switch (ty.zigTypeTag(zcu)) {
32 .@"struct" => {
33 const bit_size = ty.bitSize(zcu);
34 if (ty.containerLayout(zcu) == .@"packed") {
35 if (bit_size > 64) return .memory;
36 return .byval;
37 }
38 if (bit_size > max_byval_size) return .memory;
39 const float_count = countFloats(ty, zcu, &maybe_float_bits);
40 if (float_count <= byval_float_count) return .byval;
41
42 const fields = ty.structFieldCount(zcu);
43 var i: u32 = 0;
44 while (i < fields) : (i += 1) {
45 const field_ty = ty.fieldType(i, zcu);
46 const field_alignment = ty.fieldAlignment(i, zcu);
47 const field_size = field_ty.bitSize(zcu);
48 if (field_size > 32 or field_alignment.compare(.gt, .@"32")) {
49 return Class.arrSize(bit_size, 64);
50 }
51 }
52 return Class.arrSize(bit_size, 32);
53 },
54 .@"union" => {
55 const bit_size = ty.bitSize(zcu);
56 const union_obj = zcu.typeToUnion(ty).?;
57 if (union_obj.flagsUnordered(ip).layout == .@"packed") {
58 if (bit_size > 64) return .memory;
59 return .byval;
60 }
61 if (bit_size > max_byval_size) return .memory;
62 const float_count = countFloats(ty, zcu, &maybe_float_bits);
63 if (float_count <= byval_float_count) return .byval;
64
65 for (union_obj.field_types.get(ip), 0..) |field_ty, field_index| {
66 if (Type.fromInterned(field_ty).bitSize(zcu) > 32 or
67 ty.fieldAlignment(field_index, zcu).compare(.gt, .@"32"))
68 {
69 return Class.arrSize(bit_size, 64);
70 }
71 }
72 return Class.arrSize(bit_size, 32);
73 },
74 .bool, .float => return .byval,
75 .int => {
76 // TODO this is incorrect for _BitInt(128) but implementing
77 // this correctly makes implementing compiler-rt impossible.
78 // const bit_size = ty.bitSize(zcu);
79 // if (bit_size > 64) return .memory;
80 return .byval;
81 },
82 .@"enum", .error_set => {
83 const bit_size = ty.bitSize(zcu);
84 if (bit_size > 64) return .memory;
85 return .byval;
86 },
87 .vector => {
88 const bit_size = ty.bitSize(zcu);
89 // TODO is this controlled by a cpu feature?
90 if (ctx == .ret and bit_size > 128) return .memory;
91 if (bit_size > 512) return .memory;
92 return .byval;
93 },
94 .optional => {
95 assert(ty.isPtrLikeOptional(zcu));
96 return .byval;
97 },
98 .pointer => {
99 assert(!ty.isSlice(zcu));
100 return .byval;
101 },
102 .error_union,
103 .frame,
104 .@"anyframe",
105 .noreturn,
106 .void,
107 .type,
108 .comptime_float,
109 .comptime_int,
110 .undefined,
111 .null,
112 .@"fn",
113 .@"opaque",
114 .enum_literal,
115 .array,
116 => unreachable,
117 }
118}
119
120const byval_float_count = 4;
121fn countFloats(ty: Type, zcu: *Zcu, maybe_float_bits: *?u16) u32 {
122 const ip = &zcu.intern_pool;
123 const target = zcu.getTarget();
124 const invalid = std.math.maxInt(u32);
125 switch (ty.zigTypeTag(zcu)) {
126 .@"union" => {
127 const union_obj = zcu.typeToUnion(ty).?;
128 var max_count: u32 = 0;
129 for (union_obj.field_types.get(ip)) |field_ty| {
130 const field_count = countFloats(Type.fromInterned(field_ty), zcu, maybe_float_bits);
131 if (field_count == invalid) return invalid;
132 if (field_count > max_count) max_count = field_count;
133 if (max_count > byval_float_count) return invalid;
134 }
135 return max_count;
136 },
137 .@"struct" => {
138 const fields_len = ty.structFieldCount(zcu);
139 var count: u32 = 0;
140 var i: u32 = 0;
141 while (i < fields_len) : (i += 1) {
142 const field_ty = ty.fieldType(i, zcu);
143 const field_count = countFloats(field_ty, zcu, maybe_float_bits);
144 if (field_count == invalid) return invalid;
145 count += field_count;
146 if (count > byval_float_count) return invalid;
147 }
148 return count;
149 },
150 .float => {
151 const float_bits = maybe_float_bits.* orelse {
152 const float_bits = ty.floatBits(target);
153 if (float_bits != 32 and float_bits != 64) return invalid;
154 maybe_float_bits.* = float_bits;
155 return 1;
156 };
157 if (ty.floatBits(target) == float_bits) return 1;
158 return invalid;
159 },
160 .void => return 0,
161 else => return invalid,
162 }
163}
src/codegen/llvm.zig+4-4
...@@ -21,11 +21,11 @@ const Air = @import("../Air.zig");...@@ -21,11 +21,11 @@ const Air = @import("../Air.zig");
21const Value = @import("../Value.zig");21const Value = @import("../Value.zig");
22const Type = @import("../Type.zig");22const Type = @import("../Type.zig");
23const x86_64_abi = @import("../arch/x86_64/abi.zig");23const x86_64_abi = @import("../arch/x86_64/abi.zig");
24const wasm_c_abi = @import("../arch/wasm/abi.zig");24const wasm_c_abi = @import("wasm/abi.zig");
25const aarch64_c_abi = @import("../arch/aarch64/abi.zig");25const aarch64_c_abi = @import("aarch64/abi.zig");
26const arm_c_abi = @import("../arch/arm/abi.zig");26const arm_c_abi = @import("arm/abi.zig");
27const riscv_c_abi = @import("../arch/riscv64/abi.zig");27const riscv_c_abi = @import("../arch/riscv64/abi.zig");
28const mips_c_abi = @import("../arch/mips/abi.zig");28const mips_c_abi = @import("mips/abi.zig");
29const dev = @import("../dev.zig");29const dev = @import("../dev.zig");
3030
31const target_util = @import("../target.zig");31const target_util = @import("../target.zig");
src/codegen/mips/abi.zig created+84
...@@ -0,0 +1,84 @@
1const std = @import("std");
2const Type = @import("../../Type.zig");
3const Zcu = @import("../../Zcu.zig");
4const assert = std.debug.assert;
5
6pub const Class = union(enum) {
7 memory,
8 byval,
9 i32_array: u8,
10};
11
12pub const Context = enum { ret, arg };
13
14pub fn classifyType(ty: Type, zcu: *Zcu, ctx: Context) Class {
15 const target = zcu.getTarget();
16 std.debug.assert(ty.hasRuntimeBitsIgnoreComptime(zcu));
17
18 const max_direct_size = target.ptrBitWidth() * 2;
19 switch (ty.zigTypeTag(zcu)) {
20 .@"struct" => {
21 const bit_size = ty.bitSize(zcu);
22 if (ty.containerLayout(zcu) == .@"packed") {
23 if (bit_size > max_direct_size) return .memory;
24 return .byval;
25 }
26 if (bit_size > max_direct_size) return .memory;
27 // TODO: for bit_size <= 32 using byval is more correct, but that needs inreg argument attribute
28 const count = @as(u8, @intCast(std.mem.alignForward(u64, bit_size, 32) / 32));
29 return .{ .i32_array = count };
30 },
31 .@"union" => {
32 const bit_size = ty.bitSize(zcu);
33 if (ty.containerLayout(zcu) == .@"packed") {
34 if (bit_size > max_direct_size) return .memory;
35 return .byval;
36 }
37 if (bit_size > max_direct_size) return .memory;
38
39 return .byval;
40 },
41 .bool => return .byval,
42 .float => return .byval,
43 .int, .@"enum", .error_set => {
44 return .byval;
45 },
46 .vector => {
47 const elem_type = ty.elemType2(zcu);
48 switch (elem_type.zigTypeTag(zcu)) {
49 .bool, .int => {
50 const bit_size = ty.bitSize(zcu);
51 if (ctx == .ret and bit_size > 128) return .memory;
52 if (bit_size > 512) return .memory;
53 // TODO: byval vector arguments with non power of 2 size need inreg attribute
54 return .byval;
55 },
56 .float => return .memory,
57 else => unreachable,
58 }
59 },
60 .optional => {
61 std.debug.assert(ty.isPtrLikeOptional(zcu));
62 return .byval;
63 },
64 .pointer => {
65 std.debug.assert(!ty.isSlice(zcu));
66 return .byval;
67 },
68 .error_union,
69 .frame,
70 .@"anyframe",
71 .noreturn,
72 .void,
73 .type,
74 .comptime_float,
75 .comptime_int,
76 .undefined,
77 .null,
78 .@"fn",
79 .@"opaque",
80 .enum_literal,
81 .array,
82 => unreachable,
83 }
84}
src/codegen/wasm/abi.zig created+87
...@@ -0,0 +1,87 @@
1//! Classifies Zig types to follow the C-ABI for Wasm.
2//! The convention for Wasm's C-ABI can be found at the tool-conventions repo:
3//! https://github.com/WebAssembly/tool-conventions/blob/main/BasicCABI.md
4//! When not targeting the C-ABI, Zig is allowed to do derail from this convention.
5//! Note: Above mentioned document is not an official specification, therefore called a convention.
6
7const std = @import("std");
8const Target = std.Target;
9const assert = std.debug.assert;
10
11const Type = @import("../../Type.zig");
12const Zcu = @import("../../Zcu.zig");
13
14/// Defines how to pass a type as part of a function signature,
15/// both for parameters as well as return values.
16pub const Class = union(enum) {
17 direct: Type,
18 indirect,
19};
20
21/// Classifies a given Zig type to determine how they must be passed
22/// or returned as value within a wasm function.
23pub fn classifyType(ty: Type, zcu: *const Zcu) Class {
24 const ip = &zcu.intern_pool;
25 assert(ty.hasRuntimeBitsIgnoreComptime(zcu));
26 switch (ty.zigTypeTag(zcu)) {
27 .int, .@"enum", .error_set => return .{ .direct = ty },
28 .float => return .{ .direct = ty },
29 .bool => return .{ .direct = ty },
30 .vector => return .{ .direct = ty },
31 .array => return .indirect,
32 .optional => {
33 assert(ty.isPtrLikeOptional(zcu));
34 return .{ .direct = ty };
35 },
36 .pointer => {
37 assert(!ty.isSlice(zcu));
38 return .{ .direct = ty };
39 },
40 .@"struct" => {
41 const struct_type = zcu.typeToStruct(ty).?;
42 if (struct_type.layout == .@"packed") {
43 return .{ .direct = ty };
44 }
45 if (struct_type.field_types.len > 1) {
46 // The struct type is non-scalar.
47 return .indirect;
48 }
49 const field_ty = Type.fromInterned(struct_type.field_types.get(ip)[0]);
50 const explicit_align = struct_type.fieldAlign(ip, 0);
51 if (explicit_align != .none) {
52 if (explicit_align.compareStrict(.gt, field_ty.abiAlignment(zcu)))
53 return .indirect;
54 }
55 return classifyType(field_ty, zcu);
56 },
57 .@"union" => {
58 const union_obj = zcu.typeToUnion(ty).?;
59 if (union_obj.flagsUnordered(ip).layout == .@"packed") {
60 return .{ .direct = ty };
61 }
62 const layout = ty.unionGetLayout(zcu);
63 assert(layout.tag_size == 0);
64 if (union_obj.field_types.len > 1) return .indirect;
65 const first_field_ty = Type.fromInterned(union_obj.field_types.get(ip)[0]);
66 return classifyType(first_field_ty, zcu);
67 },
68 .error_union,
69 .frame,
70 .@"anyframe",
71 .noreturn,
72 .void,
73 .type,
74 .comptime_float,
75 .comptime_int,
76 .undefined,
77 .null,
78 .@"fn",
79 .@"opaque",
80 .enum_literal,
81 => unreachable,
82 }
83}
84
85pub fn lowerAsDoubleI64(scalar_ty: Type, zcu: *const Zcu) bool {
86 return scalar_ty.bitSize(zcu) > 64;
87}
src/link/Wasm.zig+1-1
...@@ -31,7 +31,7 @@ const mem = std.mem;...@@ -31,7 +31,7 @@ const mem = std.mem;
3131
32const Mir = @import("../arch/wasm/Mir.zig");32const Mir = @import("../arch/wasm/Mir.zig");
33const CodeGen = @import("../arch/wasm/CodeGen.zig");33const CodeGen = @import("../arch/wasm/CodeGen.zig");
34const abi = @import("../arch/wasm/abi.zig");34const abi = @import("../codegen/wasm/abi.zig");
35const Compilation = @import("../Compilation.zig");35const Compilation = @import("../Compilation.zig");
36const Dwarf = @import("Dwarf.zig");36const Dwarf = @import("Dwarf.zig");
37const InternPool = @import("../InternPool.zig");37const InternPool = @import("../InternPool.zig");