| ... | ... | @@ -28,10 +28,15 @@ literal_relocs: std.ArrayListUnmanaged(codegen.aarch64.Mir.Reloc.Literal), |
| 28 | 28 | |
| 29 | 29 | // Stack Frame |
| 30 | 30 | returns: bool, |
| 31 | | va_list: struct { |
| 32 | | __stack: Value.Indirect, |
| 33 | | __gr_top: Value.Indirect, |
| 34 | | __vr_top: Value.Indirect, |
| 31 | va_list: union(enum) { |
| 32 | other: Value.Indirect, |
| 33 | sysv: struct { |
| 34 | __stack: Value.Indirect, |
| 35 | __gr_top: Value.Indirect, |
| 36 | __vr_top: Value.Indirect, |
| 37 | __gr_offs: i32, |
| 38 | __vr_offs: i32, |
| 39 | }, |
| 35 | 40 | }, |
| 36 | 41 | stack_size: u24, |
| 37 | 42 | stack_align: InternPool.Alignment, |
| ... | ... | @@ -408,13 +413,7 @@ pub fn analyze(isel: *Select, air_body: []const Air.Inst.Index) !void { |
| 408 | 413 | |
| 409 | 414 | air_body_index += 1; |
| 410 | 415 | }, |
| 411 | | .breakpoint, |
| 412 | | .dbg_stmt, |
| 413 | | .dbg_empty_stmt, |
| 414 | | .dbg_var_ptr, |
| 415 | | .dbg_var_val, |
| 416 | | .dbg_arg_inline, |
| 417 | | => { |
| 416 | .breakpoint, .dbg_stmt, .dbg_empty_stmt, .dbg_var_ptr, .dbg_var_val, .dbg_arg_inline, .c_va_end => { |
| 418 | 417 | air_body_index += 1; |
| 419 | 418 | air_inst_index = air_body[air_body_index]; |
| 420 | 419 | continue :air_tag air_tags[@intFromEnum(air_inst_index)]; |
| ... | ... | @@ -428,23 +427,43 @@ pub fn analyze(isel: *Select, air_body: []const Air.Inst.Index) !void { |
| 428 | 427 | const extra = isel.air.extraData(Air.Call, pl_op.payload); |
| 429 | 428 | const args: []const Air.Inst.Ref = @ptrCast(isel.air.extra.items[extra.end..][0..extra.data.args_len]); |
| 430 | 429 | isel.saved_registers.insert(.lr); |
| 430 | const callee_ty = isel.air.typeOf(pl_op.operand, ip); |
| 431 | const func_info = switch (ip.indexToKey(callee_ty.toIntern())) { |
| 432 | else => unreachable, |
| 433 | .func_type => |func_type| func_type, |
| 434 | .ptr_type => |ptr_type| ip.indexToKey(ptr_type.child).func_type, |
| 435 | }; |
| 431 | 436 | |
| 432 | 437 | try isel.analyzeUse(pl_op.operand); |
| 433 | 438 | var param_it: CallAbiIterator = .init; |
| 434 | | for (args) |arg| { |
| 439 | for (args, 0..) |arg, arg_index| { |
| 435 | 440 | const restore_values_len = isel.values.items.len; |
| 436 | 441 | defer isel.values.shrinkRetainingCapacity(restore_values_len); |
| 437 | | const param_vi = try param_it.param(isel, isel.air.typeOf(arg, ip)) orelse continue; |
| 438 | | const param_parent = param_vi.parent(isel); |
| 439 | | switch (switch (param_parent) { |
| 440 | | .unallocated, .stack_slot => param_parent, |
| 442 | const param_vi = param_vi: { |
| 443 | const param_ty = isel.air.typeOf(arg, ip); |
| 444 | if (arg_index >= func_info.param_types.len) { |
| 445 | assert(func_info.is_var_args); |
| 446 | switch (isel.va_list) { |
| 447 | .other => break :param_vi try param_it.nonSysvVarArg(isel, param_ty), |
| 448 | .sysv => {}, |
| 449 | } |
| 450 | } |
| 451 | break :param_vi try param_it.param(isel, param_ty); |
| 452 | } orelse continue; |
| 453 | defer param_vi.deref(isel); |
| 454 | const passed_vi = switch (param_vi.parent(isel)) { |
| 455 | .unallocated, .stack_slot => param_vi, |
| 441 | 456 | .value, .constant => unreachable, |
| 442 | | .address => |address_vi| address_vi.parent(isel), |
| 443 | | }) { |
| 457 | .address => |address_vi| address_vi, |
| 458 | }; |
| 459 | switch (passed_vi.parent(isel)) { |
| 444 | 460 | .unallocated => {}, |
| 445 | 461 | .stack_slot => |stack_slot| { |
| 446 | 462 | assert(stack_slot.base == .sp); |
| 447 | | isel.stack_size = @max(isel.stack_size, stack_slot.offset); |
| 463 | isel.stack_size = @max( |
| 464 | isel.stack_size, |
| 465 | stack_slot.offset + @as(u24, @intCast(passed_vi.size(isel))), |
| 466 | ); |
| 448 | 467 | }, |
| 449 | 468 | .value, .constant, .address => unreachable, |
| 450 | 469 | } |
| ... | ... | @@ -802,7 +821,7 @@ pub fn analyze(isel: *Select, air_body: []const Air.Inst.Index) !void { |
| 802 | 821 | air_inst_index = air_body[air_body_index]; |
| 803 | 822 | continue :air_tag air_tags[@intFromEnum(air_inst_index)]; |
| 804 | 823 | }, |
| 805 | | .set_err_return_trace, .c_va_end => { |
| 824 | .set_err_return_trace => { |
| 806 | 825 | const un_op = air_data[@intFromEnum(air_inst_index)].un_op; |
| 807 | 826 | |
| 808 | 827 | try isel.analyzeUse(un_op); |
| ... | ... | @@ -2474,6 +2493,7 @@ pub fn body(isel: *Select, air_body: []const Air.Inst.Index) error{ OutOfMemory, |
| 2474 | 2493 | } |
| 2475 | 2494 | if (air.next()) |next_air_tag| continue :air_tag next_air_tag; |
| 2476 | 2495 | }, |
| 2496 | .inferred_alloc, .inferred_alloc_comptime => unreachable, |
| 2477 | 2497 | .assembly => { |
| 2478 | 2498 | const ty_pl = air.data(air.inst_index).ty_pl; |
| 2479 | 2499 | const extra = isel.air.extraData(Air.Asm, ty_pl.payload); |
| ... | ... | @@ -3389,6 +3409,12 @@ pub fn body(isel: *Select, air_body: []const Air.Inst.Index) error{ OutOfMemory, |
| 3389 | 3409 | const pl_op = air.data(air.inst_index).pl_op; |
| 3390 | 3410 | const extra = isel.air.extraData(Air.Call, pl_op.payload); |
| 3391 | 3411 | const args: []const Air.Inst.Ref = @ptrCast(isel.air.extra.items[extra.end..][0..extra.data.args_len]); |
| 3412 | const callee_ty = isel.air.typeOf(pl_op.operand, ip); |
| 3413 | const func_info = switch (ip.indexToKey(callee_ty.toIntern())) { |
| 3414 | else => unreachable, |
| 3415 | .func_type => |func_type| func_type, |
| 3416 | .ptr_type => |ptr_type| ip.indexToKey(ptr_type.child).func_type, |
| 3417 | }; |
| 3392 | 3418 | |
| 3393 | 3419 | try call.prepareReturn(isel); |
| 3394 | 3420 | const maybe_def_ret_vi = isel.live_values.fetchRemove(air.inst_index); |
| ... | ... | @@ -3455,41 +3481,50 @@ pub fn body(isel: *Select, air_body: []const Air.Inst.Index) error{ OutOfMemory, |
| 3455 | 3481 | ret_addr_vi.hint(isel).?, |
| 3456 | 3482 | ); |
| 3457 | 3483 | var param_it: CallAbiIterator = .init; |
| 3458 | | for (args) |arg| { |
| 3459 | | const param_vi = try param_it.param(isel, isel.air.typeOf(arg, ip)) orelse continue; |
| 3484 | for (args, 0..) |arg, arg_index| { |
| 3485 | const param_ty = isel.air.typeOf(arg, ip); |
| 3486 | const param_vi = param_vi: { |
| 3487 | if (arg_index >= func_info.param_types.len) { |
| 3488 | assert(func_info.is_var_args); |
| 3489 | switch (isel.va_list) { |
| 3490 | .other => break :param_vi try param_it.nonSysvVarArg(isel, param_ty), |
| 3491 | .sysv => {}, |
| 3492 | } |
| 3493 | } |
| 3494 | break :param_vi try param_it.param(isel, param_ty); |
| 3495 | } orelse continue; |
| 3460 | 3496 | defer param_vi.deref(isel); |
| 3461 | 3497 | const arg_vi = try isel.use(arg); |
| 3462 | | const passed_vi = switch (param_vi.parent(isel)) { |
| 3463 | | .unallocated, .stack_slot => param_vi, |
| 3464 | | .value, .constant => unreachable, |
| 3465 | | .address => |address_vi| { |
| 3466 | | try call.paramAddress(isel, arg_vi, address_vi.hint(isel).?); |
| 3467 | | continue; |
| 3498 | switch (param_vi.parent(isel)) { |
| 3499 | .unallocated => if (param_vi.hint(isel)) |param_ra| { |
| 3500 | try call.paramLiveOut(isel, arg_vi, param_ra); |
| 3501 | } else { |
| 3502 | var param_part_it = param_vi.parts(isel); |
| 3503 | var arg_part_it = arg_vi.parts(isel); |
| 3504 | if (arg_part_it.only()) |_| { |
| 3505 | try isel.values.ensureUnusedCapacity(gpa, param_part_it.remaining); |
| 3506 | arg_vi.setParts(isel, param_part_it.remaining); |
| 3507 | while (param_part_it.next()) |param_part_vi| _ = arg_vi.addPart( |
| 3508 | isel, |
| 3509 | param_part_vi.get(isel).offset_from_parent, |
| 3510 | param_part_vi.size(isel), |
| 3511 | ); |
| 3512 | param_part_it = param_vi.parts(isel); |
| 3513 | arg_part_it = arg_vi.parts(isel); |
| 3514 | } |
| 3515 | while (param_part_it.next()) |param_part_vi| { |
| 3516 | const arg_part_vi = arg_part_it.next().?; |
| 3517 | assert(arg_part_vi.get(isel).offset_from_parent == |
| 3518 | param_part_vi.get(isel).offset_from_parent); |
| 3519 | assert(arg_part_vi.size(isel) == param_part_vi.size(isel)); |
| 3520 | try call.paramLiveOut(isel, arg_part_vi, param_part_vi.hint(isel).?); |
| 3521 | } |
| 3468 | 3522 | }, |
| 3469 | | }; |
| 3470 | | if (passed_vi.hint(isel)) |param_ra| { |
| 3471 | | try call.paramLiveOut(isel, arg_vi, param_ra); |
| 3472 | | } else { |
| 3473 | | var param_part_it = passed_vi.parts(isel); |
| 3474 | | var arg_part_it = arg_vi.parts(isel); |
| 3475 | | if (arg_part_it.only()) |_| { |
| 3476 | | try isel.values.ensureUnusedCapacity(gpa, param_part_it.remaining); |
| 3477 | | arg_vi.setParts(isel, param_part_it.remaining); |
| 3478 | | while (param_part_it.next()) |param_part_vi| _ = arg_vi.addPart( |
| 3479 | | isel, |
| 3480 | | param_part_vi.get(isel).offset_from_parent, |
| 3481 | | param_part_vi.size(isel), |
| 3482 | | ); |
| 3483 | | param_part_it = passed_vi.parts(isel); |
| 3484 | | arg_part_it = arg_vi.parts(isel); |
| 3485 | | } |
| 3486 | | while (param_part_it.next()) |param_part_vi| { |
| 3487 | | const arg_part_vi = arg_part_it.next().?; |
| 3488 | | assert(arg_part_vi.get(isel).offset_from_parent == |
| 3489 | | param_part_vi.get(isel).offset_from_parent); |
| 3490 | | assert(arg_part_vi.size(isel) == param_part_vi.size(isel)); |
| 3491 | | try call.paramLiveOut(isel, arg_part_vi, param_part_vi.hint(isel).?); |
| 3492 | | } |
| 3523 | .stack_slot => |stack_slot| try arg_vi.store(isel, param_ty, stack_slot.base, .{ |
| 3524 | .offset = @intCast(stack_slot.offset), |
| 3525 | }), |
| 3526 | .value, .constant => unreachable, |
| 3527 | .address => |address_vi| try call.paramAddress(isel, arg_vi, address_vi.hint(isel).?), |
| 3493 | 3528 | } |
| 3494 | 3529 | } |
| 3495 | 3530 | try call.finishParams(isel); |
| ... | ... | @@ -4828,9 +4863,7 @@ pub fn body(isel: *Select, air_body: []const Air.Inst.Index) error{ OutOfMemory, |
| 4828 | 4863 | |
| 4829 | 4864 | if (air.next()) |next_air_tag| continue :air_tag next_air_tag; |
| 4830 | 4865 | }, |
| 4831 | | .dbg_stmt => { |
| 4832 | | if (air.next()) |next_air_tag| continue :air_tag next_air_tag; |
| 4833 | | }, |
| 4866 | .dbg_stmt => if (air.next()) |next_air_tag| continue :air_tag next_air_tag, |
| 4834 | 4867 | .dbg_empty_stmt => { |
| 4835 | 4868 | try isel.emit(.nop()); |
| 4836 | 4869 | if (air.next()) |next_air_tag| continue :air_tag next_air_tag; |
| ... | ... | @@ -7079,6 +7112,7 @@ pub fn body(isel: *Select, air_body: []const Air.Inst.Index) error{ OutOfMemory, |
| 7079 | 7112 | } |
| 7080 | 7113 | if (air.next()) |next_air_tag| continue :air_tag next_air_tag; |
| 7081 | 7114 | }, |
| 7115 | .wasm_memory_size, .wasm_memory_grow => unreachable, |
| 7082 | 7116 | .cmp_lt_errors_len => { |
| 7083 | 7117 | if (isel.live_values.fetchRemove(air.inst_index)) |is_vi| unused: { |
| 7084 | 7118 | defer is_vi.value.deref(isel); |
| ... | ... | @@ -7135,16 +7169,266 @@ pub fn body(isel: *Select, air_body: []const Air.Inst.Index) error{ OutOfMemory, |
| 7135 | 7169 | } |
| 7136 | 7170 | if (air.next()) |next_air_tag| continue :air_tag next_air_tag; |
| 7137 | 7171 | }, |
| 7138 | | .inferred_alloc, |
| 7139 | | .inferred_alloc_comptime, |
| 7140 | | .int_from_float_safe, |
| 7141 | | .int_from_float_optimized_safe, |
| 7142 | | .wasm_memory_size, |
| 7143 | | .wasm_memory_grow, |
| 7144 | | .work_item_id, |
| 7145 | | .work_group_size, |
| 7146 | | .work_group_id, |
| 7147 | | => unreachable, |
| 7172 | .c_va_arg => { |
| 7173 | const maybe_arg_vi = isel.live_values.fetchRemove(air.inst_index); |
| 7174 | defer if (maybe_arg_vi) |arg_vi| arg_vi.value.deref(isel); |
| 7175 | const ty_op = air.data(air.inst_index).ty_op; |
| 7176 | const ty = ty_op.ty.toType(); |
| 7177 | var param_it: CallAbiIterator = .init; |
| 7178 | const param_vi = try param_it.param(isel, ty); |
| 7179 | defer param_vi.?.deref(isel); |
| 7180 | const passed_vi = switch (param_vi.?.parent(isel)) { |
| 7181 | .unallocated => param_vi.?, |
| 7182 | .stack_slot, .value, .constant => unreachable, |
| 7183 | .address => |address_vi| address_vi, |
| 7184 | }; |
| 7185 | const passed_size: u5 = @intCast(passed_vi.alignment(isel).forward(passed_vi.size(isel))); |
| 7186 | const passed_is_vector = passed_vi.isVector(isel); |
| 7187 | |
| 7188 | const va_list_ptr_vi = try isel.use(ty_op.operand); |
| 7189 | const va_list_ptr_mat = try va_list_ptr_vi.matReg(isel); |
| 7190 | const offs_ra = try isel.allocIntReg(); |
| 7191 | defer isel.freeReg(offs_ra); |
| 7192 | const stack_ra = try isel.allocIntReg(); |
| 7193 | defer isel.freeReg(stack_ra); |
| 7194 | |
| 7195 | var part_vis: [2]Value.Index = undefined; |
| 7196 | var arg_part_ras: [2]?Register.Alias = @splat(null); |
| 7197 | const parts_len = parts_len: { |
| 7198 | var parts_len: u2 = 0; |
| 7199 | var part_it = passed_vi.parts(isel); |
| 7200 | while (part_it.next()) |part_vi| : (parts_len += 1) { |
| 7201 | part_vis[parts_len] = part_vi; |
| 7202 | const arg_vi = maybe_arg_vi orelse continue; |
| 7203 | const part_offset, const part_size = part_vi.position(isel); |
| 7204 | var arg_part_it = arg_vi.value.field(ty, part_offset, part_size); |
| 7205 | const arg_part_vi = try arg_part_it.only(isel); |
| 7206 | arg_part_ras[parts_len] = try arg_part_vi.?.defReg(isel); |
| 7207 | } |
| 7208 | break :parts_len parts_len; |
| 7209 | }; |
| 7210 | |
| 7211 | const done_label = isel.instructions.items.len; |
| 7212 | try isel.emit(.str(stack_ra.x(), .{ .unsigned_offset = .{ |
| 7213 | .base = va_list_ptr_mat.ra.x(), |
| 7214 | .offset = 0, |
| 7215 | } })); |
| 7216 | try isel.emit(switch (parts_len) { |
| 7217 | else => unreachable, |
| 7218 | 1 => if (arg_part_ras[0]) |arg_part_ra| switch (part_vis[0].size(isel)) { |
| 7219 | else => unreachable, |
| 7220 | 1 => if (arg_part_ra.isVector()) .ldr(arg_part_ra.b(), .{ .post_index = .{ |
| 7221 | .base = stack_ra.x(), |
| 7222 | .index = passed_size, |
| 7223 | } }) else switch (part_vis[0].signedness(isel)) { |
| 7224 | .signed => .ldrsb(arg_part_ra.w(), .{ .post_index = .{ |
| 7225 | .base = stack_ra.x(), |
| 7226 | .index = passed_size, |
| 7227 | } }), |
| 7228 | .unsigned => .ldrb(arg_part_ra.w(), .{ .post_index = .{ |
| 7229 | .base = stack_ra.x(), |
| 7230 | .index = passed_size, |
| 7231 | } }), |
| 7232 | }, |
| 7233 | 2 => if (arg_part_ra.isVector()) .ldr(arg_part_ra.h(), .{ .post_index = .{ |
| 7234 | .base = stack_ra.x(), |
| 7235 | .index = passed_size, |
| 7236 | } }) else switch (part_vis[0].signedness(isel)) { |
| 7237 | .signed => .ldrsh(arg_part_ra.w(), .{ .post_index = .{ |
| 7238 | .base = stack_ra.x(), |
| 7239 | .index = passed_size, |
| 7240 | } }), |
| 7241 | .unsigned => .ldrh(arg_part_ra.w(), .{ .post_index = .{ |
| 7242 | .base = stack_ra.x(), |
| 7243 | .index = passed_size, |
| 7244 | } }), |
| 7245 | }, |
| 7246 | 4 => .ldr(if (arg_part_ra.isVector()) arg_part_ra.s() else arg_part_ra.w(), .{ .post_index = .{ |
| 7247 | .base = stack_ra.x(), |
| 7248 | .index = passed_size, |
| 7249 | } }), |
| 7250 | 8 => .ldr(if (arg_part_ra.isVector()) arg_part_ra.d() else arg_part_ra.x(), .{ .post_index = .{ |
| 7251 | .base = stack_ra.x(), |
| 7252 | .index = passed_size, |
| 7253 | } }), |
| 7254 | 16 => .ldr(arg_part_ra.q(), .{ .post_index = .{ |
| 7255 | .base = stack_ra.x(), |
| 7256 | .index = passed_size, |
| 7257 | } }), |
| 7258 | } else .add(stack_ra.x(), stack_ra.x(), .{ .immediate = passed_size }), |
| 7259 | 2 => if (arg_part_ras[0] != null or arg_part_ras[1] != null) .ldp( |
| 7260 | @as(Register.Alias, arg_part_ras[0] orelse .zr).x(), |
| 7261 | @as(Register.Alias, arg_part_ras[1] orelse .zr).x(), |
| 7262 | .{ .post_index = .{ |
| 7263 | .base = stack_ra.x(), |
| 7264 | .index = passed_size, |
| 7265 | } }, |
| 7266 | ) else .add(stack_ra.x(), stack_ra.x(), .{ .immediate = passed_size }), |
| 7267 | }); |
| 7268 | try isel.emit(.ldr(stack_ra.x(), .{ .unsigned_offset = .{ |
| 7269 | .base = va_list_ptr_mat.ra.x(), |
| 7270 | .offset = 0, |
| 7271 | } })); |
| 7272 | switch (isel.va_list) { |
| 7273 | .other => {}, |
| 7274 | .sysv => { |
| 7275 | const stack_label = isel.instructions.items.len; |
| 7276 | try isel.emit(.b( |
| 7277 | @intCast((isel.instructions.items.len + 1 - done_label) << 2), |
| 7278 | )); |
| 7279 | switch (parts_len) { |
| 7280 | else => unreachable, |
| 7281 | 1 => if (arg_part_ras[0]) |arg_part_ra| try isel.emit(switch (part_vis[0].size(isel)) { |
| 7282 | else => unreachable, |
| 7283 | 1 => if (arg_part_ra.isVector()) .ldr(arg_part_ra.b(), .{ .extended_register = .{ |
| 7284 | .base = stack_ra.x(), |
| 7285 | .index = offs_ra.w(), |
| 7286 | .extend = .{ .sxtw = 0 }, |
| 7287 | } }) else switch (part_vis[0].signedness(isel)) { |
| 7288 | .signed => .ldrsb(arg_part_ra.w(), .{ .extended_register = .{ |
| 7289 | .base = stack_ra.x(), |
| 7290 | .index = offs_ra.w(), |
| 7291 | .extend = .{ .sxtw = 0 }, |
| 7292 | } }), |
| 7293 | .unsigned => .ldrb(arg_part_ra.w(), .{ .extended_register = .{ |
| 7294 | .base = stack_ra.x(), |
| 7295 | .index = offs_ra.w(), |
| 7296 | .extend = .{ .sxtw = 0 }, |
| 7297 | } }), |
| 7298 | }, |
| 7299 | 2 => if (arg_part_ra.isVector()) .ldr(arg_part_ra.h(), .{ .extended_register = .{ |
| 7300 | .base = stack_ra.x(), |
| 7301 | .index = offs_ra.w(), |
| 7302 | .extend = .{ .sxtw = 0 }, |
| 7303 | } }) else switch (part_vis[0].signedness(isel)) { |
| 7304 | .signed => .ldrsh(arg_part_ra.w(), .{ .extended_register = .{ |
| 7305 | .base = stack_ra.x(), |
| 7306 | .index = offs_ra.w(), |
| 7307 | .extend = .{ .sxtw = 0 }, |
| 7308 | } }), |
| 7309 | .unsigned => .ldrh(arg_part_ra.w(), .{ .extended_register = .{ |
| 7310 | .base = stack_ra.x(), |
| 7311 | .index = offs_ra.w(), |
| 7312 | .extend = .{ .sxtw = 0 }, |
| 7313 | } }), |
| 7314 | }, |
| 7315 | 4 => .ldr(if (arg_part_ra.isVector()) arg_part_ra.s() else arg_part_ra.w(), .{ .extended_register = .{ |
| 7316 | .base = stack_ra.x(), |
| 7317 | .index = offs_ra.w(), |
| 7318 | .extend = .{ .sxtw = 0 }, |
| 7319 | } }), |
| 7320 | 8 => .ldr(if (arg_part_ra.isVector()) arg_part_ra.d() else arg_part_ra.x(), .{ .extended_register = .{ |
| 7321 | .base = stack_ra.x(), |
| 7322 | .index = offs_ra.w(), |
| 7323 | .extend = .{ .sxtw = 0 }, |
| 7324 | } }), |
| 7325 | 16 => .ldr(arg_part_ra.q(), .{ .extended_register = .{ |
| 7326 | .base = stack_ra.x(), |
| 7327 | .index = offs_ra.w(), |
| 7328 | .extend = .{ .sxtw = 0 }, |
| 7329 | } }), |
| 7330 | }), |
| 7331 | 2 => if (arg_part_ras[0] != null or arg_part_ras[1] != null) { |
| 7332 | try isel.emit(.ldp( |
| 7333 | @as(Register.Alias, arg_part_ras[0] orelse .zr).x(), |
| 7334 | @as(Register.Alias, arg_part_ras[1] orelse .zr).x(), |
| 7335 | .{ .base = stack_ra.x() }, |
| 7336 | )); |
| 7337 | try isel.emit(.add(stack_ra.x(), stack_ra.x(), .{ .extended_register = .{ |
| 7338 | .register = offs_ra.w(), |
| 7339 | .extend = .{ .sxtw = 0 }, |
| 7340 | } })); |
| 7341 | }, |
| 7342 | } |
| 7343 | try isel.emit(.ldr(stack_ra.x(), .{ .unsigned_offset = .{ |
| 7344 | .base = va_list_ptr_mat.ra.x(), |
| 7345 | .offset = if (passed_is_vector) 16 else 8, |
| 7346 | } })); |
| 7347 | try isel.emit(.@"b."( |
| 7348 | .gt, |
| 7349 | @intCast((isel.instructions.items.len + 1 - stack_label) << 2), |
| 7350 | )); |
| 7351 | try isel.emit(.str(stack_ra.w(), .{ .unsigned_offset = .{ |
| 7352 | .base = va_list_ptr_mat.ra.x(), |
| 7353 | .offset = if (passed_is_vector) 28 else 24, |
| 7354 | } })); |
| 7355 | try isel.emit(.adds(stack_ra.w(), offs_ra.w(), .{ .immediate = passed_size })); |
| 7356 | try isel.emit(.tbz( |
| 7357 | offs_ra.w(), |
| 7358 | 31, |
| 7359 | @intCast((isel.instructions.items.len + 1 - stack_label) << 2), |
| 7360 | )); |
| 7361 | try isel.emit(.ldr(offs_ra.w(), .{ .unsigned_offset = .{ |
| 7362 | .base = va_list_ptr_mat.ra.x(), |
| 7363 | .offset = if (passed_is_vector) 28 else 24, |
| 7364 | } })); |
| 7365 | }, |
| 7366 | } |
| 7367 | try va_list_ptr_mat.finish(isel); |
| 7368 | if (air.next()) |next_air_tag| continue :air_tag next_air_tag; |
| 7369 | }, |
| 7370 | .c_va_copy => { |
| 7371 | if (isel.live_values.fetchRemove(air.inst_index)) |va_list_vi| { |
| 7372 | defer va_list_vi.value.deref(isel); |
| 7373 | const ty_op = air.data(air.inst_index).ty_op; |
| 7374 | const va_list_ptr_vi = try isel.use(ty_op.operand); |
| 7375 | const va_list_ptr_mat = try va_list_ptr_vi.matReg(isel); |
| 7376 | _ = try va_list_vi.value.load(isel, ty_op.ty.toType(), va_list_ptr_mat.ra, .{}); |
| 7377 | try va_list_ptr_mat.finish(isel); |
| 7378 | } |
| 7379 | if (air.next()) |next_air_tag| continue :air_tag next_air_tag; |
| 7380 | }, |
| 7381 | .c_va_end => if (air.next()) |next_air_tag| continue :air_tag next_air_tag, |
| 7382 | .c_va_start => { |
| 7383 | if (isel.live_values.fetchRemove(air.inst_index)) |va_list_vi| { |
| 7384 | defer va_list_vi.value.deref(isel); |
| 7385 | const ty = air.data(air.inst_index).ty; |
| 7386 | switch (isel.va_list) { |
| 7387 | .other => |va_list| if (try va_list_vi.value.defReg(isel)) |va_list_ra| try isel.emit(.add( |
| 7388 | va_list_ra.x(), |
| 7389 | va_list.base.x(), |
| 7390 | .{ .immediate = @intCast(va_list.offset) }, |
| 7391 | )), |
| 7392 | .sysv => |va_list| { |
| 7393 | var vr_offs_it = va_list_vi.value.field(ty, 28, 4); |
| 7394 | const vr_offs_vi = try vr_offs_it.only(isel); |
| 7395 | if (try vr_offs_vi.?.defReg(isel)) |vr_offs_ra| try isel.movImmediate( |
| 7396 | vr_offs_ra.w(), |
| 7397 | @as(u32, @bitCast(va_list.__vr_offs)), |
| 7398 | ); |
| 7399 | var gr_offs_it = va_list_vi.value.field(ty, 24, 4); |
| 7400 | const gr_offs_vi = try gr_offs_it.only(isel); |
| 7401 | if (try gr_offs_vi.?.defReg(isel)) |gr_offs_ra| try isel.movImmediate( |
| 7402 | gr_offs_ra.w(), |
| 7403 | @as(u32, @bitCast(va_list.__gr_offs)), |
| 7404 | ); |
| 7405 | var vr_top_it = va_list_vi.value.field(ty, 16, 8); |
| 7406 | const vr_top_vi = try vr_top_it.only(isel); |
| 7407 | if (try vr_top_vi.?.defReg(isel)) |vr_top_ra| try isel.emit(.add( |
| 7408 | vr_top_ra.x(), |
| 7409 | va_list.__vr_top.base.x(), |
| 7410 | .{ .immediate = @intCast(va_list.__vr_top.offset) }, |
| 7411 | )); |
| 7412 | var gr_top_it = va_list_vi.value.field(ty, 8, 8); |
| 7413 | const gr_top_vi = try gr_top_it.only(isel); |
| 7414 | if (try gr_top_vi.?.defReg(isel)) |gr_top_ra| try isel.emit(.add( |
| 7415 | gr_top_ra.x(), |
| 7416 | va_list.__gr_top.base.x(), |
| 7417 | .{ .immediate = @intCast(va_list.__gr_top.offset) }, |
| 7418 | )); |
| 7419 | var stack_it = va_list_vi.value.field(ty, 0, 8); |
| 7420 | const stack_vi = try stack_it.only(isel); |
| 7421 | if (try stack_vi.?.defReg(isel)) |stack_ra| try isel.emit(.add( |
| 7422 | stack_ra.x(), |
| 7423 | va_list.__stack.base.x(), |
| 7424 | .{ .immediate = @intCast(va_list.__stack.offset) }, |
| 7425 | )); |
| 7426 | }, |
| 7427 | } |
| 7428 | } |
| 7429 | if (air.next()) |next_air_tag| continue :air_tag next_air_tag; |
| 7430 | }, |
| 7431 | .work_item_id, .work_group_size, .work_group_id => unreachable, |
| 7148 | 7432 | } |
| 7149 | 7433 | assert(air.body_index == 0); |
| 7150 | 7434 | } |
| ... | ... | @@ -7225,7 +7509,7 @@ pub fn verify(isel: *Select, check_values: bool) void { |
| 7225 | 7509 | pub fn layout( |
| 7226 | 7510 | isel: *Select, |
| 7227 | 7511 | incoming: CallAbiIterator, |
| 7228 | | have_va: bool, |
| 7512 | is_sysv_var_args: bool, |
| 7229 | 7513 | saved_gra_len: u7, |
| 7230 | 7514 | saved_vra_len: u7, |
| 7231 | 7515 | mod: *const Package.Module, |
| ... | ... | @@ -7236,8 +7520,6 @@ pub fn layout( |
| 7236 | 7520 | wip_mir_log.debug("{f}<body>:\n", .{nav.fqn.fmt(ip)}); |
| 7237 | 7521 | |
| 7238 | 7522 | const stack_size: u24 = @intCast(InternPool.Alignment.@"16".forward(isel.stack_size)); |
| 7239 | | const stack_size_lo: u12 = @truncate(stack_size >> 0); |
| 7240 | | const stack_size_hi: u12 = @truncate(stack_size >> 12); |
| 7241 | 7523 | |
| 7242 | 7524 | var saves_buf: [10 + 8 + 8 + 2 + 8]struct { |
| 7243 | 7525 | class: enum { integer, vector }, |
| ... | ... | @@ -7315,7 +7597,7 @@ pub fn layout( |
| 7315 | 7597 | |
| 7316 | 7598 | // incoming vr arguments |
| 7317 | 7599 | save_ra = if (mod.strip) incoming.nsrn else CallAbiIterator.nsrn_start; |
| 7318 | | while (save_ra != if (have_va) CallAbiIterator.nsrn_end else incoming.nsrn) : (save_ra = @enumFromInt(@intFromEnum(save_ra) + 1)) { |
| 7600 | while (save_ra != if (is_sysv_var_args) CallAbiIterator.nsrn_end else incoming.nsrn) : (save_ra = @enumFromInt(@intFromEnum(save_ra) + 1)) { |
| 7319 | 7601 | saves_size = std.mem.alignForward(u10, saves_size, 16); |
| 7320 | 7602 | saves_buf[saves_len] = .{ |
| 7321 | 7603 | .class = .vector, |
| ... | ... | @@ -7370,7 +7652,7 @@ pub fn layout( |
| 7370 | 7652 | 1 => saves_size += 8, |
| 7371 | 7653 | } |
| 7372 | 7654 | save_ra = if (mod.strip) incoming.ngrn else CallAbiIterator.ngrn_start; |
| 7373 | | while (save_ra != if (have_va) CallAbiIterator.ngrn_end else incoming.ngrn) : (save_ra = @enumFromInt(@intFromEnum(save_ra) + 1)) { |
| 7655 | while (save_ra != if (is_sysv_var_args) CallAbiIterator.ngrn_end else incoming.ngrn) : (save_ra = @enumFromInt(@intFromEnum(save_ra) + 1)) { |
| 7374 | 7656 | saves_size = std.mem.alignForward(u10, saves_size, 8); |
| 7375 | 7657 | saves_buf[saves_len] = .{ |
| 7376 | 7658 | .class = .integer, |
| ... | ... | @@ -7434,6 +7716,8 @@ pub fn layout( |
| 7434 | 7716 | .fp |
| 7435 | 7717 | else |
| 7436 | 7718 | .ip0; |
| 7719 | const stack_size_lo: u12 = @truncate(stack_size >> 0); |
| 7720 | const stack_size_hi: u12 = @truncate(stack_size >> 12); |
| 7437 | 7721 | if (mod.stack_check) { |
| 7438 | 7722 | if (stack_size_hi > 2) { |
| 7439 | 7723 | try isel.movImmediate(.ip1, stack_size_hi); |
| ... | ... | @@ -7481,6 +7765,7 @@ pub fn layout( |
| 7481 | 7765 | if (isel.returns) { |
| 7482 | 7766 | try isel.emit(.ret(.lr)); |
| 7483 | 7767 | var save_index: usize = 0; |
| 7768 | var first_offset: ?u10 = null; |
| 7484 | 7769 | while (save_index < saves.len) { |
| 7485 | 7770 | if (save_index + 2 <= saves.len and saves[save_index + 1].needs_restore and |
| 7486 | 7771 | saves[save_index + 0].class == saves[save_index + 1].class and |
| ... | ... | @@ -7489,46 +7774,51 @@ pub fn layout( |
| 7489 | 7774 | try isel.emit(.ldp( |
| 7490 | 7775 | saves[save_index + 0].register, |
| 7491 | 7776 | saves[save_index + 1].register, |
| 7492 | | switch (saves[save_index + 0].offset) { |
| 7493 | | 0 => .{ .post_index = .{ |
| 7494 | | .base = .sp, |
| 7495 | | .index = @intCast(saves_size), |
| 7496 | | } }, |
| 7497 | | else => |offset| .{ .signed_offset = .{ |
| 7777 | if (first_offset) |offset| .{ .signed_offset = .{ |
| 7778 | .base = .sp, |
| 7779 | .offset = @intCast(saves[save_index + 0].offset - offset), |
| 7780 | } } else form: { |
| 7781 | first_offset = @intCast(saves[save_index + 0].offset); |
| 7782 | break :form .{ .post_index = .{ |
| 7498 | 7783 | .base = .sp, |
| 7499 | | .offset = @intCast(offset), |
| 7500 | | } }, |
| 7784 | .index = @intCast(saves_size - first_offset.?), |
| 7785 | } }; |
| 7501 | 7786 | }, |
| 7502 | 7787 | )); |
| 7503 | 7788 | save_index += 2; |
| 7504 | 7789 | } else if (saves[save_index].needs_restore) { |
| 7505 | 7790 | try isel.emit(.ldr( |
| 7506 | 7791 | saves[save_index].register, |
| 7507 | | switch (saves[save_index].offset) { |
| 7508 | | 0 => .{ .post_index = .{ |
| 7509 | | .base = .sp, |
| 7510 | | .index = @intCast(saves_size), |
| 7511 | | } }, |
| 7512 | | else => |offset| .{ .unsigned_offset = .{ |
| 7792 | if (first_offset) |offset| .{ .unsigned_offset = .{ |
| 7793 | .base = .sp, |
| 7794 | .offset = saves[save_index + 0].offset - offset, |
| 7795 | } } else form: { |
| 7796 | const offset = saves[save_index + 0].offset; |
| 7797 | first_offset = offset; |
| 7798 | break :form .{ .post_index = .{ |
| 7513 | 7799 | .base = .sp, |
| 7514 | | .offset = @intCast(offset), |
| 7515 | | } }, |
| 7800 | .index = @intCast(saves_size - offset), |
| 7801 | } }; |
| 7516 | 7802 | }, |
| 7517 | 7803 | )); |
| 7518 | 7804 | save_index += 1; |
| 7519 | 7805 | } else save_index += 1; |
| 7520 | 7806 | } |
| 7521 | | if (isel.stack_align != .@"16" or (stack_size_lo > 0 and stack_size_hi > 0)) { |
| 7522 | | try isel.emit(switch (frame_record_offset) { |
| 7523 | | 0 => .add(.sp, .fp, .{ .immediate = 0 }), |
| 7524 | | else => |offset| .sub(.sp, .fp, .{ .immediate = offset }), |
| 7525 | | }); |
| 7807 | const offset = stack_size + first_offset.?; |
| 7808 | const offset_lo: u12 = @truncate(offset >> 0); |
| 7809 | const offset_hi: u12 = @truncate(offset >> 12); |
| 7810 | if (isel.stack_align != .@"16" or (offset_lo > 0 and offset_hi > 0)) { |
| 7811 | const fp_offset = @as(i11, first_offset.?) - frame_record_offset; |
| 7812 | try isel.emit(if (fp_offset >= 0) |
| 7813 | .add(.sp, .fp, .{ .immediate = @intCast(fp_offset) }) |
| 7814 | else |
| 7815 | .sub(.sp, .fp, .{ .immediate = @intCast(-fp_offset) })); |
| 7526 | 7816 | } else { |
| 7527 | | if (stack_size_hi > 0) try isel.emit(.add(.sp, .sp, .{ |
| 7528 | | .shifted_immediate = .{ .immediate = stack_size_hi, .lsl = .@"12" }, |
| 7817 | if (offset_hi > 0) try isel.emit(.add(.sp, .sp, .{ |
| 7818 | .shifted_immediate = .{ .immediate = offset_hi, .lsl = .@"12" }, |
| 7529 | 7819 | })); |
| 7530 | | if (stack_size_lo > 0) try isel.emit(.add(.sp, .sp, .{ |
| 7531 | | .immediate = stack_size_lo, |
| 7820 | if (offset_lo > 0) try isel.emit(.add(.sp, .sp, .{ |
| 7821 | .immediate = offset_lo, |
| 7532 | 7822 | })); |
| 7533 | 7823 | } |
| 7534 | 7824 | wip_mir_log.debug("{f}<epilogue>:\n", .{nav.fqn.fmt(ip)}); |
| ... | ... | @@ -9493,6 +9783,11 @@ pub const Value = struct { |
| 9493 | 9783 | return it.vi; |
| 9494 | 9784 | } |
| 9495 | 9785 | |
| 9786 | pub fn peek(it: PartIterator) ?Value.Index { |
| 9787 | var it_mut = it; |
| 9788 | return it_mut.next(); |
| 9789 | } |
| 9790 | |
| 9496 | 9791 | pub fn only(it: PartIterator) ?Value.Index { |
| 9497 | 9792 | return if (it.remaining == 1) it.vi else null; |
| 9498 | 9793 | } |
| ... | ... | @@ -11607,6 +11902,16 @@ pub const CallAbiIterator = struct { |
| 11607 | 11902 | return wip_vi.ref(isel); |
| 11608 | 11903 | } |
| 11609 | 11904 | |
| 11905 | pub fn nonSysvVarArg(it: *CallAbiIterator, isel: *Select, ty: ZigType) !?Value.Index { |
| 11906 | const ngrn = it.ngrn; |
| 11907 | defer it.ngrn = ngrn; |
| 11908 | it.ngrn = ngrn_end; |
| 11909 | const nsrn = it.nsrn; |
| 11910 | defer it.nsrn = nsrn; |
| 11911 | it.nsrn = nsrn_end; |
| 11912 | return it.param(isel, ty); |
| 11913 | } |
| 11914 | |
| 11610 | 11915 | pub fn ret(it: *CallAbiIterator, isel: *Select, ty: ZigType) !?Value.Index { |
| 11611 | 11916 | const wip_vi = try it.param(isel, ty) orelse return null; |
| 11612 | 11917 | switch (wip_vi.parent(isel)) { |
Jacob Young <jacobly@ziglang.org> 2025-07-27 06:50:20-04:00
src