diff --git a/src/codegen/llvm.zig b/src/codegen/llvm.zig index 74e9b24127e9ec94cf4c7868be535d85c1b273f4..e58a229ac201a45925d03ef75e5a97ab02cbfc1f 100644 --- a/src/codegen/llvm.zig +++ b/src/codegen/llvm.zig @@ -2355,25 +2355,21 @@ pub const Object = struct { if (fn_info.return_type == .noreturn_type) try attributes.addFnAttr(.noreturn, &o.builder); - var it = iterateParamTypes(o, fn_info.cc, fn_info.param_types); - if (try fnReturnStrat(o, fn_info.cc, .fromInterned(fn_info.return_type)) == .sret) { + const ret_strat = try fnReturnStrat(o, fn_info.cc, .fromInterned(fn_info.return_type)); + var it = iterateParamTypes(o, fn_info.cc, fn_info.param_types, ret_strat); + if (ret_strat == .sret) { try o.addSRetFnAttributes( attributes, try o.lowerType(.fromInterned(fn_info.return_type), .in_memory), Type.fromInterned(fn_info.return_type).abiAlignment(zcu).toLlvm(), .declaration, ); - it.llvm_index += 1; } else if (ccAbiPromoteInt(fn_info.cc, zcu, Type.fromInterned(fn_info.return_type))) |s| switch (s) { .signed => try attributes.addRetAttr(.signext, &o.builder), .unsigned => try attributes.addRetAttr(.zeroext, &o.builder), }; - const err_return_tracing = fn_info.cc == .auto and zcu.comp.config.any_error_tracing; - if (err_return_tracing) { - try attributes.addParamAttr(it.llvm_index, .nonnull, &o.builder); - it.llvm_index += 1; - } + if (it.errorReturnTraceIndex()) |i| try attributes.addParamAttr(i, .nonnull, &o.builder); var remaining_inreg_int = cc_info.inreg_int_params; var remaining_inreg_float = cc_info.inreg_float_params; @@ -3111,17 +3107,14 @@ pub const Object = struct { var llvm_params: std.ArrayList(Builder.Type) = .empty; defer llvm_params.deinit(o.gpa); - if (ret_strat == .sret) { - try llvm_params.append(o.gpa, .ptr); - } - - if (fn_info.cc == .auto and zcu.comp.config.any_error_tracing) { + var it = iterateParamTypes(o, fn_info.cc, fn_info.param_types, ret_strat); + if (it.sretIndex() != null) try llvm_params.append(o.gpa, .ptr); + if (it.errorReturnTraceIndex() != null) { // First parameter is a pointer to `std.lang.StackTrace`. const llvm_ptr_ty = try o.builder.ptrType(toLlvmAddressSpace(.generic, target)); try llvm_params.append(o.gpa, llvm_ptr_ty); } - var it = iterateParamTypes(o, fn_info.cc, fn_info.param_types); while (try it.next()) |lowering| switch (lowering) { .no_bits => continue, .byval => { diff --git a/src/codegen/llvm/FuncGen.zig b/src/codegen/llvm/FuncGen.zig index 4a776752e8342365bf07ea87595083048aff0903..7b59ddf03652c8d678b2e9755526f9f5258d3d94 100644 --- a/src/codegen/llvm/FuncGen.zig +++ b/src/codegen/llvm/FuncGen.zig @@ -189,24 +189,15 @@ pub fn genMainBody(fg: *FuncGen) TodoError!void { const fn_ty: Type = .fromInterned(ip.getNav(fg.nav_index).resolved.?.type); const fn_info = zcu.typeToFunc(fn_ty).?; const param_types = fn_info.param_types.get(ip); + const ret_strat = try fnReturnStrat(o, fn_info.cc, .fromInterned(fn_info.return_type)); - var it = iterateParamTypes(o, fn_info.cc, fn_info.param_types.get(ip)); + var it = iterateParamTypes(o, fn_info.cc, param_types, ret_strat); // Populate `fg.ret_ptr`... - fg.ret_ptr = switch (try fnReturnStrat(o, fn_info.cc, .fromInterned(fn_info.return_type))) { - .sret => rp: { - defer it.llvm_index += 1; - break :rp fg.wip.arg(it.llvm_index); - }, - else => .none, - }; + fg.ret_ptr = if (it.sretIndex()) |i| fg.wip.arg(i) else .none; // ...and `fg.err_ret_trace`... - if (fn_info.cc == .auto and comp.config.any_error_tracing) { - fg.err_ret_trace = fg.wip.arg(it.llvm_index); - it.llvm_index += 1; - } else { - fg.err_ret_trace = .none; - } + fg.err_ret_trace = if (it.errorReturnTraceIndex()) |i| fg.wip.arg(i) else .none; + // ...and as for `fg.args`, we'll put all of the arguments into this ArrayList, and once that's // done we'll use its buffer as `fg.args`. var args: std.ArrayList(Builder.Value) = .empty; @@ -816,13 +807,12 @@ fn buildCall( }, }; - const err_return_tracing = fn_info.cc == .auto and zcu.comp.config.any_error_tracing; - if (err_return_tracing) { + var it = iterateParamTypes(o, fn_info.cc, fn_info.param_types, ret_strat); + if (it.errorReturnTraceIndex() != null) { assert(fg.err_ret_trace != .none); try llvm_args.append(fg.gpa, fg.err_ret_trace); } - var it = iterateParamTypes(o, fn_info.cc, fn_info.param_types); while (try it.nextCall(arg_types)) |lowering| { const arg_ty: Type = .fromInterned(arg_types[it.zig_index - 1]); const arg_val = arg_values[it.zig_index - 1]; @@ -922,9 +912,7 @@ fn buildCall( { // Add argument attributes. - it = iterateParamTypes(o, fn_info.cc, fn_info.param_types); - it.llvm_index += @intFromBool(ret_strat == .sret); - it.llvm_index += @intFromBool(err_return_tracing); + it = iterateParamTypes(o, fn_info.cc, fn_info.param_types, ret_strat); var remaining_inreg_int = cc_info.inreg_int_params; var remaining_inreg_float = cc_info.inreg_float_params; while (try it.next()) |lowering| switch (lowering) { @@ -7015,6 +7003,24 @@ fn typeOfIndex(fg: *FuncGen, inst: Air.Inst.Index) Type { } const ParamTypeIterator = struct { + const Regs = struct { + gpr: u8 = 0, + fpr: u8 = 0, + + pub fn tryTake(self: *Regs, taking: Regs) bool { + if (self.gpr < taking.gpr or self.fpr < taking.fpr) return false; + + self.gpr -= taking.gpr; + self.fpr -= taking.fpr; + + return true; + } + + pub fn total(self: Regs) u8 { + return self.gpr + self.fpr; + } + }; + object: *Object, cc: std.lang.CallingConvention, param_types: []const InternPool.Index, @@ -7024,6 +7030,9 @@ const ParamTypeIterator = struct { types_buffer: [8]Builder.Type, offsets_buffer: [9]u64, byval_attr: ?Object.Byval, + free_regs: Regs, + has_sret: bool, + has_err_ret_trace: bool, const Lowering = union(enum) { no_bits, @@ -7393,110 +7402,128 @@ const ParamTypeIterator = struct { const o = it.object; const zcu = o.zcu; const target = zcu.getTarget(); + + var used_regs: ParamTypeIterator.Regs = .{ .gpr = 0, .fpr = 0 }; const classes = x86_64_abi.classifySystemV(ty, zcu, target, .arg); - var types_len: u32 = 0; - const classes_len = for (classes, 0..) |class, class_index| switch (class) { + const classes_len = std.mem.findScalar(x86_64_abi.Class, classes[0..], .none) orelse classes.len; + const early_lowering: ?Lowering = for (classes[0..classes_len], 0..) |class, index| switch (class) { .integer => { - it.types_buffer[types_len] = try o.builder.intType(@min(8 * ty.abiSize(zcu) - 64 * class_index, 64)); - it.offsets_buffer[types_len] = 8 * class_index; - types_len += 1; + it.types_buffer[used_regs.total()] = try o.builder.intType(@min(8 * ty.abiSize(zcu) - 64 * index, 64)); + it.offsets_buffer[used_regs.total()] = 8 * index; + used_regs.gpr += 1; }, .sse => { - it.types_buffer[types_len] = .double; - it.offsets_buffer[types_len] = 8 * class_index; - types_len += 1; + it.types_buffer[used_regs.total()] = .double; + it.offsets_buffer[used_regs.total()] = 8 * index; + used_regs.fpr += 1; }, .sseup => { - if (it.types_buffer[types_len - 1] == .double) { - if (ty.isVector(zcu)) { - it.zig_index += 1; - it.llvm_index += 1; - return .byval; - } - it.types_buffer[types_len - 1] = .fp128; + if (it.types_buffer[used_regs.total() - 1] == .double) { + if (ty.isVector(zcu)) break .byval; + it.types_buffer[used_regs.total() - 1] = .fp128; } else { - it.types_buffer[types_len] = .double; - it.offsets_buffer[types_len] = 8 * class_index; - types_len += 1; + it.types_buffer[used_regs.total()] = .double; + it.offsets_buffer[used_regs.total()] = 8 * index; + used_regs.fpr += 1; } }, .float => { - it.types_buffer[types_len] = .float; - it.offsets_buffer[types_len] = 8 * class_index; - types_len += 1; + it.types_buffer[used_regs.total()] = .float; + it.offsets_buffer[used_regs.total()] = 8 * index; + used_regs.fpr += 1; }, .float_combine => { - it.types_buffer[types_len] = try it.object.builder.vectorType(.normal, 2, .float); - it.offsets_buffer[types_len] = 8 * class_index; - types_len += 1; + it.types_buffer[used_regs.total()] = try it.object.builder.vectorType(.normal, 2, .float); + it.offsets_buffer[used_regs.total()] = 8 * index; + used_regs.fpr += 1; }, - .x87 => { - it.zig_index += 1; - it.llvm_index += 1; - it.byval_attr = .{}; - return .byref; + .x87, .memory => break .byref, + .integer_per_element => { + used_regs.gpr += @intCast(@min(ty.vectorLen(zcu), it.free_regs.gpr)); + break .byval; }, - .x87up => unreachable, - .none => break class_index, - .memory => { - it.zig_index += 1; - it.llvm_index += 1; - it.byval_attr = .{}; - return .byref; - }, - .win_i128 => unreachable, // windows only + .none, + .x87up, + // Only produced by `classifyWindows`; see `next_x86_64_win`. + .win_i128, .bool_vector_mask, - .integer_per_element, .sse_per_element, + // Only produced for `ctx == .ret`; this classifies with `.arg`. .sse_sse_x87_per_qword, .sse_per_xword, .sse_per_yword, .sse_per_zword, - => { - it.zig_index += 1; - it.llvm_index += 1; - return .byval; + => unreachable, + } else null; + + const fits = it.free_regs.tryTake(used_regs); + + const len = used_regs.total(); + it.offsets_buffer[len] = 8 * classes_len; + it.types_len = len; + + const lowering: Lowering = early_lowering orelse switch (len) { + 0 => unreachable, + 1 => single: { + if (isByRef(ty, zcu)) break :single .multiple_llvm_types; + + const reg_ty = it.types_buffer[0]; + const llvm_ty = try o.lowerType(ty, .as_value); + break :single if (reg_ty == llvm_ty or (reg_ty == .i64 and llvm_ty.isPointer(&o.builder))) + .byval + else + .multiple_llvm_types; }, - } else classes.len; - if (types_len > 1) { - if (it.llvm_index + classes_len > 6) { - it.zig_index += 1; - it.llvm_index += 1; - it.byval_attr = .{}; - return .byref; - } - } else if (!isByRef(ty, zcu)) { - const llvm_ty = try o.lowerType(ty, .as_value); - if (it.types_buffer[0] == llvm_ty or - (it.types_buffer[0] == .i64 and llvm_ty.isPointer(&o.builder))) - { - it.zig_index += 1; - it.llvm_index += 1; - return .byval; - } - } - it.offsets_buffer[types_len] = 8 * classes_len; - it.types_len = types_len; - it.llvm_index += types_len; + else => if (fits) .multiple_llvm_types else .byref, + }; + it.zig_index += 1; - return .multiple_llvm_types; + it.llvm_index += if (lowering == .multiple_llvm_types) len else 1; + + if (lowering == .byref) it.byval_attr = .{}; + + return lowering; + } + + pub fn sretIndex(it: ParamTypeIterator) ?u32 { + return if (it.has_sret) 0 else null; + } + + pub fn errorReturnTraceIndex(it: ParamTypeIterator) ?u32 { + return if (it.has_err_ret_trace) + if (it.sretIndex() != null) 1 else 0 + else + null; } }; pub fn iterateParamTypes( object: *Object, cc: std.lang.CallingConvention, param_types: []const InternPool.Index, + ret_strat: FnReturnStrat, ) ParamTypeIterator { + const has_sret = ret_strat == .sret; + const has_err_ret_trace = cc == .auto and object.zcu.comp.config.any_error_tracing; + return .{ .object = object, .cc = cc, .param_types = param_types, .zig_index = 0, - .llvm_index = 0, + .llvm_index = @as(u32, @intFromBool(has_sret)) + @intFromBool(has_err_ret_trace), .types_len = undefined, .types_buffer = undefined, .offsets_buffer = undefined, .byval_attr = null, + .free_regs = switch (cc) { + .x86_64_sysv, .x86_64_x32 => .{ + .gpr = 6 - @as(u8, @intFromBool(has_sret)), + .fpr = 8, + }, + else => undefined, + }, + .has_sret = has_sret, + .has_err_ret_trace = has_err_ret_trace, }; } diff --git a/test/c_abi/cfuncs.c b/test/c_abi/cfuncs.c index 327bc04ceb34148541b994bb894f04bea85bf210..6a045e31f00166922f2e73ba8e2116d6d950a74f 100644 --- a/test/c_abi/cfuncs.c +++ b/test/c_abi/cfuncs.c @@ -15834,6 +15834,13 @@ struct BigStruct { void zig_big_struct(struct BigStruct); +struct Struct_u64_u64_u64_u64 zig_sret_gpr_straddle( + struct Struct_u64_u64, struct Struct_u64_u64, struct Struct_u64_u64); +uint64_t zig_sse_then_gpr(double, double, double, double, double, double, + struct Struct_u64_u64); +double zig_sse_budget_straddle(double, double, double, double, double, double, + double, double, struct Struct_f64_f64); + union BigUnion { struct BigStruct a; }; @@ -16277,6 +16284,64 @@ struct SplitStructMixed c_ret_split_struct_mixed(void) { return s; } +struct Struct_u64_u64_u64_u64 c_sret_gpr_straddle( + struct Struct_u64_u64 p, struct Struct_u64_u64 q, struct Struct_u64_u64 r) { + assert_or_panic(p.a == 1); + assert_or_panic(p.b == 2); + assert_or_panic(q.a == 3); + assert_or_panic(q.b == 4); + assert_or_panic(r.a == 5); + assert_or_panic(r.b == 6); + struct Struct_u64_u64_u64_u64 res = {p.a, q.b, r.a, r.b}; + return res; +} + +uint64_t c_sse_then_gpr(double a, double b, double c, double d, double e, double f, + struct Struct_u64_u64 s) { + assert_or_panic(a == 1.0); + assert_or_panic(b == 2.0); + assert_or_panic(c == 3.0); + assert_or_panic(d == 4.0); + assert_or_panic(e == 5.0); + assert_or_panic(f == 6.0); + return s.a + s.b; +} + +double c_sse_budget_straddle(double a, double b, double c, double d, double e, double f, + double g, double h, struct Struct_f64_f64 s) { + assert_or_panic(a == 1.0); + assert_or_panic(b == 2.0); + assert_or_panic(c == 3.0); + assert_or_panic(d == 4.0); + assert_or_panic(e == 5.0); + assert_or_panic(f == 6.0); + assert_or_panic(g == 7.0); + assert_or_panic(h == 8.0); + return s.a + s.b; +} + +void c_test_sret_gpr_straddle(void) { + struct Struct_u64_u64 p = {10, 20}; + struct Struct_u64_u64 q = {30, 40}; + struct Struct_u64_u64 r = {50, 60}; + struct Struct_u64_u64_u64_u64 res = zig_sret_gpr_straddle(p, q, r); + assert_or_panic(res.a == 10); + assert_or_panic(res.b == 40); + assert_or_panic(res.c == 50); + assert_or_panic(res.d == 60); +} + +void c_test_sse_then_gpr(void) { + struct Struct_u64_u64 s = {100, 23}; + assert_or_panic(zig_sse_then_gpr(10.0, 20.0, 30.0, 40.0, 50.0, 60.0, s) == 123); +} + +void c_test_sse_budget_straddle(void) { + struct Struct_f64_f64 s = {100.0, 23.0}; + assert_or_panic(zig_sse_budget_straddle(10.0, 20.0, 30.0, 40.0, 50.0, 60.0, + 70.0, 80.0, s) == 123.0); +} + struct BigStruct c_big_struct_both(struct BigStruct x) { assert_or_panic(x.a == 1); assert_or_panic(x.b == 2); diff --git a/test/c_abi/main.zig b/test/c_abi/main.zig index a481a015117276e8abf7c4393f6d85cf60dd20fb..cbd9994a0019c8c2ece7753f7d754badbcf2d024 100644 --- a/test/c_abi/main.zig +++ b/test/c_abi/main.zig @@ -17285,6 +17285,93 @@ export fn zig_big_struct_both(x: BigStruct) BigStruct { return s; } +extern fn c_sret_gpr_straddle(Struct_u64_u64, Struct_u64_u64, Struct_u64_u64) Struct_u64_u64_u64_u64; +extern fn c_test_sret_gpr_straddle() void; + +test "sret with integer parameters straddling the register budget" { + if (builtin.cpu.arch == .hexagon) return error.SkipZigTest; + if (builtin.cpu.arch.isPowerPC32()) return error.SkipZigTest; + + const p: Struct_u64_u64 = .{ .a = 1, .b = 2 }; + const q: Struct_u64_u64 = .{ .a = 3, .b = 4 }; + const r: Struct_u64_u64 = .{ .a = 5, .b = 6 }; + const res = c_sret_gpr_straddle(p, q, r); + try expect(res.a == 1); + try expect(res.b == 4); + try expect(res.c == 5); + try expect(res.d == 6); + c_test_sret_gpr_straddle(); +} + +export fn zig_sret_gpr_straddle( + p: Struct_u64_u64, + q: Struct_u64_u64, + r: Struct_u64_u64, +) Struct_u64_u64_u64_u64 { + expect(p.a == 10) catch @panic("test failure: zig_sret_gpr_straddle p.a"); + expect(p.b == 20) catch @panic("test failure: zig_sret_gpr_straddle p.b"); + expect(q.a == 30) catch @panic("test failure: zig_sret_gpr_straddle q.a"); + expect(q.b == 40) catch @panic("test failure: zig_sret_gpr_straddle q.b"); + expect(r.a == 50) catch @panic("test failure: zig_sret_gpr_straddle r.a"); + expect(r.b == 60) catch @panic("test failure: zig_sret_gpr_straddle r.b"); + return .{ .a = p.a, .b = q.b, .c = r.a, .d = r.b }; +} + +extern fn c_sse_then_gpr(f64, f64, f64, f64, f64, f64, Struct_u64_u64) u64; +extern fn c_test_sse_then_gpr() void; + +test "integer aggregate after the float parameter registers" { + if (builtin.cpu.arch.isPowerPC32()) return error.SkipZigTest; + + const s: Struct_u64_u64 = .{ .a = 100, .b = 23 }; + try expect(c_sse_then_gpr(1.0, 2.0, 3.0, 4.0, 5.0, 6.0, s) == 123); + c_test_sse_then_gpr(); +} + +export fn zig_sse_then_gpr(a: f64, b: f64, c: f64, d: f64, e: f64, f: f64, s: Struct_u64_u64) u64 { + expect(a == 10.0) catch @panic("test failure: zig_sse_then_gpr a"); + expect(b == 20.0) catch @panic("test failure: zig_sse_then_gpr b"); + expect(c == 30.0) catch @panic("test failure: zig_sse_then_gpr c"); + expect(d == 40.0) catch @panic("test failure: zig_sse_then_gpr d"); + expect(e == 50.0) catch @panic("test failure: zig_sse_then_gpr e"); + expect(f == 60.0) catch @panic("test failure: zig_sse_then_gpr f"); + return s.a +% s.b; +} + +extern fn c_sse_budget_straddle(f64, f64, f64, f64, f64, f64, f64, f64, Struct_f64_f64) f64; +extern fn c_test_sse_budget_straddle() void; + +test "float aggregate straddling the float register budget" { + if (builtin.cpu.arch.isPowerPC()) return error.SkipZigTest; + if (builtin.cpu.arch.isRiscv32()) return error.SkipZigTest; + + const s: Struct_f64_f64 = .{ .a = 100.0, .b = 23.0 }; + try expect(c_sse_budget_straddle(1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, s) == 123.0); + c_test_sse_budget_straddle(); +} + +export fn zig_sse_budget_straddle( + a: f64, + b: f64, + c: f64, + d: f64, + e: f64, + f: f64, + g: f64, + h: f64, + s: Struct_f64_f64, +) f64 { + expect(a == 10.0) catch @panic("test failure: zig_sse_budget_straddle a"); + expect(b == 20.0) catch @panic("test failure: zig_sse_budget_straddle b"); + expect(c == 30.0) catch @panic("test failure: zig_sse_budget_straddle c"); + expect(d == 40.0) catch @panic("test failure: zig_sse_budget_straddle d"); + expect(e == 50.0) catch @panic("test failure: zig_sse_budget_straddle e"); + expect(f == 60.0) catch @panic("test failure: zig_sse_budget_straddle f"); + expect(g == 70.0) catch @panic("test failure: zig_sse_budget_straddle g"); + expect(h == 80.0) catch @panic("test failure: zig_sse_budget_straddle h"); + return s.a + s.b; +} + export fn zig_ret_bool() bool { return true; }