authorgravatar for mlugg@mlugg.co.ukMatthew Lugg <mlugg@mlugg.co.uk> 2026-05-08 10:26:10+01:00
committergravatar for mlugg@mlugg.co.ukMatthew Lugg <mlugg@mlugg.co.uk> 2026-05-09 09:35:44+02:00
loga76ce771082c3bf126e9ed745607bc8dadb56c61
tree2f29e17cee9f033b4ee702c16acec8cc2ddf2bf0
parentad1b746e2887d25a6e279762ada516e683db13da

llvm: fix lowering of x86 fastcall and vectorcall

LLVM requires these calling conventions to specify `inreg` attributes on all parameters which are passed via register.

4 files changed, 165 insertions(+), 73 deletions(-)

src/codegen/llvm.zig+81-70
......@@ -2718,73 +2718,60 @@ pub const Object = struct {
27182718 @panic("TODO: LLVM backend lower async function");
27192719 }
27202720
2721 {
2722 const cc_info = toLlvmCallConv(fn_info.cc, target).?;
2721 const cc_info = toLlvmCallConv(fn_info.cc, target).?;
27232722
2724 function_index.setCallConv(cc_info.llvm_cc, &o.builder);
2723 function_index.setCallConv(cc_info.llvm_cc, &o.builder);
27252724
2726 if (cc_info.align_stack) {
2727 try attributes.addFnAttr(.{ .alignstack = .wrap(.fromByteUnits(target.stackAlignment())) }, &o.builder);
2728 } else {
2729 _ = try attributes.removeFnAttr(.alignstack);
2730 }
2731
2732 if (cc_info.naked) {
2733 try attributes.addFnAttr(.naked, &o.builder);
2734 } else {
2735 _ = try attributes.removeFnAttr(.naked);
2736 }
2725 if (cc_info.align_stack) {
2726 try attributes.addFnAttr(.{ .alignstack = .wrap(.fromByteUnits(target.stackAlignment())) }, &o.builder);
2727 }
27372728
2738 for (0..cc_info.inreg_param_count) |param_idx| {
2739 try attributes.addParamAttr(param_idx, .inreg, &o.builder);
2740 }
2741 for (cc_info.inreg_param_count..std.math.maxInt(u2)) |param_idx| {
2742 _ = try attributes.removeParamAttr(param_idx, .inreg);
2743 }
2729 if (cc_info.naked) {
2730 try attributes.addFnAttr(.naked, &o.builder);
2731 }
27442732
2745 switch (fn_info.cc) {
2746 inline .riscv64_interrupt,
2747 .riscv32_interrupt,
2748 .mips_interrupt,
2749 .mips64_interrupt,
2750 => |info| {
2751 try attributes.addFnAttr(.{ .string = .{
2752 .kind = try o.builder.string("interrupt"),
2753 .value = try o.builder.string(@tagName(info.mode)),
2754 } }, &o.builder);
2755 },
2756 .arm_interrupt,
2757 => |info| {
2758 try attributes.addFnAttr(.{ .string = .{
2759 .kind = try o.builder.string("interrupt"),
2760 .value = try o.builder.string(switch (info.type) {
2761 .generic => "",
2762 .irq => "IRQ",
2763 .fiq => "FIQ",
2764 .swi => "SWI",
2765 .abort => "ABORT",
2766 .undef => "UNDEF",
2767 }),
2768 } }, &o.builder);
2769 },
2770 // these function attributes serve as a backup against any mistakes LLVM makes.
2771 // clang sets both the function's calling convention and the function attributes
2772 // in its backend, so future patches to the AVR backend could end up checking only one,
2773 // possibly breaking our support. it's safer to just emit both.
2774 .avr_interrupt, .avr_signal, .csky_interrupt => {
2775 try attributes.addFnAttr(.{ .string = .{
2776 .kind = try o.builder.string(switch (fn_info.cc) {
2777 .avr_interrupt,
2778 .csky_interrupt,
2779 => "interrupt",
2780 .avr_signal => "signal",
2781 else => unreachable,
2782 }),
2783 .value = .empty,
2784 } }, &o.builder);
2785 },
2786 else => {},
2787 }
2733 switch (fn_info.cc) {
2734 inline .riscv64_interrupt,
2735 .riscv32_interrupt,
2736 .mips_interrupt,
2737 .mips64_interrupt,
2738 => |info| {
2739 try attributes.addFnAttr(.{ .string = .{
2740 .kind = try o.builder.string("interrupt"),
2741 .value = try o.builder.string(@tagName(info.mode)),
2742 } }, &o.builder);
2743 },
2744 .arm_interrupt,
2745 => |info| {
2746 try attributes.addFnAttr(.{ .string = .{
2747 .kind = try o.builder.string("interrupt"),
2748 .value = try o.builder.string(switch (info.type) {
2749 .generic => "",
2750 .irq => "IRQ",
2751 .fiq => "FIQ",
2752 .swi => "SWI",
2753 .abort => "ABORT",
2754 .undef => "UNDEF",
2755 }),
2756 } }, &o.builder);
2757 },
2758 // these function attributes serve as a backup against any mistakes LLVM makes.
2759 // clang sets both the function's calling convention and the function attributes
2760 // in its backend, so future patches to the AVR backend could end up checking only one,
2761 // possibly breaking our support. it's safer to just emit both.
2762 .avr_interrupt, .avr_signal, .csky_interrupt => {
2763 try attributes.addFnAttr(.{ .string = .{
2764 .kind = try o.builder.string(switch (fn_info.cc) {
2765 .avr_interrupt,
2766 .csky_interrupt,
2767 => "interrupt",
2768 .avr_signal => "signal",
2769 else => unreachable,
2770 }),
2771 .value = .empty,
2772 } }, &o.builder);
2773 },
2774 else => {},
27882775 }
27892776
27902777 // Function attributes that are independent of analysis results of the function body.
......@@ -2821,6 +2808,10 @@ pub const Object = struct {
28212808 try attributes.addParamAttr(it.llvm_index, .nonnull, &o.builder);
28222809 it.llvm_index += 1;
28232810 }
2811
2812 var remaining_inreg_int = cc_info.inreg_int_params;
2813 var remaining_inreg_float = cc_info.inreg_float_params;
2814
28242815 while (try it.next()) |lowering| switch (lowering) {
28252816 .byval => {
28262817 const param_index = it.zig_index - 1;
......@@ -2828,6 +2819,21 @@ pub const Object = struct {
28282819 if (!isByRef(param_ty, zcu)) {
28292820 try o.addByValParamAttrs(pt, &attributes, param_ty, param_index, fn_info, it.llvm_index - 1);
28302821 }
2822
2823 if (remaining_inreg_int > 0 and
2824 (param_ty.isPtrAtRuntime(zcu) or
2825 (param_ty.isAbiInt(zcu) and param_ty.abiSize(zcu) <= Type.usize.abiSize(zcu))))
2826 {
2827 try attributes.addParamAttr(it.llvm_index - 1, .inreg, &o.builder);
2828 remaining_inreg_int -= 1;
2829 }
2830
2831 if (remaining_inreg_float > 0 and
2832 param_ty.zigTypeTag(zcu) == .float)
2833 {
2834 try attributes.addParamAttr(it.llvm_index - 1, .inreg, &o.builder);
2835 remaining_inreg_float -= 1;
2836 }
28312837 },
28322838 .byref => {
28332839 const param_ty: Type = .fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);
......@@ -4379,15 +4385,19 @@ const CallingConventionInfo = struct {
43794385 align_stack: bool,
43804386 /// Whether the function needs a `naked` attribute.
43814387 naked: bool,
4382 /// How many leading parameters to apply the `inreg` attribute to.
4383 inreg_param_count: u2 = 0,
4388 /// How many leading register-sized integer parameters to apply the `inreg` attribute to.
4389 inreg_int_params: u2 = 0,
4390 /// How many leading floating-point parameters to apply the `inreg` attribute to.
4391 inreg_float_params: u3 = 0,
43844392};
43854393
43864394pub fn toLlvmCallConv(cc: std.lang.CallingConvention, target: *const std.Target) ?CallingConventionInfo {
43874395 const llvm_cc = toLlvmCallConvTag(cc, target) orelse return null;
4388 const incoming_stack_alignment: ?u64, const register_params: u2 = switch (cc) {
4396 const incoming_stack_alignment: ?u64, const inreg_int_params: u2, const inreg_float_params: u3 = switch (cc) {
4397 .x86_fastcall => |opts| .{ opts.incoming_stack_alignment, 2, 0 },
4398 .x86_vectorcall => |opts| .{ opts.incoming_stack_alignment, 2, 6 },
43894399 inline else => |pl| switch (@TypeOf(pl)) {
4390 void => .{ null, 0 },
4400 void => .{ null, 0, 0 },
43914401 std.lang.CallingConvention.ArcInterruptOptions,
43924402 std.lang.CallingConvention.ArmInterruptOptions,
43934403 std.lang.CallingConvention.RiscvInterruptOptions,
......@@ -4395,8 +4405,8 @@ pub fn toLlvmCallConv(cc: std.lang.CallingConvention, target: *const std.Target)
43954405 std.lang.CallingConvention.MicroblazeInterruptOptions,
43964406 std.lang.CallingConvention.MipsInterruptOptions,
43974407 std.lang.CallingConvention.CommonOptions,
4398 => .{ pl.incoming_stack_alignment, 0 },
4399 std.lang.CallingConvention.X86RegparmOptions => .{ pl.incoming_stack_alignment, pl.register_params },
4408 => .{ pl.incoming_stack_alignment, 0, 0 },
4409 std.lang.CallingConvention.X86RegparmOptions => .{ pl.incoming_stack_alignment, pl.register_params, 0 },
44004410 else => @compileError("TODO: toLlvmCallConv" ++ @tagName(pl)),
44014411 },
44024412 };
......@@ -4407,7 +4417,8 @@ pub fn toLlvmCallConv(cc: std.lang.CallingConvention, target: *const std.Target)
44074417 break :need_align a < normal_stack_align;
44084418 } else false,
44094419 .naked = cc == .naked,
4410 .inreg_param_count = register_params,
4420 .inreg_int_params = inreg_int_params,
4421 .inreg_float_params = inreg_float_params,
44114422 };
44124423}
44134424pub fn toLlvmCallConvTag(cc_tag: std.lang.CallingConvention.Tag, target: *const std.Target) ?Builder.CallConv {
src/codegen/llvm/FuncGen.zig+20-1
......@@ -767,11 +767,15 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier
767767 },
768768 };
769769
770 const cc_info = llvm.toLlvmCallConv(fn_info.cc, target).?;
771
770772 {
771773 // Add argument attributes.
772774 it = iterateParamTypes(o, fn_info);
773775 it.llvm_index += @intFromBool(sret);
774776 it.llvm_index += @intFromBool(err_return_tracing);
777 var remaining_inreg_int = cc_info.inreg_int_params;
778 var remaining_inreg_float = cc_info.inreg_float_params;
775779 while (try it.next()) |lowering| switch (lowering) {
776780 .byval => {
777781 const param_index = it.zig_index - 1;
......@@ -779,6 +783,21 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier
779783 if (!isByRef(param_ty, zcu)) {
780784 try o.addByValParamAttrs(pt, &attributes, param_ty, param_index, fn_info, it.llvm_index - 1);
781785 }
786
787 if (remaining_inreg_int > 0 and
788 (param_ty.isPtrAtRuntime(zcu) or
789 (param_ty.isAbiInt(zcu) and param_ty.abiSize(zcu) <= Type.usize.abiSize(zcu))))
790 {
791 try attributes.addParamAttr(it.llvm_index - 1, .inreg, &o.builder);
792 remaining_inreg_int -= 1;
793 }
794
795 if (remaining_inreg_float > 0 and
796 param_ty.zigTypeTag(zcu) == .float)
797 {
798 try attributes.addParamAttr(it.llvm_index - 1, .inreg, &o.builder);
799 remaining_inreg_float -= 1;
800 }
782801 },
783802 .byref => {
784803 const param_index = it.zig_index - 1;
......@@ -833,7 +852,7 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier
833852 .always_tail => .musttail,
834853 .no_suspend, .always_inline, .compile_time => unreachable,
835854 },
836 llvm.toLlvmCallConvTag(fn_info.cc, target).?,
855 cc_info.llvm_cc,
837856 try attributes.finish(&o.builder),
838857 try o.lowerType(zig_fn_ty),
839858 llvm_fn,
test/c_abi/cfuncs.c+27
......@@ -5892,3 +5892,30 @@ struct byval_tail_callsite_attr_Rect {
58925892double c_byval_tail_callsite_attr(struct byval_tail_callsite_attr_Rect in) {
58935893 return in.size.width;
58945894}
5895
5896#ifdef __i386__
5897void __attribute__((fastcall)) zig_fastcall_check(int a, float b, void *c, double d, int e);
5898void __attribute__((fastcall)) c_fastcall_check(int a, float b, void *c, double d, int e) {
5899 assert_or_panic(a == 1);
5900 assert_or_panic(b == 2.0);
5901 assert_or_panic((uintptr_t)c == 3);
5902 assert_or_panic(d == 4.0);
5903 assert_or_panic(e == 5);
5904 zig_fastcall_check(a, b, c, d, e);
5905}
5906
5907void __attribute__((vectorcall)) zig_vectorcall_check(int a, float b, double c, void *d, float e, double f, double g, float h, float i, int j);
5908void __attribute__((vectorcall)) c_vectorcall_check(int a, float b, double c, void *d, float e, double f, double g, float h, float i, int j) {
5909 assert_or_panic(a == 1);
5910 assert_or_panic(b == 2.0);
5911 assert_or_panic(c == 3.0);
5912 assert_or_panic((uintptr_t)d == 4);
5913 assert_or_panic(e == 5.0);
5914 assert_or_panic(f == 6.0);
5915 assert_or_panic(g == 7.0);
5916 assert_or_panic(h == 8.0);
5917 assert_or_panic(i == 9.0);
5918 assert_or_panic(j == 10);
5919 zig_vectorcall_check(a, b, c, d, e, f, g, h, i, j);
5920}
5921#endif
test/c_abi/main.zig+37-2
......@@ -6075,7 +6075,7 @@ test "Stdcall ABI big union" {
60756075}
60766076
60776077extern fn c_explict_win64(ByRef) callconv(.{ .x86_64_win = .{} }) ByRef;
6078test "explicit SysV calling convention" {
6078test "explicit Win64 calling convention" {
60796079 if (builtin.cpu.arch != .x86_64) return error.SkipZigTest;
60806080
60816081 const res = c_explict_win64(.{ .val = 1, .arr = undefined });
......@@ -6083,7 +6083,7 @@ test "explicit SysV calling convention" {
60836083}
60846084
60856085extern fn c_explict_sys_v(ByRef) callconv(.{ .x86_64_sysv = .{} }) ByRef;
6086test "explicit Win64 calling convention" {
6086test "explicit SysV calling convention" {
60876087 if (builtin.cpu.arch != .x86_64) return error.SkipZigTest;
60886088
60896089 const res = c_explict_sys_v(.{ .val = 1, .arr = undefined });
......@@ -6147,3 +6147,38 @@ test "byval tail callsite attribute" {
61476147 };
61486148 try expect(v.run() == 3.0);
61496149}
6150
6151test "x86 fastcall calling convention" {
6152 if (builtin.cpu.arch != .x86) return error.SkipZigTest;
6153 const static = struct {
6154 extern fn c_fastcall_check(a: c_int, b: f32, c: *anyopaque, d: f64, e: c_int) callconv(.{ .x86_fastcall = .{} }) void;
6155 export fn zig_fastcall_check(a: c_int, b: f32, c: *anyopaque, d: f64, e: c_int) callconv(.{ .x86_fastcall = .{} }) void {
6156 if (a != 1) @panic("test failure");
6157 if (b != 2.0) @panic("test failure");
6158 if (@intFromPtr(c) != 3) @panic("test failure");
6159 if (d != 4.0) @panic("test failure");
6160 if (e != 5) @panic("test failure");
6161 }
6162 };
6163 static.c_fastcall_check(1, 2.0, @ptrFromInt(3), 4.0, 5);
6164}
6165
6166test "x86 vectorcall calling convention" {
6167 if (builtin.cpu.arch != .x86) return error.SkipZigTest;
6168 const static = struct {
6169 extern fn c_vectorcall_check(a: c_int, b: f32, c: f64, d: *anyopaque, e: f32, f: f64, g: f64, h: f32, i: f32, j: c_int) callconv(.{ .x86_vectorcall = .{} }) void;
6170 export fn zig_vectorcall_check(a: c_int, b: f32, c: f64, d: *anyopaque, e: f32, f: f64, g: f64, h: f32, i: f32, j: c_int) callconv(.{ .x86_vectorcall = .{} }) void {
6171 if (a != 1) @panic("test failure");
6172 if (b != 2.0) @panic("test failure");
6173 if (c != 3.0) @panic("test failure");
6174 if (@intFromPtr(d) != 4) @panic("test failure");
6175 if (e != 5.0) @panic("test failure");
6176 if (f != 6.0) @panic("test failure");
6177 if (g != 7.0) @panic("test failure");
6178 if (h != 8.0) @panic("test failure");
6179 if (i != 9.0) @panic("test failure");
6180 if (j != 10) @panic("test failure");
6181 }
6182 };
6183 static.c_vectorcall_check(1, 2.0, 3.0, @ptrFromInt(4), 5.0, 6.0, 7.0, 8.0, 9.0, 10);
6184}