authorgravatar for mlugg@mlugg.co.ukMatthew Lugg <mlugg@mlugg.co.uk> 2026-05-08 10:26:10+01:00
committergravatar for mlugg@mlugg.co.ukMatthew Lugg <mlugg@mlugg.co.uk> 2026-05-09 09:35:44+02:00
loga76ce771082c3bf126e9ed745607bc8dadb56c61
tree2f29e17cee9f033b4ee702c16acec8cc2ddf2bf0
parentad1b746e2887d25a6e279762ada516e683db13da

llvm: fix lowering of x86 fastcall and vectorcall

LLVM requires these calling conventions to specify `inreg` attributes on all parameters which are passed via register.

4 files changed, 165 insertions(+), 73 deletions(-)

src/codegen/llvm.zig+81-70
...@@ -2718,73 +2718,60 @@ pub const Object = struct {...@@ -2718,73 +2718,60 @@ pub const Object = struct {
2718 @panic("TODO: LLVM backend lower async function");2718 @panic("TODO: LLVM backend lower async function");
2719 }2719 }
27202720
2721 {2721 const cc_info = toLlvmCallConv(fn_info.cc, target).?;
2722 const cc_info = toLlvmCallConv(fn_info.cc, target).?;
27232722
2724 function_index.setCallConv(cc_info.llvm_cc, &o.builder);2723 function_index.setCallConv(cc_info.llvm_cc, &o.builder);
27252724
2726 if (cc_info.align_stack) {2725 if (cc_info.align_stack) {
2727 try attributes.addFnAttr(.{ .alignstack = .wrap(.fromByteUnits(target.stackAlignment())) }, &o.builder);2726 try attributes.addFnAttr(.{ .alignstack = .wrap(.fromByteUnits(target.stackAlignment())) }, &o.builder);
2728 } else {2727 }
2729 _ = try attributes.removeFnAttr(.alignstack);
2730 }
2731
2732 if (cc_info.naked) {
2733 try attributes.addFnAttr(.naked, &o.builder);
2734 } else {
2735 _ = try attributes.removeFnAttr(.naked);
2736 }
27372728
2738 for (0..cc_info.inreg_param_count) |param_idx| {2729 if (cc_info.naked) {
2739 try attributes.addParamAttr(param_idx, .inreg, &o.builder);2730 try attributes.addFnAttr(.naked, &o.builder);
2740 }2731 }
2741 for (cc_info.inreg_param_count..std.math.maxInt(u2)) |param_idx| {
2742 _ = try attributes.removeParamAttr(param_idx, .inreg);
2743 }
27442732
2745 switch (fn_info.cc) {2733 switch (fn_info.cc) {
2746 inline .riscv64_interrupt,2734 inline .riscv64_interrupt,
2747 .riscv32_interrupt,2735 .riscv32_interrupt,
2748 .mips_interrupt,2736 .mips_interrupt,
2749 .mips64_interrupt,2737 .mips64_interrupt,
2750 => |info| {2738 => |info| {
2751 try attributes.addFnAttr(.{ .string = .{2739 try attributes.addFnAttr(.{ .string = .{
2752 .kind = try o.builder.string("interrupt"),2740 .kind = try o.builder.string("interrupt"),
2753 .value = try o.builder.string(@tagName(info.mode)),2741 .value = try o.builder.string(@tagName(info.mode)),
2754 } }, &o.builder);2742 } }, &o.builder);
2755 },2743 },
2756 .arm_interrupt,2744 .arm_interrupt,
2757 => |info| {2745 => |info| {
2758 try attributes.addFnAttr(.{ .string = .{2746 try attributes.addFnAttr(.{ .string = .{
2759 .kind = try o.builder.string("interrupt"),2747 .kind = try o.builder.string("interrupt"),
2760 .value = try o.builder.string(switch (info.type) {2748 .value = try o.builder.string(switch (info.type) {
2761 .generic => "",2749 .generic => "",
2762 .irq => "IRQ",2750 .irq => "IRQ",
2763 .fiq => "FIQ",2751 .fiq => "FIQ",
2764 .swi => "SWI",2752 .swi => "SWI",
2765 .abort => "ABORT",2753 .abort => "ABORT",
2766 .undef => "UNDEF",2754 .undef => "UNDEF",
2767 }),2755 }),
2768 } }, &o.builder);2756 } }, &o.builder);
2769 },2757 },
2770 // these function attributes serve as a backup against any mistakes LLVM makes.2758 // these function attributes serve as a backup against any mistakes LLVM makes.
2771 // clang sets both the function's calling convention and the function attributes2759 // clang sets both the function's calling convention and the function attributes
2772 // in its backend, so future patches to the AVR backend could end up checking only one,2760 // in its backend, so future patches to the AVR backend could end up checking only one,
2773 // possibly breaking our support. it's safer to just emit both.2761 // possibly breaking our support. it's safer to just emit both.
2774 .avr_interrupt, .avr_signal, .csky_interrupt => {2762 .avr_interrupt, .avr_signal, .csky_interrupt => {
2775 try attributes.addFnAttr(.{ .string = .{2763 try attributes.addFnAttr(.{ .string = .{
2776 .kind = try o.builder.string(switch (fn_info.cc) {2764 .kind = try o.builder.string(switch (fn_info.cc) {
2777 .avr_interrupt,2765 .avr_interrupt,
2778 .csky_interrupt,2766 .csky_interrupt,
2779 => "interrupt",2767 => "interrupt",
2780 .avr_signal => "signal",2768 .avr_signal => "signal",
2781 else => unreachable,2769 else => unreachable,
2782 }),2770 }),
2783 .value = .empty,2771 .value = .empty,
2784 } }, &o.builder);2772 } }, &o.builder);
2785 },2773 },
2786 else => {},2774 else => {},
2787 }
2788 }2775 }
27892776
2790 // Function attributes that are independent of analysis results of the function body.2777 // Function attributes that are independent of analysis results of the function body.
...@@ -2821,6 +2808,10 @@ pub const Object = struct {...@@ -2821,6 +2808,10 @@ pub const Object = struct {
2821 try attributes.addParamAttr(it.llvm_index, .nonnull, &o.builder);2808 try attributes.addParamAttr(it.llvm_index, .nonnull, &o.builder);
2822 it.llvm_index += 1;2809 it.llvm_index += 1;
2823 }2810 }
2811
2812 var remaining_inreg_int = cc_info.inreg_int_params;
2813 var remaining_inreg_float = cc_info.inreg_float_params;
2814
2824 while (try it.next()) |lowering| switch (lowering) {2815 while (try it.next()) |lowering| switch (lowering) {
2825 .byval => {2816 .byval => {
2826 const param_index = it.zig_index - 1;2817 const param_index = it.zig_index - 1;
...@@ -2828,6 +2819,21 @@ pub const Object = struct {...@@ -2828,6 +2819,21 @@ pub const Object = struct {
2828 if (!isByRef(param_ty, zcu)) {2819 if (!isByRef(param_ty, zcu)) {
2829 try o.addByValParamAttrs(pt, &attributes, param_ty, param_index, fn_info, it.llvm_index - 1);2820 try o.addByValParamAttrs(pt, &attributes, param_ty, param_index, fn_info, it.llvm_index - 1);
2830 }2821 }
2822
2823 if (remaining_inreg_int > 0 and
2824 (param_ty.isPtrAtRuntime(zcu) or
2825 (param_ty.isAbiInt(zcu) and param_ty.abiSize(zcu) <= Type.usize.abiSize(zcu))))
2826 {
2827 try attributes.addParamAttr(it.llvm_index - 1, .inreg, &o.builder);
2828 remaining_inreg_int -= 1;
2829 }
2830
2831 if (remaining_inreg_float > 0 and
2832 param_ty.zigTypeTag(zcu) == .float)
2833 {
2834 try attributes.addParamAttr(it.llvm_index - 1, .inreg, &o.builder);
2835 remaining_inreg_float -= 1;
2836 }
2831 },2837 },
2832 .byref => {2838 .byref => {
2833 const param_ty: Type = .fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);2839 const param_ty: Type = .fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);
...@@ -4379,15 +4385,19 @@ const CallingConventionInfo = struct {...@@ -4379,15 +4385,19 @@ const CallingConventionInfo = struct {
4379 align_stack: bool,4385 align_stack: bool,
4380 /// Whether the function needs a `naked` attribute.4386 /// Whether the function needs a `naked` attribute.
4381 naked: bool,4387 naked: bool,
4382 /// How many leading parameters to apply the `inreg` attribute to.4388 /// How many leading register-sized integer parameters to apply the `inreg` attribute to.
4383 inreg_param_count: u2 = 0,4389 inreg_int_params: u2 = 0,
4390 /// How many leading floating-point parameters to apply the `inreg` attribute to.
4391 inreg_float_params: u3 = 0,
4384};4392};
43854393
4386pub fn toLlvmCallConv(cc: std.lang.CallingConvention, target: *const std.Target) ?CallingConventionInfo {4394pub fn toLlvmCallConv(cc: std.lang.CallingConvention, target: *const std.Target) ?CallingConventionInfo {
4387 const llvm_cc = toLlvmCallConvTag(cc, target) orelse return null;4395 const llvm_cc = toLlvmCallConvTag(cc, target) orelse return null;
4388 const incoming_stack_alignment: ?u64, const register_params: u2 = switch (cc) {4396 const incoming_stack_alignment: ?u64, const inreg_int_params: u2, const inreg_float_params: u3 = switch (cc) {
4397 .x86_fastcall => |opts| .{ opts.incoming_stack_alignment, 2, 0 },
4398 .x86_vectorcall => |opts| .{ opts.incoming_stack_alignment, 2, 6 },
4389 inline else => |pl| switch (@TypeOf(pl)) {4399 inline else => |pl| switch (@TypeOf(pl)) {
4390 void => .{ null, 0 },4400 void => .{ null, 0, 0 },
4391 std.lang.CallingConvention.ArcInterruptOptions,4401 std.lang.CallingConvention.ArcInterruptOptions,
4392 std.lang.CallingConvention.ArmInterruptOptions,4402 std.lang.CallingConvention.ArmInterruptOptions,
4393 std.lang.CallingConvention.RiscvInterruptOptions,4403 std.lang.CallingConvention.RiscvInterruptOptions,
...@@ -4395,8 +4405,8 @@ pub fn toLlvmCallConv(cc: std.lang.CallingConvention, target: *const std.Target)...@@ -4395,8 +4405,8 @@ pub fn toLlvmCallConv(cc: std.lang.CallingConvention, target: *const std.Target)
4395 std.lang.CallingConvention.MicroblazeInterruptOptions,4405 std.lang.CallingConvention.MicroblazeInterruptOptions,
4396 std.lang.CallingConvention.MipsInterruptOptions,4406 std.lang.CallingConvention.MipsInterruptOptions,
4397 std.lang.CallingConvention.CommonOptions,4407 std.lang.CallingConvention.CommonOptions,
4398 => .{ pl.incoming_stack_alignment, 0 },4408 => .{ pl.incoming_stack_alignment, 0, 0 },
4399 std.lang.CallingConvention.X86RegparmOptions => .{ pl.incoming_stack_alignment, pl.register_params },4409 std.lang.CallingConvention.X86RegparmOptions => .{ pl.incoming_stack_alignment, pl.register_params, 0 },
4400 else => @compileError("TODO: toLlvmCallConv" ++ @tagName(pl)),4410 else => @compileError("TODO: toLlvmCallConv" ++ @tagName(pl)),
4401 },4411 },
4402 };4412 };
...@@ -4407,7 +4417,8 @@ pub fn toLlvmCallConv(cc: std.lang.CallingConvention, target: *const std.Target)...@@ -4407,7 +4417,8 @@ pub fn toLlvmCallConv(cc: std.lang.CallingConvention, target: *const std.Target)
4407 break :need_align a < normal_stack_align;4417 break :need_align a < normal_stack_align;
4408 } else false,4418 } else false,
4409 .naked = cc == .naked,4419 .naked = cc == .naked,
4410 .inreg_param_count = register_params,4420 .inreg_int_params = inreg_int_params,
4421 .inreg_float_params = inreg_float_params,
4411 };4422 };
4412}4423}
4413pub fn toLlvmCallConvTag(cc_tag: std.lang.CallingConvention.Tag, target: *const std.Target) ?Builder.CallConv {4424pub fn toLlvmCallConvTag(cc_tag: std.lang.CallingConvention.Tag, target: *const std.Target) ?Builder.CallConv {
src/codegen/llvm/FuncGen.zig+20-1
...@@ -767,11 +767,15 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier...@@ -767,11 +767,15 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier
767 },767 },
768 };768 };
769769
770 const cc_info = llvm.toLlvmCallConv(fn_info.cc, target).?;
771
770 {772 {
771 // Add argument attributes.773 // Add argument attributes.
772 it = iterateParamTypes(o, fn_info);774 it = iterateParamTypes(o, fn_info);
773 it.llvm_index += @intFromBool(sret);775 it.llvm_index += @intFromBool(sret);
774 it.llvm_index += @intFromBool(err_return_tracing);776 it.llvm_index += @intFromBool(err_return_tracing);
777 var remaining_inreg_int = cc_info.inreg_int_params;
778 var remaining_inreg_float = cc_info.inreg_float_params;
775 while (try it.next()) |lowering| switch (lowering) {779 while (try it.next()) |lowering| switch (lowering) {
776 .byval => {780 .byval => {
777 const param_index = it.zig_index - 1;781 const param_index = it.zig_index - 1;
...@@ -779,6 +783,21 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier...@@ -779,6 +783,21 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier
779 if (!isByRef(param_ty, zcu)) {783 if (!isByRef(param_ty, zcu)) {
780 try o.addByValParamAttrs(pt, &attributes, param_ty, param_index, fn_info, it.llvm_index - 1);784 try o.addByValParamAttrs(pt, &attributes, param_ty, param_index, fn_info, it.llvm_index - 1);
781 }785 }
786
787 if (remaining_inreg_int > 0 and
788 (param_ty.isPtrAtRuntime(zcu) or
789 (param_ty.isAbiInt(zcu) and param_ty.abiSize(zcu) <= Type.usize.abiSize(zcu))))
790 {
791 try attributes.addParamAttr(it.llvm_index - 1, .inreg, &o.builder);
792 remaining_inreg_int -= 1;
793 }
794
795 if (remaining_inreg_float > 0 and
796 param_ty.zigTypeTag(zcu) == .float)
797 {
798 try attributes.addParamAttr(it.llvm_index - 1, .inreg, &o.builder);
799 remaining_inreg_float -= 1;
800 }
782 },801 },
783 .byref => {802 .byref => {
784 const param_index = it.zig_index - 1;803 const param_index = it.zig_index - 1;
...@@ -833,7 +852,7 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier...@@ -833,7 +852,7 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier
833 .always_tail => .musttail,852 .always_tail => .musttail,
834 .no_suspend, .always_inline, .compile_time => unreachable,853 .no_suspend, .always_inline, .compile_time => unreachable,
835 },854 },
836 llvm.toLlvmCallConvTag(fn_info.cc, target).?,855 cc_info.llvm_cc,
837 try attributes.finish(&o.builder),856 try attributes.finish(&o.builder),
838 try o.lowerType(zig_fn_ty),857 try o.lowerType(zig_fn_ty),
839 llvm_fn,858 llvm_fn,
test/c_abi/cfuncs.c+27
...@@ -5892,3 +5892,30 @@ struct byval_tail_callsite_attr_Rect {...@@ -5892,3 +5892,30 @@ struct byval_tail_callsite_attr_Rect {
5892double c_byval_tail_callsite_attr(struct byval_tail_callsite_attr_Rect in) {5892double c_byval_tail_callsite_attr(struct byval_tail_callsite_attr_Rect in) {
5893 return in.size.width;5893 return in.size.width;
5894}5894}
5895
5896#ifdef __i386__
5897void __attribute__((fastcall)) zig_fastcall_check(int a, float b, void *c, double d, int e);
5898void __attribute__((fastcall)) c_fastcall_check(int a, float b, void *c, double d, int e) {
5899 assert_or_panic(a == 1);
5900 assert_or_panic(b == 2.0);
5901 assert_or_panic((uintptr_t)c == 3);
5902 assert_or_panic(d == 4.0);
5903 assert_or_panic(e == 5);
5904 zig_fastcall_check(a, b, c, d, e);
5905}
5906
5907void __attribute__((vectorcall)) zig_vectorcall_check(int a, float b, double c, void *d, float e, double f, double g, float h, float i, int j);
5908void __attribute__((vectorcall)) c_vectorcall_check(int a, float b, double c, void *d, float e, double f, double g, float h, float i, int j) {
5909 assert_or_panic(a == 1);
5910 assert_or_panic(b == 2.0);
5911 assert_or_panic(c == 3.0);
5912 assert_or_panic((uintptr_t)d == 4);
5913 assert_or_panic(e == 5.0);
5914 assert_or_panic(f == 6.0);
5915 assert_or_panic(g == 7.0);
5916 assert_or_panic(h == 8.0);
5917 assert_or_panic(i == 9.0);
5918 assert_or_panic(j == 10);
5919 zig_vectorcall_check(a, b, c, d, e, f, g, h, i, j);
5920}
5921#endif
test/c_abi/main.zig+37-2
...@@ -6075,7 +6075,7 @@ test "Stdcall ABI big union" {...@@ -6075,7 +6075,7 @@ test "Stdcall ABI big union" {
6075}6075}
60766076
6077extern fn c_explict_win64(ByRef) callconv(.{ .x86_64_win = .{} }) ByRef;6077extern fn c_explict_win64(ByRef) callconv(.{ .x86_64_win = .{} }) ByRef;
6078test "explicit SysV calling convention" {6078test "explicit Win64 calling convention" {
6079 if (builtin.cpu.arch != .x86_64) return error.SkipZigTest;6079 if (builtin.cpu.arch != .x86_64) return error.SkipZigTest;
60806080
6081 const res = c_explict_win64(.{ .val = 1, .arr = undefined });6081 const res = c_explict_win64(.{ .val = 1, .arr = undefined });
...@@ -6083,7 +6083,7 @@ test "explicit SysV calling convention" {...@@ -6083,7 +6083,7 @@ test "explicit SysV calling convention" {
6083}6083}
60846084
6085extern fn c_explict_sys_v(ByRef) callconv(.{ .x86_64_sysv = .{} }) ByRef;6085extern fn c_explict_sys_v(ByRef) callconv(.{ .x86_64_sysv = .{} }) ByRef;
6086test "explicit Win64 calling convention" {6086test "explicit SysV calling convention" {
6087 if (builtin.cpu.arch != .x86_64) return error.SkipZigTest;6087 if (builtin.cpu.arch != .x86_64) return error.SkipZigTest;
60886088
6089 const res = c_explict_sys_v(.{ .val = 1, .arr = undefined });6089 const res = c_explict_sys_v(.{ .val = 1, .arr = undefined });
...@@ -6147,3 +6147,38 @@ test "byval tail callsite attribute" {...@@ -6147,3 +6147,38 @@ test "byval tail callsite attribute" {
6147 };6147 };
6148 try expect(v.run() == 3.0);6148 try expect(v.run() == 3.0);
6149}6149}
6150
6151test "x86 fastcall calling convention" {
6152 if (builtin.cpu.arch != .x86) return error.SkipZigTest;
6153 const static = struct {
6154 extern fn c_fastcall_check(a: c_int, b: f32, c: *anyopaque, d: f64, e: c_int) callconv(.{ .x86_fastcall = .{} }) void;
6155 export fn zig_fastcall_check(a: c_int, b: f32, c: *anyopaque, d: f64, e: c_int) callconv(.{ .x86_fastcall = .{} }) void {
6156 if (a != 1) @panic("test failure");
6157 if (b != 2.0) @panic("test failure");
6158 if (@intFromPtr(c) != 3) @panic("test failure");
6159 if (d != 4.0) @panic("test failure");
6160 if (e != 5) @panic("test failure");
6161 }
6162 };
6163 static.c_fastcall_check(1, 2.0, @ptrFromInt(3), 4.0, 5);
6164}
6165
6166test "x86 vectorcall calling convention" {
6167 if (builtin.cpu.arch != .x86) return error.SkipZigTest;
6168 const static = struct {
6169 extern fn c_vectorcall_check(a: c_int, b: f32, c: f64, d: *anyopaque, e: f32, f: f64, g: f64, h: f32, i: f32, j: c_int) callconv(.{ .x86_vectorcall = .{} }) void;
6170 export fn zig_vectorcall_check(a: c_int, b: f32, c: f64, d: *anyopaque, e: f32, f: f64, g: f64, h: f32, i: f32, j: c_int) callconv(.{ .x86_vectorcall = .{} }) void {
6171 if (a != 1) @panic("test failure");
6172 if (b != 2.0) @panic("test failure");
6173 if (c != 3.0) @panic("test failure");
6174 if (@intFromPtr(d) != 4) @panic("test failure");
6175 if (e != 5.0) @panic("test failure");
6176 if (f != 6.0) @panic("test failure");
6177 if (g != 7.0) @panic("test failure");
6178 if (h != 8.0) @panic("test failure");
6179 if (i != 9.0) @panic("test failure");
6180 if (j != 10) @panic("test failure");
6181 }
6182 };
6183 static.c_vectorcall_check(1, 2.0, 3.0, @ptrFromInt(4), 5.0, 6.0, 7.0, 8.0, 9.0, 10);
6184}