authorgravatar for alex@alexrp.comAlex Rønne Petersen <alex@alexrp.com> 2025-02-23 00:51:40+01:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2025-02-23 00:51:40+01:00
log5e203e157b0188a57ce43b876ab4d1877f1dfea1
tree17440ee9d22fb5e83edcbd846a471e76c1a92ac6
parent813312f0e871d436581c5d0be8700ee04aa2fb97
parent21eb75749722668632da55499831eedc62120d61
signaturebadge-check Signed by PGP key B5690EEEBB952194

Merge pull request #22903 from alexrp/llvm-nobuiltin-memcpy-inline

`llvm`: Use inline variants of `memcpy`/`memset` intrinsics when using `-fno-builtin`

2 files changed, 83 insertions(+), 18 deletions(-)

src/codegen/llvm.zig+64-10
......@@ -5771,6 +5771,7 @@ pub const FuncGen = struct {
57715771 try o.builder.intValue(.i8, 0xaa),
57725772 len,
57735773 if (ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal,
5774 self.ng.ownerModule().no_builtin,
57745775 );
57755776 const owner_mod = self.ng.ownerModule();
57765777 if (owner_mod.valgrind) {
......@@ -5821,6 +5822,7 @@ pub const FuncGen = struct {
58215822 try o.builder.intValue(.i8, 0xaa),
58225823 len,
58235824 .normal,
5825 self.ng.ownerModule().no_builtin,
58245826 );
58255827 const owner_mod = self.ng.ownerModule();
58265828 if (owner_mod.valgrind) {
......@@ -5902,7 +5904,7 @@ pub const FuncGen = struct {
59025904 const result_alignment = va_list_ty.abiAlignment(pt.zcu).toLlvm();
59035905 const dest_list = try self.buildAllocaWorkaround(va_list_ty, result_alignment);
59045906
5905 _ = try self.wip.callIntrinsic(.normal, .none, .va_copy, &.{}, &.{ dest_list, src_list }, "");
5907 _ = try self.wip.callIntrinsic(.normal, .none, .va_copy, &.{dest_list.typeOfWip(&self.wip)}, &.{ dest_list, src_list }, "");
59065908 return if (isByRef(va_list_ty, zcu))
59075909 dest_list
59085910 else
......@@ -5913,7 +5915,7 @@ pub const FuncGen = struct {
59135915 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
59145916 const src_list = try self.resolveInst(un_op);
59155917
5916 _ = try self.wip.callIntrinsic(.normal, .none, .va_end, &.{}, &.{src_list}, "");
5918 _ = try self.wip.callIntrinsic(.normal, .none, .va_end, &.{src_list.typeOfWip(&self.wip)}, &.{src_list}, "");
59175919 return .none;
59185920 }
59195921
......@@ -5927,7 +5929,7 @@ pub const FuncGen = struct {
59275929 const result_alignment = va_list_ty.abiAlignment(pt.zcu).toLlvm();
59285930 const dest_list = try self.buildAllocaWorkaround(va_list_ty, result_alignment);
59295931
5930 _ = try self.wip.callIntrinsic(.normal, .none, .va_start, &.{}, &.{dest_list}, "");
5932 _ = try self.wip.callIntrinsic(.normal, .none, .va_start, &.{dest_list.typeOfWip(&self.wip)}, &.{dest_list}, "");
59315933 return if (isByRef(va_list_ty, zcu))
59325934 dest_list
59335935 else
......@@ -9734,6 +9736,7 @@ pub const FuncGen = struct {
97349736 if (safety) try o.builder.intValue(.i8, 0xaa) else try o.builder.undefValue(.i8),
97359737 len,
97369738 if (ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal,
9739 self.ng.ownerModule().no_builtin,
97379740 );
97389741 if (safety and owner_mod.valgrind) {
97399742 try self.valgrindMarkUndef(dest_ptr, len);
......@@ -10041,9 +10044,22 @@ pub const FuncGen = struct {
1004110044 try o.builder.undefValue(.i8);
1004210045 const len = try self.sliceOrArrayLenInBytes(dest_slice, ptr_ty);
1004310046 if (intrinsic_len0_traps) {
10044 try self.safeWasmMemset(dest_ptr, fill_byte, len, dest_ptr_align, access_kind);
10047 try self.safeWasmMemset(
10048 dest_ptr,
10049 fill_byte,
10050 len,
10051 dest_ptr_align,
10052 access_kind,
10053 );
1004510054 } else {
10046 _ = try self.wip.callMemSet(dest_ptr, dest_ptr_align, fill_byte, len, access_kind);
10055 _ = try self.wip.callMemSet(
10056 dest_ptr,
10057 dest_ptr_align,
10058 fill_byte,
10059 len,
10060 access_kind,
10061 self.ng.ownerModule().no_builtin,
10062 );
1004710063 }
1004810064 const owner_mod = self.ng.ownerModule();
1004910065 if (safety and owner_mod.valgrind) {
......@@ -10060,9 +10076,22 @@ pub const FuncGen = struct {
1006010076 const fill_byte = try o.builder.intValue(.i8, byte_val);
1006110077 const len = try self.sliceOrArrayLenInBytes(dest_slice, ptr_ty);
1006210078 if (intrinsic_len0_traps) {
10063 try self.safeWasmMemset(dest_ptr, fill_byte, len, dest_ptr_align, access_kind);
10079 try self.safeWasmMemset(
10080 dest_ptr,
10081 fill_byte,
10082 len,
10083 dest_ptr_align,
10084 access_kind,
10085 );
1006410086 } else {
10065 _ = try self.wip.callMemSet(dest_ptr, dest_ptr_align, fill_byte, len, access_kind);
10087 _ = try self.wip.callMemSet(
10088 dest_ptr,
10089 dest_ptr_align,
10090 fill_byte,
10091 len,
10092 access_kind,
10093 self.ng.ownerModule().no_builtin,
10094 );
1006610095 }
1006710096 return .none;
1006810097 }
......@@ -10077,9 +10106,22 @@ pub const FuncGen = struct {
1007710106 const len = try self.sliceOrArrayLenInBytes(dest_slice, ptr_ty);
1007810107
1007910108 if (intrinsic_len0_traps) {
10080 try self.safeWasmMemset(dest_ptr, fill_byte, len, dest_ptr_align, access_kind);
10109 try self.safeWasmMemset(
10110 dest_ptr,
10111 fill_byte,
10112 len,
10113 dest_ptr_align,
10114 access_kind,
10115 );
1008110116 } else {
10082 _ = try self.wip.callMemSet(dest_ptr, dest_ptr_align, fill_byte, len, access_kind);
10117 _ = try self.wip.callMemSet(
10118 dest_ptr,
10119 dest_ptr_align,
10120 fill_byte,
10121 len,
10122 access_kind,
10123 self.ng.ownerModule().no_builtin,
10124 );
1008310125 }
1008410126 return .none;
1008510127 }
......@@ -10131,6 +10173,7 @@ pub const FuncGen = struct {
1013110173 elem_abi_align.toLlvm(),
1013210174 try o.builder.intValue(llvm_usize_ty, elem_abi_size),
1013310175 access_kind,
10176 self.ng.ownerModule().no_builtin,
1013410177 );
1013510178 } else _ = try self.wip.store(access_kind, value, it_ptr.toValue(), it_ptr_align);
1013610179 const next_ptr = try self.wip.gep(.inbounds, elem_llvm_ty, it_ptr.toValue(), &.{
......@@ -10158,7 +10201,14 @@ pub const FuncGen = struct {
1015810201 const end_block = try self.wip.block(2, "MemsetTrapEnd");
1015910202 _ = try self.wip.brCond(cond, memset_block, end_block, .none);
1016010203 self.wip.cursor = .{ .block = memset_block };
10161 _ = try self.wip.callMemSet(dest_ptr, dest_ptr_align, fill_byte, len, access_kind);
10204 _ = try self.wip.callMemSet(
10205 dest_ptr,
10206 dest_ptr_align,
10207 fill_byte,
10208 len,
10209 access_kind,
10210 self.ng.ownerModule().no_builtin,
10211 );
1016210212 _ = try self.wip.br(end_block);
1016310213 self.wip.cursor = .{ .block = end_block };
1016410214 }
......@@ -10200,6 +10250,7 @@ pub const FuncGen = struct {
1020010250 src_ptr_ty.ptrAlignment(zcu).toLlvm(),
1020110251 len,
1020210252 access_kind,
10253 self.ng.ownerModule().no_builtin,
1020310254 );
1020410255 _ = try self.wip.br(end_block);
1020510256 self.wip.cursor = .{ .block = end_block };
......@@ -10213,6 +10264,7 @@ pub const FuncGen = struct {
1021310264 src_ptr_ty.ptrAlignment(zcu).toLlvm(),
1021410265 len,
1021510266 access_kind,
10267 self.ng.ownerModule().no_builtin,
1021610268 );
1021710269 return .none;
1021810270 }
......@@ -11346,6 +11398,7 @@ pub const FuncGen = struct {
1134611398 ptr_alignment,
1134711399 try o.builder.intValue(try o.lowerType(Type.usize), size_bytes),
1134811400 access_kind,
11401 fg.ng.ownerModule().no_builtin,
1134911402 );
1135011403 return result_ptr;
1135111404 }
......@@ -11513,6 +11566,7 @@ pub const FuncGen = struct {
1151311566 elem_ty.abiAlignment(zcu).toLlvm(),
1151411567 try o.builder.intValue(try o.lowerType(Type.usize), elem_ty.abiSize(zcu)),
1151511568 access_kind,
11569 self.ng.ownerModule().no_builtin,
1151611570 );
1151711571 }
1151811572
src/codegen/llvm/Builder.zig+19-8
......@@ -2634,6 +2634,7 @@ pub const Intrinsic = enum {
26342634 cos,
26352635 pow,
26362636 exp,
2637 exp10,
26372638 exp2,
26382639 ldexp,
26392640 frexp,
......@@ -2801,22 +2802,22 @@ pub const Intrinsic = enum {
28012802 .va_start = .{
28022803 .ret_len = 0,
28032804 .params = &.{
2804 .{ .kind = .{ .type = .ptr } },
2805 .{ .kind = .overloaded },
28052806 },
28062807 .attrs = &.{ .nocallback, .nofree, .nosync, .nounwind, .willreturn },
28072808 },
28082809 .va_end = .{
28092810 .ret_len = 0,
28102811 .params = &.{
2811 .{ .kind = .{ .type = .ptr } },
2812 .{ .kind = .overloaded },
28122813 },
28132814 .attrs = &.{ .nocallback, .nofree, .nosync, .nounwind, .willreturn },
28142815 },
28152816 .va_copy = .{
28162817 .ret_len = 0,
28172818 .params = &.{
2818 .{ .kind = .{ .type = .ptr } },
2819 .{ .kind = .{ .type = .ptr } },
2819 .{ .kind = .overloaded },
2820 .{ .kind = .{ .matches = 0 } },
28202821 },
28212822 .attrs = &.{ .nocallback, .nofree, .nosync, .nounwind, .willreturn },
28222823 },
......@@ -2929,7 +2930,7 @@ pub const Intrinsic = enum {
29292930 .params = &.{
29302931 .{ .kind = .overloaded, .attrs = &.{ .@"noalias", .nocapture, .writeonly } },
29312932 .{ .kind = .overloaded, .attrs = &.{ .@"noalias", .nocapture, .readonly } },
2932 .{ .kind = .overloaded, .attrs = &.{.immarg} },
2933 .{ .kind = .overloaded },
29332934 .{ .kind = .{ .type = .i1 }, .attrs = &.{.immarg} },
29342935 },
29352936 .attrs = &.{ .nocallback, .nofree, .nounwind, .willreturn, .{ .memory = .{ .argmem = .readwrite } } },
......@@ -2959,7 +2960,7 @@ pub const Intrinsic = enum {
29592960 .params = &.{
29602961 .{ .kind = .overloaded, .attrs = &.{ .nocapture, .writeonly } },
29612962 .{ .kind = .{ .type = .i8 } },
2962 .{ .kind = .overloaded, .attrs = &.{.immarg} },
2963 .{ .kind = .overloaded },
29632964 .{ .kind = .{ .type = .i1 }, .attrs = &.{.immarg} },
29642965 },
29652966 .attrs = &.{ .nocallback, .nofree, .nounwind, .willreturn, .{ .memory = .{ .argmem = .write } } },
......@@ -3022,6 +3023,14 @@ pub const Intrinsic = enum {
30223023 },
30233024 .attrs = &.{ .nocallback, .nofree, .nosync, .nounwind, .speculatable, .willreturn, .{ .memory = Attribute.Memory.all(.none) } },
30243025 },
3026 .exp10 = .{
3027 .ret_len = 1,
3028 .params = &.{
3029 .{ .kind = .overloaded },
3030 .{ .kind = .{ .matches = 0 } },
3031 },
3032 .attrs = &.{ .nocallback, .nofree, .nosync, .nounwind, .speculatable, .willreturn, .{ .memory = Attribute.Memory.all(.none) } },
3033 },
30253034 .ldexp = .{
30263035 .ret_len = 1,
30273036 .params = &.{
......@@ -6093,6 +6102,7 @@ pub const WipFunction = struct {
60936102 src_align: Alignment,
60946103 len: Value,
60956104 kind: MemoryAccessKind,
6105 @"inline": bool,
60966106 ) Allocator.Error!Instruction.Index {
60976107 var dst_attrs = [_]Attribute.Index{try self.builder.attr(.{ .@"align" = dst_align })};
60986108 var src_attrs = [_]Attribute.Index{try self.builder.attr(.{ .@"align" = src_align })};
......@@ -6104,7 +6114,7 @@ pub const WipFunction = struct {
61046114 try self.builder.attrs(&dst_attrs),
61056115 try self.builder.attrs(&src_attrs),
61066116 }),
6107 .memcpy,
6117 if (@"inline") .@"memcpy.inline" else .memcpy,
61086118 &.{ dst.typeOfWip(self), src.typeOfWip(self), len.typeOfWip(self) },
61096119 &.{ dst, src, len, switch (kind) {
61106120 .normal => Value.false,
......@@ -6122,12 +6132,13 @@ pub const WipFunction = struct {
61226132 val: Value,
61236133 len: Value,
61246134 kind: MemoryAccessKind,
6135 @"inline": bool,
61256136 ) Allocator.Error!Instruction.Index {
61266137 var dst_attrs = [_]Attribute.Index{try self.builder.attr(.{ .@"align" = dst_align })};
61276138 const value = try self.callIntrinsic(
61286139 .normal,
61296140 try self.builder.fnAttrs(&.{ .none, .none, try self.builder.attrs(&dst_attrs) }),
6130 .memset,
6141 if (@"inline") .@"memset.inline" else .memset,
61316142 &.{ dst.typeOfWip(self), len.typeOfWip(self) },
61326143 &.{ dst, val, len, switch (kind) {
61336144 .normal => Value.false,