authorgravatar for 4678790+dweiller@users.noreply.github.comDominic <4678790+dweiller@users.noreply.github.com> 2025-02-11 15:01:35+11:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-02-11 17:40:20-08:00
log6dc1a4db7f9561dd2b84a99159b3ad7b55958717
treedec4044e1316dcffba1d1a2b346f307e89dd8d62
parentc2a3d8cbb935ae68a5bd95d45c65b0f7f32974b0

compiler-rt: fix memcpy generating recursive calls

When using the LLVM backend, array copies were lowered as calls to `llvm.memcpy.*` builtin which could cause recursive calls to memcpy to be generated (observed with `-target x86_64-linux -mcpu x86_64+avx512vl --debug-rt`). By instead performing these small fixed-size copies with integers or vectors the LLVM backend does not generate calls to the `llvm.memcpy` builtin, and so (with `-fno-builtin`) recursive calls to memcpy will not be generated by LLVM. The assertions and (test build) runtime safety have been removed as they may cause (mutually) recursive calls to memcpy in debug builds since the panic handler generates calls to llvm.memcpy.

1 files changed, 38 insertions(+), 17 deletions(-)

lib/compiler_rt/memcpy.zig+38-17
......@@ -34,7 +34,7 @@ comptime {
3434}
3535
3636fn memcpySmall(noalias dest: ?[*]u8, noalias src: ?[*]const u8, len: usize) callconv(.C) ?[*]u8 {
37 @setRuntimeSafety(builtin.is_test);
37 @setRuntimeSafety(false);
3838
3939 for (0..len) |i| {
4040 dest.?[i] = src.?[i];
......@@ -44,7 +44,7 @@ fn memcpySmall(noalias dest: ?[*]u8, noalias src: ?[*]const u8, len: usize) call
4444}
4545
4646fn memcpyFast(noalias dest: ?[*]u8, noalias src: ?[*]const u8, len: usize) callconv(.C) ?[*]u8 {
47 @setRuntimeSafety(builtin.is_test);
47 @setRuntimeSafety(false);
4848
4949 const small_limit = 2 * @sizeOf(Element);
5050
......@@ -78,7 +78,7 @@ inline fn copyLessThan16(
7878 src: [*]const u8,
7979 len: usize,
8080) void {
81 @setRuntimeSafety(builtin.is_test);
81 @setRuntimeSafety(false);
8282 if (len < 4) {
8383 if (len == 0) return;
8484 dest[0] = src[0];
......@@ -95,7 +95,7 @@ inline fn copy16ToSmallLimit(
9595 src: [*]const u8,
9696 len: usize,
9797) bool {
98 @setRuntimeSafety(builtin.is_test);
98 @setRuntimeSafety(false);
9999 inline for (2..(std.math.log2(small_limit) + 1) / 2 + 1) |p| {
100100 const limit = 1 << (2 * p);
101101 if (len < limit) {
......@@ -111,10 +111,9 @@ inline fn copyForwards(
111111 noalias src: [*]const u8,
112112 len: usize,
113113) void {
114 @setRuntimeSafety(builtin.is_test);
115 assert(len >= 2 * @sizeOf(Element));
114 @setRuntimeSafety(false);
116115
117 dest[0..@sizeOf(Element)].* = src[0..@sizeOf(Element)].*;
116 copyFixedLength(dest, src, @sizeOf(Element));
118117 const alignment_offset = @alignOf(Element) - @intFromPtr(src) % @alignOf(Element);
119118 const n = len - alignment_offset;
120119 const d = dest + alignment_offset;
......@@ -125,7 +124,7 @@ inline fn copyForwards(
125124 // copy last `@sizeOf(Element)` bytes unconditionally, since block copy
126125 // methods only copy a multiple of `@sizeOf(Element)` bytes.
127126 const offset = len - @sizeOf(Element);
128 dest[offset..][0..@sizeOf(Element)].* = src[offset..][0..@sizeOf(Element)].*;
127 copyFixedLength(dest + offset, src + offset, @sizeOf(Element));
129128}
130129
131130inline fn copyBlocksAlignedSource(
......@@ -144,7 +143,7 @@ inline fn copyBlocks(
144143 noalias src: anytype,
145144 max_bytes: usize,
146145) void {
147 @setRuntimeSafety(builtin.is_test);
146 @setRuntimeSafety(false);
148147
149148 const T = @typeInfo(@TypeOf(dest)).pointer.child;
150149 comptime assert(T == @typeInfo(@TypeOf(src)).pointer.child);
......@@ -156,6 +155,31 @@ inline fn copyBlocks(
156155 }
157156}
158157
158inline fn copyFixedLength(
159 noalias dest: [*]u8,
160 noalias src: [*]const u8,
161 comptime len: comptime_int,
162) void {
163 @setRuntimeSafety(false);
164 comptime assert(std.math.isPowerOfTwo(len));
165
166 const T = if (len >= @sizeOf(Element))
167 Element
168 else if (len > @sizeOf(usize))
169 @Vector(len, u8)
170 else
171 @Type(.{ .int = .{ .signedness = .unsigned, .bits = len * 8 } });
172
173 const loop_count = @divExact(len, @sizeOf(T));
174
175 const d: [*]align(1) T = @ptrCast(dest);
176 const s: [*]align(1) const T = @ptrCast(src);
177
178 inline for (0..loop_count) |i| {
179 d[i] = s[i];
180 }
181}
182
159183/// copy `len` bytes from `src` to `dest`; `len` must be in the range
160184/// `[copy_len, 4 * copy_len)`.
161185inline fn copyRange4(
......@@ -164,10 +188,8 @@ inline fn copyRange4(
164188 noalias src: [*]const u8,
165189 len: usize,
166190) void {
167 @setRuntimeSafety(builtin.is_test);
191 @setRuntimeSafety(false);
168192 comptime assert(std.math.isPowerOfTwo(copy_len));
169 assert(len >= copy_len);
170 assert(len < 4 * copy_len);
171193
172194 const a = len & (copy_len * 2);
173195 const b = a / 2;
......@@ -175,11 +197,10 @@ inline fn copyRange4(
175197 const last = len - copy_len;
176198 const pen = last - b;
177199
178 const Int = @Type(.{ .int = .{ .signedness = .unsigned, .bits = copy_len * 8 } });
179 @as(*align(1) Int, @ptrCast(dest[0..copy_len])).* = @as(*align(1) const Int, @ptrCast(src[0..copy_len])).*;
180 @as(*align(1) Int, @ptrCast(dest[b..][0..copy_len])).* = @as(*align(1) const Int, @ptrCast(src[b..][0..copy_len])).*;
181 @as(*align(1) Int, @ptrCast(dest[pen..][0..copy_len])).* = @as(*align(1) const Int, @ptrCast(src[pen..][0..copy_len])).*;
182 @as(*align(1) Int, @ptrCast(dest[last..][0..copy_len])).* = @as(*align(1) const Int, @ptrCast(src[last..][0..copy_len])).*;
200 copyFixedLength(dest, src, copy_len);
201 copyFixedLength(dest + b, src + b, copy_len);
202 copyFixedLength(dest + pen, src + pen, copy_len);
203 copyFixedLength(dest + last, src + last, copy_len);
183204}
184205
185206test "memcpy" {