authorgravatar for 4678790+dweiller@users.noreply.github.comDominic <4678790+dweiller@users.noreply.github.com> 2025-01-26 03:54:23+11:00
committergravatar for 4678790+dweiller@users.noreply.github.comDominic <4678790+dweiller@users.noreply.github.com> 2025-01-30 19:56:57+11:00
log7e7c36fb5736d345e791d1656b285cb40f4e97a6
tree18db50b3e642496dcb468a27baa94b860b4662fb
parent63f5a80b7146cfe7b9034be683b4a8dfc29a1a25

compiler-rt: remove manual unroll code from memmove

1 files changed, 10 insertions(+), 48 deletions(-)

lib/compiler_rt/memmove.zig+10-48
......@@ -40,9 +40,7 @@ fn memmoveSmall(opt_dest: ?[*]u8, opt_src: ?[*]const u8, len: usize) callconv(.C
4040
4141fn memmoveFast(dest: ?[*]u8, src: ?[*]u8, len: usize) callconv(.C) ?[*]u8 {
4242 @setRuntimeSafety(builtin.is_test);
43 const unroll_count = 1;
44 comptime assert(std.math.isPowerOfTwo(unroll_count));
45 const small_limit = @max(2 * @sizeOf(Element), unroll_count * @sizeOf(Element));
43 const small_limit = @max(2 * @sizeOf(Element), @sizeOf(Element));
4644
4745 if (copySmallLength(small_limit, dest.?, src.?, len)) return dest;
4846
......@@ -50,9 +48,9 @@ fn memmoveFast(dest: ?[*]u8, src: ?[*]u8, len: usize) callconv(.C) ?[*]u8 {
5048 const src_address = @intFromPtr(src);
5149
5250 if (src_address < dest_address and src_address + len > dest_address) {
53 copyBackwards(unroll_count, dest.?, src.?, len);
51 copyBackwards(dest.?, src.?, len);
5452 } else {
55 copyForwards(unroll_count, dest.?, src.?, len);
53 copyForwards(dest.?, src.?, len);
5654 }
5755
5856 return dest;
......@@ -145,14 +143,12 @@ inline fn copyRange4(
145143}
146144
147145inline fn copyForwards(
148 comptime unroll_count: comptime_int,
149146 dest: [*]u8,
150147 src: [*]const u8,
151148 len: usize,
152149) void {
153150 @setRuntimeSafety(builtin.is_test);
154151 assert(len >= 2 * @sizeOf(Element));
155 assert(len >= unroll_count * @sizeOf(Element));
156152
157153 const head = src[0..@sizeOf(Element)].*;
158154 const tail = src[len - @sizeOf(Element) ..][0..@sizeOf(Element)].*;
......@@ -161,7 +157,7 @@ inline fn copyForwards(
161157 const d = dest + alignment_offset;
162158 const s = src + alignment_offset;
163159
164 copyBlocksAlignedSource(@ptrCast(d), @alignCast(@ptrCast(s)), n, unroll_count);
160 copyBlocksAlignedSource(@ptrCast(d), @alignCast(@ptrCast(s)), n);
165161
166162 // copy last `copy_size` bytes unconditionally, since block copy
167163 // methods only copy a multiple of `copy_size` bytes.
......@@ -173,53 +169,31 @@ inline fn copyBlocksAlignedSource(
173169 dest: [*]align(1) Element,
174170 src: [*]const Element,
175171 max_bytes: usize,
176 comptime unroll_count: comptime_int,
177172) void {
178 copyBlocks(dest, src, max_bytes, unroll_count);
173 copyBlocks(dest, src, max_bytes);
179174}
180175
181176/// Copies the largest multiple of `@sizeOf(T)` bytes from `src` to `dest`,
182177/// that is less than `max_bytes` where `T` is the child type of `src` and
183/// `dest`; `max_bytes` must be at least `@sizeOf(T)`. The primary copy loop
184/// will be unrolled to perform `unroll_count` copies per iteration.
178/// `dest`; `max_bytes` must be at least `@sizeOf(T)`.
185179inline fn copyBlocks(
186180 dest: anytype,
187181 src: anytype,
188182 max_bytes: usize,
189 comptime unroll_count: comptime_int,
190183) void {
191184 @setRuntimeSafety(builtin.is_test);
192 comptime assert(unroll_count > 0);
193185
194186 const T = @typeInfo(@TypeOf(dest)).pointer.child;
195187 comptime assert(T == @typeInfo(@TypeOf(src)).pointer.child);
196188
197 const loop_count = max_bytes / (@sizeOf(T) * unroll_count);
189 const loop_count = max_bytes / @sizeOf(T);
198190
199 // save tail since it can overlap with `dest `in main copy loop
200 const tail_start = (max_bytes / @sizeOf(T)) - (unroll_count - 1);
201 const st = src[tail_start..][0 .. unroll_count - 1];
202 var tail_data: [unroll_count - 1]Element = undefined;
203 inline for (&tail_data, st) |*d, s| {
204 d.* = s;
205 }
206
207 for (0..loop_count) |i| {
208 const du = dest[i * unroll_count ..][0..unroll_count];
209 const su = src[i * unroll_count ..][0..unroll_count];
210 inline for (du, su) |*d, s| {
211 d.* = s;
212 }
213 }
214
215 const dt = dest[tail_start..][0 .. unroll_count - 1];
216 inline for (dt, tail_data) |*d, s| {
191 for (dest[0..loop_count], src[0..loop_count]) |*d, s| {
217192 d.* = s;
218193 }
219194}
220195
221196inline fn copyBackwards(
222 comptime unroll_count: comptime_int,
223197 dest: [*]u8,
224198 src: [*]const u8,
225199 len: usize,
......@@ -227,30 +201,18 @@ inline fn copyBackwards(
227201 const end_bytes = src[len - @sizeOf(Element) ..][0..@sizeOf(Element)].*;
228202 const start_bytes = src[0..@sizeOf(Element)].*;
229203
230 const tail_dest: [*]Element = @ptrFromInt(std.mem.alignForward(usize, @intFromPtr(dest), @alignOf(Element)));
231 const tail_src: [*]align(1) const Element = @ptrCast(src + (@intFromPtr(tail_dest) - @intFromPtr(dest)));
232 const tail_bytes: [unroll_count - 1]Element = tail_src[0 .. unroll_count - 1].*;
233
234204 const d_addr: usize = std.mem.alignBackward(usize, @intFromPtr(dest) + len, @alignOf(Element));
235205 const d: [*]Element = @ptrFromInt(d_addr);
236206 const n = d_addr - @intFromPtr(dest);
237207 const s: [*]align(1) const Element = @ptrCast(src + n);
238208
239 const loop_count = n / (unroll_count * @sizeOf(Element));
209 const loop_count = n / @sizeOf(Element);
240210 var i: usize = 1;
241211 while (i < loop_count + 1) : (i += 1) {
242 const du = d - (i * unroll_count);
243 const su = s - (i * unroll_count);
244 inline for (0..unroll_count) |j| {
245 du[unroll_count - 1 - j] = su[unroll_count - 1 - j];
246 }
212 (d - i)[0] = (s - i)[0];
247213 }
248214
249 inline for (tail_dest[0 .. unroll_count - 1], tail_bytes) |*dt, st| {
250 dt.* = st;
251 }
252215 dest[0..@sizeOf(Element)].* = start_bytes;
253
254216 dest[len - @sizeOf(Element) ..][0..@sizeOf(Element)].* = end_bytes;
255217}
256218