authorgravatar for 4678790+dweiller@users.noreply.github.comDominic <4678790+dweiller@users.noreply.github.com> 2025-01-26 03:54:23+11:00
committergravatar for 4678790+dweiller@users.noreply.github.comDominic <4678790+dweiller@users.noreply.github.com> 2025-01-30 19:56:57+11:00
log7e7c36fb5736d345e791d1656b285cb40f4e97a6
tree18db50b3e642496dcb468a27baa94b860b4662fb
parent63f5a80b7146cfe7b9034be683b4a8dfc29a1a25

compiler-rt: remove manual unroll code from memmove

1 files changed, 10 insertions(+), 48 deletions(-)

lib/compiler_rt/memmove.zig+10-48
...@@ -40,9 +40,7 @@ fn memmoveSmall(opt_dest: ?[*]u8, opt_src: ?[*]const u8, len: usize) callconv(.C...@@ -40,9 +40,7 @@ fn memmoveSmall(opt_dest: ?[*]u8, opt_src: ?[*]const u8, len: usize) callconv(.C
4040
41fn memmoveFast(dest: ?[*]u8, src: ?[*]u8, len: usize) callconv(.C) ?[*]u8 {41fn memmoveFast(dest: ?[*]u8, src: ?[*]u8, len: usize) callconv(.C) ?[*]u8 {
42 @setRuntimeSafety(builtin.is_test);42 @setRuntimeSafety(builtin.is_test);
43 const unroll_count = 1;43 const small_limit = @max(2 * @sizeOf(Element), @sizeOf(Element));
44 comptime assert(std.math.isPowerOfTwo(unroll_count));
45 const small_limit = @max(2 * @sizeOf(Element), unroll_count * @sizeOf(Element));
4644
47 if (copySmallLength(small_limit, dest.?, src.?, len)) return dest;45 if (copySmallLength(small_limit, dest.?, src.?, len)) return dest;
4846
...@@ -50,9 +48,9 @@ fn memmoveFast(dest: ?[*]u8, src: ?[*]u8, len: usize) callconv(.C) ?[*]u8 {...@@ -50,9 +48,9 @@ fn memmoveFast(dest: ?[*]u8, src: ?[*]u8, len: usize) callconv(.C) ?[*]u8 {
50 const src_address = @intFromPtr(src);48 const src_address = @intFromPtr(src);
5149
52 if (src_address < dest_address and src_address + len > dest_address) {50 if (src_address < dest_address and src_address + len > dest_address) {
53 copyBackwards(unroll_count, dest.?, src.?, len);51 copyBackwards(dest.?, src.?, len);
54 } else {52 } else {
55 copyForwards(unroll_count, dest.?, src.?, len);53 copyForwards(dest.?, src.?, len);
56 }54 }
5755
58 return dest;56 return dest;
...@@ -145,14 +143,12 @@ inline fn copyRange4(...@@ -145,14 +143,12 @@ inline fn copyRange4(
145}143}
146144
147inline fn copyForwards(145inline fn copyForwards(
148 comptime unroll_count: comptime_int,
149 dest: [*]u8,146 dest: [*]u8,
150 src: [*]const u8,147 src: [*]const u8,
151 len: usize,148 len: usize,
152) void {149) void {
153 @setRuntimeSafety(builtin.is_test);150 @setRuntimeSafety(builtin.is_test);
154 assert(len >= 2 * @sizeOf(Element));151 assert(len >= 2 * @sizeOf(Element));
155 assert(len >= unroll_count * @sizeOf(Element));
156152
157 const head = src[0..@sizeOf(Element)].*;153 const head = src[0..@sizeOf(Element)].*;
158 const tail = src[len - @sizeOf(Element) ..][0..@sizeOf(Element)].*;154 const tail = src[len - @sizeOf(Element) ..][0..@sizeOf(Element)].*;
...@@ -161,7 +157,7 @@ inline fn copyForwards(...@@ -161,7 +157,7 @@ inline fn copyForwards(
161 const d = dest + alignment_offset;157 const d = dest + alignment_offset;
162 const s = src + alignment_offset;158 const s = src + alignment_offset;
163159
164 copyBlocksAlignedSource(@ptrCast(d), @alignCast(@ptrCast(s)), n, unroll_count);160 copyBlocksAlignedSource(@ptrCast(d), @alignCast(@ptrCast(s)), n);
165161
166 // copy last `copy_size` bytes unconditionally, since block copy162 // copy last `copy_size` bytes unconditionally, since block copy
167 // methods only copy a multiple of `copy_size` bytes.163 // methods only copy a multiple of `copy_size` bytes.
...@@ -173,53 +169,31 @@ inline fn copyBlocksAlignedSource(...@@ -173,53 +169,31 @@ inline fn copyBlocksAlignedSource(
173 dest: [*]align(1) Element,169 dest: [*]align(1) Element,
174 src: [*]const Element,170 src: [*]const Element,
175 max_bytes: usize,171 max_bytes: usize,
176 comptime unroll_count: comptime_int,
177) void {172) void {
178 copyBlocks(dest, src, max_bytes, unroll_count);173 copyBlocks(dest, src, max_bytes);
179}174}
180175
181/// Copies the largest multiple of `@sizeOf(T)` bytes from `src` to `dest`,176/// Copies the largest multiple of `@sizeOf(T)` bytes from `src` to `dest`,
182/// that is less than `max_bytes` where `T` is the child type of `src` and177/// that is less than `max_bytes` where `T` is the child type of `src` and
183/// `dest`; `max_bytes` must be at least `@sizeOf(T)`. The primary copy loop178/// `dest`; `max_bytes` must be at least `@sizeOf(T)`.
184/// will be unrolled to perform `unroll_count` copies per iteration.
185inline fn copyBlocks(179inline fn copyBlocks(
186 dest: anytype,180 dest: anytype,
187 src: anytype,181 src: anytype,
188 max_bytes: usize,182 max_bytes: usize,
189 comptime unroll_count: comptime_int,
190) void {183) void {
191 @setRuntimeSafety(builtin.is_test);184 @setRuntimeSafety(builtin.is_test);
192 comptime assert(unroll_count > 0);
193185
194 const T = @typeInfo(@TypeOf(dest)).pointer.child;186 const T = @typeInfo(@TypeOf(dest)).pointer.child;
195 comptime assert(T == @typeInfo(@TypeOf(src)).pointer.child);187 comptime assert(T == @typeInfo(@TypeOf(src)).pointer.child);
196188
197 const loop_count = max_bytes / (@sizeOf(T) * unroll_count);189 const loop_count = max_bytes / @sizeOf(T);
198190
199 // save tail since it can overlap with `dest `in main copy loop191 for (dest[0..loop_count], src[0..loop_count]) |*d, s| {
200 const tail_start = (max_bytes / @sizeOf(T)) - (unroll_count - 1);
201 const st = src[tail_start..][0 .. unroll_count - 1];
202 var tail_data: [unroll_count - 1]Element = undefined;
203 inline for (&tail_data, st) |*d, s| {
204 d.* = s;
205 }
206
207 for (0..loop_count) |i| {
208 const du = dest[i * unroll_count ..][0..unroll_count];
209 const su = src[i * unroll_count ..][0..unroll_count];
210 inline for (du, su) |*d, s| {
211 d.* = s;
212 }
213 }
214
215 const dt = dest[tail_start..][0 .. unroll_count - 1];
216 inline for (dt, tail_data) |*d, s| {
217 d.* = s;192 d.* = s;
218 }193 }
219}194}
220195
221inline fn copyBackwards(196inline fn copyBackwards(
222 comptime unroll_count: comptime_int,
223 dest: [*]u8,197 dest: [*]u8,
224 src: [*]const u8,198 src: [*]const u8,
225 len: usize,199 len: usize,
...@@ -227,30 +201,18 @@ inline fn copyBackwards(...@@ -227,30 +201,18 @@ inline fn copyBackwards(
227 const end_bytes = src[len - @sizeOf(Element) ..][0..@sizeOf(Element)].*;201 const end_bytes = src[len - @sizeOf(Element) ..][0..@sizeOf(Element)].*;
228 const start_bytes = src[0..@sizeOf(Element)].*;202 const start_bytes = src[0..@sizeOf(Element)].*;
229203
230 const tail_dest: [*]Element = @ptrFromInt(std.mem.alignForward(usize, @intFromPtr(dest), @alignOf(Element)));
231 const tail_src: [*]align(1) const Element = @ptrCast(src + (@intFromPtr(tail_dest) - @intFromPtr(dest)));
232 const tail_bytes: [unroll_count - 1]Element = tail_src[0 .. unroll_count - 1].*;
233
234 const d_addr: usize = std.mem.alignBackward(usize, @intFromPtr(dest) + len, @alignOf(Element));204 const d_addr: usize = std.mem.alignBackward(usize, @intFromPtr(dest) + len, @alignOf(Element));
235 const d: [*]Element = @ptrFromInt(d_addr);205 const d: [*]Element = @ptrFromInt(d_addr);
236 const n = d_addr - @intFromPtr(dest);206 const n = d_addr - @intFromPtr(dest);
237 const s: [*]align(1) const Element = @ptrCast(src + n);207 const s: [*]align(1) const Element = @ptrCast(src + n);
238208
239 const loop_count = n / (unroll_count * @sizeOf(Element));209 const loop_count = n / @sizeOf(Element);
240 var i: usize = 1;210 var i: usize = 1;
241 while (i < loop_count + 1) : (i += 1) {211 while (i < loop_count + 1) : (i += 1) {
242 const du = d - (i * unroll_count);212 (d - i)[0] = (s - i)[0];
243 const su = s - (i * unroll_count);
244 inline for (0..unroll_count) |j| {
245 du[unroll_count - 1 - j] = su[unroll_count - 1 - j];
246 }
247 }213 }
248214
249 inline for (tail_dest[0 .. unroll_count - 1], tail_bytes) |*dt, st| {
250 dt.* = st;
251 }
252 dest[0..@sizeOf(Element)].* = start_bytes;215 dest[0..@sizeOf(Element)].* = start_bytes;
253
254 dest[len - @sizeOf(Element) ..][0..@sizeOf(Element)].* = end_bytes;216 dest[len - @sizeOf(Element) ..][0..@sizeOf(Element)].* = end_bytes;
255}217}
256218