authorgravatar for 4678790+dweiller@users.noreply.github.comDominic <4678790+dweiller@users.noreply.github.com> 2024-12-05 23:54:51+11:00
committergravatar for 4678790+dweiller@users.noreply.github.comDominic <4678790+dweiller@users.noreply.github.com> 2025-01-30 19:56:57+11:00
log4db01eb13072f3ff98802a42448cba092e6837ac
tree653ed374e92f2d82937c1ae8ab1d7dfe1e4b3fe1
parent7cef585f554fdb263a80479f0556fbfba28a7fb8

compiler-rt: optimize memmove

3 files changed, 231 insertions(+), 112 deletions(-)

lib/compiler_rt/common.zig+11
...@@ -16,6 +16,17 @@ else...@@ -16,6 +16,17 @@ else
16pub const visibility: std.builtin.SymbolVisibility =16pub const visibility: std.builtin.SymbolVisibility =
17 if (builtin.target.isWasm() and linkage != .internal) .hidden else .default;17 if (builtin.target.isWasm() and linkage != .internal) .hidden else .default;
1818
19pub const PreferredLoadStoreElement = element: {
20 if (std.simd.suggestVectorLength(u8)) |vec_size| {
21 const Vec = @Vector(vec_size, u8);
22
23 if (@sizeOf(Vec) == vec_size and std.math.isPowerOfTwo(vec_size)) {
24 break :element Vec;
25 }
26 }
27 break :element usize;
28};
29
19pub const want_aeabi = switch (builtin.abi) {30pub const want_aeabi = switch (builtin.abi) {
20 .eabi,31 .eabi,
21 .eabihf,32 .eabihf,
lib/compiler_rt/memcpy.zig+1-10
...@@ -18,16 +18,7 @@ comptime {...@@ -18,16 +18,7 @@ comptime {
18 }18 }
19}19}
2020
21const Element = Element: {21const Element = common.PreferredLoadStoreElement;
22 if (std.simd.suggestVectorLength(u8)) |vec_size| {
23 const Vec = @Vector(vec_size, u8);
24
25 if (@sizeOf(Vec) == vec_size and std.math.isPowerOfTwo(vec_size)) {
26 break :Element Vec;
27 }
28 }
29 break :Element usize;
30};
3122
32comptime {23comptime {
33 assert(std.math.isPowerOfTwo(@sizeOf(Element)));24 assert(std.math.isPowerOfTwo(@sizeOf(Element)));
lib/compiler_rt/memmove.zig+219-102
...@@ -1,6 +1,10 @@...@@ -1,6 +1,10 @@
1const std = @import("std");1const std = @import("std");
2const common = @import("./common.zig");2const common = @import("./common.zig");
3const builtin = @import("builtin");3const builtin = @import("builtin");
4const assert = std.debug.assert;
5const memcpy = @import("memcpy.zig");
6
7const Element = common.PreferredLoadStoreElement;
48
5comptime {9comptime {
6 if (builtin.object_format != .c) {10 if (builtin.object_format != .c) {
...@@ -34,137 +38,250 @@ fn memmoveSmall(opt_dest: ?[*]u8, opt_src: ?[*]const u8, len: usize) callconv(.C...@@ -34,137 +38,250 @@ fn memmoveSmall(opt_dest: ?[*]u8, opt_src: ?[*]const u8, len: usize) callconv(.C
34 return dest;38 return dest;
35}39}
3640
37pub fn memmoveFast(opt_dest: ?[*]u8, opt_src: ?[*]const u8, len: usize) callconv(.C) ?[*]u8 {41fn memmoveFast(dest: ?[*]u8, src: ?[*]u8, len: usize) callconv(.C) ?[*]u8 {
38 // a port of https://github.com/facebook/folly/blob/1c8bc50e88804e2a7361a57cd9b551dd10f6c5fd/folly/memcpy.S42 @setRuntimeSafety(builtin.is_test);
39 if (len == 0) {43 const unroll_count = 1;
40 @branchHint(.unlikely);44 comptime assert(std.math.isPowerOfTwo(unroll_count));
41 return opt_dest;45 const small_limit = @max(2 * @sizeOf(Element), unroll_count * @sizeOf(Element));
46
47 if (copySmallLength(small_limit, dest.?, src.?, len)) return dest;
48
49 const dest_address = @intFromPtr(dest);
50 const src_address = @intFromPtr(src);
51
52 if (src_address < dest_address and src_address + len > dest_address) {
53 copyBackwards(unroll_count, dest.?, src.?, len);
54 } else {
55 copyForwards(unroll_count, dest.?, src.?, len);
42 }56 }
4357
44 const dest = opt_dest.?;58 return dest;
45 const src = opt_src.?;59}
4660
47 if (len < 8) {61inline fn copySmallLength(
48 @branchHint(.unlikely);62 comptime small_limit: comptime_int,
49 if (len == 1) {63 dest: [*]u8,
50 @branchHint(.unlikely);64 src: [*]const u8,
51 dest[0] = src[0];65 len: usize,
52 } else if (len >= 4) {66) bool {
53 @branchHint(.unlikely);67 if (len < 16) {
54 blockCopy(dest, src, 4, len);68 copyLessThan16(dest, src, len);
55 } else {69 return true;
56 blockCopy(dest, src, 2, len);
57 }
58 return dest;
59 }70 }
6071
61 if (len > 32) {72 if (comptime 2 < (std.math.log2(small_limit) + 1) / 2) {
62 @branchHint(.unlikely);73 if (copy16ToSmallLimit(small_limit, dest, src, len)) return true;
63 if (len > 256) {
64 @branchHint(.unlikely);
65 copyMove(dest, src, len);
66 return dest;
67 }
68 copyLong(dest, src, len);
69 return dest;
70 }74 }
7175
72 if (len > 16) {76 return false;
73 @branchHint(.unlikely);77}
74 blockCopy(dest, src, 16, len);78
75 return dest;79inline fn copyLessThan16(
80 dest: [*]u8,
81 src: [*]const u8,
82 len: usize,
83) void {
84 @setRuntimeSafety(builtin.is_test);
85 if (len < 4) {
86 if (len == 0) return;
87 const b = len / 2;
88 const d0 = src[0];
89 const db = src[b];
90 const de = src[len - 1];
91 dest[0] = d0;
92 dest[b] = db;
93 dest[len - 1] = de;
94 return;
95 }
96 copyRange4(4, dest, src, len);
97}
98
99inline fn copy16ToSmallLimit(
100 comptime small_limit: comptime_int,
101 dest: [*]u8,
102 src: [*]const u8,
103 len: usize,
104) bool {
105 @setRuntimeSafety(builtin.is_test);
106 inline for (2..(std.math.log2(small_limit) + 1) / 2 + 1) |p| {
107 const limit = 1 << (2 * p);
108 if (len < limit) {
109 copyRange4(limit / 4, dest, src, len);
110 return true;
111 }
76 }112 }
113 return false;
114}
77115
78 blockCopy(dest, src, 8, len);116/// copy `len` bytes from `src` to `dest`; `len` must be in the range
117/// `[copy_len, 4 * copy_len)`.
118inline fn copyRange4(
119 comptime copy_len: comptime_int,
120 dest: [*]u8,
121 src: [*]const u8,
122 len: usize,
123) void {
124 @setRuntimeSafety(builtin.is_test);
125 comptime assert(std.math.isPowerOfTwo(copy_len));
126 assert(len >= copy_len);
127 assert(len < 4 * copy_len);
79128
80 return dest;129 const a = len & (copy_len * 2);
130 const b = a / 2;
131
132 const last = len - copy_len;
133 const pen = last - b;
134
135 const d0 = src[0..copy_len].*;
136 const d1 = src[b..][0..copy_len].*;
137 const d2 = src[pen..][0..copy_len].*;
138 const d3 = src[last..][0..copy_len].*;
139
140 dest[0..copy_len].* = d0;
141 dest[b..][0..copy_len].* = d1;
142 dest[pen..][0..copy_len].* = d2;
143 dest[last..][0..copy_len].* = d3;
144}
145
146inline fn copyForwards(
147 comptime unroll_count: comptime_int,
148 dest: [*]u8,
149 src: [*]const u8,
150 len: usize,
151) void {
152 @setRuntimeSafety(builtin.is_test);
153 assert(len >= 2 * @sizeOf(Element));
154 assert(len >= unroll_count * @sizeOf(Element));
155
156 const head = src[0..@sizeOf(Element)].*;
157 const tail = src[len - @sizeOf(Element) ..][0..@sizeOf(Element)].*;
158 const alignment_offset = @alignOf(Element) - @intFromPtr(src) % @alignOf(Element);
159 const n = len - alignment_offset;
160 const d = dest + alignment_offset;
161 const s = src + alignment_offset;
162
163 copyBlocksAlignedSource(@ptrCast(d), @alignCast(@ptrCast(s)), n, unroll_count);
164
165 // copy last `copy_size` bytes unconditionally, since block copy
166 // methods only copy a multiple of `copy_size` bytes.
167 dest[len - @sizeOf(Element) ..][0..@sizeOf(Element)].* = tail;
168 dest[0..@sizeOf(Element)].* = head;
81}169}
82170
83inline fn blockCopy(dest: [*]u8, src: [*]const u8, block_size: comptime_int, len: usize) void {171inline fn copyBlocksAlignedSource(
84 const first = @as(*align(1) const @Vector(block_size, u8), src[0..block_size]).*;172 dest: [*]align(1) Element,
85 const second = @as(*align(1) const @Vector(block_size, u8), src[len - block_size ..][0..block_size]).*;173 src: [*]const Element,
86 dest[0..block_size].* = first;174 max_bytes: usize,
87 dest[len - block_size ..][0..block_size].* = second;175 comptime unroll_count: comptime_int,
176) void {
177 copyBlocks(dest, src, max_bytes, unroll_count);
88}178}
89179
90inline fn copyLong(dest: [*]u8, src: [*]const u8, len: usize) void {180/// Copies the largest multiple of `@sizeOf(T)` bytes from `src` to `dest`,
91 var array: [8]@Vector(32, u8) = undefined;181/// that is less than `max_bytes` where `T` is the child type of `src` and
182/// `dest`; `max_bytes` must be at least `@sizeOf(T)`. The primary copy loop
183/// will be unrolled to perform `unroll_count` copies per iteration.
184inline fn copyBlocks(
185 dest: anytype,
186 src: anytype,
187 max_bytes: usize,
188 comptime unroll_count: comptime_int,
189) void {
190 @setRuntimeSafety(builtin.is_test);
191 comptime assert(unroll_count > 0);
92192
93 inline for (.{ 64, 128, 192, 256 }, 0..) |N, i| {193 const T = @typeInfo(@TypeOf(dest)).pointer.child;
94 array[i * 2] = src[(N / 2) - 32 ..][0..32].*;194 comptime assert(T == @typeInfo(@TypeOf(src)).pointer.child);
95 array[(i * 2) + 1] = src[len - N / 2 ..][0..32].*;
96195
97 if (len <= N) {196 const loop_count = max_bytes / (@sizeOf(T) * unroll_count);
98 @branchHint(.unlikely);197
99 for (0..i + 1) |j| {198 // save tail since it can overlap with `dest `in main copy loop
100 dest[j * 32 ..][0..32].* = array[j * 2];199 const tail_start = (max_bytes / @sizeOf(T)) - (unroll_count - 1);
101 dest[len - ((j * 32) + 32) ..][0..32].* = array[(j * 2) + 1];200 const st = src[tail_start..][0 .. unroll_count - 1];
102 }201 var tail_data: [unroll_count - 1]Element = undefined;
103 return;202 inline for (&tail_data, st) |*d, s| {
203 d.* = s;
204 }
205
206 for (0..loop_count) |i| {
207 const du = dest[i * unroll_count ..][0..unroll_count];
208 const su = src[i * unroll_count ..][0..unroll_count];
209 inline for (du, su) |*d, s| {
210 d.* = s;
104 }211 }
105 }212 }
106}
107213
108inline fn copyMove(dest: [*]u8, src: [*]const u8, len: usize) void {214 const dt = dest[tail_start..][0 .. unroll_count - 1];
109 if (@intFromPtr(src) >= @intFromPtr(dest)) {215 inline for (dt, tail_data) |*d, s| {
110 @branchHint(.unlikely);216 d.* = s;
111 copyForward(dest, src, len);
112 } else if (@intFromPtr(src) + len > @intFromPtr(dest)) {
113 @branchHint(.unlikely);
114 overlapBwd(dest, src, len);
115 } else {
116 copyForward(dest, src, len);
117 }217 }
118}218}
119219
120inline fn copyForward(dest: [*]u8, src: [*]const u8, len: usize) void {220inline fn copyBackwards(
121 const tail: @Vector(32, u8) = src[len - 32 ..][0..32].*;221 comptime unroll_count: comptime_int,
222 dest: [*]u8,
223 src: [*]const u8,
224 len: usize,
225) void {
226 const end_bytes = src[len - @sizeOf(Element) ..][0..@sizeOf(Element)].*;
227 const start_bytes = src[0..@sizeOf(Element)].*;
228
229 const tail_dest: [*]Element = @ptrFromInt(std.mem.alignForward(usize, @intFromPtr(dest), @alignOf(Element)));
230 const tail_src: [*]align(1) const Element = @ptrCast(src + (@intFromPtr(tail_dest) - @intFromPtr(dest)));
231 const tail_bytes: [unroll_count - 1]Element = tail_src[0 .. unroll_count - 1].*;
122232
123 const N: usize = len & ~@as(usize, 127);233 const d_addr: usize = std.mem.alignBackward(usize, @intFromPtr(dest) + len, @alignOf(Element));
124 var i: usize = 0;234 const d: [*]Element = @ptrFromInt(d_addr);
235 const n = d_addr - @intFromPtr(dest);
236 const s: [*]align(1) const Element = @ptrCast(src + n);
125237
126 while (i < N) : (i += 128) {238 const loop_count = n / (unroll_count * @sizeOf(Element));
127 dest[i..][0..32].* = src[i..][0..32].*;239 var i: usize = 1;
128 dest[i + 32 ..][0..32].* = src[i + 32 ..][0..32].*;240 while (i < loop_count + 1) : (i += 1) {
129 dest[i + 64 ..][0..32].* = src[i + 64 ..][0..32].*;241 const du = d - (i * unroll_count);
130 dest[i + 96 ..][0..32].* = src[i + 96 ..][0..32].*;242 const su = s - (i * unroll_count);
243 inline for (0..unroll_count) |j| {
244 du[unroll_count - 1 - j] = su[unroll_count - 1 - j];
245 }
131 }246 }
132247
133 if (len - i <= 32) {248 inline for (tail_dest[0 .. unroll_count - 1], tail_bytes) |*dt, st| {
134 @branchHint(.unlikely);249 dt.* = st;
135 dest[len - 32 ..][0..32].* = tail;
136 } else {
137 copyLong(dest[i..], src[i..], len - i);
138 }250 }
251 dest[0..@sizeOf(Element)].* = start_bytes;
252
253 dest[len - @sizeOf(Element) ..][0..@sizeOf(Element)].* = end_bytes;
139}254}
140255
141inline fn overlapBwd(dest: [*]u8, src: [*]const u8, len: usize) void {256test memmoveFast {
142 var array: [5]@Vector(32, u8) = undefined;257 const max_len = 1024;
143 array[0] = src[len - 32 ..][0..32].*;258 var buffer: [max_len + @alignOf(Element) - 1]u8 = undefined;
144 inline for (1..5) |i| array[i] = src[(i - 1) << 5 ..][0..32].*;259 for (&buffer, 0..) |*b, i| {
145260 b.* = @intCast(i % 97);
146 const end: usize = (@intFromPtr(dest) + len - 32) & 31;
147 const range = len - end;
148 var s = src + range;
149 var d = dest + range;
150
151 while (@intFromPtr(s) > @intFromPtr(src + 128)) {
152 // zig fmt: off
153 const first = @as(*align(1) const @Vector(32, u8), @ptrCast(s - 32)).*;
154 const second = @as(*align(1) const @Vector(32, u8), @ptrCast(s - 64)).*;
155 const third = @as(*align(1) const @Vector(32, u8), @ptrCast(s - 96)).*;
156 const fourth = @as(*align(1) const @Vector(32, u8), @ptrCast(s - 128)).*;
157
158 @as(*align(32) @Vector(32, u8), @alignCast(@ptrCast(d - 32))).* = first;
159 @as(*align(32) @Vector(32, u8), @alignCast(@ptrCast(d - 64))).* = second;
160 @as(*align(32) @Vector(32, u8), @alignCast(@ptrCast(d - 96))).* = third;
161 @as(*align(32) @Vector(32, u8), @alignCast(@ptrCast(d - 128))).* = fourth;
162 // zig fmt: on
163
164 s -= 128;
165 d -= 128;
166 }261 }
167262
168 inline for (array[1..], 0..) |vec, i| dest[i * 32 ..][0..32].* = vec;263 var move_buffer: [max_len + @alignOf(Element) - 1]u8 align(@alignOf(Element)) = undefined;
169 dest[len - 32 ..][0..32].* = array[0];264
265 for (0..max_len) |copy_len| {
266 for (0..@alignOf(Element)) |s_offset| {
267 for (0..@alignOf(Element)) |d_offset| {
268 for (&move_buffer, buffer) |*d, s| {
269 d.* = s;
270 }
271 const dest = move_buffer[d_offset..][0..copy_len];
272 const src = move_buffer[s_offset..][0..copy_len];
273 _ = memmoveFast(dest.ptr, src.ptr, copy_len);
274 std.testing.expectEqualSlices(u8, buffer[s_offset..][0..copy_len], dest) catch |e| {
275 std.debug.print(
276 "error occured with source offset {d} and destination offset {d}\n",
277 .{
278 s_offset,
279 d_offset,
280 },
281 );
282 return e;
283 };
284 }
285 }
286 }
170}287}