authorgravatar for 4678790+dweiller@users.noreply.github.comDominic <4678790+dweiller@users.noreply.github.com> 2025-01-20 17:43:42+11:00
committergravatar for 4678790+dweiller@users.noreply.github.comDominic <4678790+dweiller@users.noreply.github.com> 2025-01-20 18:29:15+11:00
logb7a887f0fb7d166ad93eae62ecd93d1ed173297e
tree3ea5cd898f588e66f038d7a64454557cb6220057
parentd6e1166f1a93e8ed5dd0a74cc0b0daeae5c050a0

compiler-rt: optimize memcpy

The new memcpy function aims to be more generic than the previous implementation which was adapted from an implementation optimized for x86_64 avx2 machines. Even on x86_64 avx2 machines this implementation should be generally be faster due to fewer branches in the small length cases and generating less machine code. Note that the new memcpy function no longer acts as a memmove.

1 files changed, 183 insertions(+), 2 deletions(-)

lib/compiler_rt/memcpy.zig+183-2
......@@ -1,4 +1,5 @@
11const std = @import("std");
2const assert = std.debug.assert;
23const common = @import("./common.zig");
34const builtin = @import("builtin");
45
......@@ -17,6 +18,19 @@ comptime {
1718 }
1819}
1920
21const Element = if (std.simd.suggestVectorLength(u8)) |vec_size|
22 @Type(.{ .vector = .{
23 .child = u8,
24 .len = vec_size,
25 } })
26else
27 usize;
28
29comptime {
30 assert(@sizeOf(Element) >= @alignOf(Element));
31 assert(std.math.isPowerOfTwo(@sizeOf(Element)));
32}
33
2034fn memcpySmall(noalias dest: ?[*]u8, noalias src: ?[*]const u8, len: usize) callconv(.C) ?[*]u8 {
2135 @setRuntimeSafety(builtin.is_test);
2236
......@@ -27,6 +41,173 @@ fn memcpySmall(noalias dest: ?[*]u8, noalias src: ?[*]const u8, len: usize) call
2741 return dest;
2842}
2943
30fn memcpyFast(opt_dest: ?[*]u8, opt_src: ?[*]const u8, len: usize) callconv(.C) ?[*]u8 {
31 return @call(.always_inline, @import("memmove.zig").memmove, .{ opt_dest, opt_src, len });
44fn memcpyFast(noalias dest: ?[*]u8, noalias src: ?[*]const u8, len: usize) callconv(.C) ?[*]u8 {
45 @setRuntimeSafety(builtin.is_test);
46
47 const small_limit = 2 * @sizeOf(Element);
48
49 if (copySmallLength(small_limit, dest.?, src.?, len)) return dest;
50
51 copyForwards(dest.?, src.?, len);
52
53 return dest;
54}
55
56inline fn copySmallLength(
57 comptime small_limit: comptime_int,
58 dest: [*]u8,
59 src: [*]const u8,
60 len: usize,
61) bool {
62 if (len < 16) {
63 copyLessThan16(dest, src, len);
64 return true;
65 }
66
67 if (comptime 2 < (std.math.log2(small_limit) + 1) / 2) {
68 if (copy16ToSmallLimit(small_limit, dest, src, len)) return true;
69 }
70
71 return false;
72}
73
74inline fn copyLessThan16(
75 dest: [*]u8,
76 src: [*]const u8,
77 len: usize,
78) void {
79 @setRuntimeSafety(builtin.is_test);
80 if (len < 4) {
81 if (len == 0) return;
82 dest[0] = src[0];
83 dest[len / 2] = src[len / 2];
84 dest[len - 1] = src[len - 1];
85 return;
86 }
87 copyRange4(4, dest, src, len);
88}
89
90inline fn copy16ToSmallLimit(
91 comptime small_limit: comptime_int,
92 dest: [*]u8,
93 src: [*]const u8,
94 len: usize,
95) bool {
96 @setRuntimeSafety(builtin.is_test);
97 inline for (2..(std.math.log2(small_limit) + 1) / 2 + 1) |p| {
98 const limit = 1 << (2 * p);
99 if (len < limit) {
100 copyRange4(limit / 4, dest, src, len);
101 return true;
102 }
103 }
104 return false;
105}
106
107inline fn copyForwards(
108 noalias dest: [*]u8,
109 noalias src: [*]const u8,
110 len: usize,
111) void {
112 @setRuntimeSafety(builtin.is_test);
113 assert(len >= 2 * @sizeOf(Element));
114
115 dest[0..@sizeOf(Element)].* = src[0..@sizeOf(Element)].*;
116 const alignment_offset = @alignOf(Element) - @intFromPtr(src) % @alignOf(Element);
117 const n = len - alignment_offset;
118 const d = dest + alignment_offset;
119 const s = src + alignment_offset;
120
121 copyBlocksAlignedSource(@ptrCast(d), @alignCast(@ptrCast(s)), n);
122
123 // copy last `@sizeOf(Element)` bytes unconditionally, since block copy
124 // methods only copy a multiple of `@sizeOf(Element)` bytes.
125 const offset = len - @sizeOf(Element);
126 dest[offset..][0..@sizeOf(Element)].* = src[offset..][0..@sizeOf(Element)].*;
127}
128
129inline fn copyBlocksAlignedSource(
130 noalias dest: [*]align(1) Element,
131 noalias src: [*]const Element,
132 max_bytes: usize,
133) void {
134 copyBlocks(dest, src, max_bytes);
135}
136
137/// Copies the largest multiple of `@sizeOf(T)` bytes from `src` to `dest`,
138/// that is less than `max_bytes` where `T` is the child type of `src` and
139/// `dest`.
140inline fn copyBlocks(
141 noalias dest: anytype,
142 noalias src: anytype,
143 max_bytes: usize,
144) void {
145 @setRuntimeSafety(builtin.is_test);
146
147 const T = @typeInfo(@TypeOf(dest)).pointer.child;
148 comptime assert(T == @typeInfo(@TypeOf(src)).pointer.child);
149
150 const loop_count = max_bytes / @sizeOf(T);
151
152 for (dest[0..loop_count], src[0..loop_count]) |*d, s| {
153 d.* = s;
154 }
155}
156
157/// copy `len` bytes from `src` to `dest`; `len` must be in the range
158/// `[copy_len, 4 * copy_len)`.
159inline fn copyRange4(
160 comptime copy_len: comptime_int,
161 noalias dest: [*]u8,
162 noalias src: [*]const u8,
163 len: usize,
164) void {
165 @setRuntimeSafety(builtin.is_test);
166 comptime assert(std.math.isPowerOfTwo(copy_len));
167 assert(len >= copy_len);
168 assert(len < 4 * copy_len);
169
170 const a = len & (copy_len * 2);
171 const b = a / 2;
172
173 const last = len - copy_len;
174 const pen = last - b;
175
176 dest[0..copy_len].* = src[0..copy_len].*;
177 dest[b..][0..copy_len].* = src[b..][0..copy_len].*;
178 dest[pen..][0..copy_len].* = src[pen..][0..copy_len].*;
179 dest[last..][0..copy_len].* = src[last..][0..copy_len].*;
180}
181
182test {
183 const S = struct {
184 fn testFunc(comptime copy_func: anytype) !void {
185 const max_len = 1024;
186 var buffer: [max_len + @alignOf(Element) - 1]u8 align(@alignOf(Element)) = undefined;
187 for (&buffer, 0..) |*b, i| {
188 b.* = @intCast(i % 97);
189 }
190 var dest: [max_len + @alignOf(Element) - 1]u8 align(@alignOf(Element)) = undefined;
191
192 for (0..max_len) |copy_len| {
193 for (0..@alignOf(Element)) |s_offset| {
194 for (0..@alignOf(Element)) |d_offset| {
195 @memset(&dest, 0xff);
196 const s = buffer[s_offset..][0..copy_len];
197 const d = dest[d_offset..][0..copy_len];
198 _ = copy_func(@ptrCast(d.ptr), @ptrCast(s.ptr), s.len);
199 std.testing.expectEqualSlices(u8, s, d) catch |e| {
200 std.debug.print("error encountered for length={d}, s_offset={d}, d_offset={d}\n", .{
201 copy_len, s_offset, d_offset,
202 });
203 return e;
204 };
205 }
206 }
207 }
208 }
209 };
210
211 try S.testFunc(memcpySmall);
212 try S.testFunc(memcpyFast);
32213}