authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-01-21 13:05:14-05:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2025-01-21 13:05:14-05:00
log18fcb3b5e82d84e50b951373203f48325b2feada
treecc9aa163cfdbeb3b03d7d4266b492e7880e5b1fc
parentd652dd065858c754f6d744beec3e7e5bc4ec058b
parentb7a887f0fb7d166ad93eae62ecd93d1ed173297e
signaturebadge-check Signed by PGP key B5690EEEBB952194

Merge pull request #18912 from dweiller/memcpy-opt

optimized memcpy

3 files changed, 347 insertions(+), 112 deletions(-)

lib/compiler_rt.zig+1
...@@ -233,6 +233,7 @@ comptime {...@@ -233,6 +233,7 @@ comptime {
233233
234 _ = @import("compiler_rt/memcpy.zig");234 _ = @import("compiler_rt/memcpy.zig");
235 _ = @import("compiler_rt/memset.zig");235 _ = @import("compiler_rt/memset.zig");
236 _ = @import("compiler_rt/memmove.zig");
236 _ = @import("compiler_rt/memcmp.zig");237 _ = @import("compiler_rt/memcmp.zig");
237 _ = @import("compiler_rt/bcmp.zig");238 _ = @import("compiler_rt/bcmp.zig");
238 _ = @import("compiler_rt/ssp.zig");239 _ = @import("compiler_rt/ssp.zig");
lib/compiler_rt/memcpy.zig+176-112
...@@ -1,149 +1,213 @@...@@ -1,149 +1,213 @@
1const std = @import("std");1const std = @import("std");
2const assert = std.debug.assert;
2const common = @import("./common.zig");3const common = @import("./common.zig");
3const builtin = @import("builtin");4const builtin = @import("builtin");
45
5comptime {6comptime {
6 if (builtin.object_format != .c) {7 if (builtin.object_format != .c) {
7 @export(&memcpy, .{ .name = "memcpy", .linkage = common.linkage, .visibility = common.visibility });8 const export_options: std.builtin.ExportOptions = .{
8 @export(&memmove, .{ .name = "memmove", .linkage = common.linkage, .visibility = common.visibility });9 .name = "memcpy",
10 .linkage = common.linkage,
11 .visibility = common.visibility,
12 };
13
14 if (builtin.mode == .ReleaseSmall)
15 @export(&memcpySmall, export_options)
16 else
17 @export(&memcpyFast, export_options);
9 }18 }
10}19}
1120
12fn memcpy(noalias opt_dest: ?[*]u8, noalias opt_src: ?[*]const u8, len: usize) callconv(.C) ?[*]u8 {21const Element = if (std.simd.suggestVectorLength(u8)) |vec_size|
13 return memmove(opt_dest, opt_src, len);22 @Type(.{ .vector = .{
23 .child = u8,
24 .len = vec_size,
25 } })
26else
27 usize;
28
29comptime {
30 assert(@sizeOf(Element) >= @alignOf(Element));
31 assert(std.math.isPowerOfTwo(@sizeOf(Element)));
14}32}
1533
16fn memmove(opt_dest: ?[*]u8, opt_src: ?[*]const u8, len: usize) callconv(.C) ?[*]u8 {34fn memcpySmall(noalias dest: ?[*]u8, noalias src: ?[*]const u8, len: usize) callconv(.C) ?[*]u8 {
17 // a port of https://github.com/facebook/folly/blob/1c8bc50e88804e2a7361a57cd9b551dd10f6c5fd/folly/memcpy.S35 @setRuntimeSafety(builtin.is_test);
18 if (len == 0) {
19 @branchHint(.unlikely);
20 return opt_dest;
21 }
2236
23 const dest = opt_dest.?;37 for (0..len) |i| {
24 const src = opt_src.?;38 dest.?[i] = src.?[i];
25
26 if (len < 8) {
27 @branchHint(.unlikely);
28 if (len == 1) {
29 @branchHint(.unlikely);
30 dest[0] = src[0];
31 } else if (len >= 4) {
32 @branchHint(.unlikely);
33 blockCopy(dest, src, 4, len);
34 } else {
35 blockCopy(dest, src, 2, len);
36 }
37 return dest;
38 }39 }
3940
40 if (len > 32) {41 return dest;
41 @branchHint(.unlikely);42}
42 if (len > 256) {
43 @branchHint(.unlikely);
44 copyMove(dest, src, len);
45 return dest;
46 }
47 copyLong(dest, src, len);
48 return dest;
49 }
5043
51 if (len > 16) {44fn memcpyFast(noalias dest: ?[*]u8, noalias src: ?[*]const u8, len: usize) callconv(.C) ?[*]u8 {
52 @branchHint(.unlikely);45 @setRuntimeSafety(builtin.is_test);
53 blockCopy(dest, src, 16, len);46
54 return dest;47 const small_limit = 2 * @sizeOf(Element);
55 }
5648
57 blockCopy(dest, src, 8, len);49 if (copySmallLength(small_limit, dest.?, src.?, len)) return dest;
50
51 copyForwards(dest.?, src.?, len);
5852
59 return dest;53 return dest;
60}54}
6155
62inline fn blockCopy(dest: [*]u8, src: [*]const u8, block_size: comptime_int, len: usize) void {56inline fn copySmallLength(
63 const first = @as(*align(1) const @Vector(block_size, u8), src[0..block_size]).*;57 comptime small_limit: comptime_int,
64 const second = @as(*align(1) const @Vector(block_size, u8), src[len - block_size ..][0..block_size]).*;58 dest: [*]u8,
65 dest[0..block_size].* = first;59 src: [*]const u8,
66 dest[len - block_size ..][0..block_size].* = second;60 len: usize,
67}61) bool {
62 if (len < 16) {
63 copyLessThan16(dest, src, len);
64 return true;
65 }
6866
69inline fn copyLong(dest: [*]u8, src: [*]const u8, len: usize) void {67 if (comptime 2 < (std.math.log2(small_limit) + 1) / 2) {
70 var array: [8]@Vector(32, u8) = undefined;68 if (copy16ToSmallLimit(small_limit, dest, src, len)) return true;
69 }
7170
72 inline for (.{ 64, 128, 192, 256 }, 0..) |N, i| {71 return false;
73 array[i * 2] = src[(N / 2) - 32 ..][0..32].*;72}
74 array[(i * 2) + 1] = src[len - N / 2 ..][0..32].*;
7573
76 if (len <= N) {74inline fn copyLessThan16(
77 @branchHint(.unlikely);75 dest: [*]u8,
78 for (0..i + 1) |j| {76 src: [*]const u8,
79 dest[j * 32 ..][0..32].* = array[j * 2];77 len: usize,
80 dest[len - ((j * 32) + 32) ..][0..32].* = array[(j * 2) + 1];78) void {
81 }79 @setRuntimeSafety(builtin.is_test);
82 return;80 if (len < 4) {
83 }81 if (len == 0) return;
82 dest[0] = src[0];
83 dest[len / 2] = src[len / 2];
84 dest[len - 1] = src[len - 1];
85 return;
84 }86 }
87 copyRange4(4, dest, src, len);
85}88}
8689
87inline fn copyMove(dest: [*]u8, src: [*]const u8, len: usize) void {90inline fn copy16ToSmallLimit(
88 if (@intFromPtr(src) >= @intFromPtr(dest)) {91 comptime small_limit: comptime_int,
89 @branchHint(.unlikely);92 dest: [*]u8,
90 copyForward(dest, src, len);93 src: [*]const u8,
91 } else if (@intFromPtr(src) + len > @intFromPtr(dest)) {94 len: usize,
92 @branchHint(.unlikely);95) bool {
93 overlapBwd(dest, src, len);96 @setRuntimeSafety(builtin.is_test);
94 } else {97 inline for (2..(std.math.log2(small_limit) + 1) / 2 + 1) |p| {
95 copyForward(dest, src, len);98 const limit = 1 << (2 * p);
99 if (len < limit) {
100 copyRange4(limit / 4, dest, src, len);
101 return true;
102 }
96 }103 }
104 return false;
97}105}
98106
99inline fn copyForward(dest: [*]u8, src: [*]const u8, len: usize) void {107inline fn copyForwards(
100 const tail: @Vector(32, u8) = src[len - 32 ..][0..32].*;108 noalias dest: [*]u8,
109 noalias src: [*]const u8,
110 len: usize,
111) void {
112 @setRuntimeSafety(builtin.is_test);
113 assert(len >= 2 * @sizeOf(Element));
114
115 dest[0..@sizeOf(Element)].* = src[0..@sizeOf(Element)].*;
116 const alignment_offset = @alignOf(Element) - @intFromPtr(src) % @alignOf(Element);
117 const n = len - alignment_offset;
118 const d = dest + alignment_offset;
119 const s = src + alignment_offset;
120
121 copyBlocksAlignedSource(@ptrCast(d), @alignCast(@ptrCast(s)), n);
122
123 // copy last `@sizeOf(Element)` bytes unconditionally, since block copy
124 // methods only copy a multiple of `@sizeOf(Element)` bytes.
125 const offset = len - @sizeOf(Element);
126 dest[offset..][0..@sizeOf(Element)].* = src[offset..][0..@sizeOf(Element)].*;
127}
101128
102 const N: usize = len & ~@as(usize, 127);129inline fn copyBlocksAlignedSource(
103 var i: usize = 0;130 noalias dest: [*]align(1) Element,
131 noalias src: [*]const Element,
132 max_bytes: usize,
133) void {
134 copyBlocks(dest, src, max_bytes);
135}
104136
105 while (i < N) : (i += 128) {137/// Copies the largest multiple of `@sizeOf(T)` bytes from `src` to `dest`,
106 dest[i..][0..32].* = src[i..][0..32].*;138/// that is less than `max_bytes` where `T` is the child type of `src` and
107 dest[i + 32 ..][0..32].* = src[i + 32 ..][0..32].*;139/// `dest`.
108 dest[i + 64 ..][0..32].* = src[i + 64 ..][0..32].*;140inline fn copyBlocks(
109 dest[i + 96 ..][0..32].* = src[i + 96 ..][0..32].*;141 noalias dest: anytype,
110 }142 noalias src: anytype,
143 max_bytes: usize,
144) void {
145 @setRuntimeSafety(builtin.is_test);
146
147 const T = @typeInfo(@TypeOf(dest)).pointer.child;
148 comptime assert(T == @typeInfo(@TypeOf(src)).pointer.child);
111149
112 if (len - i <= 32) {150 const loop_count = max_bytes / @sizeOf(T);
113 @branchHint(.unlikely);151
114 dest[len - 32 ..][0..32].* = tail;152 for (dest[0..loop_count], src[0..loop_count]) |*d, s| {
115 } else {153 d.* = s;
116 copyLong(dest[i..], src[i..], len - i);
117 }154 }
118}155}
119156
120inline fn overlapBwd(dest: [*]u8, src: [*]const u8, len: usize) void {157/// copy `len` bytes from `src` to `dest`; `len` must be in the range
121 var array: [5]@Vector(32, u8) = undefined;158/// `[copy_len, 4 * copy_len)`.
122 array[0] = src[len - 32 ..][0..32].*;159inline fn copyRange4(
123 inline for (1..5) |i| array[i] = src[(i - 1) << 5 ..][0..32].*;160 comptime copy_len: comptime_int,
124161 noalias dest: [*]u8,
125 const end: usize = (@intFromPtr(dest) + len - 32) & 31;162 noalias src: [*]const u8,
126 const range = len - end;163 len: usize,
127 var s = src + range;164) void {
128 var d = dest + range;165 @setRuntimeSafety(builtin.is_test);
129166 comptime assert(std.math.isPowerOfTwo(copy_len));
130 while (@intFromPtr(s) > @intFromPtr(src + 128)) {167 assert(len >= copy_len);
131 // zig fmt: off168 assert(len < 4 * copy_len);
132 const first = @as(*align(1) const @Vector(32, u8), @ptrCast(s - 32)).*;169
133 const second = @as(*align(1) const @Vector(32, u8), @ptrCast(s - 64)).*;170 const a = len & (copy_len * 2);
134 const third = @as(*align(1) const @Vector(32, u8), @ptrCast(s - 96)).*;171 const b = a / 2;
135 const fourth = @as(*align(1) const @Vector(32, u8), @ptrCast(s - 128)).*;172
136173 const last = len - copy_len;
137 @as(*align(32) @Vector(32, u8), @alignCast(@ptrCast(d - 32))).* = first;174 const pen = last - b;
138 @as(*align(32) @Vector(32, u8), @alignCast(@ptrCast(d - 64))).* = second;175
139 @as(*align(32) @Vector(32, u8), @alignCast(@ptrCast(d - 96))).* = third;176 dest[0..copy_len].* = src[0..copy_len].*;
140 @as(*align(32) @Vector(32, u8), @alignCast(@ptrCast(d - 128))).* = fourth;177 dest[b..][0..copy_len].* = src[b..][0..copy_len].*;
141 // zig fmt: on178 dest[pen..][0..copy_len].* = src[pen..][0..copy_len].*;
142179 dest[last..][0..copy_len].* = src[last..][0..copy_len].*;
143 s -= 128;180}
144 d -= 128;181
145 }182test {
183 const S = struct {
184 fn testFunc(comptime copy_func: anytype) !void {
185 const max_len = 1024;
186 var buffer: [max_len + @alignOf(Element) - 1]u8 align(@alignOf(Element)) = undefined;
187 for (&buffer, 0..) |*b, i| {
188 b.* = @intCast(i % 97);
189 }
190 var dest: [max_len + @alignOf(Element) - 1]u8 align(@alignOf(Element)) = undefined;
191
192 for (0..max_len) |copy_len| {
193 for (0..@alignOf(Element)) |s_offset| {
194 for (0..@alignOf(Element)) |d_offset| {
195 @memset(&dest, 0xff);
196 const s = buffer[s_offset..][0..copy_len];
197 const d = dest[d_offset..][0..copy_len];
198 _ = copy_func(@ptrCast(d.ptr), @ptrCast(s.ptr), s.len);
199 std.testing.expectEqualSlices(u8, s, d) catch |e| {
200 std.debug.print("error encountered for length={d}, s_offset={d}, d_offset={d}\n", .{
201 copy_len, s_offset, d_offset,
202 });
203 return e;
204 };
205 }
206 }
207 }
208 }
209 };
146210
147 inline for (array[1..], 0..) |vec, i| dest[i * 32 ..][0..32].* = vec;211 try S.testFunc(memcpySmall);
148 dest[len - 32 ..][0..32].* = array[0];212 try S.testFunc(memcpyFast);
149}213}
lib/compiler_rt/memmove.zig created+170
...@@ -0,0 +1,170 @@
1const std = @import("std");
2const common = @import("./common.zig");
3const builtin = @import("builtin");
4
5comptime {
6 if (builtin.object_format != .c) {
7 const export_options: std.builtin.ExportOptions = .{
8 .name = "memmove",
9 .linkage = common.linkage,
10 .visibility = common.visibility,
11 };
12
13 if (builtin.mode == .ReleaseSmall)
14 @export(&memmoveSmall, export_options)
15 else
16 @export(&memmoveFast, export_options);
17 }
18}
19
20fn memmoveSmall(opt_dest: ?[*]u8, opt_src: ?[*]const u8, len: usize) callconv(.C) ?[*]u8 {
21 const dest = opt_dest.?;
22 const src = opt_src.?;
23
24 if (@intFromPtr(dest) < @intFromPtr(src)) {
25 for (0..len) |i| {
26 dest[i] = src[i];
27 }
28 } else {
29 for (0..len) |i| {
30 dest[len - 1 - i] = src[len - 1 - i];
31 }
32 }
33
34 return dest;
35}
36
37pub fn memmoveFast(opt_dest: ?[*]u8, opt_src: ?[*]const u8, len: usize) callconv(.C) ?[*]u8 {
38 // a port of https://github.com/facebook/folly/blob/1c8bc50e88804e2a7361a57cd9b551dd10f6c5fd/folly/memcpy.S
39 if (len == 0) {
40 @branchHint(.unlikely);
41 return opt_dest;
42 }
43
44 const dest = opt_dest.?;
45 const src = opt_src.?;
46
47 if (len < 8) {
48 @branchHint(.unlikely);
49 if (len == 1) {
50 @branchHint(.unlikely);
51 dest[0] = src[0];
52 } else if (len >= 4) {
53 @branchHint(.unlikely);
54 blockCopy(dest, src, 4, len);
55 } else {
56 blockCopy(dest, src, 2, len);
57 }
58 return dest;
59 }
60
61 if (len > 32) {
62 @branchHint(.unlikely);
63 if (len > 256) {
64 @branchHint(.unlikely);
65 copyMove(dest, src, len);
66 return dest;
67 }
68 copyLong(dest, src, len);
69 return dest;
70 }
71
72 if (len > 16) {
73 @branchHint(.unlikely);
74 blockCopy(dest, src, 16, len);
75 return dest;
76 }
77
78 blockCopy(dest, src, 8, len);
79
80 return dest;
81}
82
83inline fn blockCopy(dest: [*]u8, src: [*]const u8, block_size: comptime_int, len: usize) void {
84 const first = @as(*align(1) const @Vector(block_size, u8), src[0..block_size]).*;
85 const second = @as(*align(1) const @Vector(block_size, u8), src[len - block_size ..][0..block_size]).*;
86 dest[0..block_size].* = first;
87 dest[len - block_size ..][0..block_size].* = second;
88}
89
90inline fn copyLong(dest: [*]u8, src: [*]const u8, len: usize) void {
91 var array: [8]@Vector(32, u8) = undefined;
92
93 inline for (.{ 64, 128, 192, 256 }, 0..) |N, i| {
94 array[i * 2] = src[(N / 2) - 32 ..][0..32].*;
95 array[(i * 2) + 1] = src[len - N / 2 ..][0..32].*;
96
97 if (len <= N) {
98 @branchHint(.unlikely);
99 for (0..i + 1) |j| {
100 dest[j * 32 ..][0..32].* = array[j * 2];
101 dest[len - ((j * 32) + 32) ..][0..32].* = array[(j * 2) + 1];
102 }
103 return;
104 }
105 }
106}
107
108inline fn copyMove(dest: [*]u8, src: [*]const u8, len: usize) void {
109 if (@intFromPtr(src) >= @intFromPtr(dest)) {
110 @branchHint(.unlikely);
111 copyForward(dest, src, len);
112 } else if (@intFromPtr(src) + len > @intFromPtr(dest)) {
113 @branchHint(.unlikely);
114 overlapBwd(dest, src, len);
115 } else {
116 copyForward(dest, src, len);
117 }
118}
119
120inline fn copyForward(dest: [*]u8, src: [*]const u8, len: usize) void {
121 const tail: @Vector(32, u8) = src[len - 32 ..][0..32].*;
122
123 const N: usize = len & ~@as(usize, 127);
124 var i: usize = 0;
125
126 while (i < N) : (i += 128) {
127 dest[i..][0..32].* = src[i..][0..32].*;
128 dest[i + 32 ..][0..32].* = src[i + 32 ..][0..32].*;
129 dest[i + 64 ..][0..32].* = src[i + 64 ..][0..32].*;
130 dest[i + 96 ..][0..32].* = src[i + 96 ..][0..32].*;
131 }
132
133 if (len - i <= 32) {
134 @branchHint(.unlikely);
135 dest[len - 32 ..][0..32].* = tail;
136 } else {
137 copyLong(dest[i..], src[i..], len - i);
138 }
139}
140
141inline fn overlapBwd(dest: [*]u8, src: [*]const u8, len: usize) void {
142 var array: [5]@Vector(32, u8) = undefined;
143 array[0] = src[len - 32 ..][0..32].*;
144 inline for (1..5) |i| array[i] = src[(i - 1) << 5 ..][0..32].*;
145
146 const end: usize = (@intFromPtr(dest) + len - 32) & 31;
147 const range = len - end;
148 var s = src + range;
149 var d = dest + range;
150
151 while (@intFromPtr(s) > @intFromPtr(src + 128)) {
152 // zig fmt: off
153 const first = @as(*align(1) const @Vector(32, u8), @ptrCast(s - 32)).*;
154 const second = @as(*align(1) const @Vector(32, u8), @ptrCast(s - 64)).*;
155 const third = @as(*align(1) const @Vector(32, u8), @ptrCast(s - 96)).*;
156 const fourth = @as(*align(1) const @Vector(32, u8), @ptrCast(s - 128)).*;
157
158 @as(*align(32) @Vector(32, u8), @alignCast(@ptrCast(d - 32))).* = first;
159 @as(*align(32) @Vector(32, u8), @alignCast(@ptrCast(d - 64))).* = second;
160 @as(*align(32) @Vector(32, u8), @alignCast(@ptrCast(d - 96))).* = third;
161 @as(*align(32) @Vector(32, u8), @alignCast(@ptrCast(d - 128))).* = fourth;
162 // zig fmt: on
163
164 s -= 128;
165 d -= 128;
166 }
167
168 inline for (array[1..], 0..) |vec, i| dest[i * 32 ..][0..32].* = vec;
169 dest[len - 32 ..][0..32].* = array[0];
170}