authorgravatar for 4678790+dweiller@users.noreply.github.comDominic <4678790+dweiller@users.noreply.github.com> 2025-01-20 17:16:24+11:00
committergravatar for 4678790+dweiller@users.noreply.github.comDominic <4678790+dweiller@users.noreply.github.com> 2025-01-20 17:16:24+11:00
logbf89482f23aae3524c92d385d6d8241d065d1031
tree1acb589b414ea47fe5923216a97ce0ef9eee4490
parent5cfcb015033864c769235726975ab9919c217ef9

compiler-rt: move memmove back to memmove.zig

3 files changed, 147 insertions(+), 138 deletions(-)

lib/compiler_rt.zig+1
...@@ -233,6 +233,7 @@ comptime {...@@ -233,6 +233,7 @@ comptime {
233233
234 _ = @import("compiler_rt/memcpy.zig");234 _ = @import("compiler_rt/memcpy.zig");
235 _ = @import("compiler_rt/memset.zig");235 _ = @import("compiler_rt/memset.zig");
236 _ = @import("compiler_rt/memmove.zig");
236 _ = @import("compiler_rt/memcmp.zig");237 _ = @import("compiler_rt/memcmp.zig");
237 _ = @import("compiler_rt/bcmp.zig");238 _ = @import("compiler_rt/bcmp.zig");
238 _ = @import("compiler_rt/ssp.zig");239 _ = @import("compiler_rt/ssp.zig");
lib/compiler_rt/memcpy.zig+2-138
...@@ -5,145 +5,9 @@ const builtin = @import("builtin");...@@ -5,145 +5,9 @@ const builtin = @import("builtin");
5comptime {5comptime {
6 if (builtin.object_format != .c) {6 if (builtin.object_format != .c) {
7 @export(&memcpy, .{ .name = "memcpy", .linkage = common.linkage, .visibility = common.visibility });7 @export(&memcpy, .{ .name = "memcpy", .linkage = common.linkage, .visibility = common.visibility });
8 @export(&memmove, .{ .name = "memmove", .linkage = common.linkage, .visibility = common.visibility });
9 }8 }
10}9}
1110
12fn memcpy(noalias opt_dest: ?[*]u8, noalias opt_src: ?[*]const u8, len: usize) callconv(.C) ?[*]u8 {11fn memcpy(opt_dest: ?[*]u8, opt_src: ?[*]const u8, len: usize) callconv(.C) ?[*]u8 {
13 return memmove(opt_dest, opt_src, len);12 return @call(.always_inline, @import("memmove.zig").memmove, .{ opt_dest, opt_src, len });
14}
15
16fn memmove(opt_dest: ?[*]u8, opt_src: ?[*]const u8, len: usize) callconv(.C) ?[*]u8 {
17 // a port of https://github.com/facebook/folly/blob/1c8bc50e88804e2a7361a57cd9b551dd10f6c5fd/folly/memcpy.S
18 if (len == 0) {
19 @branchHint(.unlikely);
20 return opt_dest;
21 }
22
23 const dest = opt_dest.?;
24 const src = opt_src.?;
25
26 if (len < 8) {
27 @branchHint(.unlikely);
28 if (len == 1) {
29 @branchHint(.unlikely);
30 dest[0] = src[0];
31 } else if (len >= 4) {
32 @branchHint(.unlikely);
33 blockCopy(dest, src, 4, len);
34 } else {
35 blockCopy(dest, src, 2, len);
36 }
37 return dest;
38 }
39
40 if (len > 32) {
41 @branchHint(.unlikely);
42 if (len > 256) {
43 @branchHint(.unlikely);
44 copyMove(dest, src, len);
45 return dest;
46 }
47 copyLong(dest, src, len);
48 return dest;
49 }
50
51 if (len > 16) {
52 @branchHint(.unlikely);
53 blockCopy(dest, src, 16, len);
54 return dest;
55 }
56
57 blockCopy(dest, src, 8, len);
58
59 return dest;
60}
61
62inline fn blockCopy(dest: [*]u8, src: [*]const u8, block_size: comptime_int, len: usize) void {
63 const first = @as(*align(1) const @Vector(block_size, u8), src[0..block_size]).*;
64 const second = @as(*align(1) const @Vector(block_size, u8), src[len - block_size ..][0..block_size]).*;
65 dest[0..block_size].* = first;
66 dest[len - block_size ..][0..block_size].* = second;
67}
68
69inline fn copyLong(dest: [*]u8, src: [*]const u8, len: usize) void {
70 var array: [8]@Vector(32, u8) = undefined;
71
72 inline for (.{ 64, 128, 192, 256 }, 0..) |N, i| {
73 array[i * 2] = src[(N / 2) - 32 ..][0..32].*;
74 array[(i * 2) + 1] = src[len - N / 2 ..][0..32].*;
75
76 if (len <= N) {
77 @branchHint(.unlikely);
78 for (0..i + 1) |j| {
79 dest[j * 32 ..][0..32].* = array[j * 2];
80 dest[len - ((j * 32) + 32) ..][0..32].* = array[(j * 2) + 1];
81 }
82 return;
83 }
84 }
85}
86
87inline fn copyMove(dest: [*]u8, src: [*]const u8, len: usize) void {
88 if (@intFromPtr(src) >= @intFromPtr(dest)) {
89 @branchHint(.unlikely);
90 copyForward(dest, src, len);
91 } else if (@intFromPtr(src) + len > @intFromPtr(dest)) {
92 @branchHint(.unlikely);
93 overlapBwd(dest, src, len);
94 } else {
95 copyForward(dest, src, len);
96 }
97}
98
99inline fn copyForward(dest: [*]u8, src: [*]const u8, len: usize) void {
100 const tail: @Vector(32, u8) = src[len - 32 ..][0..32].*;
101
102 const N: usize = len & ~@as(usize, 127);
103 var i: usize = 0;
104
105 while (i < N) : (i += 128) {
106 dest[i..][0..32].* = src[i..][0..32].*;
107 dest[i + 32 ..][0..32].* = src[i + 32 ..][0..32].*;
108 dest[i + 64 ..][0..32].* = src[i + 64 ..][0..32].*;
109 dest[i + 96 ..][0..32].* = src[i + 96 ..][0..32].*;
110 }
111
112 if (len - i <= 32) {
113 @branchHint(.unlikely);
114 dest[len - 32 ..][0..32].* = tail;
115 } else {
116 copyLong(dest[i..], src[i..], len - i);
117 }
118}
119
120inline fn overlapBwd(dest: [*]u8, src: [*]const u8, len: usize) void {
121 var array: [5]@Vector(32, u8) = undefined;
122 array[0] = src[len - 32 ..][0..32].*;
123 inline for (1..5) |i| array[i] = src[(i - 1) << 5 ..][0..32].*;
124
125 const end: usize = (@intFromPtr(dest) + len - 32) & 31;
126 const range = len - end;
127 var s = src + range;
128 var d = dest + range;
129
130 while (@intFromPtr(s) > @intFromPtr(src + 128)) {
131 // zig fmt: off
132 const first = @as(*align(1) const @Vector(32, u8), @ptrCast(s - 32)).*;
133 const second = @as(*align(1) const @Vector(32, u8), @ptrCast(s - 64)).*;
134 const third = @as(*align(1) const @Vector(32, u8), @ptrCast(s - 96)).*;
135 const fourth = @as(*align(1) const @Vector(32, u8), @ptrCast(s - 128)).*;
136
137 @as(*align(32) @Vector(32, u8), @alignCast(@ptrCast(d - 32))).* = first;
138 @as(*align(32) @Vector(32, u8), @alignCast(@ptrCast(d - 64))).* = second;
139 @as(*align(32) @Vector(32, u8), @alignCast(@ptrCast(d - 96))).* = third;
140 @as(*align(32) @Vector(32, u8), @alignCast(@ptrCast(d - 128))).* = fourth;
141 // zig fmt: on
142
143 s -= 128;
144 d -= 128;
145 }
146
147 inline for (array[1..], 0..) |vec, i| dest[i * 32 ..][0..32].* = vec;
148 dest[len - 32 ..][0..32].* = array[0];
149}13}
lib/compiler_rt/memmove.zig created+144
...@@ -0,0 +1,144 @@
1const std = @import("std");
2const common = @import("./common.zig");
3const builtin = @import("builtin");
4
5comptime {
6 if (builtin.object_format != .c) {
7 @export(&memmove, .{ .name = "memmove", .linkage = common.linkage, .visibility = common.visibility });
8 }
9}
10
11pub fn memmove(opt_dest: ?[*]u8, opt_src: ?[*]const u8, len: usize) callconv(.C) ?[*]u8 {
12 // a port of https://github.com/facebook/folly/blob/1c8bc50e88804e2a7361a57cd9b551dd10f6c5fd/folly/memcpy.S
13 if (len == 0) {
14 @branchHint(.unlikely);
15 return opt_dest;
16 }
17
18 const dest = opt_dest.?;
19 const src = opt_src.?;
20
21 if (len < 8) {
22 @branchHint(.unlikely);
23 if (len == 1) {
24 @branchHint(.unlikely);
25 dest[0] = src[0];
26 } else if (len >= 4) {
27 @branchHint(.unlikely);
28 blockCopy(dest, src, 4, len);
29 } else {
30 blockCopy(dest, src, 2, len);
31 }
32 return dest;
33 }
34
35 if (len > 32) {
36 @branchHint(.unlikely);
37 if (len > 256) {
38 @branchHint(.unlikely);
39 copyMove(dest, src, len);
40 return dest;
41 }
42 copyLong(dest, src, len);
43 return dest;
44 }
45
46 if (len > 16) {
47 @branchHint(.unlikely);
48 blockCopy(dest, src, 16, len);
49 return dest;
50 }
51
52 blockCopy(dest, src, 8, len);
53
54 return dest;
55}
56
57inline fn blockCopy(dest: [*]u8, src: [*]const u8, block_size: comptime_int, len: usize) void {
58 const first = @as(*align(1) const @Vector(block_size, u8), src[0..block_size]).*;
59 const second = @as(*align(1) const @Vector(block_size, u8), src[len - block_size ..][0..block_size]).*;
60 dest[0..block_size].* = first;
61 dest[len - block_size ..][0..block_size].* = second;
62}
63
64inline fn copyLong(dest: [*]u8, src: [*]const u8, len: usize) void {
65 var array: [8]@Vector(32, u8) = undefined;
66
67 inline for (.{ 64, 128, 192, 256 }, 0..) |N, i| {
68 array[i * 2] = src[(N / 2) - 32 ..][0..32].*;
69 array[(i * 2) + 1] = src[len - N / 2 ..][0..32].*;
70
71 if (len <= N) {
72 @branchHint(.unlikely);
73 for (0..i + 1) |j| {
74 dest[j * 32 ..][0..32].* = array[j * 2];
75 dest[len - ((j * 32) + 32) ..][0..32].* = array[(j * 2) + 1];
76 }
77 return;
78 }
79 }
80}
81
82inline fn copyMove(dest: [*]u8, src: [*]const u8, len: usize) void {
83 if (@intFromPtr(src) >= @intFromPtr(dest)) {
84 @branchHint(.unlikely);
85 copyForward(dest, src, len);
86 } else if (@intFromPtr(src) + len > @intFromPtr(dest)) {
87 @branchHint(.unlikely);
88 overlapBwd(dest, src, len);
89 } else {
90 copyForward(dest, src, len);
91 }
92}
93
94inline fn copyForward(dest: [*]u8, src: [*]const u8, len: usize) void {
95 const tail: @Vector(32, u8) = src[len - 32 ..][0..32].*;
96
97 const N: usize = len & ~@as(usize, 127);
98 var i: usize = 0;
99
100 while (i < N) : (i += 128) {
101 dest[i..][0..32].* = src[i..][0..32].*;
102 dest[i + 32 ..][0..32].* = src[i + 32 ..][0..32].*;
103 dest[i + 64 ..][0..32].* = src[i + 64 ..][0..32].*;
104 dest[i + 96 ..][0..32].* = src[i + 96 ..][0..32].*;
105 }
106
107 if (len - i <= 32) {
108 @branchHint(.unlikely);
109 dest[len - 32 ..][0..32].* = tail;
110 } else {
111 copyLong(dest[i..], src[i..], len - i);
112 }
113}
114
115inline fn overlapBwd(dest: [*]u8, src: [*]const u8, len: usize) void {
116 var array: [5]@Vector(32, u8) = undefined;
117 array[0] = src[len - 32 ..][0..32].*;
118 inline for (1..5) |i| array[i] = src[(i - 1) << 5 ..][0..32].*;
119
120 const end: usize = (@intFromPtr(dest) + len - 32) & 31;
121 const range = len - end;
122 var s = src + range;
123 var d = dest + range;
124
125 while (@intFromPtr(s) > @intFromPtr(src + 128)) {
126 // zig fmt: off
127 const first = @as(*align(1) const @Vector(32, u8), @ptrCast(s - 32)).*;
128 const second = @as(*align(1) const @Vector(32, u8), @ptrCast(s - 64)).*;
129 const third = @as(*align(1) const @Vector(32, u8), @ptrCast(s - 96)).*;
130 const fourth = @as(*align(1) const @Vector(32, u8), @ptrCast(s - 128)).*;
131
132 @as(*align(32) @Vector(32, u8), @alignCast(@ptrCast(d - 32))).* = first;
133 @as(*align(32) @Vector(32, u8), @alignCast(@ptrCast(d - 64))).* = second;
134 @as(*align(32) @Vector(32, u8), @alignCast(@ptrCast(d - 96))).* = third;
135 @as(*align(32) @Vector(32, u8), @alignCast(@ptrCast(d - 128))).* = fourth;
136 // zig fmt: on
137
138 s -= 128;
139 d -= 128;
140 }
141
142 inline for (array[1..], 0..) |vec, i| dest[i * 32 ..][0..32].* = vec;
143 dest[len - 32 ..][0..32].* = array[0];
144}