authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-02-06 21:47:46-08:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-02-07 14:41:49-08:00
log839c453d880e22f2f120d62332b273380a215cc5
treec4a4ca0cb67163795d8bef16c864f1acb4470924
parent60765a9ee2d66d4c2b870e726aa7e7bc2590e2b8

std.heap.SmpAllocator: eliminate the global mutex

1 files changed, 24 insertions(+), 62 deletions(-)

lib/std/heap/SmpAllocator.zig+24-62
...@@ -39,20 +39,14 @@ const Allocator = std.mem.Allocator;...@@ -39,20 +39,14 @@ const Allocator = std.mem.Allocator;
39const SmpAllocator = @This();39const SmpAllocator = @This();
40const PageAllocator = std.heap.PageAllocator;40const PageAllocator = std.heap.PageAllocator;
4141
42/// Protects the state in this struct (global state), except for `threads`
43/// which each have their own mutex.
44mutex: std.Thread.Mutex,
45next_thread_index: u32,
46cpu_count: u32,42cpu_count: u32,
47threads: [max_thread_count]Thread,43threads: [max_thread_count]Thread,
4844
49var global: SmpAllocator = .{45var global: SmpAllocator = .{
50 .mutex = .{},
51 .next_thread_index = 0,
52 .threads = @splat(.{}),46 .threads = @splat(.{}),
53 .cpu_count = 0,47 .cpu_count = 0,
54};48};
55threadlocal var thread_id: Thread.Id = .none;49threadlocal var thread_index: u32 = 0;
5650
57const max_thread_count = 128;51const max_thread_count = 128;
58const slab_len: usize = @max(std.heap.page_size_max, 256 * 1024);52const slab_len: usize = @max(std.heap.page_size_max, 256 * 1024);
...@@ -74,60 +68,22 @@ const Thread = struct {...@@ -74,60 +68,22 @@ const Thread = struct {
74 /// For each size class, points to the freed pointer.68 /// For each size class, points to the freed pointer.
75 frees: [size_class_count]usize = @splat(0),69 frees: [size_class_count]usize = @splat(0),
7670
77 /// Index into `SmpAllocator.threads`.
78 const Id = enum(usize) {
79 none = 0,
80 first = 1,
81 _,
82
83 fn fromIndex(index: usize) Id {
84 return @enumFromInt(index + 1);
85 }
86
87 fn toIndex(id: Id) usize {
88 return @intFromEnum(id) - 1;
89 }
90 };
91
92 fn lock() *Thread {71 fn lock() *Thread {
93 const id = thread_id;72 var index = thread_index;
94 if (id != .none) {73 {
95 var index = id.toIndex();74 const t = &global.threads[index];
96 {75 if (t.mutex.tryLock()) {
97 const t = &global.threads[index];76 @branchHint(.likely);
98 if (t.mutex.tryLock()) return t;77 return t;
99 }
100 const cpu_count = global.cpu_count;
101 assert(cpu_count != 0);
102 while (true) {
103 index = (index + 1) % cpu_count;
104 const t = &global.threads[index];
105 if (t.mutex.tryLock()) {
106 thread_id = .fromIndex(index);
107 return t;
108 }
109 }78 }
110 }79 }
80 const cpu_count = getCpuCount();
81 assert(cpu_count != 0);
111 while (true) {82 while (true) {
112 const thread_index = i: {83 index = (index + 1) % cpu_count;
113 global.mutex.lock();84 const t = &global.threads[index];
114 defer global.mutex.unlock();
115 const cpu_count = c: {
116 const cpu_count = global.cpu_count;
117 if (cpu_count == 0) {
118 const n: u32 = @intCast(@max(std.Thread.getCpuCount() catch max_thread_count, max_thread_count));
119 global.cpu_count = n;
120 break :c n;
121 }
122 break :c cpu_count;
123 };
124 const thread_index = global.next_thread_index;
125 global.next_thread_index = @intCast((thread_index + 1) % cpu_count);
126 break :i thread_index;
127 };
128 const t = &global.threads[thread_index];
129 if (t.mutex.tryLock()) {85 if (t.mutex.tryLock()) {
130 thread_id = .fromIndex(thread_index);86 thread_index = index;
131 return t;87 return t;
132 }88 }
133 }89 }
...@@ -138,6 +94,13 @@ const Thread = struct {...@@ -138,6 +94,13 @@ const Thread = struct {
138 }94 }
139};95};
14096
97fn getCpuCount() u32 {
98 const cpu_count = @atomicLoad(u32, &global.cpu_count, .unordered);
99 if (cpu_count != 0) return cpu_count;
100 const n: u32 = @intCast(@max(std.Thread.getCpuCount() catch max_thread_count, max_thread_count));
101 return if (@cmpxchgStrong(u32, &global.cpu_count, 0, n, .monotonic, .monotonic)) |other| other else n;
102}
103
141pub const vtable: Allocator.VTable = .{104pub const vtable: Allocator.VTable = .{
142 .alloc = alloc,105 .alloc = alloc,
143 .resize = resize,106 .resize = resize,
...@@ -159,8 +122,8 @@ fn alloc(context: *anyopaque, len: usize, alignment: mem.Alignment, ra: usize) ?...@@ -159,8 +122,8 @@ fn alloc(context: *anyopaque, len: usize, alignment: mem.Alignment, ra: usize) ?
159 }122 }
160123
161 const slot_size = slotSize(class);124 const slot_size = slotSize(class);
162 const max_search = 2;125 const max_search = 1;
163 var search_count: u32 = 0;126 var search_count: u8 = 0;
164127
165 var t = Thread.lock();128 var t = Thread.lock();
166129
...@@ -191,15 +154,14 @@ fn alloc(context: *anyopaque, len: usize, alignment: mem.Alignment, ra: usize) ?...@@ -191,15 +154,14 @@ fn alloc(context: *anyopaque, len: usize, alignment: mem.Alignment, ra: usize) ?
191 }154 }
192155
193 t.unlock();156 t.unlock();
194 t = undefined;157 const cpu_count = getCpuCount();
195 const cpu_count = global.cpu_count;
196 assert(cpu_count != 0);158 assert(cpu_count != 0);
197 var index = thread_id.toIndex();159 var index = thread_index;
198 while (true) {160 while (true) {
199 index = (index + 1) % cpu_count;161 index = (index + 1) % cpu_count;
200 t = &global.threads[index];162 t = &global.threads[index];
201 if (t.mutex.tryLock()) {163 if (t.mutex.tryLock()) {
202 thread_id = .fromIndex(index);164 thread_index = index;
203 search_count += 1;165 search_count += 1;
204 continue :outer;166 continue :outer;
205 }167 }