authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-02-06 19:48:02-08:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-02-07 14:41:49-08:00
log60765a9ee2d66d4c2b870e726aa7e7bc2590e2b8
tree95fd21b7dc85ecce7aa9a6d4ceb38c7ed48904b4
parent7360be19a461175d1a50dfb1d7c086bcc650b3c1

std.heap.SmpAllocator: implement searching on alloc

rotate a couple times before resorting to mapping more memory.

1 files changed, 47 insertions(+), 20 deletions(-)

lib/std/heap/SmpAllocator.zig+47-20
...@@ -158,27 +158,53 @@ fn alloc(context: *anyopaque, len: usize, alignment: mem.Alignment, ra: usize) ?...@@ -158,27 +158,53 @@ fn alloc(context: *anyopaque, len: usize, alignment: mem.Alignment, ra: usize) ?
158 return PageAllocator.map(len, alignment);158 return PageAllocator.map(len, alignment);
159 }159 }
160160
161 const t = Thread.lock();
162 defer t.unlock();
163
164 const slot_size = slotSize(class);161 const slot_size = slotSize(class);
162 const max_search = 2;
163 var search_count: u32 = 0;
165164
166 const top_free_ptr = t.frees[class];165 var t = Thread.lock();
167 if (top_free_ptr != 0) {
168 const node: *usize = @ptrFromInt(top_free_ptr + (slot_size - @sizeOf(usize)));
169 t.frees[class] = node.*;
170 return @ptrFromInt(top_free_ptr);
171 }
172166
173 const next_addr = t.next_addrs[class];167 outer: while (true) {
174 if (next_addr % slab_len == 0) {168 const top_free_ptr = t.frees[class];
175 const slab = PageAllocator.map(slab_len, .fromByteUnits(std.heap.pageSize())) orelse return null;169 if (top_free_ptr != 0) {
176 t.next_addrs[class] = @intFromPtr(slab) + slot_size;170 @branchHint(.likely);
177 return slab;171 defer t.unlock();
178 }172 const node: *usize = @ptrFromInt(top_free_ptr + (slot_size - @sizeOf(usize)));
173 t.frees[class] = node.*;
174 return @ptrFromInt(top_free_ptr);
175 }
179176
180 t.next_addrs[class] = next_addr + slot_size;177 const next_addr = t.next_addrs[class];
181 return @ptrFromInt(next_addr);178 if ((next_addr % slab_len) != 0) {
179 @branchHint(.likely);
180 defer t.unlock();
181 t.next_addrs[class] = next_addr + slot_size;
182 return @ptrFromInt(next_addr);
183 }
184
185 if (search_count >= max_search) {
186 @branchHint(.likely);
187 defer t.unlock();
188 const slab = PageAllocator.map(slab_len, .fromByteUnits(std.heap.pageSize())) orelse return null;
189 t.next_addrs[class] = @intFromPtr(slab) + slot_size;
190 return slab;
191 }
192
193 t.unlock();
194 t = undefined;
195 const cpu_count = global.cpu_count;
196 assert(cpu_count != 0);
197 var index = thread_id.toIndex();
198 while (true) {
199 index = (index + 1) % cpu_count;
200 t = &global.threads[index];
201 if (t.mutex.tryLock()) {
202 thread_id = .fromIndex(index);
203 search_count += 1;
204 continue :outer;
205 }
206 }
207 }
182}208}
183209
184fn resize(context: *anyopaque, memory: []u8, alignment: mem.Alignment, new_len: usize, ra: usize) bool {210fn resize(context: *anyopaque, memory: []u8, alignment: mem.Alignment, new_len: usize, ra: usize) bool {
...@@ -214,12 +240,13 @@ fn free(context: *anyopaque, memory: []u8, alignment: mem.Alignment, ra: usize)...@@ -214,12 +240,13 @@ fn free(context: *anyopaque, memory: []u8, alignment: mem.Alignment, ra: usize)
214 return PageAllocator.unmap(@alignCast(memory));240 return PageAllocator.unmap(@alignCast(memory));
215 }241 }
216242
217 const t = Thread.lock();
218 defer t.unlock();
219
220 const slot_size = slotSize(class);243 const slot_size = slotSize(class);
221 const addr = @intFromPtr(memory.ptr);244 const addr = @intFromPtr(memory.ptr);
222 const node: *usize = @ptrFromInt(addr + (slot_size - @sizeOf(usize)));245 const node: *usize = @ptrFromInt(addr + (slot_size - @sizeOf(usize)));
246
247 const t = Thread.lock();
248 defer t.unlock();
249
223 node.* = t.frees[class];250 node.* = t.frees[class];
224 t.frees[class] = addr;251 t.frees[class] = addr;
225}252}