authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-02-08 04:54:38-08:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2025-02-08 04:54:38-08:00
logea1ce2df9b20b5c91278eb0ed99a9cd0b0949e1a
treea3456166f71ca3a8dc56c99da9452ef3ffb4aa25
parent3fe981e1ad746f9e3dfa2006fc69c907c92ddce6
parent975cd9fc4ff8c12ae1f54e470b72be04d26e0837
signaturebadge-check Signed by PGP key B5690EEEBB952194

Merge pull request #22808 from ziglang/fast-gpa

introduce std.heap.SmpAllocator

11 files changed, 326 insertions(+), 43 deletions(-)

bootstrap.c+1-1
......@@ -139,7 +139,7 @@ int main(int argc, char **argv) {
139139 "pub const enable_tracy = false;\n"
140140 "pub const value_tracing = false;\n"
141141 "pub const skip_non_native = false;\n"
142 "pub const force_gpa = false;\n"
142 "pub const debug_gpa = false;\n"
143143 "pub const dev = .core;\n"
144144 "pub const value_interpret_mode = .direct;\n"
145145 , zig_version);
build.zig+3-3
......@@ -171,7 +171,7 @@ pub fn build(b: *std.Build) !void {
171171 const tracy_callstack = b.option(bool, "tracy-callstack", "Include callstack information with Tracy data. Does nothing if -Dtracy is not provided") orelse (tracy != null);
172172 const tracy_allocation = b.option(bool, "tracy-allocation", "Include allocation information with Tracy data. Does nothing if -Dtracy is not provided") orelse (tracy != null);
173173 const tracy_callstack_depth: u32 = b.option(u32, "tracy-callstack-depth", "Declare callstack depth for Tracy data. Does nothing if -Dtracy_callstack is not provided") orelse 10;
174 const force_gpa = b.option(bool, "force-gpa", "Force the compiler to use GeneralPurposeAllocator") orelse false;
174 const debug_gpa = b.option(bool, "debug-allocator", "Force the compiler to use DebugAllocator") orelse false;
175175 const link_libc = b.option(bool, "force-link-libc", "Force self-hosted compiler to link libc") orelse (enable_llvm or only_c);
176176 const sanitize_thread = b.option(bool, "sanitize-thread", "Enable thread-sanitization") orelse false;
177177 const strip = b.option(bool, "strip", "Omit debug information");
......@@ -233,7 +233,7 @@ pub fn build(b: *std.Build) !void {
233233 exe_options.addOption(bool, "llvm_has_csky", llvm_has_csky);
234234 exe_options.addOption(bool, "llvm_has_arc", llvm_has_arc);
235235 exe_options.addOption(bool, "llvm_has_xtensa", llvm_has_xtensa);
236 exe_options.addOption(bool, "force_gpa", force_gpa);
236 exe_options.addOption(bool, "debug_gpa", debug_gpa);
237237 exe_options.addOption(DevEnv, "dev", b.option(DevEnv, "dev", "Build a compiler with a reduced feature set for development of specific features") orelse if (only_c) .bootstrap else .full);
238238 exe_options.addOption(ValueInterpretMode, "value_interpret_mode", value_interpret_mode);
239239
......@@ -608,7 +608,7 @@ fn addWasiUpdateStep(b: *std.Build, version: [:0]const u8) !void {
608608
609609 exe_options.addOption(u32, "mem_leak_frames", 0);
610610 exe_options.addOption(bool, "have_llvm", false);
611 exe_options.addOption(bool, "force_gpa", false);
611 exe_options.addOption(bool, "debug_gpa", false);
612612 exe_options.addOption([:0]const u8, "version", version);
613613 exe_options.addOption(std.SemanticVersion, "semver", semver);
614614 exe_options.addOption(bool, "enable_debug_extensions", false);
lib/libc/musl/src/thread/riscv32/clone.s+2
......@@ -7,6 +7,8 @@
77.global __clone
88.type __clone, %function
99__clone:
10 andi a1, a1, -16
11
1012 # Save func and arg to stack
1113 addi a1, a1, -16
1214 sw a0, 0(a1)
lib/libc/musl/src/thread/riscv64/clone.s+2
......@@ -7,6 +7,8 @@
77.global __clone
88.type __clone, %function
99__clone:
10 andi a1, a1, -16
11
1012 # Save func and arg to stack
1113 addi a1, a1, -16
1214 sd a0, 0(a1)
lib/std/heap.zig+20-5
......@@ -9,11 +9,12 @@ const Allocator = std.mem.Allocator;
99const windows = std.os.windows;
1010
1111pub const ArenaAllocator = @import("heap/arena_allocator.zig").ArenaAllocator;
12pub const WasmAllocator = @import("heap/WasmAllocator.zig");
12pub const SmpAllocator = @import("heap/SmpAllocator.zig");
13pub const FixedBufferAllocator = @import("heap/FixedBufferAllocator.zig");
1314pub const PageAllocator = @import("heap/PageAllocator.zig");
14pub const ThreadSafeAllocator = @import("heap/ThreadSafeAllocator.zig");
1515pub const SbrkAllocator = @import("heap/sbrk_allocator.zig").SbrkAllocator;
16pub const FixedBufferAllocator = @import("heap/FixedBufferAllocator.zig");
16pub const ThreadSafeAllocator = @import("heap/ThreadSafeAllocator.zig");
17pub const WasmAllocator = @import("heap/WasmAllocator.zig");
1718
1819pub const DebugAllocatorConfig = @import("heap/debug_allocator.zig").Config;
1920pub const DebugAllocator = @import("heap/debug_allocator.zig").DebugAllocator;
......@@ -358,6 +359,11 @@ else if (builtin.target.isWasm()) .{
358359 .vtable = &PageAllocator.vtable,
359360};
360361
362pub const smp_allocator: Allocator = .{
363 .ptr = undefined,
364 .vtable = &SmpAllocator.vtable,
365};
366
361367/// This allocator is fast, small, and specific to WebAssembly. In the future,
362368/// this will be the implementation automatically selected by
363369/// `GeneralPurposeAllocator` when compiling in `ReleaseSmall` mode for wasm32
......@@ -475,7 +481,7 @@ pub fn StackFallbackAllocator(comptime size: usize) type {
475481 };
476482}
477483
478test "c_allocator" {
484test c_allocator {
479485 if (builtin.link_libc) {
480486 try testAllocator(c_allocator);
481487 try testAllocatorAligned(c_allocator);
......@@ -484,12 +490,20 @@ test "c_allocator" {
484490 }
485491}
486492
487test "raw_c_allocator" {
493test raw_c_allocator {
488494 if (builtin.link_libc) {
489495 try testAllocator(raw_c_allocator);
490496 }
491497}
492498
499test smp_allocator {
500 if (builtin.single_threaded) return;
501 try testAllocator(smp_allocator);
502 try testAllocatorAligned(smp_allocator);
503 try testAllocatorLargeAlignment(smp_allocator);
504 try testAllocatorAlignedShrink(smp_allocator);
505}
506
493507test PageAllocator {
494508 const allocator = page_allocator;
495509 try testAllocator(allocator);
......@@ -978,4 +992,5 @@ test {
978992 if (builtin.target.isWasm()) {
979993 _ = WasmAllocator;
980994 }
995 if (!builtin.single_threaded) _ = smp_allocator;
981996}
lib/std/heap/PageAllocator.zig+18-12
......@@ -16,11 +16,7 @@ pub const vtable: Allocator.VTable = .{
1616 .free = free,
1717};
1818
19fn alloc(context: *anyopaque, n: usize, alignment: mem.Alignment, ra: usize) ?[*]u8 {
20 _ = context;
21 _ = ra;
22 assert(n > 0);
23
19pub fn map(n: usize, alignment: mem.Alignment) ?[*]u8 {
2420 const page_size = std.heap.pageSize();
2521 if (n >= maxInt(usize) - page_size) return null;
2622 const alignment_bytes = alignment.toByteUnits();
......@@ -101,6 +97,13 @@ fn alloc(context: *anyopaque, n: usize, alignment: mem.Alignment, ra: usize) ?[*
10197 return result_ptr;
10298}
10399
100fn alloc(context: *anyopaque, n: usize, alignment: mem.Alignment, ra: usize) ?[*]u8 {
101 _ = context;
102 _ = ra;
103 assert(n > 0);
104 return map(n, alignment);
105}
106
104107fn resize(
105108 context: *anyopaque,
106109 memory: []u8,
......@@ -114,7 +117,7 @@ fn resize(
114117 return realloc(memory, new_len, false) != null;
115118}
116119
117pub fn remap(
120fn remap(
118121 context: *anyopaque,
119122 memory: []u8,
120123 alignment: mem.Alignment,
......@@ -127,21 +130,24 @@ pub fn remap(
127130 return realloc(memory, new_len, true);
128131}
129132
130fn free(context: *anyopaque, slice: []u8, alignment: mem.Alignment, return_address: usize) void {
133fn free(context: *anyopaque, memory: []u8, alignment: mem.Alignment, return_address: usize) void {
131134 _ = context;
132135 _ = alignment;
133136 _ = return_address;
137 return unmap(@alignCast(memory));
138}
134139
140pub fn unmap(memory: []align(page_size_min) u8) void {
135141 if (native_os == .windows) {
136 windows.VirtualFree(slice.ptr, 0, windows.MEM_RELEASE);
142 windows.VirtualFree(memory.ptr, 0, windows.MEM_RELEASE);
137143 } else {
138 const buf_aligned_len = mem.alignForward(usize, slice.len, std.heap.pageSize());
139 posix.munmap(@alignCast(slice.ptr[0..buf_aligned_len]));
144 const page_aligned_len = mem.alignForward(usize, memory.len, std.heap.pageSize());
145 posix.munmap(memory.ptr[0..page_aligned_len]);
140146 }
141147}
142148
143fn realloc(uncasted_memory: []u8, new_len: usize, may_move: bool) ?[*]u8 {
144 const memory: []align(std.heap.page_size_min) u8 = @alignCast(uncasted_memory);
149pub fn realloc(uncasted_memory: []u8, new_len: usize, may_move: bool) ?[*]u8 {
150 const memory: []align(page_size_min) u8 = @alignCast(uncasted_memory);
145151 const page_size = std.heap.pageSize();
146152 const new_size_aligned = mem.alignForward(usize, new_len, page_size);
147153
lib/std/heap/SmpAllocator.zig created+261
......@@ -0,0 +1,261 @@
1//! An allocator that is designed for ReleaseFast optimization mode, with
2//! multi-threading enabled.
3//!
4//! This allocator is a singleton; it uses global state and only one should be
5//! instantiated for the entire process.
6//!
7//! ## Basic Design
8//!
9//! Each thread gets a separate freelist, however, the data must be recoverable
10//! when the thread exits. We do not directly learn when a thread exits, so
11//! occasionally, one thread must attempt to reclaim another thread's
12//! resources.
13//!
14//! Above a certain size, those allocations are memory mapped directly, with no
15//! storage of allocation metadata. This works because the implementation
16//! refuses resizes that would move an allocation from small category to large
17//! category or vice versa.
18//!
19//! Each allocator operation checks the thread identifier from a threadlocal
20//! variable to find out which metadata in the global state to access, and
21//! attempts to grab its lock. This will usually succeed without contention,
22//! unless another thread has been assigned the same id. In the case of such
23//! contention, the thread moves on to the next thread metadata slot and
24//! repeats the process of attempting to obtain the lock.
25//!
26//! By limiting the thread-local metadata array to the same number as the CPU
27//! count, ensures that as threads are created and destroyed, they cycle
28//! through the full set of freelists.
29
30const builtin = @import("builtin");
31
32const std = @import("../std.zig");
33const assert = std.debug.assert;
34const mem = std.mem;
35const math = std.math;
36const Allocator = std.mem.Allocator;
37const SmpAllocator = @This();
38const PageAllocator = std.heap.PageAllocator;
39
40cpu_count: u32,
41threads: [max_thread_count]Thread,
42
43var global: SmpAllocator = .{
44 .threads = @splat(.{}),
45 .cpu_count = 0,
46};
47threadlocal var thread_index: u32 = 0;
48
49const max_thread_count = 128;
50const slab_len: usize = @max(std.heap.page_size_max, 64 * 1024);
51/// Because of storing free list pointers, the minimum size class is 3.
52const min_class = math.log2(@sizeOf(usize));
53const size_class_count = math.log2(slab_len) - min_class;
54/// When a freelist length exceeds this number, a `free` will rotate up to
55/// `max_free_search` times before pushing.
56const max_freelist_len: u8 = 16;
57const max_free_search = 1;
58/// Before mapping a fresh page, `alloc` will rotate this many times.
59const max_alloc_search = 1;
60
61const Thread = struct {
62 /// Avoid false sharing.
63 _: void align(std.atomic.cache_line) = {},
64
65 /// Protects the state in this struct (per-thread state).
66 ///
67 /// Threads lock this before accessing their own state in order
68 /// to support freelist reclamation.
69 mutex: std.Thread.Mutex = .{},
70
71 /// For each size class, tracks the next address to be returned from
72 /// `alloc` when the freelist is empty.
73 next_addrs: [size_class_count]usize = @splat(0),
74 /// For each size class, points to the freed pointer.
75 frees: [size_class_count]usize = @splat(0),
76 /// For each size class, tracks the number of items in the freelist.
77 freelist_lens: [size_class_count]u8 = @splat(0),
78
79 fn lock() *Thread {
80 var index = thread_index;
81 {
82 const t = &global.threads[index];
83 if (t.mutex.tryLock()) {
84 @branchHint(.likely);
85 return t;
86 }
87 }
88 const cpu_count = getCpuCount();
89 assert(cpu_count != 0);
90 while (true) {
91 index = (index + 1) % cpu_count;
92 const t = &global.threads[index];
93 if (t.mutex.tryLock()) {
94 thread_index = index;
95 return t;
96 }
97 }
98 }
99
100 fn unlock(t: *Thread) void {
101 t.mutex.unlock();
102 }
103};
104
105fn getCpuCount() u32 {
106 const cpu_count = @atomicLoad(u32, &global.cpu_count, .unordered);
107 if (cpu_count != 0) return cpu_count;
108 const n: u32 = @min(std.Thread.getCpuCount() catch max_thread_count, max_thread_count);
109 return if (@cmpxchgStrong(u32, &global.cpu_count, 0, n, .monotonic, .monotonic)) |other| other else n;
110}
111
112pub const vtable: Allocator.VTable = .{
113 .alloc = alloc,
114 .resize = resize,
115 .remap = remap,
116 .free = free,
117};
118
119comptime {
120 assert(!builtin.single_threaded); // you're holding it wrong
121}
122
123fn alloc(context: *anyopaque, len: usize, alignment: mem.Alignment, ra: usize) ?[*]u8 {
124 _ = context;
125 _ = ra;
126 const class = sizeClassIndex(len, alignment);
127 if (class >= size_class_count) {
128 @branchHint(.unlikely);
129 return PageAllocator.map(len, alignment);
130 }
131
132 const slot_size = slotSize(class);
133 assert(slab_len % slot_size == 0);
134 var search_count: u8 = 0;
135
136 var t = Thread.lock();
137
138 outer: while (true) {
139 const top_free_ptr = t.frees[class];
140 if (top_free_ptr != 0) {
141 @branchHint(.likely);
142 defer t.unlock();
143 const node: *usize = @ptrFromInt(top_free_ptr);
144 t.frees[class] = node.*;
145 t.freelist_lens[class] -|= 1;
146 return @ptrFromInt(top_free_ptr);
147 }
148
149 const next_addr = t.next_addrs[class];
150 if ((next_addr % slab_len) != 0) {
151 @branchHint(.likely);
152 defer t.unlock();
153 t.next_addrs[class] = next_addr + slot_size;
154 return @ptrFromInt(next_addr);
155 }
156
157 if (search_count >= max_alloc_search) {
158 @branchHint(.likely);
159 defer t.unlock();
160 // slab alignment here ensures the % slab len earlier catches the end of slots.
161 const slab = PageAllocator.map(slab_len, .fromByteUnits(slab_len)) orelse return null;
162 t.next_addrs[class] = @intFromPtr(slab) + slot_size;
163 t.freelist_lens[class] = 0;
164 return slab;
165 }
166
167 t.unlock();
168 const cpu_count = getCpuCount();
169 assert(cpu_count != 0);
170 var index = thread_index;
171 while (true) {
172 index = (index + 1) % cpu_count;
173 t = &global.threads[index];
174 if (t.mutex.tryLock()) {
175 thread_index = index;
176 search_count += 1;
177 continue :outer;
178 }
179 }
180 }
181}
182
183fn resize(context: *anyopaque, memory: []u8, alignment: mem.Alignment, new_len: usize, ra: usize) bool {
184 _ = context;
185 _ = ra;
186 const class = sizeClassIndex(memory.len, alignment);
187 const new_class = sizeClassIndex(new_len, alignment);
188 if (class >= size_class_count) {
189 if (new_class < size_class_count) return false;
190 return PageAllocator.realloc(memory, new_len, false) != null;
191 }
192 return new_class == class;
193}
194
195fn remap(context: *anyopaque, memory: []u8, alignment: mem.Alignment, new_len: usize, ra: usize) ?[*]u8 {
196 _ = context;
197 _ = ra;
198 const class = sizeClassIndex(memory.len, alignment);
199 const new_class = sizeClassIndex(new_len, alignment);
200 if (class >= size_class_count) {
201 if (new_class < size_class_count) return null;
202 return PageAllocator.realloc(memory, new_len, true);
203 }
204 return if (new_class == class) memory.ptr else null;
205}
206
207fn free(context: *anyopaque, memory: []u8, alignment: mem.Alignment, ra: usize) void {
208 _ = context;
209 _ = ra;
210 const class = sizeClassIndex(memory.len, alignment);
211 if (class >= size_class_count) {
212 @branchHint(.unlikely);
213 return PageAllocator.unmap(@alignCast(memory));
214 }
215
216 const node: *usize = @alignCast(@ptrCast(memory.ptr));
217 var search_count: u8 = 0;
218
219 var t = Thread.lock();
220
221 outer: while (true) {
222 const freelist_len = t.freelist_lens[class];
223 if (freelist_len < max_freelist_len) {
224 @branchHint(.likely);
225 defer t.unlock();
226 node.* = t.frees[class];
227 t.frees[class] = @intFromPtr(node);
228 return;
229 }
230
231 if (search_count >= max_free_search) {
232 defer t.unlock();
233 t.freelist_lens[class] = freelist_len +| 1;
234 node.* = t.frees[class];
235 t.frees[class] = @intFromPtr(node);
236 return;
237 }
238
239 t.unlock();
240 const cpu_count = getCpuCount();
241 assert(cpu_count != 0);
242 var index = thread_index;
243 while (true) {
244 index = (index + 1) % cpu_count;
245 t = &global.threads[index];
246 if (t.mutex.tryLock()) {
247 thread_index = index;
248 search_count += 1;
249 continue :outer;
250 }
251 }
252 }
253}
254
255fn sizeClassIndex(len: usize, alignment: mem.Alignment) usize {
256 return @max(@bitSizeOf(usize) - @clz(len - 1), @intFromEnum(alignment), min_class) - min_class;
257}
258
259fn slotSize(class: usize) usize {
260 return @as(usize, 1) << @intCast(class + min_class);
261}
lib/std/heap/WasmAllocator.zig-2
......@@ -1,5 +1,3 @@
1//! This is intended to be merged into GeneralPurposeAllocator at some point.
2
31const std = @import("../std.zig");
42const builtin = @import("builtin");
53const Allocator = std.mem.Allocator;
lib/std/heap/debug_allocator.zig-2
......@@ -851,8 +851,6 @@ pub fn DebugAllocator(comptime config: Config) type {
851851 self.mutex.lock();
852852 defer self.mutex.unlock();
853853
854 assert(old_memory.len != 0);
855
856854 const size_class_index: usize = @max(@bitSizeOf(usize) - @clz(old_memory.len - 1), @intFromEnum(alignment));
857855 if (size_class_index >= self.buckets.len) {
858856 @branchHint(.unlikely);
src/main.zig+18-17
......@@ -171,30 +171,31 @@ pub fn log(
171171 std.debug.print(prefix1 ++ prefix2 ++ format ++ "\n", args);
172172}
173173
174var general_purpose_allocator = std.heap.GeneralPurposeAllocator(.{
174var debug_allocator: std.heap.DebugAllocator(.{
175175 .stack_trace_frames = build_options.mem_leak_frames,
176}){};
176}) = .init;
177177
178178pub fn main() anyerror!void {
179179 crash_report.initialize();
180180
181 const use_gpa = (build_options.force_gpa or !builtin.link_libc) and native_os != .wasi;
182 const gpa = gpa: {
183 if (native_os == .wasi) {
184 break :gpa std.heap.wasm_allocator;
185 }
186 if (use_gpa) {
187 break :gpa general_purpose_allocator.allocator();
188 }
189 // We would prefer to use raw libc allocator here, but cannot
190 // use it if it won't support the alignment we need.
191 if (@alignOf(std.c.max_align_t) < @max(@alignOf(i128), std.atomic.cache_line)) {
192 break :gpa std.heap.c_allocator;
181 const gpa, const is_debug = gpa: {
182 if (build_options.debug_gpa) break :gpa .{ debug_allocator.allocator(), true };
183 if (native_os == .wasi) break :gpa .{ std.heap.wasm_allocator, false };
184 if (builtin.link_libc) {
185 // We would prefer to use raw libc allocator here, but cannot use
186 // it if it won't support the alignment we need.
187 if (@alignOf(std.c.max_align_t) < @max(@alignOf(i128), std.atomic.cache_line)) {
188 break :gpa .{ std.heap.c_allocator, false };
189 }
190 break :gpa .{ std.heap.raw_c_allocator, false };
193191 }
194 break :gpa std.heap.raw_c_allocator;
192 break :gpa switch (builtin.mode) {
193 .Debug, .ReleaseSafe => .{ debug_allocator.allocator(), true },
194 .ReleaseFast, .ReleaseSmall => .{ std.heap.smp_allocator, false },
195 };
195196 };
196 defer if (use_gpa) {
197 _ = general_purpose_allocator.deinit();
197 defer if (is_debug) {
198 _ = debug_allocator.deinit();
198199 };
199200 var arena_instance = std.heap.ArenaAllocator.init(gpa);
200201 defer arena_instance.deinit();
stage1/config.zig.in+1-1
......@@ -11,6 +11,6 @@ pub const enable_link_snapshots = false;
1111pub const enable_tracy = false;
1212pub const value_tracing = false;
1313pub const skip_non_native = false;
14pub const force_gpa = false;
14pub const debug_gpa = false;
1515pub const dev = .core;
1616pub const value_interpret_mode = .direct;