authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2026-02-10 19:28:43-08:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2026-02-12 13:14:51-08:00
log6ccabbd4e558dd0d56c471408c4cf29e0992dc1c
tree0756cc89b01f788c55a853ad3f521ed342a3b355
parent6744160211b2ab480e4c479e46440c76f00c1810

std: brk allocator for single-threaded mode

6 files changed, 372 insertions(+), 523 deletions(-)

lib/c/malloc.zig+3-3
...@@ -1,7 +1,7 @@...@@ -1,7 +1,7 @@
1//! Based on wrapping a stateless Zig Allocator implementation, apropriate for:1//! Based on wrapping a stateless Zig Allocator implementation, appropriate for:
2//! - ReleaseFast and ReleaseSmall optimization modes, with multi-threading2//! - ReleaseFast and ReleaseSmall optimization modes, with multi-threading
3//! enabled.3//! enabled.
4//! - WebAssembly in single-threaded mode.4//! - WebAssembly or Linux in single-threaded release modes.
5//!5//!
6//! Because the libc APIs don't have client alignment and size tracking, in6//! Because the libc APIs don't have client alignment and size tracking, in
7//! order to take advantage of Zig allocator implementations, additional7//! order to take advantage of Zig allocator implementations, additional
...@@ -40,7 +40,7 @@ const no_context: *anyopaque = undefined;...@@ -40,7 +40,7 @@ const no_context: *anyopaque = undefined;
40const no_ra: usize = undefined;40const no_ra: usize = undefined;
41const vtable = switch (builtin.cpu.arch) {41const vtable = switch (builtin.cpu.arch) {
42 .wasm32, .wasm64 => std.heap.WasmAllocator.vtable,42 .wasm32, .wasm64 => std.heap.WasmAllocator.vtable,
43 else => std.heap.SmpAllocator.vtable,43 else => if (builtin.single_threaded) std.heap.BrkAllocator.vtable else std.heap.SmpAllocator.vtable,
44};44};
4545
46/// Needed because libc memory allocators don't provide old alignment and size46/// Needed because libc memory allocators don't provide old alignment and size
lib/std/heap.zig+15-14
...@@ -13,9 +13,9 @@ pub const ArenaAllocator = @import("heap/arena_allocator.zig").ArenaAllocator;...@@ -13,9 +13,9 @@ pub const ArenaAllocator = @import("heap/arena_allocator.zig").ArenaAllocator;
13pub const SmpAllocator = @import("heap/SmpAllocator.zig");13pub const SmpAllocator = @import("heap/SmpAllocator.zig");
14pub const FixedBufferAllocator = @import("heap/FixedBufferAllocator.zig");14pub const FixedBufferAllocator = @import("heap/FixedBufferAllocator.zig");
15pub const PageAllocator = @import("heap/PageAllocator.zig");15pub const PageAllocator = @import("heap/PageAllocator.zig");
16pub const SbrkAllocator = @import("heap/sbrk_allocator.zig").SbrkAllocator;
17pub const ThreadSafeAllocator = @import("heap/ThreadSafeAllocator.zig");16pub const ThreadSafeAllocator = @import("heap/ThreadSafeAllocator.zig");
18pub const WasmAllocator = @import("heap/WasmAllocator.zig");17pub const WasmAllocator = if (builtin.single_threaded) BrkAllocator else @compileError("unimplemented");
18pub const BrkAllocator = @import("heap/BrkAllocator.zig");
1919
20pub const DebugAllocatorConfig = @import("heap/debug_allocator.zig").Config;20pub const DebugAllocatorConfig = @import("heap/debug_allocator.zig").Config;
21pub const DebugAllocator = @import("heap/debug_allocator.zig").DebugAllocator;21pub const DebugAllocator = @import("heap/debug_allocator.zig").DebugAllocator;
...@@ -356,9 +356,6 @@ pub const page_allocator: Allocator = if (@hasDecl(root, "os") and...@@ -356,9 +356,6 @@ pub const page_allocator: Allocator = if (@hasDecl(root, "os") and
356else if (builtin.target.cpu.arch.isWasm()) .{356else if (builtin.target.cpu.arch.isWasm()) .{
357 .ptr = undefined,357 .ptr = undefined,
358 .vtable = &WasmAllocator.vtable,358 .vtable = &WasmAllocator.vtable,
359} else if (builtin.target.os.tag == .plan9) .{
360 .ptr = undefined,
361 .vtable = &SbrkAllocator(std.os.plan9.sbrk).vtable,
362} else .{359} else .{
363 .ptr = undefined,360 .ptr = undefined,
364 .vtable = &PageAllocator.vtable,361 .vtable = &PageAllocator.vtable,
...@@ -369,16 +366,18 @@ pub const smp_allocator: Allocator = .{...@@ -369,16 +366,18 @@ pub const smp_allocator: Allocator = .{
369 .vtable = &SmpAllocator.vtable,366 .vtable = &SmpAllocator.vtable,
370};367};
371368
372/// This allocator is fast, small, and specific to WebAssembly. In the future,369/// This allocator is fast, small, and specific to WebAssembly.
373/// this will be the implementation automatically selected by
374/// `GeneralPurposeAllocator` when compiling in `ReleaseSmall` mode for wasm32
375/// and wasm64 architectures.
376/// Until then, it is available here to play with.
377pub const wasm_allocator: Allocator = .{370pub const wasm_allocator: Allocator = .{
378 .ptr = undefined,371 .ptr = undefined,
379 .vtable = &WasmAllocator.vtable,372 .vtable = &WasmAllocator.vtable,
380};373};
381374
375/// Supports single-threaded WebAssembly and Linux.
376pub const brk_allocator: Allocator = .{
377 .ptr = undefined,
378 .vtable = &BrkAllocator.vtable,
379};
380
382/// Returns a `StackFallbackAllocator` allocating using either a381/// Returns a `StackFallbackAllocator` allocating using either a
383/// `FixedBufferAllocator` on an array of size `size` and falling back to382/// `FixedBufferAllocator` on an array of size `size` and falling back to
384/// `fallback_allocator` if that fails.383/// `fallback_allocator` if that fails.
...@@ -1014,9 +1013,11 @@ test {...@@ -1014,9 +1013,11 @@ test {
1014 _ = GeneralPurposeAllocator;1013 _ = GeneralPurposeAllocator;
1015 _ = FixedBufferAllocator;1014 _ = FixedBufferAllocator;
1016 _ = ThreadSafeAllocator;1015 _ = ThreadSafeAllocator;
1017 _ = SbrkAllocator;1016 if (builtin.single_threaded) {
1018 if (builtin.target.cpu.arch.isWasm()) {1017 if (builtin.cpu.arch.isWasm() or (builtin.os.tag == .linux and !builtin.link_libc)) {
1019 _ = WasmAllocator;1018 _ = brk_allocator;
1019 }
1020 } else {
1021 _ = smp_allocator;
1020 }1022 }
1021 if (!builtin.single_threaded) _ = smp_allocator;
1022}1023}
lib/std/heap/BrkAllocator.zig created+350
...@@ -0,0 +1,350 @@
1//! Supports single-threaded targets that have a sbrk-like primitive which includes
2//! Linux and WebAssembly.
3//!
4//! On Linux, assumes exclusive access to the brk syscall.
5const BrkAllocator = @This();
6const builtin = @import("builtin");
7
8const std = @import("../std.zig");
9const Allocator = std.mem.Allocator;
10const Alignment = std.mem.Alignment;
11const assert = std.debug.assert;
12const math = std.math;
13const page_size_max = std.heap.page_size_max;
14
15comptime {
16 if (!builtin.single_threaded) @compileError("unsupported");
17}
18
19next_addrs: [size_class_count]usize = @splat(0),
20/// For each size class, points to the freed pointer.
21frees: [size_class_count]usize = @splat(0),
22/// For each big size class, points to the freed pointer.
23big_frees: [big_size_class_count]usize = @splat(0),
24prev_brk: usize = 0,
25
26var global: BrkAllocator = .{};
27
28pub const vtable: Allocator.VTable = .{
29 .alloc = alloc,
30 .resize = resize,
31 .remap = remap,
32 .free = free,
33};
34
35pub const Error = Allocator.Error;
36
37const max_usize = math.maxInt(usize);
38const ushift = math.Log2Int(usize);
39const bigpage_size = 64 * 1024;
40const pages_per_bigpage = bigpage_size / page_size_max;
41const bigpage_count = max_usize / bigpage_size;
42
43comptime {
44 assert(bigpage_size >= page_size_max);
45}
46
47/// Because of storing free list pointers, the minimum size class is 3.
48const min_class = math.log2(math.ceilPowerOfTwoAssert(usize, 1 + @sizeOf(usize)));
49const size_class_count = math.log2(bigpage_size) - min_class;
50/// 0 - 1 bigpage
51/// 1 - 2 bigpages
52/// 2 - 4 bigpages
53/// etc.
54const big_size_class_count = math.log2(bigpage_count);
55
56fn alloc(ctx: *anyopaque, len: usize, alignment: Alignment, return_address: usize) ?[*]u8 {
57 _ = ctx;
58 _ = return_address;
59 // Make room for the freelist next pointer.
60 const actual_len = @max(len +| @sizeOf(usize), alignment.toByteUnits());
61 const slot_size = math.ceilPowerOfTwo(usize, actual_len) catch return null;
62 const class = math.log2(slot_size) - min_class;
63 if (class < size_class_count) {
64 const addr = a: {
65 const top_free_ptr = global.frees[class];
66 if (top_free_ptr != 0) {
67 const node: *usize = @ptrFromInt(top_free_ptr + (slot_size - @sizeOf(usize)));
68 global.frees[class] = node.*;
69 break :a top_free_ptr;
70 }
71
72 const next_addr = global.next_addrs[class];
73 if (next_addr % page_size_max == 0) {
74 const addr = allocBigPages(1);
75 if (addr == 0) return null;
76 //std.debug.print("allocated fresh slot_size={d} class={d} addr=0x{x}\n", .{
77 // slot_size, class, addr,
78 //});
79 global.next_addrs[class] = addr + slot_size;
80 break :a addr;
81 } else {
82 global.next_addrs[class] = next_addr + slot_size;
83 break :a next_addr;
84 }
85 };
86 return @ptrFromInt(addr);
87 }
88 const bigpages_needed = bigPagesNeeded(actual_len);
89 return @ptrFromInt(allocBigPages(bigpages_needed));
90}
91
92fn resize(
93 ctx: *anyopaque,
94 buf: []u8,
95 alignment: Alignment,
96 new_len: usize,
97 return_address: usize,
98) bool {
99 _ = ctx;
100 _ = return_address;
101 // We don't want to move anything from one size class to another, but we
102 // can recover bytes in between powers of two.
103 const buf_align = alignment.toByteUnits();
104 const old_actual_len = @max(buf.len + @sizeOf(usize), buf_align);
105 const new_actual_len = @max(new_len +| @sizeOf(usize), buf_align);
106 const old_small_slot_size = math.ceilPowerOfTwoAssert(usize, old_actual_len);
107 const old_small_class = math.log2(old_small_slot_size) - min_class;
108 if (old_small_class < size_class_count) {
109 const new_small_slot_size = math.ceilPowerOfTwo(usize, new_actual_len) catch return false;
110 return old_small_slot_size == new_small_slot_size;
111 } else {
112 const old_bigpages_needed = bigPagesNeeded(old_actual_len);
113 const old_big_slot_pages = math.ceilPowerOfTwoAssert(usize, old_bigpages_needed);
114 const new_bigpages_needed = bigPagesNeeded(new_actual_len);
115 const new_big_slot_pages = math.ceilPowerOfTwo(usize, new_bigpages_needed) catch return false;
116 return old_big_slot_pages == new_big_slot_pages;
117 }
118}
119
120fn remap(
121 context: *anyopaque,
122 memory: []u8,
123 alignment: Alignment,
124 new_len: usize,
125 return_address: usize,
126) ?[*]u8 {
127 return if (resize(context, memory, alignment, new_len, return_address)) memory.ptr else null;
128}
129
130fn free(
131 ctx: *anyopaque,
132 buf: []u8,
133 alignment: Alignment,
134 return_address: usize,
135) void {
136 _ = ctx;
137 _ = return_address;
138 const buf_align = alignment.toByteUnits();
139 const actual_len = @max(buf.len + @sizeOf(usize), buf_align);
140 const slot_size = math.ceilPowerOfTwoAssert(usize, actual_len);
141 const class = math.log2(slot_size) - min_class;
142 const addr = @intFromPtr(buf.ptr);
143 if (class < size_class_count) {
144 const node: *usize = @ptrFromInt(addr + (slot_size - @sizeOf(usize)));
145 node.* = global.frees[class];
146 global.frees[class] = addr;
147 } else {
148 const bigpages_needed = bigPagesNeeded(actual_len);
149 const pow2_pages = math.ceilPowerOfTwoAssert(usize, bigpages_needed);
150 const big_slot_size_bytes = pow2_pages * bigpage_size;
151 const node: *usize = @ptrFromInt(addr + (big_slot_size_bytes - @sizeOf(usize)));
152 const big_class = math.log2(pow2_pages);
153 node.* = global.big_frees[big_class];
154 global.big_frees[big_class] = addr;
155 }
156}
157
158inline fn bigPagesNeeded(byte_count: usize) usize {
159 return (byte_count + (bigpage_size + (@sizeOf(usize) - 1))) / bigpage_size;
160}
161
162fn allocBigPages(n: usize) usize {
163 const pow2_pages = math.ceilPowerOfTwoAssert(usize, n);
164 const slot_size_bytes = pow2_pages * bigpage_size;
165 const class = math.log2(pow2_pages);
166
167 const top_free_ptr = global.big_frees[class];
168 if (top_free_ptr != 0) {
169 const node: *usize = @ptrFromInt(top_free_ptr + (slot_size_bytes - @sizeOf(usize)));
170 global.big_frees[class] = node.*;
171 return top_free_ptr;
172 }
173
174 if (builtin.cpu.arch.isWasm()) {
175 const page_index = @wasmMemoryGrow(0, pow2_pages * pages_per_bigpage);
176 if (page_index == -1) return 0;
177 return @as(usize, @intCast(page_index)) * page_size_max;
178 } else if (builtin.os.tag == .linux) {
179 const start_brk = s: {
180 const start_brk = global.prev_brk;
181 break :s if (start_brk == 0) std.os.linux.brk(0) else start_brk;
182 };
183 const end_brk = start_brk + pow2_pages * pages_per_bigpage * page_size_max;
184 const new_prev_brk = std.os.linux.brk(end_brk);
185 global.prev_brk = new_prev_brk;
186 if (new_prev_brk != end_brk) return 0;
187 return start_brk;
188 } else {
189 @compileError("no sbrk-like OS primitive available");
190 }
191}
192
193const test_ally: Allocator = .{
194 .ptr = undefined,
195 .vtable = &vtable,
196};
197
198test "small allocations - free in same order" {
199 var list: [513]*u64 = undefined;
200
201 var i: usize = 0;
202 while (i < 513) : (i += 1) {
203 const ptr = try test_ally.create(u64);
204 list[i] = ptr;
205 }
206
207 for (list) |ptr| {
208 test_ally.destroy(ptr);
209 }
210}
211
212test "small allocations - free in reverse order" {
213 var list: [513]*u64 = undefined;
214
215 var i: usize = 0;
216 while (i < 513) : (i += 1) {
217 const ptr = try test_ally.create(u64);
218 list[i] = ptr;
219 }
220
221 i = list.len;
222 while (i > 0) {
223 i -= 1;
224 const ptr = list[i];
225 test_ally.destroy(ptr);
226 }
227}
228
229test "large allocations" {
230 const ptr1 = try test_ally.alloc(u64, 42768);
231 const ptr2 = try test_ally.alloc(u64, 52768);
232 test_ally.free(ptr1);
233 const ptr3 = try test_ally.alloc(u64, 62768);
234 test_ally.free(ptr3);
235 test_ally.free(ptr2);
236}
237
238test "very large allocation" {
239 try std.testing.expectError(error.OutOfMemory, test_ally.alloc(u8, math.maxInt(usize)));
240}
241
242test "realloc" {
243 var slice = try test_ally.alignedAlloc(u8, .of(u32), 1);
244 defer test_ally.free(slice);
245 slice[0] = 0x12;
246
247 // This reallocation should keep its pointer address.
248 const old_slice = slice;
249 slice = try test_ally.realloc(slice, 2);
250 try std.testing.expect(old_slice.ptr == slice.ptr);
251 try std.testing.expect(slice[0] == 0x12);
252 slice[1] = 0x34;
253
254 // This requires upgrading to a larger size class
255 slice = try test_ally.realloc(slice, 17);
256 try std.testing.expect(slice[0] == 0x12);
257 try std.testing.expect(slice[1] == 0x34);
258}
259
260test "shrink" {
261 var slice = try test_ally.alloc(u8, 20);
262 defer test_ally.free(slice);
263
264 @memset(slice, 0x11);
265
266 try std.testing.expect(test_ally.resize(slice, 17));
267 slice = slice[0..17];
268
269 for (slice) |b| {
270 try std.testing.expect(b == 0x11);
271 }
272
273 try std.testing.expect(test_ally.resize(slice, 16));
274 slice = slice[0..16];
275
276 for (slice) |b| {
277 try std.testing.expect(b == 0x11);
278 }
279}
280
281test "large object - grow" {
282 if (builtin.os.tag == .linux) return error.SkipZigTest;
283
284 var slice1 = try test_ally.alloc(u8, bigpage_size * 2 - 20);
285 defer test_ally.free(slice1);
286
287 const old = slice1;
288 slice1 = try test_ally.realloc(slice1, bigpage_size * 2 - 10);
289 try std.testing.expectEqual(slice1.ptr, old.ptr);
290
291 slice1 = try test_ally.realloc(slice1, bigpage_size * 2);
292 slice1 = try test_ally.realloc(slice1, bigpage_size * 2 + 1);
293}
294
295test "realloc small object to large object" {
296 var slice = try test_ally.alloc(u8, 70);
297 defer test_ally.free(slice);
298 slice[0] = 0x12;
299 slice[60] = 0x34;
300
301 // This requires upgrading to a large object
302 const large_object_size = bigpage_size * 2 + 50;
303 slice = try test_ally.realloc(slice, large_object_size);
304 try std.testing.expect(slice[0] == 0x12);
305 try std.testing.expect(slice[60] == 0x34);
306}
307
308test "shrink large object to large object" {
309 var slice = try test_ally.alloc(u8, bigpage_size * 2 + 50);
310 defer test_ally.free(slice);
311 slice[0] = 0x12;
312 slice[60] = 0x34;
313
314 try std.testing.expect(test_ally.resize(slice, bigpage_size * 2 + 1));
315 slice = slice[0 .. bigpage_size * 2 + 1];
316 try std.testing.expect(slice[0] == 0x12);
317 try std.testing.expect(slice[60] == 0x34);
318
319 try std.testing.expect(test_ally.resize(slice, bigpage_size * 2 + 1));
320 try std.testing.expect(slice[0] == 0x12);
321 try std.testing.expect(slice[60] == 0x34);
322
323 slice = try test_ally.realloc(slice, bigpage_size * 2);
324 try std.testing.expect(slice[0] == 0x12);
325 try std.testing.expect(slice[60] == 0x34);
326}
327
328test "realloc large object to small object" {
329 var slice = try test_ally.alloc(u8, bigpage_size * 2 + 50);
330 defer test_ally.free(slice);
331 slice[0] = 0x12;
332 slice[16] = 0x34;
333
334 slice = try test_ally.realloc(slice, 19);
335 try std.testing.expect(slice[0] == 0x12);
336 try std.testing.expect(slice[16] == 0x34);
337}
338
339test "objects of size 1024 and 2048" {
340 const slice = try test_ally.alloc(u8, 1025);
341 const slice2 = try test_ally.alloc(u8, 3000);
342
343 test_ally.free(slice);
344 test_ally.free(slice2);
345}
346
347test "standard allocator tests" {
348 try std.heap.testAllocator(test_ally);
349 try std.heap.testAllocatorAligned(test_ally);
350}
lib/std/heap/WasmAllocator.zig deleted-326
...@@ -1,326 +0,0 @@
1const std = @import("../std.zig");
2const builtin = @import("builtin");
3const Allocator = std.mem.Allocator;
4const mem = std.mem;
5const assert = std.debug.assert;
6const wasm = std.wasm;
7const math = std.math;
8
9comptime {
10 if (!builtin.target.cpu.arch.isWasm()) {
11 @compileError("only available for wasm32 arch");
12 }
13 if (!builtin.single_threaded) {
14 @compileError("TODO implement support for multi-threaded wasm");
15 }
16}
17
18pub const vtable: Allocator.VTable = .{
19 .alloc = alloc,
20 .resize = resize,
21 .remap = remap,
22 .free = free,
23};
24
25pub const Error = Allocator.Error;
26
27const max_usize = math.maxInt(usize);
28const ushift = math.Log2Int(usize);
29const bigpage_size = 64 * 1024;
30const pages_per_bigpage = bigpage_size / wasm.page_size;
31const bigpage_count = max_usize / bigpage_size;
32
33/// Because of storing free list pointers, the minimum size class is 3.
34const min_class = math.log2(math.ceilPowerOfTwoAssert(usize, 1 + @sizeOf(usize)));
35const size_class_count = math.log2(bigpage_size) - min_class;
36/// 0 - 1 bigpage
37/// 1 - 2 bigpages
38/// 2 - 4 bigpages
39/// etc.
40const big_size_class_count = math.log2(bigpage_count);
41
42var next_addrs: [size_class_count]usize = @splat(0);
43/// For each size class, points to the freed pointer.
44var frees: [size_class_count]usize = @splat(0);
45/// For each big size class, points to the freed pointer.
46var big_frees: [big_size_class_count]usize = @splat(0);
47
48fn alloc(ctx: *anyopaque, len: usize, alignment: mem.Alignment, return_address: usize) ?[*]u8 {
49 _ = ctx;
50 _ = return_address;
51 // Make room for the freelist next pointer.
52 const actual_len = @max(len +| @sizeOf(usize), alignment.toByteUnits());
53 const slot_size = math.ceilPowerOfTwo(usize, actual_len) catch return null;
54 const class = math.log2(slot_size) - min_class;
55 if (class < size_class_count) {
56 const addr = a: {
57 const top_free_ptr = frees[class];
58 if (top_free_ptr != 0) {
59 const node: *usize = @ptrFromInt(top_free_ptr + (slot_size - @sizeOf(usize)));
60 frees[class] = node.*;
61 break :a top_free_ptr;
62 }
63
64 const next_addr = next_addrs[class];
65 if (next_addr % wasm.page_size == 0) {
66 const addr = allocBigPages(1);
67 if (addr == 0) return null;
68 //std.debug.print("allocated fresh slot_size={d} class={d} addr=0x{x}\n", .{
69 // slot_size, class, addr,
70 //});
71 next_addrs[class] = addr + slot_size;
72 break :a addr;
73 } else {
74 next_addrs[class] = next_addr + slot_size;
75 break :a next_addr;
76 }
77 };
78 return @ptrFromInt(addr);
79 }
80 const bigpages_needed = bigPagesNeeded(actual_len);
81 return @ptrFromInt(allocBigPages(bigpages_needed));
82}
83
84fn resize(
85 ctx: *anyopaque,
86 buf: []u8,
87 alignment: mem.Alignment,
88 new_len: usize,
89 return_address: usize,
90) bool {
91 _ = ctx;
92 _ = return_address;
93 // We don't want to move anything from one size class to another, but we
94 // can recover bytes in between powers of two.
95 const buf_align = alignment.toByteUnits();
96 const old_actual_len = @max(buf.len + @sizeOf(usize), buf_align);
97 const new_actual_len = @max(new_len +| @sizeOf(usize), buf_align);
98 const old_small_slot_size = math.ceilPowerOfTwoAssert(usize, old_actual_len);
99 const old_small_class = math.log2(old_small_slot_size) - min_class;
100 if (old_small_class < size_class_count) {
101 const new_small_slot_size = math.ceilPowerOfTwo(usize, new_actual_len) catch return false;
102 return old_small_slot_size == new_small_slot_size;
103 } else {
104 const old_bigpages_needed = bigPagesNeeded(old_actual_len);
105 const old_big_slot_pages = math.ceilPowerOfTwoAssert(usize, old_bigpages_needed);
106 const new_bigpages_needed = bigPagesNeeded(new_actual_len);
107 const new_big_slot_pages = math.ceilPowerOfTwo(usize, new_bigpages_needed) catch return false;
108 return old_big_slot_pages == new_big_slot_pages;
109 }
110}
111
112fn remap(
113 context: *anyopaque,
114 memory: []u8,
115 alignment: mem.Alignment,
116 new_len: usize,
117 return_address: usize,
118) ?[*]u8 {
119 return if (resize(context, memory, alignment, new_len, return_address)) memory.ptr else null;
120}
121
122fn free(
123 ctx: *anyopaque,
124 buf: []u8,
125 alignment: mem.Alignment,
126 return_address: usize,
127) void {
128 _ = ctx;
129 _ = return_address;
130 const buf_align = alignment.toByteUnits();
131 const actual_len = @max(buf.len + @sizeOf(usize), buf_align);
132 const slot_size = math.ceilPowerOfTwoAssert(usize, actual_len);
133 const class = math.log2(slot_size) - min_class;
134 const addr = @intFromPtr(buf.ptr);
135 if (class < size_class_count) {
136 const node: *usize = @ptrFromInt(addr + (slot_size - @sizeOf(usize)));
137 node.* = frees[class];
138 frees[class] = addr;
139 } else {
140 const bigpages_needed = bigPagesNeeded(actual_len);
141 const pow2_pages = math.ceilPowerOfTwoAssert(usize, bigpages_needed);
142 const big_slot_size_bytes = pow2_pages * bigpage_size;
143 const node: *usize = @ptrFromInt(addr + (big_slot_size_bytes - @sizeOf(usize)));
144 const big_class = math.log2(pow2_pages);
145 node.* = big_frees[big_class];
146 big_frees[big_class] = addr;
147 }
148}
149
150inline fn bigPagesNeeded(byte_count: usize) usize {
151 return (byte_count + (bigpage_size + (@sizeOf(usize) - 1))) / bigpage_size;
152}
153
154fn allocBigPages(n: usize) usize {
155 const pow2_pages = math.ceilPowerOfTwoAssert(usize, n);
156 const slot_size_bytes = pow2_pages * bigpage_size;
157 const class = math.log2(pow2_pages);
158
159 const top_free_ptr = big_frees[class];
160 if (top_free_ptr != 0) {
161 const node: *usize = @ptrFromInt(top_free_ptr + (slot_size_bytes - @sizeOf(usize)));
162 big_frees[class] = node.*;
163 return top_free_ptr;
164 }
165
166 const page_index = @wasmMemoryGrow(0, pow2_pages * pages_per_bigpage);
167 if (page_index == -1) return 0;
168 return @as(usize, @intCast(page_index)) * wasm.page_size;
169}
170
171const test_ally: Allocator = .{
172 .ptr = undefined,
173 .vtable = &vtable,
174};
175
176test "small allocations - free in same order" {
177 var list: [513]*u64 = undefined;
178
179 var i: usize = 0;
180 while (i < 513) : (i += 1) {
181 const ptr = try test_ally.create(u64);
182 list[i] = ptr;
183 }
184
185 for (list) |ptr| {
186 test_ally.destroy(ptr);
187 }
188}
189
190test "small allocations - free in reverse order" {
191 var list: [513]*u64 = undefined;
192
193 var i: usize = 0;
194 while (i < 513) : (i += 1) {
195 const ptr = try test_ally.create(u64);
196 list[i] = ptr;
197 }
198
199 i = list.len;
200 while (i > 0) {
201 i -= 1;
202 const ptr = list[i];
203 test_ally.destroy(ptr);
204 }
205}
206
207test "large allocations" {
208 const ptr1 = try test_ally.alloc(u64, 42768);
209 const ptr2 = try test_ally.alloc(u64, 52768);
210 test_ally.free(ptr1);
211 const ptr3 = try test_ally.alloc(u64, 62768);
212 test_ally.free(ptr3);
213 test_ally.free(ptr2);
214}
215
216test "very large allocation" {
217 try std.testing.expectError(error.OutOfMemory, test_ally.alloc(u8, math.maxInt(usize)));
218}
219
220test "realloc" {
221 var slice = try test_ally.alignedAlloc(u8, .of(u32), 1);
222 defer test_ally.free(slice);
223 slice[0] = 0x12;
224
225 // This reallocation should keep its pointer address.
226 const old_slice = slice;
227 slice = try test_ally.realloc(slice, 2);
228 try std.testing.expect(old_slice.ptr == slice.ptr);
229 try std.testing.expect(slice[0] == 0x12);
230 slice[1] = 0x34;
231
232 // This requires upgrading to a larger size class
233 slice = try test_ally.realloc(slice, 17);
234 try std.testing.expect(slice[0] == 0x12);
235 try std.testing.expect(slice[1] == 0x34);
236}
237
238test "shrink" {
239 var slice = try test_ally.alloc(u8, 20);
240 defer test_ally.free(slice);
241
242 @memset(slice, 0x11);
243
244 try std.testing.expect(test_ally.resize(slice, 17));
245 slice = slice[0..17];
246
247 for (slice) |b| {
248 try std.testing.expect(b == 0x11);
249 }
250
251 try std.testing.expect(test_ally.resize(slice, 16));
252 slice = slice[0..16];
253
254 for (slice) |b| {
255 try std.testing.expect(b == 0x11);
256 }
257}
258
259test "large object - grow" {
260 var slice1 = try test_ally.alloc(u8, bigpage_size * 2 - 20);
261 defer test_ally.free(slice1);
262
263 const old = slice1;
264 slice1 = try test_ally.realloc(slice1, bigpage_size * 2 - 10);
265 try std.testing.expect(slice1.ptr == old.ptr);
266
267 slice1 = try test_ally.realloc(slice1, bigpage_size * 2);
268 slice1 = try test_ally.realloc(slice1, bigpage_size * 2 + 1);
269}
270
271test "realloc small object to large object" {
272 var slice = try test_ally.alloc(u8, 70);
273 defer test_ally.free(slice);
274 slice[0] = 0x12;
275 slice[60] = 0x34;
276
277 // This requires upgrading to a large object
278 const large_object_size = bigpage_size * 2 + 50;
279 slice = try test_ally.realloc(slice, large_object_size);
280 try std.testing.expect(slice[0] == 0x12);
281 try std.testing.expect(slice[60] == 0x34);
282}
283
284test "shrink large object to large object" {
285 var slice = try test_ally.alloc(u8, bigpage_size * 2 + 50);
286 defer test_ally.free(slice);
287 slice[0] = 0x12;
288 slice[60] = 0x34;
289
290 try std.testing.expect(test_ally.resize(slice, bigpage_size * 2 + 1));
291 slice = slice[0 .. bigpage_size * 2 + 1];
292 try std.testing.expect(slice[0] == 0x12);
293 try std.testing.expect(slice[60] == 0x34);
294
295 try std.testing.expect(test_ally.resize(slice, bigpage_size * 2 + 1));
296 try std.testing.expect(slice[0] == 0x12);
297 try std.testing.expect(slice[60] == 0x34);
298
299 slice = try test_ally.realloc(slice, bigpage_size * 2);
300 try std.testing.expect(slice[0] == 0x12);
301 try std.testing.expect(slice[60] == 0x34);
302}
303
304test "realloc large object to small object" {
305 var slice = try test_ally.alloc(u8, bigpage_size * 2 + 50);
306 defer test_ally.free(slice);
307 slice[0] = 0x12;
308 slice[16] = 0x34;
309
310 slice = try test_ally.realloc(slice, 19);
311 try std.testing.expect(slice[0] == 0x12);
312 try std.testing.expect(slice[16] == 0x34);
313}
314
315test "objects of size 1024 and 2048" {
316 const slice = try test_ally.alloc(u8, 1025);
317 const slice2 = try test_ally.alloc(u8, 3000);
318
319 test_ally.free(slice);
320 test_ally.free(slice2);
321}
322
323test "standard allocator tests" {
324 try std.heap.testAllocator(test_ally);
325 try std.heap.testAllocatorAligned(test_ally);
326}
lib/std/heap/sbrk_allocator.zig deleted-180
...@@ -1,180 +0,0 @@
1const builtin = @import("builtin");
2
3const std = @import("../std.zig");
4const Io = std.Io;
5const math = std.math;
6const Allocator = std.mem.Allocator;
7const mem = std.mem;
8const heap = std.heap;
9const assert = std.debug.assert;
10
11pub fn SbrkAllocator(comptime sbrk: *const fn (n: usize) usize) type {
12 return struct {
13 pub const vtable: Allocator.VTable = .{
14 .alloc = alloc,
15 .resize = resize,
16 .remap = remap,
17 .free = free,
18 };
19
20 pub const Error = Allocator.Error;
21
22 const max_usize = math.maxInt(usize);
23 const ushift = math.Log2Int(usize);
24 const bigpage_size = 64 * 1024;
25 const pages_per_bigpage = bigpage_size / heap.pageSize();
26 const bigpage_count = max_usize / bigpage_size;
27
28 /// Because of storing free list pointers, the minimum size class is 3.
29 const min_class = math.log2(math.ceilPowerOfTwoAssert(usize, 1 + @sizeOf(usize)));
30 const size_class_count = math.log2(bigpage_size) - min_class;
31 /// 0 - 1 bigpage
32 /// 1 - 2 bigpages
33 /// 2 - 4 bigpages
34 /// etc.
35 const big_size_class_count = math.log2(bigpage_count);
36
37 var next_addrs = [1]usize{0} ** size_class_count;
38 /// For each size class, points to the freed pointer.
39 var frees = [1]usize{0} ** size_class_count;
40 /// For each big size class, points to the freed pointer.
41 var big_frees = [1]usize{0} ** big_size_class_count;
42
43 // TODO don't do the naive locking strategy
44 var mutex: Io.Mutex = .{};
45 fn alloc(ctx: *anyopaque, len: usize, alignment: mem.Alignment, return_address: usize) ?[*]u8 {
46 _ = ctx;
47 _ = return_address;
48 Io.Threaded.mutexLock(&mutex);
49 defer Io.Threaded.mutexUnlock(&mutex);
50 // Make room for the freelist next pointer.
51 const actual_len = @max(len +| @sizeOf(usize), alignment.toByteUnits());
52 const slot_size = math.ceilPowerOfTwo(usize, actual_len) catch return null;
53 const class = math.log2(slot_size) - min_class;
54 if (class < size_class_count) {
55 const addr = a: {
56 const top_free_ptr = frees[class];
57 if (top_free_ptr != 0) {
58 const node = @as(*usize, @ptrFromInt(top_free_ptr + (slot_size - @sizeOf(usize))));
59 frees[class] = node.*;
60 break :a top_free_ptr;
61 }
62
63 const next_addr = next_addrs[class];
64 if (next_addr % heap.pageSize() == 0) {
65 const addr = allocBigPages(1);
66 if (addr == 0) return null;
67 //std.debug.print("allocated fresh slot_size={d} class={d} addr=0x{x}\n", .{
68 // slot_size, class, addr,
69 //});
70 next_addrs[class] = addr + slot_size;
71 break :a addr;
72 } else {
73 next_addrs[class] = next_addr + slot_size;
74 break :a next_addr;
75 }
76 };
77 return @as([*]u8, @ptrFromInt(addr));
78 }
79 const bigpages_needed = bigPagesNeeded(actual_len);
80 const addr = allocBigPages(bigpages_needed);
81 return @as([*]u8, @ptrFromInt(addr));
82 }
83
84 fn resize(
85 ctx: *anyopaque,
86 buf: []u8,
87 alignment: mem.Alignment,
88 new_len: usize,
89 return_address: usize,
90 ) bool {
91 _ = ctx;
92 _ = return_address;
93 Io.Threaded.mutexLock(&mutex);
94 defer Io.Threaded.mutexUnlock(&mutex);
95 // We don't want to move anything from one size class to another, but we
96 // can recover bytes in between powers of two.
97 const buf_align = alignment.toByteUnits();
98 const old_actual_len = @max(buf.len + @sizeOf(usize), buf_align);
99 const new_actual_len = @max(new_len +| @sizeOf(usize), buf_align);
100 const old_small_slot_size = math.ceilPowerOfTwoAssert(usize, old_actual_len);
101 const old_small_class = math.log2(old_small_slot_size) - min_class;
102 if (old_small_class < size_class_count) {
103 const new_small_slot_size = math.ceilPowerOfTwo(usize, new_actual_len) catch return false;
104 return old_small_slot_size == new_small_slot_size;
105 } else {
106 const old_bigpages_needed = bigPagesNeeded(old_actual_len);
107 const old_big_slot_pages = math.ceilPowerOfTwoAssert(usize, old_bigpages_needed);
108 const new_bigpages_needed = bigPagesNeeded(new_actual_len);
109 const new_big_slot_pages = math.ceilPowerOfTwo(usize, new_bigpages_needed) catch return false;
110 return old_big_slot_pages == new_big_slot_pages;
111 }
112 }
113
114 fn remap(
115 context: *anyopaque,
116 memory: []u8,
117 alignment: mem.Alignment,
118 new_len: usize,
119 return_address: usize,
120 ) ?[*]u8 {
121 return if (resize(context, memory, alignment, new_len, return_address)) memory.ptr else null;
122 }
123
124 fn free(
125 ctx: *anyopaque,
126 buf: []u8,
127 alignment: mem.Alignment,
128 return_address: usize,
129 ) void {
130 _ = ctx;
131 _ = return_address;
132 Io.Threaded.mutexLock(&mutex);
133 defer Io.Threaded.mutexUnlock(&mutex);
134 const buf_align = alignment.toByteUnits();
135 const actual_len = @max(buf.len + @sizeOf(usize), buf_align);
136 const slot_size = math.ceilPowerOfTwoAssert(usize, actual_len);
137 const class = math.log2(slot_size) - min_class;
138 const addr = @intFromPtr(buf.ptr);
139 if (class < size_class_count) {
140 const node = @as(*usize, @ptrFromInt(addr + (slot_size - @sizeOf(usize))));
141 node.* = frees[class];
142 frees[class] = addr;
143 } else {
144 const bigpages_needed = bigPagesNeeded(actual_len);
145 const pow2_pages = math.ceilPowerOfTwoAssert(usize, bigpages_needed);
146 const big_slot_size_bytes = pow2_pages * bigpage_size;
147 const node = @as(*usize, @ptrFromInt(addr + (big_slot_size_bytes - @sizeOf(usize))));
148 const big_class = math.log2(pow2_pages);
149 node.* = big_frees[big_class];
150 big_frees[big_class] = addr;
151 }
152 }
153
154 inline fn bigPagesNeeded(byte_count: usize) usize {
155 return (byte_count + (bigpage_size + (@sizeOf(usize) - 1))) / bigpage_size;
156 }
157
158 fn allocBigPages(n: usize) usize {
159 const pow2_pages = math.ceilPowerOfTwoAssert(usize, n);
160 const slot_size_bytes = pow2_pages * bigpage_size;
161 const class = math.log2(pow2_pages);
162
163 const top_free_ptr = big_frees[class];
164 if (top_free_ptr != 0) {
165 const node = @as(*usize, @ptrFromInt(top_free_ptr + (slot_size_bytes - @sizeOf(usize))));
166 big_frees[class] = node.*;
167 return top_free_ptr;
168 }
169 return sbrk(pow2_pages * pages_per_bigpage * heap.pageSize());
170 }
171 };
172}
173
174test SbrkAllocator {
175 _ = SbrkAllocator(struct {
176 fn sbrk(_: usize) usize {
177 return 0;
178 }
179 }.sbrk);
180}
lib/std/os/linux.zig+4
...@@ -594,6 +594,10 @@ pub fn errno(r: usize) E {...@@ -594,6 +594,10 @@ pub fn errno(r: usize) E {
594 return @enumFromInt(int);594 return @enumFromInt(int);
595}595}
596596
597pub fn brk(addr: usize) usize {
598 return syscall1(.brk, addr);
599}
600
597pub fn dup(old: i32) usize {601pub fn dup(old: i32) usize {
598 return syscall1(.dup, @as(usize, @bitCast(@as(isize, old))));602 return syscall1(.dup, @as(usize, @bitCast(@as(isize, old))));
599}603}