authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-01-29 21:48:05-08:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-02-06 14:23:23-08:00
log95a0474dc6bf26a7a86bea4c93e06b3ae0cd37cc
tree28b25fe6b9d68d6fac276e0a06f229cd8fc6db4d
parent284de7d957037c8a7032bd6e2a95bd5f55b73666

revert GPA to before this branch

1 files changed, 51 insertions(+), 78 deletions(-)

lib/std/heap/general_purpose_allocator.zig+51-78
......@@ -48,7 +48,7 @@
4848//!
4949//! ## Basic Design:
5050//!
51//! Small allocations are divided into buckets. For a max page size of 4K:
51//! Small allocations are divided into buckets:
5252//!
5353//! ```
5454//! index obj_size
......@@ -75,9 +75,6 @@
7575//! BucketHeader, followed by "used bits", and two stack traces for each slot
7676//! (allocation trace and free trace).
7777//!
78//! The buckets array contains buckets for every size class below `page_size_max`.
79//! At runtime, only size classes below `pageSize()` will actually be used for allocations.
80//!
8178//! The "used bits" are 1 bit per slot representing whether the slot is used.
8279//! Allocations use the data to iterate to find a free slot. Frees assert that the
8380//! corresponding bit is 1 and set it to 0.
......@@ -102,13 +99,11 @@ const math = std.math;
10299const assert = std.debug.assert;
103100const mem = std.mem;
104101const Allocator = std.mem.Allocator;
105const page_size_min = std.heap.page_size_min;
106const page_size_max = std.heap.page_size_max;
107const pageSize = std.heap.pageSize;
102const page_size = std.mem.page_size;
108103const StackTrace = std.builtin.StackTrace;
109104
110105/// Integer type for pointing to slots in a small allocation
111const SlotIndex = std.meta.Int(.unsigned, math.log2(page_size_max) + 1);
106const SlotIndex = std.meta.Int(.unsigned, math.log2(page_size) + 1);
112107
113108const default_test_stack_trace_frames: usize = if (builtin.is_test) 10 else 6;
114109const default_sys_stack_trace_frames: usize = if (std.debug.sys_can_stack_trace) default_test_stack_trace_frames else 0;
......@@ -162,9 +157,6 @@ pub const Config = struct {
162157
163158pub const Check = enum { ok, leak };
164159
165var used_small_bucket_count_cache = std.atomic.Value(usize).init(0);
166var largest_used_bucket_object_size_cache = std.atomic.Value(usize).init(0);
167
168160/// Default initialization of this struct is deprecated; use `.init` instead.
169161pub fn GeneralPurposeAllocator(comptime config: Config) type {
170162 return struct {
......@@ -214,27 +206,9 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
214206
215207 pub const Error = mem.Allocator.Error;
216208
217 const small_bucket_count = math.log2(page_size_max);
209 const small_bucket_count = math.log2(page_size);
218210 const largest_bucket_object_size = 1 << (small_bucket_count - 1);
219211 const LargestSizeClassInt = std.math.IntFittingRange(0, largest_bucket_object_size);
220 fn used_small_bucket_count() usize {
221 const cached = used_small_bucket_count_cache.load(.monotonic);
222 if (cached != 0) {
223 return cached;
224 }
225 const val = math.log2(pageSize());
226 used_small_bucket_count_cache.store(val, .monotonic);
227 return val;
228 }
229 fn largest_used_bucket_object_size() usize {
230 const cached = largest_used_bucket_object_size_cache.load(.monotonic);
231 if (cached != 0) {
232 return cached;
233 }
234 const val = @as(usize, 1) << @truncate(used_small_bucket_count() - 1);
235 largest_used_bucket_object_size_cache.store(val, .monotonic);
236 return val;
237 }
238212
239213 const bucketCompare = struct {
240214 fn compare(a: *BucketHeader, b: *BucketHeader) std.math.Order {
......@@ -287,7 +261,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
287261 // * stack_trace_addresses: [N]usize, // traces_per_slot for every allocation
288262
289263 const BucketHeader = struct {
290 page: [*]align(page_size_min) u8,
264 page: [*]align(page_size) u8,
291265 alloc_cursor: SlotIndex,
292266 used_count: SlotIndex,
293267
......@@ -299,14 +273,14 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
299273 if (!config.safety) @compileError("requested size is only stored when safety is enabled");
300274 const start_ptr = @as([*]u8, @ptrCast(bucket)) + bucketRequestedSizesStart(size_class);
301275 const sizes = @as([*]LargestSizeClassInt, @ptrCast(@alignCast(start_ptr)));
302 const slot_count = @divExact(pageSize(), size_class);
276 const slot_count = @divExact(page_size, size_class);
303277 return sizes[0..slot_count];
304278 }
305279
306280 fn log2PtrAligns(bucket: *BucketHeader, size_class: usize) []u8 {
307281 if (!config.safety) @compileError("requested size is only stored when safety is enabled");
308282 const aligns_ptr = @as([*]u8, @ptrCast(bucket)) + bucketAlignsStart(size_class);
309 const slot_count = @divExact(pageSize(), size_class);
283 const slot_count = @divExact(page_size, size_class);
310284 return aligns_ptr[0..slot_count];
311285 }
312286
......@@ -338,7 +312,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
338312 /// Only valid for buckets within `empty_buckets`, and relies on the `alloc_cursor`
339313 /// of empty buckets being set to `slot_count` when they are added to `empty_buckets`
340314 fn emptyBucketSizeClass(bucket: *BucketHeader) usize {
341 return @divExact(pageSize(), bucket.alloc_cursor);
315 return @divExact(page_size, bucket.alloc_cursor);
342316 }
343317 };
344318
......@@ -381,13 +355,13 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
381355
382356 fn bucketAlignsStart(size_class: usize) usize {
383357 if (!config.safety) @compileError("requested sizes are not stored unless safety is enabled");
384 const slot_count = @divExact(pageSize(), size_class);
358 const slot_count = @divExact(page_size, size_class);
385359 return bucketRequestedSizesStart(size_class) + (@sizeOf(LargestSizeClassInt) * slot_count);
386360 }
387361
388362 fn bucketStackFramesStart(size_class: usize) usize {
389363 const unaligned_start = if (config.safety) blk: {
390 const slot_count = @divExact(pageSize(), size_class);
364 const slot_count = @divExact(page_size, size_class);
391365 break :blk bucketAlignsStart(size_class) + slot_count;
392366 } else @sizeOf(BucketHeader) + usedBitsCount(size_class);
393367 return mem.alignForward(
......@@ -398,12 +372,12 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
398372 }
399373
400374 fn bucketSize(size_class: usize) usize {
401 const slot_count = @divExact(pageSize(), size_class);
375 const slot_count = @divExact(page_size, size_class);
402376 return bucketStackFramesStart(size_class) + one_trace_size * traces_per_slot * slot_count;
403377 }
404378
405379 fn usedBitsCount(size_class: usize) usize {
406 const slot_count = @divExact(pageSize(), size_class);
380 const slot_count = @divExact(page_size, size_class);
407381 if (slot_count < 8) return 1;
408382 return @divExact(slot_count, 8);
409383 }
......@@ -442,8 +416,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
442416 pub fn detectLeaks(self: *Self) bool {
443417 var leaks = false;
444418
445 for (0..used_small_bucket_count()) |bucket_i| {
446 const buckets = &self.buckets[bucket_i];
419 for (&self.buckets, 0..) |*buckets, bucket_i| {
447420 if (buckets.root == null) continue;
448421 const size_class = @as(usize, 1) << @as(math.Log2Int(usize), @intCast(bucket_i));
449422 const used_bits_count = usedBitsCount(size_class);
......@@ -491,7 +464,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
491464 var bucket = node.key;
492465 if (config.never_unmap) {
493466 // free page that was intentionally leaked by never_unmap
494 self.backing_allocator.free(bucket.page[0..pageSize()]);
467 self.backing_allocator.free(bucket.page[0..page_size]);
495468 }
496469 // alloc_cursor was set to slot count when bucket added to empty_buckets
497470 self.freeBucket(bucket, bucket.emptyBucketSizeClass());
......@@ -558,7 +531,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
558531 fn allocSlot(self: *Self, size_class: usize, trace_addr: usize) Error!Slot {
559532 const bucket_index = math.log2(size_class);
560533 var buckets = &self.buckets[bucket_index];
561 const slot_count = @divExact(pageSize(), size_class);
534 const slot_count = @divExact(page_size, size_class);
562535 if (self.cur_buckets[bucket_index] == null or self.cur_buckets[bucket_index].?.alloc_cursor == slot_count) {
563536 const new_bucket = try self.createBucket(size_class);
564537 errdefer self.freeBucket(new_bucket, size_class);
......@@ -591,7 +564,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
591564 addr: usize,
592565 current_bucket: ?*BucketHeader,
593566 ) ?*BucketHeader {
594 const search_page: [*]align(page_size_min) u8 = @ptrFromInt(mem.alignBackward(usize, addr, pageSize()));
567 const search_page: [*]align(page_size) u8 = @ptrFromInt(mem.alignBackward(usize, addr, page_size));
595568 if (current_bucket != null and current_bucket.?.page == search_page) {
596569 return current_bucket;
597570 }
......@@ -756,14 +729,14 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
756729 assert(old_mem.len != 0);
757730
758731 const aligned_size = @max(old_mem.len, @as(usize, 1) << log2_old_align);
759 if (aligned_size > largest_used_bucket_object_size()) {
732 if (aligned_size > largest_bucket_object_size) {
760733 return self.resizeLarge(old_mem, log2_old_align, new_size, ret_addr);
761734 }
762735 const size_class_hint = math.ceilPowerOfTwoAssert(usize, aligned_size);
763736
764737 var bucket_index = math.log2(size_class_hint);
765738 var size_class: usize = size_class_hint;
766 const bucket = while (bucket_index < used_small_bucket_count()) : (bucket_index += 1) {
739 const bucket = while (bucket_index < small_bucket_count) : (bucket_index += 1) {
767740 if (searchBucket(&self.buckets[bucket_index], @intFromPtr(old_mem.ptr), self.cur_buckets[bucket_index])) |bucket| {
768741 break bucket;
769742 }
......@@ -874,7 +847,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
874847 assert(old_mem.len != 0);
875848
876849 const aligned_size = @max(old_mem.len, @as(usize, 1) << log2_old_align);
877 if (aligned_size > largest_used_bucket_object_size()) {
850 if (aligned_size > largest_bucket_object_size) {
878851 self.freeLarge(old_mem, log2_old_align, ret_addr);
879852 return;
880853 }
......@@ -882,7 +855,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
882855
883856 var bucket_index = math.log2(size_class_hint);
884857 var size_class: usize = size_class_hint;
885 const bucket = while (bucket_index < used_small_bucket_count()) : (bucket_index += 1) {
858 const bucket = while (bucket_index < small_bucket_count) : (bucket_index += 1) {
886859 if (searchBucket(&self.buckets[bucket_index], @intFromPtr(old_mem.ptr), self.cur_buckets[bucket_index])) |bucket| {
887860 break bucket;
888861 }
......@@ -971,14 +944,14 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
971944 self.cur_buckets[bucket_index] = null;
972945 }
973946 if (!config.never_unmap) {
974 self.backing_allocator.free(bucket.page[0..pageSize()]);
947 self.backing_allocator.free(bucket.page[0..page_size]);
975948 }
976949 if (!config.retain_metadata) {
977950 self.freeBucket(bucket, size_class);
978951 self.bucket_node_pool.destroy(node);
979952 } else {
980953 // move alloc_cursor to end so we can tell size_class later
981 const slot_count = @divExact(pageSize(), size_class);
954 const slot_count = @divExact(page_size, size_class);
982955 bucket.alloc_cursor = @as(SlotIndex, @truncate(slot_count));
983956 var empty_entry = self.empty_buckets.getEntryFor(node.key);
984957 empty_entry.set(node);
......@@ -1019,7 +992,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
1019992 ret_addr: usize,
1020993 ) Allocator.Error![*]u8 {
1021994 const new_aligned_size = @max(len, @as(usize, 1) << @as(Allocator.Log2Align, @intCast(log2_ptr_align)));
1022 if (new_aligned_size > largest_used_bucket_object_size()) {
995 if (new_aligned_size > largest_bucket_object_size) {
1023996 try self.large_allocations.ensureUnusedCapacity(self.backing_allocator, 1);
1024997 const ptr = self.backing_allocator.rawAlloc(len, log2_ptr_align, ret_addr) orelse
1025998 return error.OutOfMemory;
......@@ -1062,7 +1035,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
10621035 }
10631036
10641037 fn createBucket(self: *Self, size_class: usize) Error!*BucketHeader {
1065 const page = try self.backing_allocator.alignedAlloc(u8, page_size_min, pageSize());
1038 const page = try self.backing_allocator.alignedAlloc(u8, page_size, page_size);
10661039 errdefer self.backing_allocator.free(page);
10671040
10681041 const bucket_size = bucketSize(size_class);
......@@ -1206,17 +1179,17 @@ test "large object - grow" {
12061179 defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak");
12071180 const allocator = gpa.allocator();
12081181
1209 var slice1 = try allocator.alloc(u8, pageSize() * 2 - 20);
1182 var slice1 = try allocator.alloc(u8, page_size * 2 - 20);
12101183 defer allocator.free(slice1);
12111184
12121185 const old = slice1;
1213 slice1 = try allocator.realloc(slice1, pageSize() * 2 - 10);
1186 slice1 = try allocator.realloc(slice1, page_size * 2 - 10);
12141187 try std.testing.expect(slice1.ptr == old.ptr);
12151188
1216 slice1 = try allocator.realloc(slice1, pageSize() * 2);
1189 slice1 = try allocator.realloc(slice1, page_size * 2);
12171190 try std.testing.expect(slice1.ptr == old.ptr);
12181191
1219 slice1 = try allocator.realloc(slice1, pageSize() * 2 + 1);
1192 slice1 = try allocator.realloc(slice1, page_size * 2 + 1);
12201193}
12211194
12221195test "realloc small object to large object" {
......@@ -1230,7 +1203,7 @@ test "realloc small object to large object" {
12301203 slice[60] = 0x34;
12311204
12321205 // This requires upgrading to a large object
1233 const large_object_size = pageSize() * 2 + 50;
1206 const large_object_size = page_size * 2 + 50;
12341207 slice = try allocator.realloc(slice, large_object_size);
12351208 try std.testing.expect(slice[0] == 0x12);
12361209 try std.testing.expect(slice[60] == 0x34);
......@@ -1241,22 +1214,22 @@ test "shrink large object to large object" {
12411214 defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak");
12421215 const allocator = gpa.allocator();
12431216
1244 var slice = try allocator.alloc(u8, pageSize() * 2 + 50);
1217 var slice = try allocator.alloc(u8, page_size * 2 + 50);
12451218 defer allocator.free(slice);
12461219 slice[0] = 0x12;
12471220 slice[60] = 0x34;
12481221
1249 if (!allocator.resize(slice, pageSize() * 2 + 1)) return;
1250 slice = slice.ptr[0 .. pageSize() * 2 + 1];
1222 if (!allocator.resize(slice, page_size * 2 + 1)) return;
1223 slice = slice.ptr[0 .. page_size * 2 + 1];
12511224 try std.testing.expect(slice[0] == 0x12);
12521225 try std.testing.expect(slice[60] == 0x34);
12531226
1254 try std.testing.expect(allocator.resize(slice, pageSize() * 2 + 1));
1255 slice = slice[0 .. pageSize() * 2 + 1];
1227 try std.testing.expect(allocator.resize(slice, page_size * 2 + 1));
1228 slice = slice[0 .. page_size * 2 + 1];
12561229 try std.testing.expect(slice[0] == 0x12);
12571230 try std.testing.expect(slice[60] == 0x34);
12581231
1259 slice = try allocator.realloc(slice, pageSize() * 2);
1232 slice = try allocator.realloc(slice, page_size * 2);
12601233 try std.testing.expect(slice[0] == 0x12);
12611234 try std.testing.expect(slice[60] == 0x34);
12621235}
......@@ -1272,13 +1245,13 @@ test "shrink large object to large object with larger alignment" {
12721245 var fba = std.heap.FixedBufferAllocator.init(&debug_buffer);
12731246 const debug_allocator = fba.allocator();
12741247
1275 const alloc_size = pageSize() * 2 + 50;
1248 const alloc_size = page_size * 2 + 50;
12761249 var slice = try allocator.alignedAlloc(u8, 16, alloc_size);
12771250 defer allocator.free(slice);
12781251
12791252 const big_alignment: usize = switch (builtin.os.tag) {
1280 .windows => pageSize() * 32, // Windows aligns to 64K.
1281 else => pageSize() * 2,
1253 .windows => page_size * 32, // Windows aligns to 64K.
1254 else => page_size * 2,
12821255 };
12831256 // This loop allocates until we find a page that is not aligned to the big
12841257 // alignment. Then we shrink the allocation after the loop, but increase the
......@@ -1304,7 +1277,7 @@ test "realloc large object to small object" {
13041277 defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak");
13051278 const allocator = gpa.allocator();
13061279
1307 var slice = try allocator.alloc(u8, pageSize() * 2 + 50);
1280 var slice = try allocator.alloc(u8, page_size * 2 + 50);
13081281 defer allocator.free(slice);
13091282 slice[0] = 0x12;
13101283 slice[16] = 0x34;
......@@ -1346,18 +1319,18 @@ test "realloc large object to larger alignment" {
13461319 var fba = std.heap.FixedBufferAllocator.init(&debug_buffer);
13471320 const debug_allocator = fba.allocator();
13481321
1349 var slice = try allocator.alignedAlloc(u8, 16, pageSize() * 2 + 50);
1322 var slice = try allocator.alignedAlloc(u8, 16, page_size * 2 + 50);
13501323 defer allocator.free(slice);
13511324
13521325 const big_alignment: usize = switch (builtin.os.tag) {
1353 .windows => pageSize() * 32, // Windows aligns to 64K.
1354 else => pageSize() * 2,
1326 .windows => page_size * 32, // Windows aligns to 64K.
1327 else => page_size * 2,
13551328 };
13561329 // This loop allocates until we find a page that is not aligned to the big alignment.
13571330 var stuff_to_free = std.ArrayList([]align(16) u8).init(debug_allocator);
13581331 while (mem.isAligned(@intFromPtr(slice.ptr), big_alignment)) {
13591332 try stuff_to_free.append(slice);
1360 slice = try allocator.alignedAlloc(u8, 16, pageSize() * 2 + 50);
1333 slice = try allocator.alignedAlloc(u8, 16, page_size * 2 + 50);
13611334 }
13621335 while (stuff_to_free.popOrNull()) |item| {
13631336 allocator.free(item);
......@@ -1365,15 +1338,15 @@ test "realloc large object to larger alignment" {
13651338 slice[0] = 0x12;
13661339 slice[16] = 0x34;
13671340
1368 slice = try allocator.reallocAdvanced(slice, 32, pageSize() * 2 + 100);
1341 slice = try allocator.reallocAdvanced(slice, 32, page_size * 2 + 100);
13691342 try std.testing.expect(slice[0] == 0x12);
13701343 try std.testing.expect(slice[16] == 0x34);
13711344
1372 slice = try allocator.reallocAdvanced(slice, 32, pageSize() * 2 + 25);
1345 slice = try allocator.reallocAdvanced(slice, 32, page_size * 2 + 25);
13731346 try std.testing.expect(slice[0] == 0x12);
13741347 try std.testing.expect(slice[16] == 0x34);
13751348
1376 slice = try allocator.reallocAdvanced(slice, big_alignment, pageSize() * 2 + 100);
1349 slice = try allocator.reallocAdvanced(slice, big_alignment, page_size * 2 + 100);
13771350 try std.testing.expect(slice[0] == 0x12);
13781351 try std.testing.expect(slice[16] == 0x34);
13791352}
......@@ -1389,7 +1362,7 @@ test "large object shrinks to small but allocation fails during shrink" {
13891362 defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak");
13901363 const allocator = gpa.allocator();
13911364
1392 var slice = try allocator.alloc(u8, pageSize() * 2 + 50);
1365 var slice = try allocator.alloc(u8, page_size * 2 + 50);
13931366 defer allocator.free(slice);
13941367 slice[0] = 0x12;
13951368 slice[3] = 0x34;
......@@ -1460,7 +1433,7 @@ test "double frees" {
14601433 try std.testing.expect(GPA.searchBucket(&gpa.empty_buckets, @intFromPtr(small.ptr), null) != null);
14611434
14621435 // detect a large allocation double free
1463 const large = try allocator.alloc(u8, 2 * pageSize());
1436 const large = try allocator.alloc(u8, 2 * page_size);
14641437 try std.testing.expect(gpa.large_allocations.contains(@intFromPtr(large.ptr)));
14651438 try std.testing.expectEqual(gpa.large_allocations.getEntry(@intFromPtr(large.ptr)).?.value_ptr.bytes, large);
14661439 allocator.free(large);
......@@ -1469,7 +1442,7 @@ test "double frees" {
14691442
14701443 const normal_small = try allocator.alloc(u8, size_class);
14711444 defer allocator.free(normal_small);
1472 const normal_large = try allocator.alloc(u8, 2 * pageSize());
1445 const normal_large = try allocator.alloc(u8, 2 * page_size);
14731446 defer allocator.free(normal_large);
14741447
14751448 // check that flushing retained metadata doesn't disturb live allocations
......@@ -1502,8 +1475,8 @@ test "bug 9995 fix, large allocs count requested size not backing size" {
15021475 var gpa = GeneralPurposeAllocator(.{ .enable_memory_limit = true }){};
15031476 const allocator = gpa.allocator();
15041477
1505 var buf = try allocator.alignedAlloc(u8, 1, pageSize() + 1);
1506 try std.testing.expect(gpa.total_requested_bytes == pageSize() + 1);
1478 var buf = try allocator.alignedAlloc(u8, 1, page_size + 1);
1479 try std.testing.expect(gpa.total_requested_bytes == page_size + 1);
15071480 buf = try allocator.realloc(buf, 1);
15081481 try std.testing.expect(gpa.total_requested_bytes == 1);
15091482 buf = try allocator.realloc(buf, 2);