authorgravatar for squeek502@hotmail.comRyan Liptak <squeek502@hotmail.com> 2023-10-03 17:37:09-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2023-10-04 02:55:54-07:00
logec0f76c5996e88f61d376640bf36ed7feb2b0ea6
tree1f8ca77b728ca1381fb40615c2eff9817945ff6f
parent11489bb04f82cf91e23bd1980fb4cf49b5a6fc54

GeneralPurposeAllocator.searchBucket: check current bucket before searching the list

Follow up to #17383. This is a minor optimization that only matters when a small allocation is resized/free'd soon after it is allocated. The only real difference I was able to observe with this was via a synthetic benchmark that allocates a full bucket and then frees all but one of the slots, over and over in a loop: Debug build: Benchmark 1 (9 runs): gpa-degen-master.exe measurement mean ± σ min … max outliers delta wall_time 575ms ± 5.19ms 569ms … 583ms 0 ( 0%) 0% peak_rss 43.8MB ± 1.37KB 43.8MB … 43.8MB 1 (11%) 0% Benchmark 2 (10 runs): gpa-degen-search-cur.exe measurement mean ± σ min … max outliers delta wall_time 532ms ± 5.55ms 520ms … 539ms 0 ( 0%) ⚡- 7.5% ± 0.9% peak_rss 43.8MB ± 65.2KB 43.8MB … 44.0MB 1 (10%) + 0.0% ± 0.1% ReleaseFast build: Benchmark 1 (129 runs): gpa-degen-master-release.exe measurement mean ± σ min … max outliers delta wall_time 38.9ms ± 1.12ms 36.7ms … 42.4ms 8 ( 6%) 0% peak_rss 23.2MB ± 2.39KB 23.2MB … 23.2MB 0 ( 0%) 0% Benchmark 2 (151 runs): gpa-degen-search-cur-release.exe measurement mean ± σ min … max outliers delta wall_time 33.2ms ± 999us 31.9ms … 36.3ms 20 (13%) ⚡- 14.7% ± 0.6% peak_rss 23.2MB ± 2.26KB 23.2MB … 23.2MB 0 ( 0%) + 0.0% ± 0.0%

1 files changed, 14 insertions(+), 10 deletions(-)

lib/std/heap/general_purpose_allocator.zig+14-10
...@@ -541,10 +541,14 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {...@@ -541,10 +541,14 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
541 fn searchBucket(541 fn searchBucket(
542 buckets: *Buckets,542 buckets: *Buckets,
543 addr: usize,543 addr: usize,
544 current_bucket: ?*BucketHeader,
544 ) ?*BucketHeader {545 ) ?*BucketHeader {
545 const search_page = mem.alignBackward(usize, addr, page_size);546 const search_page: [*]align(page_size) u8 = @ptrFromInt(mem.alignBackward(usize, addr, page_size));
547 if (current_bucket != null and current_bucket.?.page == search_page) {
548 return current_bucket;
549 }
546 var search_header: BucketHeader = undefined;550 var search_header: BucketHeader = undefined;
547 search_header.page = @ptrFromInt(search_page);551 search_header.page = search_page;
548 const entry = buckets.getEntryFor(&search_header);552 const entry = buckets.getEntryFor(&search_header);
549 return if (entry.node) |node| node.key else null;553 return if (entry.node) |node| node.key else null;
550 }554 }
...@@ -712,7 +716,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {...@@ -712,7 +716,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
712 var bucket_index = math.log2(size_class_hint);716 var bucket_index = math.log2(size_class_hint);
713 var size_class: usize = size_class_hint;717 var size_class: usize = size_class_hint;
714 const bucket = while (bucket_index < small_bucket_count) : (bucket_index += 1) {718 const bucket = while (bucket_index < small_bucket_count) : (bucket_index += 1) {
715 if (searchBucket(&self.buckets[bucket_index], @intFromPtr(old_mem.ptr))) |bucket| {719 if (searchBucket(&self.buckets[bucket_index], @intFromPtr(old_mem.ptr), self.cur_buckets[bucket_index])) |bucket| {
716 break bucket;720 break bucket;
717 }721 }
718 size_class *= 2;722 size_class *= 2;
...@@ -720,7 +724,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {...@@ -720,7 +724,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
720 if (config.retain_metadata) {724 if (config.retain_metadata) {
721 if (!self.large_allocations.contains(@intFromPtr(old_mem.ptr))) {725 if (!self.large_allocations.contains(@intFromPtr(old_mem.ptr))) {
722 // object not in active buckets or a large allocation, so search empty buckets726 // object not in active buckets or a large allocation, so search empty buckets
723 if (searchBucket(&self.empty_buckets, @intFromPtr(old_mem.ptr))) |bucket| {727 if (searchBucket(&self.empty_buckets, @intFromPtr(old_mem.ptr), null)) |bucket| {
724 // bucket is empty so is_used below will always be false and we exit there728 // bucket is empty so is_used below will always be false and we exit there
725 break :blk bucket;729 break :blk bucket;
726 } else {730 } else {
...@@ -830,7 +834,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {...@@ -830,7 +834,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
830 var bucket_index = math.log2(size_class_hint);834 var bucket_index = math.log2(size_class_hint);
831 var size_class: usize = size_class_hint;835 var size_class: usize = size_class_hint;
832 const bucket = while (bucket_index < small_bucket_count) : (bucket_index += 1) {836 const bucket = while (bucket_index < small_bucket_count) : (bucket_index += 1) {
833 if (searchBucket(&self.buckets[bucket_index], @intFromPtr(old_mem.ptr))) |bucket| {837 if (searchBucket(&self.buckets[bucket_index], @intFromPtr(old_mem.ptr), self.cur_buckets[bucket_index])) |bucket| {
834 break bucket;838 break bucket;
835 }839 }
836 size_class *= 2;840 size_class *= 2;
...@@ -838,7 +842,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {...@@ -838,7 +842,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
838 if (config.retain_metadata) {842 if (config.retain_metadata) {
839 if (!self.large_allocations.contains(@intFromPtr(old_mem.ptr))) {843 if (!self.large_allocations.contains(@intFromPtr(old_mem.ptr))) {
840 // object not in active buckets or a large allocation, so search empty buckets844 // object not in active buckets or a large allocation, so search empty buckets
841 if (searchBucket(&self.empty_buckets, @intFromPtr(old_mem.ptr))) |bucket| {845 if (searchBucket(&self.empty_buckets, @intFromPtr(old_mem.ptr), null)) |bucket| {
842 // bucket is empty so is_used below will always be false and we exit there846 // bucket is empty so is_used below will always be false and we exit there
843 break :blk bucket;847 break :blk bucket;
844 } else {848 } else {
...@@ -1387,10 +1391,10 @@ test "double frees" {...@@ -1387,10 +1391,10 @@ test "double frees" {
1387 const index: usize = 6;1391 const index: usize = 6;
1388 const size_class: usize = @as(usize, 1) << 6;1392 const size_class: usize = @as(usize, 1) << 6;
1389 const small = try allocator.alloc(u8, size_class);1393 const small = try allocator.alloc(u8, size_class);
1390 try std.testing.expect(GPA.searchBucket(&gpa.buckets[index], @intFromPtr(small.ptr)) != null);1394 try std.testing.expect(GPA.searchBucket(&gpa.buckets[index], @intFromPtr(small.ptr), gpa.cur_buckets[index]) != null);
1391 allocator.free(small);1395 allocator.free(small);
1392 try std.testing.expect(GPA.searchBucket(&gpa.buckets[index], @intFromPtr(small.ptr)) == null);1396 try std.testing.expect(GPA.searchBucket(&gpa.buckets[index], @intFromPtr(small.ptr), gpa.cur_buckets[index]) == null);
1393 try std.testing.expect(GPA.searchBucket(&gpa.empty_buckets, @intFromPtr(small.ptr)) != null);1397 try std.testing.expect(GPA.searchBucket(&gpa.empty_buckets, @intFromPtr(small.ptr), null) != null);
13941398
1395 // detect a large allocation double free1399 // detect a large allocation double free
1396 const large = try allocator.alloc(u8, 2 * page_size);1400 const large = try allocator.alloc(u8, 2 * page_size);
...@@ -1408,7 +1412,7 @@ test "double frees" {...@@ -1408,7 +1412,7 @@ test "double frees" {
1408 // check that flushing retained metadata doesn't disturb live allocations1412 // check that flushing retained metadata doesn't disturb live allocations
1409 gpa.flushRetainedMetadata();1413 gpa.flushRetainedMetadata();
1410 try std.testing.expect(gpa.empty_buckets.root == null);1414 try std.testing.expect(gpa.empty_buckets.root == null);
1411 try std.testing.expect(GPA.searchBucket(&gpa.buckets[index], @intFromPtr(normal_small.ptr)) != null);1415 try std.testing.expect(GPA.searchBucket(&gpa.buckets[index], @intFromPtr(normal_small.ptr), gpa.cur_buckets[index]) != null);
1412 try std.testing.expect(gpa.large_allocations.contains(@intFromPtr(normal_large.ptr)));1416 try std.testing.expect(gpa.large_allocations.contains(@intFromPtr(normal_large.ptr)));
1413 try std.testing.expect(!gpa.large_allocations.contains(@intFromPtr(large.ptr)));1417 try std.testing.expect(!gpa.large_allocations.contains(@intFromPtr(large.ptr)));
1414}1418}