authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-02-05 13:31:01-08:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-02-06 14:23:23-08:00
logdef36f2e4460ec9dd500772daac100a655f019b7
tree784c1590b76f52da473fff3a3e238f0936e9d2d7
parentc8e807c44ed8c96b31a6cbe0b7f5f01a02ced270

std.heap.GeneralPurposeAllocator: usize for used_bits

improves leak checking performance.

1 files changed, 27 insertions(+), 24 deletions(-)

lib/std/heap/general_purpose_allocator.zig+27-24
......@@ -265,7 +265,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
265265
266266 /// Bucket: In memory, in order:
267267 /// * BucketHeader
268 /// * bucket_used_bits: [N]u8, // 1 bit for every slot; 1 byte for every 8 slots
268 /// * bucket_used_bits: [N]usize, // 1 bit for every slot
269269 /// -- below only exists when config.safety is true --
270270 /// * requested_sizes: [N]LargestSizeClassInt // 1 int for every slot
271271 /// * log2_ptr_aligns: [N]u8 // 1 byte for every slot
......@@ -282,10 +282,10 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
282282 return @ptrFromInt(unaligned & ~(@as(usize, @alignOf(BucketHeader)) - 1));
283283 }
284284
285 // TODO use usize instead of u8
286 fn usedBits(bucket: *BucketHeader, index: usize) *u8 {
287 // TODO avoid ptr to int
288 return @ptrFromInt(@intFromPtr(bucket) + @sizeOf(BucketHeader) + index);
285 fn usedBits(bucket: *BucketHeader, index: usize) *usize {
286 const ptr: [*]u8 = @ptrCast(bucket);
287 const bits: [*]usize = @alignCast(@ptrCast(ptr + @sizeOf(BucketHeader)));
288 return &bits[index];
289289 }
290290
291291 fn requestedSizes(bucket: *BucketHeader, slot_count: usize) []LargestSizeClassInt {
......@@ -360,7 +360,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
360360 if (!config.safety) @compileError("requested sizes are not stored unless safety is enabled");
361361 return mem.alignForward(
362362 usize,
363 @sizeOf(BucketHeader) + usedBitsCount(slot_count),
363 @sizeOf(BucketHeader) + usedBitsSize(slot_count),
364364 @alignOf(LargestSizeClassInt),
365365 );
366366 }
......@@ -374,7 +374,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
374374 const unaligned_start = if (config.safety)
375375 bucketAlignsStart(slot_count) + slot_count
376376 else
377 @sizeOf(BucketHeader) + usedBitsCount(slot_count);
377 @sizeOf(BucketHeader) + usedBitsSize(slot_count);
378378 return mem.alignForward(usize, unaligned_start, @alignOf(usize));
379379 }
380380
......@@ -404,8 +404,11 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
404404 }
405405
406406 fn usedBitsCount(slot_count: usize) usize {
407 assert(slot_count >= 8);
408 return (slot_count + 7) / 8;
407 return (slot_count + (@bitSizeOf(usize) - 1)) / @bitSizeOf(usize);
408 }
409
410 fn usedBitsSize(slot_count: usize) usize {
411 return usedBitsCount(slot_count) * @sizeOf(usize);
409412 }
410413
411414 fn detectLeaksInBucket(bucket: *BucketHeader, size_class_index: usize, used_bits_count: usize) bool {
......@@ -413,13 +416,13 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
413416 const slot_count = slot_counts[size_class_index];
414417 var leaks = false;
415418 for (0..used_bits_count) |used_bits_byte| {
416 const used_byte = bucket.usedBits(used_bits_byte).*;
417 if (used_byte != 0) {
418 for (0..8) |bit_index_usize| {
419 const bit_index: u3 = @intCast(bit_index_usize);
420 const is_used = @as(u1, @truncate(used_byte >> bit_index)) != 0;
419 const used_int = bucket.usedBits(used_bits_byte).*;
420 if (used_int != 0) {
421 for (0..@bitSizeOf(usize)) |bit_index_usize| {
422 const bit_index: Log2USize = @intCast(bit_index_usize);
423 const is_used = @as(u1, @truncate(used_int >> bit_index)) != 0;
421424 if (is_used) {
422 const slot_index: SlotIndex = @intCast(used_bits_byte * 8 + bit_index);
425 const slot_index: SlotIndex = @intCast(used_bits_byte * @bitSizeOf(usize) + bit_index);
423426 const stack_trace = bucketStackTrace(bucket, slot_count, slot_index, .alloc);
424427 const page_addr = @intFromPtr(bucket) & ~(page_size - 1);
425428 const addr = page_addr + slot_index * size_class;
......@@ -740,9 +743,9 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
740743 if (slot_index < slot_count) {
741744 @branchHint(.likely);
742745 bucket.allocated_count = slot_index + 1;
743 const used_bits_byte = bucket.usedBits(slot_index / 8);
744 const used_bit_index: u3 = @intCast(slot_index % 8);
745 used_bits_byte.* |= (@as(u8, 1) << used_bit_index);
746 const used_bits_byte = bucket.usedBits(slot_index / @bitSizeOf(usize));
747 const used_bit_index: Log2USize = @intCast(slot_index % @bitSizeOf(usize));
748 used_bits_byte.* |= (@as(usize, 1) << used_bit_index);
746749 const size_class = @as(usize, 1) << @as(Log2USize, @intCast(size_class_index));
747750 if (config.stack_trace_frames > 0) {
748751 bucket.captureStackTrace(ret_addr, slot_count, slot_index, .alloc);
......@@ -771,7 +774,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
771774 self.buckets[size_class_index] = bucket;
772775
773776 if (!config.backing_allocator_zeroes) {
774 @memset(@as([*]u8, @as(*[1]u8, bucket.usedBits(0)))[0..usedBitsCount(slot_count)], 0);
777 @memset(@as([*]usize, @as(*[1]usize, bucket.usedBits(0)))[0..usedBitsCount(slot_count)], 0);
775778 if (config.safety) @memset(bucket.requestedSizes(slot_count), 0);
776779 }
777780
......@@ -858,8 +861,8 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
858861 const page_offset = freed_addr - page_addr;
859862 const size_class = @as(usize, 1) << @as(Log2USize, @intCast(size_class_index));
860863 const slot_index: SlotIndex = @intCast(page_offset / size_class);
861 const used_byte_index = slot_index / 8;
862 const used_bit_index: u3 = @intCast(slot_index % 8);
864 const used_byte_index = slot_index / @bitSizeOf(usize);
865 const used_bit_index: Log2USize = @intCast(slot_index % @bitSizeOf(usize));
863866 const used_byte = bucket.usedBits(used_byte_index);
864867 const is_used = @as(u1, @truncate(used_byte.* >> used_bit_index)) != 0;
865868 if (!is_used) {
......@@ -916,7 +919,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
916919 bucket.captureStackTrace(return_address, slot_count, slot_index, .free);
917920 }
918921
919 used_byte.* &= ~(@as(u8, 1) << used_bit_index);
922 used_byte.* &= ~(@as(usize, 1) << used_bit_index);
920923 if (config.safety) {
921924 bucket.requestedSizes(slot_count)[slot_index] = 0;
922925 }
......@@ -953,8 +956,8 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
953956 const page_offset = memory_addr - page_addr;
954957 const size_class = @as(usize, 1) << @as(Log2USize, @intCast(size_class_index));
955958 const slot_index: SlotIndex = @intCast(page_offset / size_class);
956 const used_byte_index = slot_index / 8;
957 const used_bit_index: u3 = @intCast(slot_index % 8);
959 const used_byte_index = slot_index / @bitSizeOf(usize);
960 const used_bit_index: Log2USize = @intCast(slot_index % @bitSizeOf(usize));
958961 const used_byte = bucket.usedBits(used_byte_index);
959962 const is_used = @as(u1, @truncate(used_byte.* >> used_bit_index)) != 0;
960963 if (!is_used) {