| ... | ... | @@ -160,11 +160,12 @@ pub const Check = enum { ok, leak }; |
| 160 | 160 | pub fn GeneralPurposeAllocator(comptime config: Config) type { |
| 161 | 161 | return struct { |
| 162 | 162 | backing_allocator: Allocator = std.heap.page_allocator, |
| 163 | | buckets: [small_bucket_count]?*BucketHeader = [1]?*BucketHeader{null} ** small_bucket_count, |
| 163 | buckets: [small_bucket_count]Buckets = [1]Buckets{Buckets{}} ** small_bucket_count, |
| 164 | cur_buckets: [small_bucket_count]?*BucketHeader = [1]?*BucketHeader{null} ** small_bucket_count, |
| 164 | 165 | large_allocations: LargeAllocTable = .{}, |
| 165 | | small_allocations: if (config.safety) SmallAllocTable else void = if (config.safety) .{} else {}, |
| 166 | | empty_buckets: if (config.retain_metadata) ?*BucketHeader else void = |
| 167 | | if (config.retain_metadata) null else {}, |
| 166 | empty_buckets: if (config.retain_metadata) Buckets else void = |
| 167 | if (config.retain_metadata) Buckets{} else {}, |
| 168 | bucket_node_pool: std.heap.MemoryPool(Buckets.Node) = std.heap.MemoryPool(Buckets.Node).init(std.heap.page_allocator), |
| 168 | 169 | |
| 169 | 170 | total_requested_bytes: @TypeOf(total_requested_bytes_init) = total_requested_bytes_init, |
| 170 | 171 | requested_memory_limit: @TypeOf(requested_memory_limit_init) = requested_memory_limit_init, |
| ... | ... | @@ -196,11 +197,14 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { |
| 196 | 197 | |
| 197 | 198 | const small_bucket_count = math.log2(page_size); |
| 198 | 199 | const largest_bucket_object_size = 1 << (small_bucket_count - 1); |
| 200 | const LargestSizeClassInt = std.math.IntFittingRange(0, largest_bucket_object_size); |
| 199 | 201 | |
| 200 | | const SmallAlloc = struct { |
| 201 | | requested_size: usize, |
| 202 | | log2_ptr_align: u8, |
| 203 | | }; |
| 202 | const bucketCompare = struct { |
| 203 | fn compare(a: *BucketHeader, b: *BucketHeader) std.math.Order { |
| 204 | return std.math.order(@intFromPtr(a.page), @intFromPtr(b.page)); |
| 205 | } |
| 206 | }.compare; |
| 207 | const Buckets = std.Treap(*BucketHeader, bucketCompare); |
| 204 | 208 | |
| 205 | 209 | const LargeAlloc = struct { |
| 206 | 210 | bytes: []u8, |
| ... | ... | @@ -235,16 +239,17 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { |
| 235 | 239 | } |
| 236 | 240 | }; |
| 237 | 241 | const LargeAllocTable = std.AutoHashMapUnmanaged(usize, LargeAlloc); |
| 238 | | const SmallAllocTable = std.AutoHashMapUnmanaged(usize, SmallAlloc); |
| 239 | 242 | |
| 240 | 243 | // Bucket: In memory, in order: |
| 241 | 244 | // * BucketHeader |
| 242 | 245 | // * bucket_used_bits: [N]u8, // 1 bit for every slot; 1 byte for every 8 slots |
| 246 | // -- below only exists when config.safety is true -- |
| 247 | // * requested_sizes: [N]LargestSizeClassInt // 1 int for every slot |
| 248 | // * log2_ptr_aligns: [N]u8 // 1 byte for every slot |
| 249 | // -- above only exists when config.safety is true -- |
| 243 | 250 | // * stack_trace_addresses: [N]usize, // traces_per_slot for every allocation |
| 244 | 251 | |
| 245 | 252 | const BucketHeader = struct { |
| 246 | | prev: *BucketHeader, |
| 247 | | next: *BucketHeader, |
| 248 | 253 | page: [*]align(page_size) u8, |
| 249 | 254 | alloc_cursor: SlotIndex, |
| 250 | 255 | used_count: SlotIndex, |
| ... | ... | @@ -253,6 +258,21 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { |
| 253 | 258 | return @as(*u8, @ptrFromInt(@intFromPtr(bucket) + @sizeOf(BucketHeader) + index)); |
| 254 | 259 | } |
| 255 | 260 | |
| 261 | fn requestedSizes(bucket: *BucketHeader, size_class: usize) []LargestSizeClassInt { |
| 262 | if (!config.safety) @compileError("requested size is only stored when safety is enabled"); |
| 263 | const start_ptr = @as([*]u8, @ptrCast(bucket)) + bucketRequestedSizesStart(size_class); |
| 264 | const sizes = @as([*]LargestSizeClassInt, @ptrCast(@alignCast(start_ptr))); |
| 265 | const slot_count = @divExact(page_size, size_class); |
| 266 | return sizes[0..slot_count]; |
| 267 | } |
| 268 | |
| 269 | fn log2PtrAligns(bucket: *BucketHeader, size_class: usize) []u8 { |
| 270 | if (!config.safety) @compileError("requested size is only stored when safety is enabled"); |
| 271 | const aligns_ptr = @as([*]u8, @ptrCast(bucket)) + bucketAlignsStart(size_class); |
| 272 | const slot_count = @divExact(page_size, size_class); |
| 273 | return aligns_ptr[0..slot_count]; |
| 274 | } |
| 275 | |
| 256 | 276 | fn stackTracePtr( |
| 257 | 277 | bucket: *BucketHeader, |
| 258 | 278 | size_class: usize, |
| ... | ... | @@ -307,10 +327,29 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { |
| 307 | 327 | }; |
| 308 | 328 | } |
| 309 | 329 | |
| 310 | | fn bucketStackFramesStart(size_class: usize) usize { |
| 330 | fn bucketRequestedSizesStart(size_class: usize) usize { |
| 331 | if (!config.safety) @compileError("requested sizes are not stored unless safety is enabled"); |
| 311 | 332 | return mem.alignForward( |
| 312 | 333 | usize, |
| 313 | 334 | @sizeOf(BucketHeader) + usedBitsCount(size_class), |
| 335 | @alignOf(LargestSizeClassInt), |
| 336 | ); |
| 337 | } |
| 338 | |
| 339 | fn bucketAlignsStart(size_class: usize) usize { |
| 340 | if (!config.safety) @compileError("requested sizes are not stored unless safety is enabled"); |
| 341 | const slot_count = @divExact(page_size, size_class); |
| 342 | return bucketRequestedSizesStart(size_class) + (@sizeOf(LargestSizeClassInt) * slot_count); |
| 343 | } |
| 344 | |
| 345 | fn bucketStackFramesStart(size_class: usize) usize { |
| 346 | const unaligned_start = if (config.safety) blk: { |
| 347 | const slot_count = @divExact(page_size, size_class); |
| 348 | break :blk bucketAlignsStart(size_class) + slot_count; |
| 349 | } else @sizeOf(BucketHeader) + usedBitsCount(size_class); |
| 350 | return mem.alignForward( |
| 351 | usize, |
| 352 | unaligned_start, |
| 314 | 353 | @alignOf(usize), |
| 315 | 354 | ); |
| 316 | 355 | } |
| ... | ... | @@ -359,16 +398,15 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { |
| 359 | 398 | /// Emits log messages for leaks and then returns whether there were any leaks. |
| 360 | 399 | pub fn detectLeaks(self: *Self) bool { |
| 361 | 400 | var leaks = false; |
| 362 | | for (self.buckets, 0..) |optional_bucket, bucket_i| { |
| 363 | | const first_bucket = optional_bucket orelse continue; |
| 401 | |
| 402 | for (&self.buckets, 0..) |*buckets, bucket_i| { |
| 403 | if (buckets.root == null) continue; |
| 364 | 404 | const size_class = @as(usize, 1) << @as(math.Log2Int(usize), @intCast(bucket_i)); |
| 365 | 405 | const used_bits_count = usedBitsCount(size_class); |
| 366 | | var bucket = first_bucket; |
| 367 | | while (true) { |
| 406 | var it = buckets.inorderIterator(); |
| 407 | while (it.next()) |node| { |
| 408 | const bucket = node.key; |
| 368 | 409 | leaks = detectLeaksInBucket(bucket, size_class, used_bits_count) or leaks; |
| 369 | | bucket = bucket.next; |
| 370 | | if (bucket == first_bucket) |
| 371 | | break; |
| 372 | 410 | } |
| 373 | 411 | } |
| 374 | 412 | var it = self.large_allocations.valueIterator(); |
| ... | ... | @@ -401,22 +439,18 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { |
| 401 | 439 | } |
| 402 | 440 | } |
| 403 | 441 | // free retained metadata for small allocations |
| 404 | | if (self.empty_buckets) |first_bucket| { |
| 405 | | var bucket = first_bucket; |
| 406 | | while (true) { |
| 407 | | const prev = bucket.prev; |
| 408 | | if (config.never_unmap) { |
| 409 | | // free page that was intentionally leaked by never_unmap |
| 410 | | self.backing_allocator.free(bucket.page[0..page_size]); |
| 411 | | } |
| 412 | | // alloc_cursor was set to slot count when bucket added to empty_buckets |
| 413 | | self.freeBucket(bucket, @divExact(page_size, bucket.alloc_cursor)); |
| 414 | | bucket = prev; |
| 415 | | if (bucket == first_bucket) |
| 416 | | break; |
| 442 | var empty_it = self.empty_buckets.inorderIterator(); |
| 443 | while (empty_it.next()) |node| { |
| 444 | var bucket = node.key; |
| 445 | if (config.never_unmap) { |
| 446 | // free page that was intentionally leaked by never_unmap |
| 447 | self.backing_allocator.free(bucket.page[0..page_size]); |
| 417 | 448 | } |
| 418 | | self.empty_buckets = null; |
| 449 | // alloc_cursor was set to slot count when bucket added to empty_buckets |
| 450 | self.freeBucket(bucket, @divExact(page_size, bucket.alloc_cursor)); |
| 451 | self.bucket_node_pool.destroy(node); |
| 419 | 452 | } |
| 453 | self.empty_buckets.root = null; |
| 420 | 454 | } |
| 421 | 455 | } |
| 422 | 456 | |
| ... | ... | @@ -440,9 +474,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { |
| 440 | 474 | self.freeRetainedMetadata(); |
| 441 | 475 | } |
| 442 | 476 | self.large_allocations.deinit(self.backing_allocator); |
| 443 | | if (config.safety) { |
| 444 | | self.small_allocations.deinit(self.backing_allocator); |
| 445 | | } |
| 477 | self.bucket_node_pool.deinit(); |
| 446 | 478 | self.* = undefined; |
| 447 | 479 | return @as(Check, @enumFromInt(@intFromBool(leaks))); |
| 448 | 480 | } |
| ... | ... | @@ -469,28 +501,27 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { |
| 469 | 501 | }); |
| 470 | 502 | } |
| 471 | 503 | |
| 472 | | fn allocSlot(self: *Self, size_class: usize, trace_addr: usize) Error![*]u8 { |
| 504 | const Slot = struct { |
| 505 | bucket: *BucketHeader, |
| 506 | slot_index: usize, |
| 507 | ptr: [*]u8, |
| 508 | }; |
| 509 | |
| 510 | fn allocSlot(self: *Self, size_class: usize, trace_addr: usize) Error!Slot { |
| 473 | 511 | const bucket_index = math.log2(size_class); |
| 474 | | const first_bucket = self.buckets[bucket_index] orelse try self.createBucket( |
| 475 | | size_class, |
| 476 | | bucket_index, |
| 477 | | ); |
| 478 | | var bucket = first_bucket; |
| 512 | var buckets = &self.buckets[bucket_index]; |
| 479 | 513 | const slot_count = @divExact(page_size, size_class); |
| 480 | | while (bucket.alloc_cursor == slot_count) { |
| 481 | | const prev_bucket = bucket; |
| 482 | | bucket = prev_bucket.next; |
| 483 | | if (bucket == first_bucket) { |
| 484 | | // make a new one |
| 485 | | bucket = try self.createBucket(size_class, bucket_index); |
| 486 | | bucket.prev = prev_bucket; |
| 487 | | bucket.next = prev_bucket.next; |
| 488 | | prev_bucket.next = bucket; |
| 489 | | bucket.next.prev = bucket; |
| 490 | | } |
| 514 | if (self.cur_buckets[bucket_index] == null or self.cur_buckets[bucket_index].?.alloc_cursor == slot_count) { |
| 515 | var new_bucket = try self.createBucket(size_class); |
| 516 | errdefer self.freeBucket(new_bucket, size_class); |
| 517 | const node = try self.bucket_node_pool.create(); |
| 518 | node.key = new_bucket; |
| 519 | var entry = buckets.getEntryFor(new_bucket); |
| 520 | std.debug.assert(entry.node == null); |
| 521 | entry.set(node); |
| 522 | self.cur_buckets[bucket_index] = node.key; |
| 491 | 523 | } |
| 492 | | // change the allocator's current bucket to be this one |
| 493 | | self.buckets[bucket_index] = bucket; |
| 524 | const bucket = self.cur_buckets[bucket_index].?; |
| 494 | 525 | |
| 495 | 526 | const slot_index = bucket.alloc_cursor; |
| 496 | 527 | bucket.alloc_cursor += 1; |
| ... | ... | @@ -500,24 +531,22 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { |
| 500 | 531 | used_bits_byte.* |= (@as(u8, 1) << used_bit_index); |
| 501 | 532 | bucket.used_count += 1; |
| 502 | 533 | bucket.captureStackTrace(trace_addr, size_class, slot_index, .alloc); |
| 503 | | return bucket.page + slot_index * size_class; |
| 534 | return .{ |
| 535 | .bucket = bucket, |
| 536 | .slot_index = slot_index, |
| 537 | .ptr = bucket.page + slot_index * size_class, |
| 538 | }; |
| 504 | 539 | } |
| 505 | 540 | |
| 506 | 541 | fn searchBucket( |
| 507 | | bucket_list: ?*BucketHeader, |
| 542 | buckets: *Buckets, |
| 508 | 543 | addr: usize, |
| 509 | 544 | ) ?*BucketHeader { |
| 510 | | const first_bucket = bucket_list orelse return null; |
| 511 | | var bucket = first_bucket; |
| 512 | | while (true) { |
| 513 | | const in_bucket_range = (addr >= @intFromPtr(bucket.page) and |
| 514 | | addr < @intFromPtr(bucket.page) + page_size); |
| 515 | | if (in_bucket_range) return bucket; |
| 516 | | bucket = bucket.prev; |
| 517 | | if (bucket == first_bucket) { |
| 518 | | return null; |
| 519 | | } |
| 520 | | } |
| 545 | const search_page = mem.alignBackward(usize, addr, page_size); |
| 546 | var search_header: BucketHeader = undefined; |
| 547 | search_header.page = @ptrFromInt(search_page); |
| 548 | const entry = buckets.getEntryFor(&search_header); |
| 549 | return if (entry.node) |node| node.key else null; |
| 521 | 550 | } |
| 522 | 551 | |
| 523 | 552 | /// This function assumes the object is in the large object storage regardless |
| ... | ... | @@ -683,9 +712,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { |
| 683 | 712 | var bucket_index = math.log2(size_class_hint); |
| 684 | 713 | var size_class: usize = size_class_hint; |
| 685 | 714 | const bucket = while (bucket_index < small_bucket_count) : (bucket_index += 1) { |
| 686 | | if (searchBucket(self.buckets[bucket_index], @intFromPtr(old_mem.ptr))) |bucket| { |
| 687 | | // move bucket to head of list to optimize search for nearby allocations |
| 688 | | self.buckets[bucket_index] = bucket; |
| 715 | if (searchBucket(&self.buckets[bucket_index], @intFromPtr(old_mem.ptr))) |bucket| { |
| 689 | 716 | break bucket; |
| 690 | 717 | } |
| 691 | 718 | size_class *= 2; |
| ... | ... | @@ -693,7 +720,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { |
| 693 | 720 | if (config.retain_metadata) { |
| 694 | 721 | if (!self.large_allocations.contains(@intFromPtr(old_mem.ptr))) { |
| 695 | 722 | // object not in active buckets or a large allocation, so search empty buckets |
| 696 | | if (searchBucket(self.empty_buckets, @intFromPtr(old_mem.ptr))) |bucket| { |
| 723 | if (searchBucket(&self.empty_buckets, @intFromPtr(old_mem.ptr))) |bucket| { |
| 697 | 724 | // bucket is empty so is_used below will always be false and we exit there |
| 698 | 725 | break :blk bucket; |
| 699 | 726 | } else { |
| ... | ... | @@ -720,26 +747,27 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { |
| 720 | 747 | |
| 721 | 748 | // Definitely an in-use small alloc now. |
| 722 | 749 | if (config.safety) { |
| 723 | | const entry = self.small_allocations.getEntry(@intFromPtr(old_mem.ptr)) orelse |
| 724 | | @panic("Invalid free"); |
| 725 | | if (old_mem.len != entry.value_ptr.requested_size or log2_old_align != entry.value_ptr.log2_ptr_align) { |
| 750 | const requested_size = bucket.requestedSizes(size_class)[slot_index]; |
| 751 | if (requested_size == 0) @panic("Invalid free"); |
| 752 | const log2_ptr_align = bucket.log2PtrAligns(size_class)[slot_index]; |
| 753 | if (old_mem.len != requested_size or log2_old_align != log2_ptr_align) { |
| 726 | 754 | var addresses: [stack_n]usize = [1]usize{0} ** stack_n; |
| 727 | 755 | var free_stack_trace = StackTrace{ |
| 728 | 756 | .instruction_addresses = &addresses, |
| 729 | 757 | .index = 0, |
| 730 | 758 | }; |
| 731 | 759 | std.debug.captureStackTrace(ret_addr, &free_stack_trace); |
| 732 | | if (old_mem.len != entry.value_ptr.requested_size) { |
| 760 | if (old_mem.len != requested_size) { |
| 733 | 761 | log.err("Allocation size {d} bytes does not match resize size {d}. Allocation: {} Resize: {}", .{ |
| 734 | | entry.value_ptr.requested_size, |
| 762 | requested_size, |
| 735 | 763 | old_mem.len, |
| 736 | 764 | bucketStackTrace(bucket, size_class, slot_index, .alloc), |
| 737 | 765 | free_stack_trace, |
| 738 | 766 | }); |
| 739 | 767 | } |
| 740 | | if (log2_old_align != entry.value_ptr.log2_ptr_align) { |
| 768 | if (log2_old_align != log2_ptr_align) { |
| 741 | 769 | log.err("Allocation alignment {d} does not match resize alignment {d}. Allocation: {} Resize: {}", .{ |
| 742 | | @as(usize, 1) << @as(math.Log2Int(usize), @intCast(entry.value_ptr.log2_ptr_align)), |
| 770 | @as(usize, 1) << @as(math.Log2Int(usize), @intCast(log2_ptr_align)), |
| 743 | 771 | @as(usize, 1) << @as(math.Log2Int(usize), @intCast(log2_old_align)), |
| 744 | 772 | bucketStackTrace(bucket, size_class, slot_index, .alloc), |
| 745 | 773 | free_stack_trace, |
| ... | ... | @@ -768,8 +796,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { |
| 768 | 796 | }); |
| 769 | 797 | } |
| 770 | 798 | if (config.safety) { |
| 771 | | const entry = self.small_allocations.getEntry(@intFromPtr(old_mem.ptr)).?; |
| 772 | | entry.value_ptr.requested_size = new_size; |
| 799 | bucket.requestedSizes(size_class)[slot_index] = @intCast(new_size); |
| 773 | 800 | } |
| 774 | 801 | return true; |
| 775 | 802 | } |
| ... | ... | @@ -803,9 +830,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { |
| 803 | 830 | var bucket_index = math.log2(size_class_hint); |
| 804 | 831 | var size_class: usize = size_class_hint; |
| 805 | 832 | const bucket = while (bucket_index < small_bucket_count) : (bucket_index += 1) { |
| 806 | | if (searchBucket(self.buckets[bucket_index], @intFromPtr(old_mem.ptr))) |bucket| { |
| 807 | | // move bucket to head of list to optimize search for nearby allocations |
| 808 | | self.buckets[bucket_index] = bucket; |
| 833 | if (searchBucket(&self.buckets[bucket_index], @intFromPtr(old_mem.ptr))) |bucket| { |
| 809 | 834 | break bucket; |
| 810 | 835 | } |
| 811 | 836 | size_class *= 2; |
| ... | ... | @@ -813,7 +838,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { |
| 813 | 838 | if (config.retain_metadata) { |
| 814 | 839 | if (!self.large_allocations.contains(@intFromPtr(old_mem.ptr))) { |
| 815 | 840 | // object not in active buckets or a large allocation, so search empty buckets |
| 816 | | if (searchBucket(self.empty_buckets, @intFromPtr(old_mem.ptr))) |bucket| { |
| 841 | if (searchBucket(&self.empty_buckets, @intFromPtr(old_mem.ptr))) |bucket| { |
| 817 | 842 | // bucket is empty so is_used below will always be false and we exit there |
| 818 | 843 | break :blk bucket; |
| 819 | 844 | } else { |
| ... | ... | @@ -842,26 +867,27 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { |
| 842 | 867 | |
| 843 | 868 | // Definitely an in-use small alloc now. |
| 844 | 869 | if (config.safety) { |
| 845 | | const entry = self.small_allocations.getEntry(@intFromPtr(old_mem.ptr)) orelse |
| 846 | | @panic("Invalid free"); |
| 847 | | if (old_mem.len != entry.value_ptr.requested_size or log2_old_align != entry.value_ptr.log2_ptr_align) { |
| 870 | const requested_size = bucket.requestedSizes(size_class)[slot_index]; |
| 871 | if (requested_size == 0) @panic("Invalid free"); |
| 872 | const log2_ptr_align = bucket.log2PtrAligns(size_class)[slot_index]; |
| 873 | if (old_mem.len != requested_size or log2_old_align != log2_ptr_align) { |
| 848 | 874 | var addresses: [stack_n]usize = [1]usize{0} ** stack_n; |
| 849 | 875 | var free_stack_trace = StackTrace{ |
| 850 | 876 | .instruction_addresses = &addresses, |
| 851 | 877 | .index = 0, |
| 852 | 878 | }; |
| 853 | 879 | std.debug.captureStackTrace(ret_addr, &free_stack_trace); |
| 854 | | if (old_mem.len != entry.value_ptr.requested_size) { |
| 880 | if (old_mem.len != requested_size) { |
| 855 | 881 | log.err("Allocation size {d} bytes does not match free size {d}. Allocation: {} Free: {}", .{ |
| 856 | | entry.value_ptr.requested_size, |
| 882 | requested_size, |
| 857 | 883 | old_mem.len, |
| 858 | 884 | bucketStackTrace(bucket, size_class, slot_index, .alloc), |
| 859 | 885 | free_stack_trace, |
| 860 | 886 | }); |
| 861 | 887 | } |
| 862 | | if (log2_old_align != entry.value_ptr.log2_ptr_align) { |
| 888 | if (log2_old_align != log2_ptr_align) { |
| 863 | 889 | log.err("Allocation alignment {d} does not match free alignment {d}. Allocation: {} Free: {}", .{ |
| 864 | | @as(usize, 1) << @as(math.Log2Int(usize), @intCast(entry.value_ptr.log2_ptr_align)), |
| 890 | @as(usize, 1) << @as(math.Log2Int(usize), @intCast(log2_ptr_align)), |
| 865 | 891 | @as(usize, 1) << @as(math.Log2Int(usize), @intCast(log2_old_align)), |
| 866 | 892 | bucketStackTrace(bucket, size_class, slot_index, .alloc), |
| 867 | 893 | free_stack_trace, |
| ... | ... | @@ -879,44 +905,33 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { |
| 879 | 905 | |
| 880 | 906 | used_byte.* &= ~(@as(u8, 1) << used_bit_index); |
| 881 | 907 | bucket.used_count -= 1; |
| 908 | if (config.safety) { |
| 909 | bucket.requestedSizes(size_class)[slot_index] = 0; |
| 910 | } |
| 882 | 911 | if (bucket.used_count == 0) { |
| 883 | | if (bucket.next == bucket) { |
| 884 | | // it's the only bucket and therefore the current one |
| 885 | | self.buckets[bucket_index] = null; |
| 886 | | } else { |
| 887 | | bucket.next.prev = bucket.prev; |
| 888 | | bucket.prev.next = bucket.next; |
| 889 | | self.buckets[bucket_index] = bucket.prev; |
| 912 | var entry = self.buckets[bucket_index].getEntryFor(bucket); |
| 913 | // save the node for destruction/insertion into in empty_buckets |
| 914 | var node = entry.node.?; |
| 915 | entry.set(null); |
| 916 | if (self.cur_buckets[bucket_index] == bucket) { |
| 917 | self.cur_buckets[bucket_index] = null; |
| 890 | 918 | } |
| 891 | 919 | if (!config.never_unmap) { |
| 892 | 920 | self.backing_allocator.free(bucket.page[0..page_size]); |
| 893 | 921 | } |
| 894 | 922 | if (!config.retain_metadata) { |
| 895 | 923 | self.freeBucket(bucket, size_class); |
| 924 | self.bucket_node_pool.destroy(node); |
| 896 | 925 | } else { |
| 897 | 926 | // move alloc_cursor to end so we can tell size_class later |
| 898 | 927 | const slot_count = @divExact(page_size, size_class); |
| 899 | 928 | bucket.alloc_cursor = @as(SlotIndex, @truncate(slot_count)); |
| 900 | | if (self.empty_buckets) |prev_bucket| { |
| 901 | | // empty_buckets is ordered newest to oldest through prev so that if |
| 902 | | // config.never_unmap is false and backing_allocator reuses freed memory |
| 903 | | // then searchBuckets will always return the newer, relevant bucket |
| 904 | | bucket.prev = prev_bucket; |
| 905 | | bucket.next = prev_bucket.next; |
| 906 | | prev_bucket.next = bucket; |
| 907 | | bucket.next.prev = bucket; |
| 908 | | } else { |
| 909 | | bucket.prev = bucket; |
| 910 | | bucket.next = bucket; |
| 911 | | } |
| 912 | | self.empty_buckets = bucket; |
| 929 | var empty_entry = self.empty_buckets.getEntryFor(node.key); |
| 930 | empty_entry.set(node); |
| 913 | 931 | } |
| 914 | 932 | } else { |
| 915 | 933 | @memset(old_mem, undefined); |
| 916 | 934 | } |
| 917 | | if (config.safety) { |
| 918 | | assert(self.small_allocations.remove(@intFromPtr(old_mem.ptr))); |
| 919 | | } |
| 920 | 935 | if (config.verbose_log) { |
| 921 | 936 | log.info("small free {d} bytes at {*}", .{ old_mem.len, old_mem.ptr }); |
| 922 | 937 | } |
| ... | ... | @@ -980,23 +995,19 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { |
| 980 | 995 | return slice.ptr; |
| 981 | 996 | } |
| 982 | 997 | |
| 983 | | if (config.safety) { |
| 984 | | try self.small_allocations.ensureUnusedCapacity(self.backing_allocator, 1); |
| 985 | | } |
| 986 | 998 | const new_size_class = math.ceilPowerOfTwoAssert(usize, new_aligned_size); |
| 987 | | const ptr = try self.allocSlot(new_size_class, ret_addr); |
| 999 | const slot = try self.allocSlot(new_size_class, ret_addr); |
| 988 | 1000 | if (config.safety) { |
| 989 | | const gop = self.small_allocations.getOrPutAssumeCapacity(@intFromPtr(ptr)); |
| 990 | | gop.value_ptr.requested_size = len; |
| 991 | | gop.value_ptr.log2_ptr_align = log2_ptr_align; |
| 1001 | slot.bucket.requestedSizes(new_size_class)[slot.slot_index] = @intCast(len); |
| 1002 | slot.bucket.log2PtrAligns(new_size_class)[slot.slot_index] = log2_ptr_align; |
| 992 | 1003 | } |
| 993 | 1004 | if (config.verbose_log) { |
| 994 | | log.info("small alloc {d} bytes at {*}", .{ len, ptr }); |
| 1005 | log.info("small alloc {d} bytes at {*}", .{ len, slot.ptr }); |
| 995 | 1006 | } |
| 996 | | return ptr; |
| 1007 | return slot.ptr; |
| 997 | 1008 | } |
| 998 | 1009 | |
| 999 | | fn createBucket(self: *Self, size_class: usize, bucket_index: usize) Error!*BucketHeader { |
| 1010 | fn createBucket(self: *Self, size_class: usize) Error!*BucketHeader { |
| 1000 | 1011 | const page = try self.backing_allocator.alignedAlloc(u8, page_size, page_size); |
| 1001 | 1012 | errdefer self.backing_allocator.free(page); |
| 1002 | 1013 | |
| ... | ... | @@ -1004,15 +1015,16 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { |
| 1004 | 1015 | const bucket_bytes = try self.backing_allocator.alignedAlloc(u8, @alignOf(BucketHeader), bucket_size); |
| 1005 | 1016 | const ptr = @as(*BucketHeader, @ptrCast(bucket_bytes.ptr)); |
| 1006 | 1017 | ptr.* = BucketHeader{ |
| 1007 | | .prev = ptr, |
| 1008 | | .next = ptr, |
| 1009 | 1018 | .page = page.ptr, |
| 1010 | 1019 | .alloc_cursor = 0, |
| 1011 | 1020 | .used_count = 0, |
| 1012 | 1021 | }; |
| 1013 | | self.buckets[bucket_index] = ptr; |
| 1014 | 1022 | // Set the used bits to all zeroes |
| 1015 | 1023 | @memset(@as([*]u8, @as(*[1]u8, ptr.usedBits(0)))[0..usedBitsCount(size_class)], 0); |
| 1024 | if (config.safety) { |
| 1025 | // Set the requested sizes to zeroes |
| 1026 | @memset(mem.sliceAsBytes(ptr.requestedSizes(size_class)), 0); |
| 1027 | } |
| 1016 | 1028 | return ptr; |
| 1017 | 1029 | } |
| 1018 | 1030 | }; |
| ... | ... | @@ -1375,10 +1387,10 @@ test "double frees" { |
| 1375 | 1387 | const index: usize = 6; |
| 1376 | 1388 | const size_class: usize = @as(usize, 1) << 6; |
| 1377 | 1389 | const small = try allocator.alloc(u8, size_class); |
| 1378 | | try std.testing.expect(GPA.searchBucket(gpa.buckets[index], @intFromPtr(small.ptr)) != null); |
| 1390 | try std.testing.expect(GPA.searchBucket(&gpa.buckets[index], @intFromPtr(small.ptr)) != null); |
| 1379 | 1391 | allocator.free(small); |
| 1380 | | try std.testing.expect(GPA.searchBucket(gpa.buckets[index], @intFromPtr(small.ptr)) == null); |
| 1381 | | try std.testing.expect(GPA.searchBucket(gpa.empty_buckets, @intFromPtr(small.ptr)) != null); |
| 1392 | try std.testing.expect(GPA.searchBucket(&gpa.buckets[index], @intFromPtr(small.ptr)) == null); |
| 1393 | try std.testing.expect(GPA.searchBucket(&gpa.empty_buckets, @intFromPtr(small.ptr)) != null); |
| 1382 | 1394 | |
| 1383 | 1395 | // detect a large allocation double free |
| 1384 | 1396 | const large = try allocator.alloc(u8, 2 * page_size); |
| ... | ... | @@ -1395,8 +1407,8 @@ test "double frees" { |
| 1395 | 1407 | |
| 1396 | 1408 | // check that flushing retained metadata doesn't disturb live allocations |
| 1397 | 1409 | gpa.flushRetainedMetadata(); |
| 1398 | | try std.testing.expect(gpa.empty_buckets == null); |
| 1399 | | try std.testing.expect(GPA.searchBucket(gpa.buckets[index], @intFromPtr(normal_small.ptr)) != null); |
| 1410 | try std.testing.expect(gpa.empty_buckets.root == null); |
| 1411 | try std.testing.expect(GPA.searchBucket(&gpa.buckets[index], @intFromPtr(normal_small.ptr)) != null); |
| 1400 | 1412 | try std.testing.expect(gpa.large_allocations.contains(@intFromPtr(normal_large.ptr))); |
| 1401 | 1413 | try std.testing.expect(!gpa.large_allocations.contains(@intFromPtr(large.ptr))); |
| 1402 | 1414 | } |
Andrew Kelley <andrew@ziglang.org> 2023-10-03 10:58:48-07:00
GitHub <noreply@github.com> 2023-10-03 10:58:48-07:00
Signed by PGP key 4AEE18F83AFDEB23
lib