| ... | @@ -160,11 +160,12 @@ pub const Check = enum { ok, leak }; | ... | @@ -160,11 +160,12 @@ pub const Check = enum { ok, leak }; |
| 160 | pub fn GeneralPurposeAllocator(comptime config: Config) type { | 160 | pub fn GeneralPurposeAllocator(comptime config: Config) type { |
| 161 | return struct { | 161 | return struct { |
| 162 | backing_allocator: Allocator = std.heap.page_allocator, | 162 | backing_allocator: Allocator = std.heap.page_allocator, |
| 163 | buckets: [small_bucket_count]?*BucketHeader = [1]?*BucketHeader{null} ** small_bucket_count, | 163 | buckets: [small_bucket_count]Buckets = [1]Buckets{Buckets{}} ** small_bucket_count, |
| | 164 | cur_buckets: [small_bucket_count]?*BucketHeader = [1]?*BucketHeader{null} ** small_bucket_count, |
| 164 | large_allocations: LargeAllocTable = .{}, | 165 | large_allocations: LargeAllocTable = .{}, |
| 165 | small_allocations: if (config.safety) SmallAllocTable else void = if (config.safety) .{} else {}, | 166 | empty_buckets: if (config.retain_metadata) Buckets else void = |
| 166 | empty_buckets: if (config.retain_metadata) ?*BucketHeader else void = | 167 | if (config.retain_metadata) Buckets{} else {}, |
| 167 | if (config.retain_metadata) null else {}, | 168 | bucket_node_pool: std.heap.MemoryPool(Buckets.Node) = std.heap.MemoryPool(Buckets.Node).init(std.heap.page_allocator), |
| 168 | | 169 | |
| 169 | total_requested_bytes: @TypeOf(total_requested_bytes_init) = total_requested_bytes_init, | 170 | total_requested_bytes: @TypeOf(total_requested_bytes_init) = total_requested_bytes_init, |
| 170 | requested_memory_limit: @TypeOf(requested_memory_limit_init) = requested_memory_limit_init, | 171 | requested_memory_limit: @TypeOf(requested_memory_limit_init) = requested_memory_limit_init, |
| ... | @@ -196,11 +197,14 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { | ... | @@ -196,11 +197,14 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { |
| 196 | | 197 | |
| 197 | const small_bucket_count = math.log2(page_size); | 198 | const small_bucket_count = math.log2(page_size); |
| 198 | const largest_bucket_object_size = 1 << (small_bucket_count - 1); | 199 | const largest_bucket_object_size = 1 << (small_bucket_count - 1); |
| | 200 | const LargestSizeClassInt = std.math.IntFittingRange(0, largest_bucket_object_size); |
| 199 | | 201 | |
| 200 | const SmallAlloc = struct { | 202 | const bucketCompare = struct { |
| 201 | requested_size: usize, | 203 | fn compare(a: *BucketHeader, b: *BucketHeader) std.math.Order { |
| 202 | log2_ptr_align: u8, | 204 | return std.math.order(@intFromPtr(a.page), @intFromPtr(b.page)); |
| 203 | }; | 205 | } |
| | 206 | }.compare; |
| | 207 | const Buckets = std.Treap(*BucketHeader, bucketCompare); |
| 204 | | 208 | |
| 205 | const LargeAlloc = struct { | 209 | const LargeAlloc = struct { |
| 206 | bytes: []u8, | 210 | bytes: []u8, |
| ... | @@ -235,16 +239,17 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { | ... | @@ -235,16 +239,17 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { |
| 235 | } | 239 | } |
| 236 | }; | 240 | }; |
| 237 | const LargeAllocTable = std.AutoHashMapUnmanaged(usize, LargeAlloc); | 241 | const LargeAllocTable = std.AutoHashMapUnmanaged(usize, LargeAlloc); |
| 238 | const SmallAllocTable = std.AutoHashMapUnmanaged(usize, SmallAlloc); | | |
| 239 | | 242 | |
| 240 | // Bucket: In memory, in order: | 243 | // Bucket: In memory, in order: |
| 241 | // * BucketHeader | 244 | // * BucketHeader |
| 242 | // * bucket_used_bits: [N]u8, // 1 bit for every slot; 1 byte for every 8 slots | 245 | // * bucket_used_bits: [N]u8, // 1 bit for every slot; 1 byte for every 8 slots |
| | 246 | // -- below only exists when config.safety is true -- |
| | 247 | // * requested_sizes: [N]LargestSizeClassInt // 1 int for every slot |
| | 248 | // * log2_ptr_aligns: [N]u8 // 1 byte for every slot |
| | 249 | // -- above only exists when config.safety is true -- |
| 243 | // * stack_trace_addresses: [N]usize, // traces_per_slot for every allocation | 250 | // * stack_trace_addresses: [N]usize, // traces_per_slot for every allocation |
| 244 | | 251 | |
| 245 | const BucketHeader = struct { | 252 | const BucketHeader = struct { |
| 246 | prev: *BucketHeader, | | |
| 247 | next: *BucketHeader, | | |
| 248 | page: [*]align(page_size) u8, | 253 | page: [*]align(page_size) u8, |
| 249 | alloc_cursor: SlotIndex, | 254 | alloc_cursor: SlotIndex, |
| 250 | used_count: SlotIndex, | 255 | used_count: SlotIndex, |
| ... | @@ -253,6 +258,21 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { | ... | @@ -253,6 +258,21 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { |
| 253 | return @as(*u8, @ptrFromInt(@intFromPtr(bucket) + @sizeOf(BucketHeader) + index)); | 258 | return @as(*u8, @ptrFromInt(@intFromPtr(bucket) + @sizeOf(BucketHeader) + index)); |
| 254 | } | 259 | } |
| 255 | | 260 | |
| | 261 | fn requestedSizes(bucket: *BucketHeader, size_class: usize) []LargestSizeClassInt { |
| | 262 | if (!config.safety) @compileError("requested size is only stored when safety is enabled"); |
| | 263 | const start_ptr = @as([*]u8, @ptrCast(bucket)) + bucketRequestedSizesStart(size_class); |
| | 264 | const sizes = @as([*]LargestSizeClassInt, @ptrCast(@alignCast(start_ptr))); |
| | 265 | const slot_count = @divExact(page_size, size_class); |
| | 266 | return sizes[0..slot_count]; |
| | 267 | } |
| | 268 | |
| | 269 | fn log2PtrAligns(bucket: *BucketHeader, size_class: usize) []u8 { |
| | 270 | if (!config.safety) @compileError("requested size is only stored when safety is enabled"); |
| | 271 | const aligns_ptr = @as([*]u8, @ptrCast(bucket)) + bucketAlignsStart(size_class); |
| | 272 | const slot_count = @divExact(page_size, size_class); |
| | 273 | return aligns_ptr[0..slot_count]; |
| | 274 | } |
| | 275 | |
| 256 | fn stackTracePtr( | 276 | fn stackTracePtr( |
| 257 | bucket: *BucketHeader, | 277 | bucket: *BucketHeader, |
| 258 | size_class: usize, | 278 | size_class: usize, |
| ... | @@ -307,10 +327,29 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { | ... | @@ -307,10 +327,29 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { |
| 307 | }; | 327 | }; |
| 308 | } | 328 | } |
| 309 | | 329 | |
| 310 | fn bucketStackFramesStart(size_class: usize) usize { | 330 | fn bucketRequestedSizesStart(size_class: usize) usize { |
| | 331 | if (!config.safety) @compileError("requested sizes are not stored unless safety is enabled"); |
| 311 | return mem.alignForward( | 332 | return mem.alignForward( |
| 312 | usize, | 333 | usize, |
| 313 | @sizeOf(BucketHeader) + usedBitsCount(size_class), | 334 | @sizeOf(BucketHeader) + usedBitsCount(size_class), |
| | 335 | @alignOf(LargestSizeClassInt), |
| | 336 | ); |
| | 337 | } |
| | 338 | |
| | 339 | fn bucketAlignsStart(size_class: usize) usize { |
| | 340 | if (!config.safety) @compileError("requested sizes are not stored unless safety is enabled"); |
| | 341 | const slot_count = @divExact(page_size, size_class); |
| | 342 | return bucketRequestedSizesStart(size_class) + (@sizeOf(LargestSizeClassInt) * slot_count); |
| | 343 | } |
| | 344 | |
| | 345 | fn bucketStackFramesStart(size_class: usize) usize { |
| | 346 | const unaligned_start = if (config.safety) blk: { |
| | 347 | const slot_count = @divExact(page_size, size_class); |
| | 348 | break :blk bucketAlignsStart(size_class) + slot_count; |
| | 349 | } else @sizeOf(BucketHeader) + usedBitsCount(size_class); |
| | 350 | return mem.alignForward( |
| | 351 | usize, |
| | 352 | unaligned_start, |
| 314 | @alignOf(usize), | 353 | @alignOf(usize), |
| 315 | ); | 354 | ); |
| 316 | } | 355 | } |
| ... | @@ -359,16 +398,15 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { | ... | @@ -359,16 +398,15 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { |
| 359 | /// Emits log messages for leaks and then returns whether there were any leaks. | 398 | /// Emits log messages for leaks and then returns whether there were any leaks. |
| 360 | pub fn detectLeaks(self: *Self) bool { | 399 | pub fn detectLeaks(self: *Self) bool { |
| 361 | var leaks = false; | 400 | var leaks = false; |
| 362 | for (self.buckets, 0..) |optional_bucket, bucket_i| { | 401 | |
| 363 | const first_bucket = optional_bucket orelse continue; | 402 | for (&self.buckets, 0..) |*buckets, bucket_i| { |
| | 403 | if (buckets.root == null) continue; |
| 364 | const size_class = @as(usize, 1) << @as(math.Log2Int(usize), @intCast(bucket_i)); | 404 | const size_class = @as(usize, 1) << @as(math.Log2Int(usize), @intCast(bucket_i)); |
| 365 | const used_bits_count = usedBitsCount(size_class); | 405 | const used_bits_count = usedBitsCount(size_class); |
| 366 | var bucket = first_bucket; | 406 | var it = buckets.inorderIterator(); |
| 367 | while (true) { | 407 | while (it.next()) |node| { |
| | 408 | const bucket = node.key; |
| 368 | leaks = detectLeaksInBucket(bucket, size_class, used_bits_count) or leaks; | 409 | leaks = detectLeaksInBucket(bucket, size_class, used_bits_count) or leaks; |
| 369 | bucket = bucket.next; | | |
| 370 | if (bucket == first_bucket) | | |
| 371 | break; | | |
| 372 | } | 410 | } |
| 373 | } | 411 | } |
| 374 | var it = self.large_allocations.valueIterator(); | 412 | var it = self.large_allocations.valueIterator(); |
| ... | @@ -401,22 +439,18 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { | ... | @@ -401,22 +439,18 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { |
| 401 | } | 439 | } |
| 402 | } | 440 | } |
| 403 | // free retained metadata for small allocations | 441 | // free retained metadata for small allocations |
| 404 | if (self.empty_buckets) |first_bucket| { | 442 | var empty_it = self.empty_buckets.inorderIterator(); |
| 405 | var bucket = first_bucket; | 443 | while (empty_it.next()) |node| { |
| 406 | while (true) { | 444 | var bucket = node.key; |
| 407 | const prev = bucket.prev; | 445 | if (config.never_unmap) { |
| 408 | if (config.never_unmap) { | 446 | // free page that was intentionally leaked by never_unmap |
| 409 | // free page that was intentionally leaked by never_unmap | 447 | self.backing_allocator.free(bucket.page[0..page_size]); |
| 410 | self.backing_allocator.free(bucket.page[0..page_size]); | | |
| 411 | } | | |
| 412 | // alloc_cursor was set to slot count when bucket added to empty_buckets | | |
| 413 | self.freeBucket(bucket, @divExact(page_size, bucket.alloc_cursor)); | | |
| 414 | bucket = prev; | | |
| 415 | if (bucket == first_bucket) | | |
| 416 | break; | | |
| 417 | } | 448 | } |
| 418 | self.empty_buckets = null; | 449 | // alloc_cursor was set to slot count when bucket added to empty_buckets |
| | 450 | self.freeBucket(bucket, @divExact(page_size, bucket.alloc_cursor)); |
| | 451 | self.bucket_node_pool.destroy(node); |
| 419 | } | 452 | } |
| | 453 | self.empty_buckets.root = null; |
| 420 | } | 454 | } |
| 421 | } | 455 | } |
| 422 | | 456 | |
| ... | @@ -440,9 +474,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { | ... | @@ -440,9 +474,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { |
| 440 | self.freeRetainedMetadata(); | 474 | self.freeRetainedMetadata(); |
| 441 | } | 475 | } |
| 442 | self.large_allocations.deinit(self.backing_allocator); | 476 | self.large_allocations.deinit(self.backing_allocator); |
| 443 | if (config.safety) { | 477 | self.bucket_node_pool.deinit(); |
| 444 | self.small_allocations.deinit(self.backing_allocator); | | |
| 445 | } | | |
| 446 | self.* = undefined; | 478 | self.* = undefined; |
| 447 | return @as(Check, @enumFromInt(@intFromBool(leaks))); | 479 | return @as(Check, @enumFromInt(@intFromBool(leaks))); |
| 448 | } | 480 | } |
| ... | @@ -469,28 +501,27 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { | ... | @@ -469,28 +501,27 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { |
| 469 | }); | 501 | }); |
| 470 | } | 502 | } |
| 471 | | 503 | |
| 472 | fn allocSlot(self: *Self, size_class: usize, trace_addr: usize) Error![*]u8 { | 504 | const Slot = struct { |
| | 505 | bucket: *BucketHeader, |
| | 506 | slot_index: usize, |
| | 507 | ptr: [*]u8, |
| | 508 | }; |
| | 509 | |
| | 510 | fn allocSlot(self: *Self, size_class: usize, trace_addr: usize) Error!Slot { |
| 473 | const bucket_index = math.log2(size_class); | 511 | const bucket_index = math.log2(size_class); |
| 474 | const first_bucket = self.buckets[bucket_index] orelse try self.createBucket( | 512 | var buckets = &self.buckets[bucket_index]; |
| 475 | size_class, | | |
| 476 | bucket_index, | | |
| 477 | ); | | |
| 478 | var bucket = first_bucket; | | |
| 479 | const slot_count = @divExact(page_size, size_class); | 513 | const slot_count = @divExact(page_size, size_class); |
| 480 | while (bucket.alloc_cursor == slot_count) { | 514 | if (self.cur_buckets[bucket_index] == null or self.cur_buckets[bucket_index].?.alloc_cursor == slot_count) { |
| 481 | const prev_bucket = bucket; | 515 | var new_bucket = try self.createBucket(size_class); |
| 482 | bucket = prev_bucket.next; | 516 | errdefer self.freeBucket(new_bucket, size_class); |
| 483 | if (bucket == first_bucket) { | 517 | const node = try self.bucket_node_pool.create(); |
| 484 | // make a new one | 518 | node.key = new_bucket; |
| 485 | bucket = try self.createBucket(size_class, bucket_index); | 519 | var entry = buckets.getEntryFor(new_bucket); |
| 486 | bucket.prev = prev_bucket; | 520 | std.debug.assert(entry.node == null); |
| 487 | bucket.next = prev_bucket.next; | 521 | entry.set(node); |
| 488 | prev_bucket.next = bucket; | 522 | self.cur_buckets[bucket_index] = node.key; |
| 489 | bucket.next.prev = bucket; | | |
| 490 | } | | |
| 491 | } | 523 | } |
| 492 | // change the allocator's current bucket to be this one | 524 | const bucket = self.cur_buckets[bucket_index].?; |
| 493 | self.buckets[bucket_index] = bucket; | | |
| 494 | | 525 | |
| 495 | const slot_index = bucket.alloc_cursor; | 526 | const slot_index = bucket.alloc_cursor; |
| 496 | bucket.alloc_cursor += 1; | 527 | bucket.alloc_cursor += 1; |
| ... | @@ -500,24 +531,22 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { | ... | @@ -500,24 +531,22 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { |
| 500 | used_bits_byte.* |= (@as(u8, 1) << used_bit_index); | 531 | used_bits_byte.* |= (@as(u8, 1) << used_bit_index); |
| 501 | bucket.used_count += 1; | 532 | bucket.used_count += 1; |
| 502 | bucket.captureStackTrace(trace_addr, size_class, slot_index, .alloc); | 533 | bucket.captureStackTrace(trace_addr, size_class, slot_index, .alloc); |
| 503 | return bucket.page + slot_index * size_class; | 534 | return .{ |
| | 535 | .bucket = bucket, |
| | 536 | .slot_index = slot_index, |
| | 537 | .ptr = bucket.page + slot_index * size_class, |
| | 538 | }; |
| 504 | } | 539 | } |
| 505 | | 540 | |
| 506 | fn searchBucket( | 541 | fn searchBucket( |
| 507 | bucket_list: ?*BucketHeader, | 542 | buckets: *Buckets, |
| 508 | addr: usize, | 543 | addr: usize, |
| 509 | ) ?*BucketHeader { | 544 | ) ?*BucketHeader { |
| 510 | const first_bucket = bucket_list orelse return null; | 545 | const search_page = mem.alignBackward(usize, addr, page_size); |
| 511 | var bucket = first_bucket; | 546 | var search_header: BucketHeader = undefined; |
| 512 | while (true) { | 547 | search_header.page = @ptrFromInt(search_page); |
| 513 | const in_bucket_range = (addr >= @intFromPtr(bucket.page) and | 548 | const entry = buckets.getEntryFor(&search_header); |
| 514 | addr < @intFromPtr(bucket.page) + page_size); | 549 | return if (entry.node) |node| node.key else null; |
| 515 | if (in_bucket_range) return bucket; | | |
| 516 | bucket = bucket.prev; | | |
| 517 | if (bucket == first_bucket) { | | |
| 518 | return null; | | |
| 519 | } | | |
| 520 | } | | |
| 521 | } | 550 | } |
| 522 | | 551 | |
| 523 | /// This function assumes the object is in the large object storage regardless | 552 | /// This function assumes the object is in the large object storage regardless |
| ... | @@ -683,9 +712,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { | ... | @@ -683,9 +712,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { |
| 683 | var bucket_index = math.log2(size_class_hint); | 712 | var bucket_index = math.log2(size_class_hint); |
| 684 | var size_class: usize = size_class_hint; | 713 | var size_class: usize = size_class_hint; |
| 685 | const bucket = while (bucket_index < small_bucket_count) : (bucket_index += 1) { | 714 | const bucket = while (bucket_index < small_bucket_count) : (bucket_index += 1) { |
| 686 | if (searchBucket(self.buckets[bucket_index], @intFromPtr(old_mem.ptr))) |bucket| { | 715 | if (searchBucket(&self.buckets[bucket_index], @intFromPtr(old_mem.ptr))) |bucket| { |
| 687 | // move bucket to head of list to optimize search for nearby allocations | | |
| 688 | self.buckets[bucket_index] = bucket; | | |
| 689 | break bucket; | 716 | break bucket; |
| 690 | } | 717 | } |
| 691 | size_class *= 2; | 718 | size_class *= 2; |
| ... | @@ -693,7 +720,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { | ... | @@ -693,7 +720,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { |
| 693 | if (config.retain_metadata) { | 720 | if (config.retain_metadata) { |
| 694 | if (!self.large_allocations.contains(@intFromPtr(old_mem.ptr))) { | 721 | if (!self.large_allocations.contains(@intFromPtr(old_mem.ptr))) { |
| 695 | // object not in active buckets or a large allocation, so search empty buckets | 722 | // object not in active buckets or a large allocation, so search empty buckets |
| 696 | if (searchBucket(self.empty_buckets, @intFromPtr(old_mem.ptr))) |bucket| { | 723 | if (searchBucket(&self.empty_buckets, @intFromPtr(old_mem.ptr))) |bucket| { |
| 697 | // bucket is empty so is_used below will always be false and we exit there | 724 | // bucket is empty so is_used below will always be false and we exit there |
| 698 | break :blk bucket; | 725 | break :blk bucket; |
| 699 | } else { | 726 | } else { |
| ... | @@ -720,26 +747,27 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { | ... | @@ -720,26 +747,27 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { |
| 720 | | 747 | |
| 721 | // Definitely an in-use small alloc now. | 748 | // Definitely an in-use small alloc now. |
| 722 | if (config.safety) { | 749 | if (config.safety) { |
| 723 | const entry = self.small_allocations.getEntry(@intFromPtr(old_mem.ptr)) orelse | 750 | const requested_size = bucket.requestedSizes(size_class)[slot_index]; |
| 724 | @panic("Invalid free"); | 751 | if (requested_size == 0) @panic("Invalid free"); |
| 725 | if (old_mem.len != entry.value_ptr.requested_size or log2_old_align != entry.value_ptr.log2_ptr_align) { | 752 | const log2_ptr_align = bucket.log2PtrAligns(size_class)[slot_index]; |
| | 753 | if (old_mem.len != requested_size or log2_old_align != log2_ptr_align) { |
| 726 | var addresses: [stack_n]usize = [1]usize{0} ** stack_n; | 754 | var addresses: [stack_n]usize = [1]usize{0} ** stack_n; |
| 727 | var free_stack_trace = StackTrace{ | 755 | var free_stack_trace = StackTrace{ |
| 728 | .instruction_addresses = &addresses, | 756 | .instruction_addresses = &addresses, |
| 729 | .index = 0, | 757 | .index = 0, |
| 730 | }; | 758 | }; |
| 731 | std.debug.captureStackTrace(ret_addr, &free_stack_trace); | 759 | std.debug.captureStackTrace(ret_addr, &free_stack_trace); |
| 732 | if (old_mem.len != entry.value_ptr.requested_size) { | 760 | if (old_mem.len != requested_size) { |
| 733 | log.err("Allocation size {d} bytes does not match resize size {d}. Allocation: {} Resize: {}", .{ | 761 | log.err("Allocation size {d} bytes does not match resize size {d}. Allocation: {} Resize: {}", .{ |
| 734 | entry.value_ptr.requested_size, | 762 | requested_size, |
| 735 | old_mem.len, | 763 | old_mem.len, |
| 736 | bucketStackTrace(bucket, size_class, slot_index, .alloc), | 764 | bucketStackTrace(bucket, size_class, slot_index, .alloc), |
| 737 | free_stack_trace, | 765 | free_stack_trace, |
| 738 | }); | 766 | }); |
| 739 | } | 767 | } |
| 740 | if (log2_old_align != entry.value_ptr.log2_ptr_align) { | 768 | if (log2_old_align != log2_ptr_align) { |
| 741 | log.err("Allocation alignment {d} does not match resize alignment {d}. Allocation: {} Resize: {}", .{ | 769 | log.err("Allocation alignment {d} does not match resize alignment {d}. Allocation: {} Resize: {}", .{ |
| 742 | @as(usize, 1) << @as(math.Log2Int(usize), @intCast(entry.value_ptr.log2_ptr_align)), | 770 | @as(usize, 1) << @as(math.Log2Int(usize), @intCast(log2_ptr_align)), |
| 743 | @as(usize, 1) << @as(math.Log2Int(usize), @intCast(log2_old_align)), | 771 | @as(usize, 1) << @as(math.Log2Int(usize), @intCast(log2_old_align)), |
| 744 | bucketStackTrace(bucket, size_class, slot_index, .alloc), | 772 | bucketStackTrace(bucket, size_class, slot_index, .alloc), |
| 745 | free_stack_trace, | 773 | free_stack_trace, |
| ... | @@ -768,8 +796,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { | ... | @@ -768,8 +796,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { |
| 768 | }); | 796 | }); |
| 769 | } | 797 | } |
| 770 | if (config.safety) { | 798 | if (config.safety) { |
| 771 | const entry = self.small_allocations.getEntry(@intFromPtr(old_mem.ptr)).?; | 799 | bucket.requestedSizes(size_class)[slot_index] = @intCast(new_size); |
| 772 | entry.value_ptr.requested_size = new_size; | | |
| 773 | } | 800 | } |
| 774 | return true; | 801 | return true; |
| 775 | } | 802 | } |
| ... | @@ -803,9 +830,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { | ... | @@ -803,9 +830,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { |
| 803 | var bucket_index = math.log2(size_class_hint); | 830 | var bucket_index = math.log2(size_class_hint); |
| 804 | var size_class: usize = size_class_hint; | 831 | var size_class: usize = size_class_hint; |
| 805 | const bucket = while (bucket_index < small_bucket_count) : (bucket_index += 1) { | 832 | const bucket = while (bucket_index < small_bucket_count) : (bucket_index += 1) { |
| 806 | if (searchBucket(self.buckets[bucket_index], @intFromPtr(old_mem.ptr))) |bucket| { | 833 | if (searchBucket(&self.buckets[bucket_index], @intFromPtr(old_mem.ptr))) |bucket| { |
| 807 | // move bucket to head of list to optimize search for nearby allocations | | |
| 808 | self.buckets[bucket_index] = bucket; | | |
| 809 | break bucket; | 834 | break bucket; |
| 810 | } | 835 | } |
| 811 | size_class *= 2; | 836 | size_class *= 2; |
| ... | @@ -813,7 +838,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { | ... | @@ -813,7 +838,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { |
| 813 | if (config.retain_metadata) { | 838 | if (config.retain_metadata) { |
| 814 | if (!self.large_allocations.contains(@intFromPtr(old_mem.ptr))) { | 839 | if (!self.large_allocations.contains(@intFromPtr(old_mem.ptr))) { |
| 815 | // object not in active buckets or a large allocation, so search empty buckets | 840 | // object not in active buckets or a large allocation, so search empty buckets |
| 816 | if (searchBucket(self.empty_buckets, @intFromPtr(old_mem.ptr))) |bucket| { | 841 | if (searchBucket(&self.empty_buckets, @intFromPtr(old_mem.ptr))) |bucket| { |
| 817 | // bucket is empty so is_used below will always be false and we exit there | 842 | // bucket is empty so is_used below will always be false and we exit there |
| 818 | break :blk bucket; | 843 | break :blk bucket; |
| 819 | } else { | 844 | } else { |
| ... | @@ -842,26 +867,27 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { | ... | @@ -842,26 +867,27 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { |
| 842 | | 867 | |
| 843 | // Definitely an in-use small alloc now. | 868 | // Definitely an in-use small alloc now. |
| 844 | if (config.safety) { | 869 | if (config.safety) { |
| 845 | const entry = self.small_allocations.getEntry(@intFromPtr(old_mem.ptr)) orelse | 870 | const requested_size = bucket.requestedSizes(size_class)[slot_index]; |
| 846 | @panic("Invalid free"); | 871 | if (requested_size == 0) @panic("Invalid free"); |
| 847 | if (old_mem.len != entry.value_ptr.requested_size or log2_old_align != entry.value_ptr.log2_ptr_align) { | 872 | const log2_ptr_align = bucket.log2PtrAligns(size_class)[slot_index]; |
| | 873 | if (old_mem.len != requested_size or log2_old_align != log2_ptr_align) { |
| 848 | var addresses: [stack_n]usize = [1]usize{0} ** stack_n; | 874 | var addresses: [stack_n]usize = [1]usize{0} ** stack_n; |
| 849 | var free_stack_trace = StackTrace{ | 875 | var free_stack_trace = StackTrace{ |
| 850 | .instruction_addresses = &addresses, | 876 | .instruction_addresses = &addresses, |
| 851 | .index = 0, | 877 | .index = 0, |
| 852 | }; | 878 | }; |
| 853 | std.debug.captureStackTrace(ret_addr, &free_stack_trace); | 879 | std.debug.captureStackTrace(ret_addr, &free_stack_trace); |
| 854 | if (old_mem.len != entry.value_ptr.requested_size) { | 880 | if (old_mem.len != requested_size) { |
| 855 | log.err("Allocation size {d} bytes does not match free size {d}. Allocation: {} Free: {}", .{ | 881 | log.err("Allocation size {d} bytes does not match free size {d}. Allocation: {} Free: {}", .{ |
| 856 | entry.value_ptr.requested_size, | 882 | requested_size, |
| 857 | old_mem.len, | 883 | old_mem.len, |
| 858 | bucketStackTrace(bucket, size_class, slot_index, .alloc), | 884 | bucketStackTrace(bucket, size_class, slot_index, .alloc), |
| 859 | free_stack_trace, | 885 | free_stack_trace, |
| 860 | }); | 886 | }); |
| 861 | } | 887 | } |
| 862 | if (log2_old_align != entry.value_ptr.log2_ptr_align) { | 888 | if (log2_old_align != log2_ptr_align) { |
| 863 | log.err("Allocation alignment {d} does not match free alignment {d}. Allocation: {} Free: {}", .{ | 889 | log.err("Allocation alignment {d} does not match free alignment {d}. Allocation: {} Free: {}", .{ |
| 864 | @as(usize, 1) << @as(math.Log2Int(usize), @intCast(entry.value_ptr.log2_ptr_align)), | 890 | @as(usize, 1) << @as(math.Log2Int(usize), @intCast(log2_ptr_align)), |
| 865 | @as(usize, 1) << @as(math.Log2Int(usize), @intCast(log2_old_align)), | 891 | @as(usize, 1) << @as(math.Log2Int(usize), @intCast(log2_old_align)), |
| 866 | bucketStackTrace(bucket, size_class, slot_index, .alloc), | 892 | bucketStackTrace(bucket, size_class, slot_index, .alloc), |
| 867 | free_stack_trace, | 893 | free_stack_trace, |
| ... | @@ -879,44 +905,33 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { | ... | @@ -879,44 +905,33 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { |
| 879 | | 905 | |
| 880 | used_byte.* &= ~(@as(u8, 1) << used_bit_index); | 906 | used_byte.* &= ~(@as(u8, 1) << used_bit_index); |
| 881 | bucket.used_count -= 1; | 907 | bucket.used_count -= 1; |
| | 908 | if (config.safety) { |
| | 909 | bucket.requestedSizes(size_class)[slot_index] = 0; |
| | 910 | } |
| 882 | if (bucket.used_count == 0) { | 911 | if (bucket.used_count == 0) { |
| 883 | if (bucket.next == bucket) { | 912 | var entry = self.buckets[bucket_index].getEntryFor(bucket); |
| 884 | // it's the only bucket and therefore the current one | 913 | // save the node for destruction/insertion into in empty_buckets |
| 885 | self.buckets[bucket_index] = null; | 914 | var node = entry.node.?; |
| 886 | } else { | 915 | entry.set(null); |
| 887 | bucket.next.prev = bucket.prev; | 916 | if (self.cur_buckets[bucket_index] == bucket) { |
| 888 | bucket.prev.next = bucket.next; | 917 | self.cur_buckets[bucket_index] = null; |
| 889 | self.buckets[bucket_index] = bucket.prev; | | |
| 890 | } | 918 | } |
| 891 | if (!config.never_unmap) { | 919 | if (!config.never_unmap) { |
| 892 | self.backing_allocator.free(bucket.page[0..page_size]); | 920 | self.backing_allocator.free(bucket.page[0..page_size]); |
| 893 | } | 921 | } |
| 894 | if (!config.retain_metadata) { | 922 | if (!config.retain_metadata) { |
| 895 | self.freeBucket(bucket, size_class); | 923 | self.freeBucket(bucket, size_class); |
| | 924 | self.bucket_node_pool.destroy(node); |
| 896 | } else { | 925 | } else { |
| 897 | // move alloc_cursor to end so we can tell size_class later | 926 | // move alloc_cursor to end so we can tell size_class later |
| 898 | const slot_count = @divExact(page_size, size_class); | 927 | const slot_count = @divExact(page_size, size_class); |
| 899 | bucket.alloc_cursor = @as(SlotIndex, @truncate(slot_count)); | 928 | bucket.alloc_cursor = @as(SlotIndex, @truncate(slot_count)); |
| 900 | if (self.empty_buckets) |prev_bucket| { | 929 | var empty_entry = self.empty_buckets.getEntryFor(node.key); |
| 901 | // empty_buckets is ordered newest to oldest through prev so that if | 930 | empty_entry.set(node); |
| 902 | // config.never_unmap is false and backing_allocator reuses freed memory | | |
| 903 | // then searchBuckets will always return the newer, relevant bucket | | |
| 904 | bucket.prev = prev_bucket; | | |
| 905 | bucket.next = prev_bucket.next; | | |
| 906 | prev_bucket.next = bucket; | | |
| 907 | bucket.next.prev = bucket; | | |
| 908 | } else { | | |
| 909 | bucket.prev = bucket; | | |
| 910 | bucket.next = bucket; | | |
| 911 | } | | |
| 912 | self.empty_buckets = bucket; | | |
| 913 | } | 931 | } |
| 914 | } else { | 932 | } else { |
| 915 | @memset(old_mem, undefined); | 933 | @memset(old_mem, undefined); |
| 916 | } | 934 | } |
| 917 | if (config.safety) { | | |
| 918 | assert(self.small_allocations.remove(@intFromPtr(old_mem.ptr))); | | |
| 919 | } | | |
| 920 | if (config.verbose_log) { | 935 | if (config.verbose_log) { |
| 921 | log.info("small free {d} bytes at {*}", .{ old_mem.len, old_mem.ptr }); | 936 | log.info("small free {d} bytes at {*}", .{ old_mem.len, old_mem.ptr }); |
| 922 | } | 937 | } |
| ... | @@ -980,23 +995,19 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { | ... | @@ -980,23 +995,19 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { |
| 980 | return slice.ptr; | 995 | return slice.ptr; |
| 981 | } | 996 | } |
| 982 | | 997 | |
| 983 | if (config.safety) { | | |
| 984 | try self.small_allocations.ensureUnusedCapacity(self.backing_allocator, 1); | | |
| 985 | } | | |
| 986 | const new_size_class = math.ceilPowerOfTwoAssert(usize, new_aligned_size); | 998 | const new_size_class = math.ceilPowerOfTwoAssert(usize, new_aligned_size); |
| 987 | const ptr = try self.allocSlot(new_size_class, ret_addr); | 999 | const slot = try self.allocSlot(new_size_class, ret_addr); |
| 988 | if (config.safety) { | 1000 | if (config.safety) { |
| 989 | const gop = self.small_allocations.getOrPutAssumeCapacity(@intFromPtr(ptr)); | 1001 | slot.bucket.requestedSizes(new_size_class)[slot.slot_index] = @intCast(len); |
| 990 | gop.value_ptr.requested_size = len; | 1002 | slot.bucket.log2PtrAligns(new_size_class)[slot.slot_index] = log2_ptr_align; |
| 991 | gop.value_ptr.log2_ptr_align = log2_ptr_align; | | |
| 992 | } | 1003 | } |
| 993 | if (config.verbose_log) { | 1004 | if (config.verbose_log) { |
| 994 | log.info("small alloc {d} bytes at {*}", .{ len, ptr }); | 1005 | log.info("small alloc {d} bytes at {*}", .{ len, slot.ptr }); |
| 995 | } | 1006 | } |
| 996 | return ptr; | 1007 | return slot.ptr; |
| 997 | } | 1008 | } |
| 998 | | 1009 | |
| 999 | fn createBucket(self: *Self, size_class: usize, bucket_index: usize) Error!*BucketHeader { | 1010 | fn createBucket(self: *Self, size_class: usize) Error!*BucketHeader { |
| 1000 | const page = try self.backing_allocator.alignedAlloc(u8, page_size, page_size); | 1011 | const page = try self.backing_allocator.alignedAlloc(u8, page_size, page_size); |
| 1001 | errdefer self.backing_allocator.free(page); | 1012 | errdefer self.backing_allocator.free(page); |
| 1002 | | 1013 | |
| ... | @@ -1004,15 +1015,16 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { | ... | @@ -1004,15 +1015,16 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { |
| 1004 | const bucket_bytes = try self.backing_allocator.alignedAlloc(u8, @alignOf(BucketHeader), bucket_size); | 1015 | const bucket_bytes = try self.backing_allocator.alignedAlloc(u8, @alignOf(BucketHeader), bucket_size); |
| 1005 | const ptr = @as(*BucketHeader, @ptrCast(bucket_bytes.ptr)); | 1016 | const ptr = @as(*BucketHeader, @ptrCast(bucket_bytes.ptr)); |
| 1006 | ptr.* = BucketHeader{ | 1017 | ptr.* = BucketHeader{ |
| 1007 | .prev = ptr, | | |
| 1008 | .next = ptr, | | |
| 1009 | .page = page.ptr, | 1018 | .page = page.ptr, |
| 1010 | .alloc_cursor = 0, | 1019 | .alloc_cursor = 0, |
| 1011 | .used_count = 0, | 1020 | .used_count = 0, |
| 1012 | }; | 1021 | }; |
| 1013 | self.buckets[bucket_index] = ptr; | | |
| 1014 | // Set the used bits to all zeroes | 1022 | // Set the used bits to all zeroes |
| 1015 | @memset(@as([*]u8, @as(*[1]u8, ptr.usedBits(0)))[0..usedBitsCount(size_class)], 0); | 1023 | @memset(@as([*]u8, @as(*[1]u8, ptr.usedBits(0)))[0..usedBitsCount(size_class)], 0); |
| | 1024 | if (config.safety) { |
| | 1025 | // Set the requested sizes to zeroes |
| | 1026 | @memset(mem.sliceAsBytes(ptr.requestedSizes(size_class)), 0); |
| | 1027 | } |
| 1016 | return ptr; | 1028 | return ptr; |
| 1017 | } | 1029 | } |
| 1018 | }; | 1030 | }; |
| ... | @@ -1375,10 +1387,10 @@ test "double frees" { | ... | @@ -1375,10 +1387,10 @@ test "double frees" { |
| 1375 | const index: usize = 6; | 1387 | const index: usize = 6; |
| 1376 | const size_class: usize = @as(usize, 1) << 6; | 1388 | const size_class: usize = @as(usize, 1) << 6; |
| 1377 | const small = try allocator.alloc(u8, size_class); | 1389 | const small = try allocator.alloc(u8, size_class); |
| 1378 | try std.testing.expect(GPA.searchBucket(gpa.buckets[index], @intFromPtr(small.ptr)) != null); | 1390 | try std.testing.expect(GPA.searchBucket(&gpa.buckets[index], @intFromPtr(small.ptr)) != null); |
| 1379 | allocator.free(small); | 1391 | allocator.free(small); |
| 1380 | try std.testing.expect(GPA.searchBucket(gpa.buckets[index], @intFromPtr(small.ptr)) == null); | 1392 | try std.testing.expect(GPA.searchBucket(&gpa.buckets[index], @intFromPtr(small.ptr)) == null); |
| 1381 | try std.testing.expect(GPA.searchBucket(gpa.empty_buckets, @intFromPtr(small.ptr)) != null); | 1393 | try std.testing.expect(GPA.searchBucket(&gpa.empty_buckets, @intFromPtr(small.ptr)) != null); |
| 1382 | | 1394 | |
| 1383 | // detect a large allocation double free | 1395 | // detect a large allocation double free |
| 1384 | const large = try allocator.alloc(u8, 2 * page_size); | 1396 | const large = try allocator.alloc(u8, 2 * page_size); |
| ... | @@ -1395,8 +1407,8 @@ test "double frees" { | ... | @@ -1395,8 +1407,8 @@ test "double frees" { |
| 1395 | | 1407 | |
| 1396 | // check that flushing retained metadata doesn't disturb live allocations | 1408 | // check that flushing retained metadata doesn't disturb live allocations |
| 1397 | gpa.flushRetainedMetadata(); | 1409 | gpa.flushRetainedMetadata(); |
| 1398 | try std.testing.expect(gpa.empty_buckets == null); | 1410 | try std.testing.expect(gpa.empty_buckets.root == null); |
| 1399 | try std.testing.expect(GPA.searchBucket(gpa.buckets[index], @intFromPtr(normal_small.ptr)) != null); | 1411 | try std.testing.expect(GPA.searchBucket(&gpa.buckets[index], @intFromPtr(normal_small.ptr)) != null); |
| 1400 | try std.testing.expect(gpa.large_allocations.contains(@intFromPtr(normal_large.ptr))); | 1412 | try std.testing.expect(gpa.large_allocations.contains(@intFromPtr(normal_large.ptr))); |
| 1401 | try std.testing.expect(!gpa.large_allocations.contains(@intFromPtr(large.ptr))); | 1413 | try std.testing.expect(!gpa.large_allocations.contains(@intFromPtr(large.ptr))); |
| 1402 | } | 1414 | } |
Andrew Kelley <andrew@ziglang.org> 2023-10-03 10:58:48-07:00
GitHub <noreply@github.com> 2023-10-03 10:58:48-07:00
Signed by PGP key 4AEE18F83AFDEB23
lib