| ... | ... | @@ -160,9 +160,16 @@ pub const Config = struct { |
| 160 | 160 | |
| 161 | 161 | /// This is a temporary debugging trick you can use to turn segfaults into more helpful |
| 162 | 162 | /// logged error messages with stack trace details. The downside is that every allocation |
| 163 | | /// will be leaked! |
| 163 | /// will be leaked, unless used with retain_metadata! |
| 164 | 164 | never_unmap: bool = false, |
| 165 | 165 | |
| 166 | /// This is a temporary debugging aid that retains metadata about allocations indefinitely. |
| 167 | /// This allows a greater range of double frees to be reported. All metadata is freed when |
| 168 | /// deinit is called. When used with never_unmap, deliberately leaked memory is also freed |
| 169 | /// during deinit. Currently should be used with never_unmap to avoid segfaults. |
| 170 | /// TODO https://github.com/ziglang/zig/issues/4298 will allow use without never_unmap |
| 171 | retain_metadata: bool = false, |
| 172 | |
| 166 | 173 | /// Enables emitting info messages with the size and address of every allocation. |
| 167 | 174 | verbose_log: bool = false, |
| 168 | 175 | }; |
| ... | ... | @@ -176,6 +183,8 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { |
| 176 | 183 | backing_allocator: *Allocator = std.heap.page_allocator, |
| 177 | 184 | buckets: [small_bucket_count]?*BucketHeader = [1]?*BucketHeader{null} ** small_bucket_count, |
| 178 | 185 | large_allocations: LargeAllocTable = .{}, |
| 186 | empty_buckets: if (config.retain_metadata) ?*BucketHeader else void = |
| 187 | if (config.retain_metadata) null else {}, |
| 179 | 188 | |
| 180 | 189 | total_requested_bytes: @TypeOf(total_requested_bytes_init) = total_requested_bytes_init, |
| 181 | 190 | requested_memory_limit: @TypeOf(requested_memory_limit_init) = requested_memory_limit_init, |
| ... | ... | @@ -205,22 +214,34 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { |
| 205 | 214 | |
| 206 | 215 | const LargeAlloc = struct { |
| 207 | 216 | bytes: []u8, |
| 208 | | stack_addresses: [stack_n]usize, |
| 217 | stack_addresses: [trace_n][stack_n]usize, |
| 218 | freed: if (config.retain_metadata) bool else void, |
| 219 | ptr_align: if (config.never_unmap and config.retain_metadata) u29 else void, |
| 220 | |
| 221 | const trace_n = if (config.retain_metadata) traces_per_slot else 1; |
| 209 | 222 | |
| 210 | | fn dumpStackTrace(self: *LargeAlloc) void { |
| 211 | | std.debug.dumpStackTrace(self.getStackTrace()); |
| 223 | fn dumpStackTrace(self: *LargeAlloc, trace_kind: TraceKind) void { |
| 224 | std.debug.dumpStackTrace(self.getStackTrace(trace_kind)); |
| 212 | 225 | } |
| 213 | 226 | |
| 214 | | fn getStackTrace(self: *LargeAlloc) std.builtin.StackTrace { |
| 227 | fn getStackTrace(self: *LargeAlloc, trace_kind: TraceKind) std.builtin.StackTrace { |
| 228 | assert(@enumToInt(trace_kind) < trace_n); |
| 229 | const stack_addresses = &self.stack_addresses[@enumToInt(trace_kind)]; |
| 215 | 230 | var len: usize = 0; |
| 216 | | while (len < stack_n and self.stack_addresses[len] != 0) { |
| 231 | while (len < stack_n and stack_addresses[len] != 0) { |
| 217 | 232 | len += 1; |
| 218 | 233 | } |
| 219 | 234 | return .{ |
| 220 | | .instruction_addresses = &self.stack_addresses, |
| 235 | .instruction_addresses = stack_addresses, |
| 221 | 236 | .index = len, |
| 222 | 237 | }; |
| 223 | 238 | } |
| 239 | |
| 240 | fn captureStackTrace(self: *LargeAlloc, ret_addr: usize, trace_kind: TraceKind) void { |
| 241 | assert(@enumToInt(trace_kind) < trace_n); |
| 242 | const stack_addresses = &self.stack_addresses[@enumToInt(trace_kind)]; |
| 243 | collectStackTrace(ret_addr, stack_addresses); |
| 244 | } |
| 224 | 245 | }; |
| 225 | 246 | const LargeAllocTable = std.AutoHashMapUnmanaged(usize, LargeAlloc); |
| 226 | 247 | |
| ... | ... | @@ -348,16 +369,71 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { |
| 348 | 369 | } |
| 349 | 370 | var it = self.large_allocations.valueIterator(); |
| 350 | 371 | while (it.next()) |large_alloc| { |
| 372 | if (config.retain_metadata and large_alloc.freed) continue; |
| 351 | 373 | log.err("memory address 0x{x} leaked: {s}", .{ |
| 352 | | @ptrToInt(large_alloc.bytes.ptr), large_alloc.getStackTrace(), |
| 374 | @ptrToInt(large_alloc.bytes.ptr), large_alloc.getStackTrace(.alloc), |
| 353 | 375 | }); |
| 354 | 376 | leaks = true; |
| 355 | 377 | } |
| 356 | 378 | return leaks; |
| 357 | 379 | } |
| 358 | 380 | |
| 381 | fn freeBucket(self: *Self, bucket: *BucketHeader, size_class: usize) void { |
| 382 | const bucket_size = bucketSize(size_class); |
| 383 | const bucket_slice = @ptrCast([*]align(@alignOf(BucketHeader)) u8, bucket)[0..bucket_size]; |
| 384 | self.backing_allocator.free(bucket_slice); |
| 385 | } |
| 386 | |
| 387 | fn freeRetainedMetadata(self: *Self) void { |
| 388 | if (config.retain_metadata) { |
| 389 | if (config.never_unmap) { |
| 390 | // free large allocations that were intentionally leaked by never_unmap |
| 391 | var it = self.large_allocations.iterator(); |
| 392 | while (it.next()) |large| { |
| 393 | if (large.value_ptr.freed) { |
| 394 | _ = self.backing_allocator.resizeFn(self.backing_allocator, large.value_ptr.bytes, |
| 395 | large.value_ptr.ptr_align, 0, 0, @returnAddress()) catch unreachable; |
| 396 | } |
| 397 | } |
| 398 | } |
| 399 | // free retained metadata for small allocations |
| 400 | if (self.empty_buckets) |first_bucket| { |
| 401 | var bucket = first_bucket; |
| 402 | while (true) { |
| 403 | const prev = bucket.prev; |
| 404 | if (config.never_unmap) { |
| 405 | // free page that was intentionally leaked by never_unmap |
| 406 | self.backing_allocator.free(bucket.page[0..page_size]); |
| 407 | } |
| 408 | // alloc_cursor was set to slot count when bucket added to empty_buckets |
| 409 | self.freeBucket(bucket, @divExact(page_size, bucket.alloc_cursor)); |
| 410 | bucket = prev; |
| 411 | if (bucket == first_bucket) |
| 412 | break; |
| 413 | } |
| 414 | self.empty_buckets = null; |
| 415 | } |
| 416 | } |
| 417 | } |
| 418 | |
| 419 | pub usingnamespace if (config.retain_metadata) struct { |
| 420 | pub fn flushRetainedMetadata(self: *Self) void { |
| 421 | self.freeRetainedMetadata(); |
| 422 | // also remove entries from large_allocations |
| 423 | var it = self.large_allocations.iterator(); |
| 424 | while (it.next()) |large| { |
| 425 | if (large.value_ptr.freed) { |
| 426 | _ = self.large_allocations.remove(@ptrToInt(large.value_ptr.bytes.ptr)); |
| 427 | } |
| 428 | } |
| 429 | } |
| 430 | } else struct {}; |
| 431 | |
| 359 | 432 | pub fn deinit(self: *Self) bool { |
| 360 | 433 | const leaks = if (config.safety) self.detectLeaks() else false; |
| 434 | if (config.retain_metadata) { |
| 435 | self.freeRetainedMetadata(); |
| 436 | } |
| 361 | 437 | self.large_allocations.deinit(self.backing_allocator); |
| 362 | 438 | self.* = undefined; |
| 363 | 439 | return leaks; |
| ... | ... | @@ -373,6 +449,18 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { |
| 373 | 449 | std.debug.captureStackTrace(first_trace_addr, &stack_trace); |
| 374 | 450 | } |
| 375 | 451 | |
| 452 | fn reportDoubleFree(ret_addr: usize, alloc_stack_trace: StackTrace, free_stack_trace: StackTrace) void { |
| 453 | var addresses: [stack_n]usize = [1]usize{0} ** stack_n; |
| 454 | var second_free_stack_trace = StackTrace{ |
| 455 | .instruction_addresses = &addresses, |
| 456 | .index = 0, |
| 457 | }; |
| 458 | std.debug.captureStackTrace(ret_addr, &second_free_stack_trace); |
| 459 | log.err("Double free detected. Allocation: {s} First free: {s} Second free: {s}", .{ |
| 460 | alloc_stack_trace, free_stack_trace, second_free_stack_trace, |
| 461 | }); |
| 462 | } |
| 463 | |
| 376 | 464 | fn allocSlot(self: *Self, size_class: usize, trace_addr: usize) Error![*]u8 { |
| 377 | 465 | const bucket_index = math.log2(size_class); |
| 378 | 466 | const first_bucket = self.buckets[bucket_index] orelse try self.createBucket( |
| ... | ... | @@ -408,11 +496,10 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { |
| 408 | 496 | } |
| 409 | 497 | |
| 410 | 498 | fn searchBucket( |
| 411 | | self: *Self, |
| 412 | | bucket_index: usize, |
| 499 | bucket_list: ?*BucketHeader, |
| 413 | 500 | addr: usize, |
| 414 | 501 | ) ?*BucketHeader { |
| 415 | | const first_bucket = self.buckets[bucket_index] orelse return null; |
| 502 | const first_bucket = bucket_list orelse return null; |
| 416 | 503 | var bucket = first_bucket; |
| 417 | 504 | while (true) { |
| 418 | 505 | const in_bucket_range = (addr >= @ptrToInt(bucket.page) and |
| ... | ... | @@ -422,7 +509,6 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { |
| 422 | 509 | if (bucket == first_bucket) { |
| 423 | 510 | return null; |
| 424 | 511 | } |
| 425 | | self.buckets[bucket_index] = bucket; |
| 426 | 512 | } |
| 427 | 513 | } |
| 428 | 514 | |
| ... | ... | @@ -444,6 +530,25 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { |
| 444 | 530 | } |
| 445 | 531 | }; |
| 446 | 532 | |
| 533 | if (config.retain_metadata and entry.value_ptr.freed) { |
| 534 | if (config.safety) { |
| 535 | reportDoubleFree(ret_addr, |
| 536 | entry.value_ptr.getStackTrace(.alloc), |
| 537 | entry.value_ptr.getStackTrace(.free) |
| 538 | ); |
| 539 | if (new_size == 0) { |
| 540 | // Recoverable. Restore self.total_requested_bytes if needed. |
| 541 | if (config.enable_memory_limit) { |
| 542 | self.total_requested_bytes += old_mem.len; |
| 543 | } |
| 544 | return @as(usize, 0); |
| 545 | } |
| 546 | @panic("Unrecoverable double free"); |
| 547 | } else { |
| 548 | unreachable; |
| 549 | } |
| 550 | } |
| 551 | |
| 447 | 552 | if (config.safety and old_mem.len != entry.value_ptr.bytes.len) { |
| 448 | 553 | var addresses: [stack_n]usize = [1]usize{0} ** stack_n; |
| 449 | 554 | var free_stack_trace = StackTrace{ |
| ... | ... | @@ -454,19 +559,25 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { |
| 454 | 559 | log.err("Allocation size {d} bytes does not match free size {d}. Allocation: {s} Free: {s}", .{ |
| 455 | 560 | entry.value_ptr.bytes.len, |
| 456 | 561 | old_mem.len, |
| 457 | | entry.value_ptr.getStackTrace(), |
| 562 | entry.value_ptr.getStackTrace(.alloc), |
| 458 | 563 | free_stack_trace, |
| 459 | 564 | }); |
| 460 | 565 | } |
| 461 | 566 | |
| 462 | | const result_len = try self.backing_allocator.resizeFn(self.backing_allocator, old_mem, old_align, new_size, len_align, ret_addr); |
| 567 | const result_len = if (config.never_unmap and new_size == 0) 0 else |
| 568 | try self.backing_allocator.resizeFn(self.backing_allocator, old_mem, old_align, new_size, len_align, ret_addr); |
| 463 | 569 | |
| 464 | 570 | if (result_len == 0) { |
| 465 | 571 | if (config.verbose_log) { |
| 466 | 572 | log.info("large free {d} bytes at {*}", .{ old_mem.len, old_mem.ptr }); |
| 467 | 573 | } |
| 468 | 574 | |
| 469 | | assert(self.large_allocations.remove(@ptrToInt(old_mem.ptr))); |
| 575 | if (!config.retain_metadata) { |
| 576 | assert(self.large_allocations.remove(@ptrToInt(old_mem.ptr))); |
| 577 | } else { |
| 578 | entry.value_ptr.freed = true; |
| 579 | entry.value_ptr.captureStackTrace(ret_addr, .free); |
| 580 | } |
| 470 | 581 | return 0; |
| 471 | 582 | } |
| 472 | 583 | |
| ... | ... | @@ -476,7 +587,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { |
| 476 | 587 | }); |
| 477 | 588 | } |
| 478 | 589 | entry.value_ptr.bytes = old_mem.ptr[0..result_len]; |
| 479 | | collectStackTrace(ret_addr, &entry.value_ptr.stack_addresses); |
| 590 | entry.value_ptr.captureStackTrace(ret_addr, .alloc); |
| 480 | 591 | return result_len; |
| 481 | 592 | } |
| 482 | 593 | |
| ... | ... | @@ -520,11 +631,24 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { |
| 520 | 631 | var bucket_index = math.log2(size_class_hint); |
| 521 | 632 | var size_class: usize = size_class_hint; |
| 522 | 633 | const bucket = while (bucket_index < small_bucket_count) : (bucket_index += 1) { |
| 523 | | if (self.searchBucket(bucket_index, @ptrToInt(old_mem.ptr))) |bucket| { |
| 634 | if (searchBucket(self.buckets[bucket_index], @ptrToInt(old_mem.ptr))) |bucket| { |
| 635 | // move bucket to head of list to optimize search for nearby allocations |
| 636 | self.buckets[bucket_index] = bucket; |
| 524 | 637 | break bucket; |
| 525 | 638 | } |
| 526 | 639 | size_class *= 2; |
| 527 | | } else { |
| 640 | } else blk: { |
| 641 | if (config.retain_metadata) { |
| 642 | if (!self.large_allocations.contains(@ptrToInt(old_mem.ptr))) { |
| 643 | // object not in active buckets or a large allocation, so search empty buckets |
| 644 | if (searchBucket(self.empty_buckets, @ptrToInt(old_mem.ptr))) |bucket| { |
| 645 | // bucket is empty so is_used below will always be false and we exit there |
| 646 | break :blk bucket; |
| 647 | } else { |
| 648 | @panic("Invalid free"); |
| 649 | } |
| 650 | } |
| 651 | } |
| 528 | 652 | return self.resizeLarge(old_mem, old_align, new_size, len_align, ret_addr); |
| 529 | 653 | }; |
| 530 | 654 | const byte_offset = @ptrToInt(old_mem.ptr) - @ptrToInt(bucket.page); |
| ... | ... | @@ -535,19 +659,10 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { |
| 535 | 659 | const is_used = @truncate(u1, used_byte.* >> used_bit_index) != 0; |
| 536 | 660 | if (!is_used) { |
| 537 | 661 | if (config.safety) { |
| 538 | | const alloc_stack_trace = bucketStackTrace(bucket, size_class, slot_index, .alloc); |
| 539 | | const free_stack_trace = bucketStackTrace(bucket, size_class, slot_index, .free); |
| 540 | | var addresses: [stack_n]usize = [1]usize{0} ** stack_n; |
| 541 | | var second_free_stack_trace = StackTrace{ |
| 542 | | .instruction_addresses = &addresses, |
| 543 | | .index = 0, |
| 544 | | }; |
| 545 | | std.debug.captureStackTrace(ret_addr, &second_free_stack_trace); |
| 546 | | log.err("Double free detected. Allocation: {s} First free: {s} Second free: {s}", .{ |
| 547 | | alloc_stack_trace, |
| 548 | | free_stack_trace, |
| 549 | | second_free_stack_trace, |
| 550 | | }); |
| 662 | reportDoubleFree(ret_addr, |
| 663 | bucketStackTrace(bucket, size_class, slot_index, .alloc), |
| 664 | bucketStackTrace(bucket, size_class, slot_index, .free) |
| 665 | ); |
| 551 | 666 | if (new_size == 0) { |
| 552 | 667 | // Recoverable. Restore self.total_requested_bytes if needed, as we |
| 553 | 668 | // don't return an error value so the errdefer above does not run. |
| ... | ... | @@ -579,9 +694,26 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { |
| 579 | 694 | if (!config.never_unmap) { |
| 580 | 695 | self.backing_allocator.free(bucket.page[0..page_size]); |
| 581 | 696 | } |
| 582 | | const bucket_size = bucketSize(size_class); |
| 583 | | const bucket_slice = @ptrCast([*]align(@alignOf(BucketHeader)) u8, bucket)[0..bucket_size]; |
| 584 | | self.backing_allocator.free(bucket_slice); |
| 697 | if (!config.retain_metadata) { |
| 698 | self.freeBucket(bucket, size_class); |
| 699 | } else { |
| 700 | // move alloc_cursor to end so we can tell size_class later |
| 701 | const slot_count = @divExact(page_size, size_class); |
| 702 | bucket.alloc_cursor = @truncate(SlotIndex, slot_count); |
| 703 | if (self.empty_buckets) |prev_bucket| { |
| 704 | // empty_buckets is ordered newest to oldest through prev so that if |
| 705 | // config.never_unmap is false and backing_allocator reuses freed memory |
| 706 | // then searchBuckets will always return the newer, relevant bucket |
| 707 | bucket.prev = prev_bucket; |
| 708 | bucket.next = prev_bucket.next; |
| 709 | prev_bucket.next = bucket; |
| 710 | bucket.next.prev = bucket; |
| 711 | } else { |
| 712 | bucket.prev = bucket; |
| 713 | bucket.next = bucket; |
| 714 | } |
| 715 | self.empty_buckets = bucket; |
| 716 | } |
| 585 | 717 | } else { |
| 586 | 718 | @memset(old_mem.ptr, undefined, old_mem.len); |
| 587 | 719 | } |
| ... | ... | @@ -644,9 +776,20 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { |
| 644 | 776 | } |
| 645 | 777 | |
| 646 | 778 | const gop = self.large_allocations.getOrPutAssumeCapacity(@ptrToInt(slice.ptr)); |
| 647 | | assert(!gop.found_existing); // This would mean the kernel double-mapped pages. |
| 779 | if (config.retain_metadata and !config.never_unmap) { |
| 780 | // Backing allocator may be reusing memory that we're retaining metadata for |
| 781 | assert(!gop.found_existing or gop.value_ptr.freed); |
| 782 | } else { |
| 783 | assert(!gop.found_existing); // This would mean the kernel double-mapped pages. |
| 784 | } |
| 648 | 785 | gop.value_ptr.bytes = slice; |
| 649 | | collectStackTrace(ret_addr, &gop.value_ptr.stack_addresses); |
| 786 | gop.value_ptr.captureStackTrace(ret_addr, .alloc); |
| 787 | if (config.retain_metadata) { |
| 788 | gop.value_ptr.freed = false; |
| 789 | if (config.never_unmap) { |
| 790 | gop.value_ptr.ptr_align = ptr_align; |
| 791 | } |
| 792 | } |
| 650 | 793 | |
| 651 | 794 | if (config.verbose_log) { |
| 652 | 795 | log.info("large alloc {d} bytes at {*}", .{ slice.len, slice.ptr }); |
| ... | ... | @@ -1015,3 +1158,43 @@ test "setting a memory cap" { |
| 1015 | 1158 | try std.testing.expect(gpa.total_requested_bytes == 1010); |
| 1016 | 1159 | allocator.free(exact); |
| 1017 | 1160 | } |
| 1161 | |
| 1162 | test "double frees" { |
| 1163 | // use a GPA to back a GPA to check for leaks of the latter's metadata |
| 1164 | var backing_gpa = GeneralPurposeAllocator(.{ .safety = true }){}; |
| 1165 | defer std.testing.expect(!backing_gpa.deinit()) catch @panic("leak"); |
| 1166 | |
| 1167 | const GPA = GeneralPurposeAllocator(.{ .safety = true, .never_unmap = true, .retain_metadata = true }); |
| 1168 | var gpa = GPA{ .backing_allocator = &backing_gpa.allocator }; |
| 1169 | defer std.testing.expect(!gpa.deinit()) catch @panic("leak"); |
| 1170 | const allocator = &gpa.allocator; |
| 1171 | |
| 1172 | // detect a small allocation double free, even though bucket is emptied |
| 1173 | const index: usize = 6; |
| 1174 | const size_class: usize = @as(usize, 1) << 6; |
| 1175 | const small = try allocator.alloc(u8, size_class); |
| 1176 | try std.testing.expect(GPA.searchBucket(gpa.buckets[index], @ptrToInt(small.ptr)) != null); |
| 1177 | allocator.free(small); |
| 1178 | try std.testing.expect(GPA.searchBucket(gpa.buckets[index], @ptrToInt(small.ptr)) == null); |
| 1179 | try std.testing.expect(GPA.searchBucket(gpa.empty_buckets, @ptrToInt(small.ptr)) != null); |
| 1180 | |
| 1181 | // detect a large allocation double free |
| 1182 | const large = try allocator.alloc(u8, 2 * page_size); |
| 1183 | try std.testing.expect(gpa.large_allocations.contains(@ptrToInt(large.ptr))); |
| 1184 | try std.testing.expectEqual(gpa.large_allocations.getEntry(@ptrToInt(large.ptr)).?.value_ptr.bytes, large); |
| 1185 | allocator.free(large); |
| 1186 | try std.testing.expect(gpa.large_allocations.contains(@ptrToInt(large.ptr))); |
| 1187 | try std.testing.expect(gpa.large_allocations.getEntry(@ptrToInt(large.ptr)).?.value_ptr.freed); |
| 1188 | |
| 1189 | const normal_small = try allocator.alloc(u8, size_class); |
| 1190 | defer allocator.free(normal_small); |
| 1191 | const normal_large = try allocator.alloc(u8, 2 * page_size); |
| 1192 | defer allocator.free(normal_large); |
| 1193 | |
| 1194 | // check that flushing retained metadata doesn't disturb live allocations |
| 1195 | gpa.flushRetainedMetadata(); |
| 1196 | try std.testing.expect(gpa.empty_buckets == null); |
| 1197 | try std.testing.expect(GPA.searchBucket(gpa.buckets[index], @ptrToInt(normal_small.ptr)) != null); |
| 1198 | try std.testing.expect(gpa.large_allocations.contains(@ptrToInt(normal_large.ptr))); |
| 1199 | try std.testing.expect(!gpa.large_allocations.contains(@ptrToInt(large.ptr))); |
| 1200 | } |
Matthew Borkowski <matthew.h.borkowski@gmail.com> 2021-05-11 23:54:11-04:00
Veikka Tuominen <git@vexu.eu> 2021-06-09 12:29:50+03:00
lib