| ... | @@ -163,6 +163,10 @@ pub const Config = struct { | ... | @@ -163,6 +163,10 @@ pub const Config = struct { |
| 163 | /// Tell whether the backing allocator returns already-zeroed memory. | 163 | /// Tell whether the backing allocator returns already-zeroed memory. |
| 164 | backing_allocator_zeroes: bool = true, | 164 | backing_allocator_zeroes: bool = true, |
| 165 | | 165 | |
| | 166 | /// When resizing an allocation, refresh the stack trace with the resize |
| | 167 | /// callsite. Comes with a performance penalty. |
| | 168 | resize_stack_traces: bool = false, |
| | 169 | |
| 166 | /// Magic value that distinguishes allocations owned by this allocator from | 170 | /// Magic value that distinguishes allocations owned by this allocator from |
| 167 | /// other regions of memory. | 171 | /// other regions of memory. |
| 168 | canary: usize = @truncate(0x9232a6ff85dff10f), | 172 | canary: usize = @truncate(0x9232a6ff85dff10f), |
| ... | @@ -554,6 +558,12 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { | ... | @@ -554,6 +558,12 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { |
| 554 | }); | 558 | }); |
| 555 | } | 559 | } |
| 556 | | 560 | |
| | 561 | // If this would move the allocation into a small size class, |
| | 562 | // refuse the request, because it would require creating small |
| | 563 | // allocation metadata. |
| | 564 | const new_size_class_index: usize = @max(@bitSizeOf(usize) - @clz(new_size - 1), @intFromEnum(alignment)); |
| | 565 | if (new_size_class_index < self.buckets.len) return null; |
| | 566 | |
| 557 | // Do memory limit accounting with requested sizes rather than what | 567 | // Do memory limit accounting with requested sizes rather than what |
| 558 | // backing_allocator returns because if we want to return | 568 | // backing_allocator returns because if we want to return |
| 559 | // error.OutOfMemory, we have to leave allocation untouched, and | 569 | // error.OutOfMemory, we have to leave allocation untouched, and |
| ... | @@ -598,7 +608,8 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { | ... | @@ -598,7 +608,8 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { |
| 598 | }); | 608 | }); |
| 599 | } | 609 | } |
| 600 | entry.value_ptr.bytes = resized_ptr[0..new_size]; | 610 | entry.value_ptr.bytes = resized_ptr[0..new_size]; |
| 601 | entry.value_ptr.captureStackTrace(ret_addr, .alloc); | 611 | if (config.resize_stack_traces) |
| | 612 | entry.value_ptr.captureStackTrace(ret_addr, .alloc); |
| 602 | | 613 | |
| 603 | // Update the key of the hash map if the memory was relocated. | 614 | // Update the key of the hash map if the memory was relocated. |
| 604 | if (resized_ptr != old_mem.ptr) { | 615 | if (resized_ptr != old_mem.ptr) { |
| ... | @@ -791,12 +802,16 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { | ... | @@ -791,12 +802,16 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { |
| 791 | new_len: usize, | 802 | new_len: usize, |
| 792 | return_address: usize, | 803 | return_address: usize, |
| 793 | ) bool { | 804 | ) bool { |
| 794 | _ = context; | 805 | const self: *Self = @ptrCast(@alignCast(context)); |
| 795 | _ = memory; | 806 | self.mutex.lock(); |
| 796 | _ = alignment; | 807 | defer self.mutex.unlock(); |
| 797 | _ = new_len; | 808 | |
| 798 | _ = return_address; | 809 | const size_class_index: usize = @max(@bitSizeOf(usize) - @clz(memory.len - 1), @intFromEnum(alignment)); |
| 799 | return false; | 810 | if (size_class_index >= self.buckets.len) { |
| | 811 | return self.resizeLarge(memory, alignment, new_len, return_address, false) != null; |
| | 812 | } else { |
| | 813 | return resizeSmall(self, memory, alignment, new_len, return_address, size_class_index); |
| | 814 | } |
| 800 | } | 815 | } |
| 801 | | 816 | |
| 802 | fn remap( | 817 | fn remap( |
| ... | @@ -806,12 +821,16 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { | ... | @@ -806,12 +821,16 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { |
| 806 | new_len: usize, | 821 | new_len: usize, |
| 807 | return_address: usize, | 822 | return_address: usize, |
| 808 | ) ?[*]u8 { | 823 | ) ?[*]u8 { |
| 809 | _ = context; | 824 | const self: *Self = @ptrCast(@alignCast(context)); |
| 810 | _ = memory; | 825 | self.mutex.lock(); |
| 811 | _ = alignment; | 826 | defer self.mutex.unlock(); |
| 812 | _ = new_len; | 827 | |
| 813 | _ = return_address; | 828 | const size_class_index: usize = @max(@bitSizeOf(usize) - @clz(memory.len - 1), @intFromEnum(alignment)); |
| 814 | return null; | 829 | if (size_class_index >= self.buckets.len) { |
| | 830 | return self.resizeLarge(memory, alignment, new_len, return_address, true); |
| | 831 | } else { |
| | 832 | return if (resizeSmall(self, memory, alignment, new_len, return_address, size_class_index)) memory.ptr else null; |
| | 833 | } |
| 815 | } | 834 | } |
| 816 | | 835 | |
| 817 | fn free( | 836 | fn free( |
| ... | @@ -894,8 +913,10 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { | ... | @@ -894,8 +913,10 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { |
| 894 | self.total_requested_bytes -= old_memory.len; | 913 | self.total_requested_bytes -= old_memory.len; |
| 895 | } | 914 | } |
| 896 | | 915 | |
| 897 | // Capture stack trace to be the "first free", in case a double free happens. | 916 | if (config.stack_trace_frames > 0) { |
| 898 | bucket.captureStackTrace(return_address, slot_count, slot_index, .free); | 917 | // Capture stack trace to be the "first free", in case a double free happens. |
| | 918 | bucket.captureStackTrace(return_address, slot_count, slot_index, .free); |
| | 919 | } |
| 899 | | 920 | |
| 900 | used_byte.* &= ~(@as(u8, 1) << used_bit_index); | 921 | used_byte.* &= ~(@as(u8, 1) << used_bit_index); |
| 901 | if (config.safety) { | 922 | if (config.safety) { |
| ... | @@ -915,6 +936,91 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { | ... | @@ -915,6 +936,91 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { |
| 915 | log.info("small free {d} bytes at {*}", .{ old_memory.len, old_memory.ptr }); | 936 | log.info("small free {d} bytes at {*}", .{ old_memory.len, old_memory.ptr }); |
| 916 | } | 937 | } |
| 917 | } | 938 | } |
| | 939 | |
| | 940 | fn resizeSmall( |
| | 941 | self: *Self, |
| | 942 | memory: []u8, |
| | 943 | alignment: mem.Alignment, |
| | 944 | new_len: usize, |
| | 945 | return_address: usize, |
| | 946 | size_class_index: usize, |
| | 947 | ) bool { |
| | 948 | const new_size_class_index: usize = @max(@bitSizeOf(usize) - @clz(new_len - 1), @intFromEnum(alignment)); |
| | 949 | if (!config.safety) return new_size_class_index == size_class_index; |
| | 950 | const slot_count = slot_counts[size_class_index]; |
| | 951 | const memory_addr = @intFromPtr(memory.ptr); |
| | 952 | const page_addr = memory_addr & ~(page_size - 1); |
| | 953 | const bucket: *BucketHeader = .fromPage(page_addr, slot_count); |
| | 954 | if (bucket.canary != config.canary) @panic("Invalid free"); |
| | 955 | const page_offset = memory_addr - page_addr; |
| | 956 | const size_class = @as(usize, 1) << @as(u6, @intCast(size_class_index)); |
| | 957 | const slot_index: SlotIndex = @intCast(page_offset / size_class); |
| | 958 | const used_byte_index = slot_index / 8; |
| | 959 | const used_bit_index: u3 = @intCast(slot_index % 8); |
| | 960 | const used_byte = bucket.usedBits(used_byte_index); |
| | 961 | const is_used = @as(u1, @truncate(used_byte.* >> used_bit_index)) != 0; |
| | 962 | if (!is_used) { |
| | 963 | reportDoubleFree( |
| | 964 | return_address, |
| | 965 | bucketStackTrace(bucket, slot_count, slot_index, .alloc), |
| | 966 | bucketStackTrace(bucket, slot_count, slot_index, .free), |
| | 967 | ); |
| | 968 | // Recoverable since this is a free. |
| | 969 | return false; |
| | 970 | } |
| | 971 | |
| | 972 | // Definitely an in-use small alloc now. |
| | 973 | const requested_size = bucket.requestedSizes(slot_count)[slot_index]; |
| | 974 | if (requested_size == 0) @panic("Invalid free"); |
| | 975 | const slot_alignment = bucket.log2PtrAligns(slot_count)[slot_index]; |
| | 976 | if (memory.len != requested_size or alignment != slot_alignment) { |
| | 977 | var addresses: [stack_n]usize = [1]usize{0} ** stack_n; |
| | 978 | var free_stack_trace: StackTrace = .{ |
| | 979 | .instruction_addresses = &addresses, |
| | 980 | .index = 0, |
| | 981 | }; |
| | 982 | std.debug.captureStackTrace(return_address, &free_stack_trace); |
| | 983 | if (memory.len != requested_size) { |
| | 984 | log.err("Allocation size {d} bytes does not match free size {d}. Allocation: {} Free: {}", .{ |
| | 985 | requested_size, |
| | 986 | memory.len, |
| | 987 | bucketStackTrace(bucket, slot_count, slot_index, .alloc), |
| | 988 | free_stack_trace, |
| | 989 | }); |
| | 990 | } |
| | 991 | if (alignment != slot_alignment) { |
| | 992 | log.err("Allocation alignment {d} does not match free alignment {d}. Allocation: {} Free: {}", .{ |
| | 993 | slot_alignment.toByteUnits(), |
| | 994 | alignment.toByteUnits(), |
| | 995 | bucketStackTrace(bucket, slot_count, slot_index, .alloc), |
| | 996 | free_stack_trace, |
| | 997 | }); |
| | 998 | } |
| | 999 | } |
| | 1000 | |
| | 1001 | if (new_size_class_index != size_class_index) return false; |
| | 1002 | |
| | 1003 | const prev_req_bytes = self.total_requested_bytes; |
| | 1004 | if (config.enable_memory_limit) { |
| | 1005 | const new_req_bytes = prev_req_bytes - memory.len + new_len; |
| | 1006 | if (new_req_bytes > prev_req_bytes and new_req_bytes > self.requested_memory_limit) { |
| | 1007 | return false; |
| | 1008 | } |
| | 1009 | self.total_requested_bytes = new_req_bytes; |
| | 1010 | } |
| | 1011 | |
| | 1012 | if (memory.len > new_len) @memset(memory[new_len..], undefined); |
| | 1013 | if (config.verbose_log) |
| | 1014 | log.info("small resize {d} bytes at {*} to {d}", .{ memory.len, memory.ptr, new_len }); |
| | 1015 | |
| | 1016 | if (config.safety) |
| | 1017 | bucket.requestedSizes(slot_count)[slot_index] = @intCast(new_len); |
| | 1018 | |
| | 1019 | if (config.resize_stack_traces) |
| | 1020 | bucket.captureStackTrace(return_address, slot_count, slot_index, .alloc); |
| | 1021 | |
| | 1022 | return true; |
| | 1023 | } |
| 918 | }; | 1024 | }; |
| 919 | } | 1025 | } |
| 920 | | 1026 | |
| ... | @@ -1023,12 +1129,8 @@ test "shrink" { | ... | @@ -1023,12 +1129,8 @@ test "shrink" { |
| 1023 | try std.testing.expect(b == 0x11); | 1129 | try std.testing.expect(b == 0x11); |
| 1024 | } | 1130 | } |
| 1025 | | 1131 | |
| 1026 | try std.testing.expect(allocator.resize(slice, 16)); | 1132 | // Does not cross size class boundaries when shrinking. |
| 1027 | slice = slice[0..16]; | 1133 | try std.testing.expect(!allocator.resize(slice, 16)); |
| 1028 | | | |
| 1029 | for (slice) |b| { | | |
| 1030 | try std.testing.expect(b == 0x11); | | |
| 1031 | } | | |
| 1032 | } | 1134 | } |
| 1033 | | 1135 | |
| 1034 | test "large object - grow" { | 1136 | test "large object - grow" { |
| ... | @@ -1212,14 +1314,14 @@ test "realloc large object to larger alignment" { | ... | @@ -1212,14 +1314,14 @@ test "realloc large object to larger alignment" { |
| 1212 | try std.testing.expect(slice[16] == 0x34); | 1314 | try std.testing.expect(slice[16] == 0x34); |
| 1213 | } | 1315 | } |
| 1214 | | 1316 | |
| 1215 | test "large object shrinks to small but allocation fails during shrink" { | 1317 | test "large object rejects shrinking to small" { |
| 1216 | if (builtin.target.isWasm()) { | 1318 | if (builtin.target.isWasm()) { |
| 1217 | // Not expected to pass on targets that do not have memory mapping. | 1319 | // Not expected to pass on targets that do not have memory mapping. |
| 1218 | return error.SkipZigTest; | 1320 | return error.SkipZigTest; |
| 1219 | } | 1321 | } |
| 1220 | | 1322 | |
| 1221 | var failing_allocator = std.testing.FailingAllocator.init(std.heap.page_allocator, .{ .fail_index = 3 }); | 1323 | var failing_allocator = std.testing.FailingAllocator.init(std.heap.page_allocator, .{ .fail_index = 3 }); |
| 1222 | var gpa = GeneralPurposeAllocator(.{}){ .backing_allocator = failing_allocator.allocator() }; | 1324 | var gpa: GeneralPurposeAllocator(.{}) = .{ .backing_allocator = failing_allocator.allocator() }; |
| 1223 | defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); | 1325 | defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); |
| 1224 | const allocator = gpa.allocator(); | 1326 | const allocator = gpa.allocator(); |
| 1225 | | 1327 | |
| ... | @@ -1228,10 +1330,7 @@ test "large object shrinks to small but allocation fails during shrink" { | ... | @@ -1228,10 +1330,7 @@ test "large object shrinks to small but allocation fails during shrink" { |
| 1228 | slice[0] = 0x12; | 1330 | slice[0] = 0x12; |
| 1229 | slice[3] = 0x34; | 1331 | slice[3] = 0x34; |
| 1230 | | 1332 | |
| 1231 | // Next allocation will fail in the backing allocator of the GeneralPurposeAllocator | 1333 | try std.testing.expect(!allocator.resize(slice, 4)); |
| 1232 | | | |
| 1233 | try std.testing.expect(allocator.resize(slice, 4)); | | |
| 1234 | slice = slice[0..4]; | | |
| 1235 | try std.testing.expect(slice[0] == 0x12); | 1334 | try std.testing.expect(slice[0] == 0x12); |
| 1236 | try std.testing.expect(slice[3] == 0x34); | 1335 | try std.testing.expect(slice[3] == 0x34); |
| 1237 | } | 1336 | } |
| ... | @@ -1274,17 +1373,16 @@ test "setting a memory cap" { | ... | @@ -1274,17 +1373,16 @@ test "setting a memory cap" { |
| 1274 | allocator.free(exact); | 1373 | allocator.free(exact); |
| 1275 | } | 1374 | } |
| 1276 | | 1375 | |
| 1277 | test "bug 9995 fix, large allocs count requested size not backing size" { | 1376 | test "large allocations count requested size not backing size" { |
| 1278 | // with AtLeast, buffer likely to be larger than requested, especially when shrinking | 1377 | var gpa: GeneralPurposeAllocator(.{ .enable_memory_limit = true }) = .{}; |
| 1279 | var gpa = GeneralPurposeAllocator(.{ .enable_memory_limit = true }){}; | | |
| 1280 | const allocator = gpa.allocator(); | 1378 | const allocator = gpa.allocator(); |
| 1281 | | 1379 | |
| 1282 | var buf = try allocator.alignedAlloc(u8, 1, page_size + 1); | 1380 | var buf = try allocator.alignedAlloc(u8, 1, page_size + 1); |
| 1283 | try std.testing.expect(gpa.total_requested_bytes == page_size + 1); | 1381 | try std.testing.expectEqual(page_size + 1, gpa.total_requested_bytes); |
| 1284 | buf = try allocator.realloc(buf, 1); | 1382 | buf = try allocator.realloc(buf, 1); |
| 1285 | try std.testing.expect(gpa.total_requested_bytes == 1); | 1383 | try std.testing.expectEqual(1, gpa.total_requested_bytes); |
| 1286 | buf = try allocator.realloc(buf, 2); | 1384 | buf = try allocator.realloc(buf, 2); |
| 1287 | try std.testing.expect(gpa.total_requested_bytes == 2); | 1385 | try std.testing.expectEqual(2, gpa.total_requested_bytes); |
| 1288 | } | 1386 | } |
| 1289 | | 1387 | |
| 1290 | test "retain metadata and never unmap" { | 1388 | test "retain metadata and never unmap" { |
Andrew Kelley <andrew@ziglang.org> 2025-02-05 00:18:43-08:00
lib