| ... | ... | @@ -163,6 +163,10 @@ pub const Config = struct { |
| 163 | 163 | /// Tell whether the backing allocator returns already-zeroed memory. |
| 164 | 164 | backing_allocator_zeroes: bool = true, |
| 165 | 165 | |
| 166 | /// When resizing an allocation, refresh the stack trace with the resize |
| 167 | /// callsite. Comes with a performance penalty. |
| 168 | resize_stack_traces: bool = false, |
| 169 | |
| 166 | 170 | /// Magic value that distinguishes allocations owned by this allocator from |
| 167 | 171 | /// other regions of memory. |
| 168 | 172 | canary: usize = @truncate(0x9232a6ff85dff10f), |
| ... | ... | @@ -554,6 +558,12 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { |
| 554 | 558 | }); |
| 555 | 559 | } |
| 556 | 560 | |
| 561 | // If this would move the allocation into a small size class, |
| 562 | // refuse the request, because it would require creating small |
| 563 | // allocation metadata. |
| 564 | const new_size_class_index: usize = @max(@bitSizeOf(usize) - @clz(new_size - 1), @intFromEnum(alignment)); |
| 565 | if (new_size_class_index < self.buckets.len) return null; |
| 566 | |
| 557 | 567 | // Do memory limit accounting with requested sizes rather than what |
| 558 | 568 | // backing_allocator returns because if we want to return |
| 559 | 569 | // error.OutOfMemory, we have to leave allocation untouched, and |
| ... | ... | @@ -598,7 +608,8 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { |
| 598 | 608 | }); |
| 599 | 609 | } |
| 600 | 610 | entry.value_ptr.bytes = resized_ptr[0..new_size]; |
| 601 | | entry.value_ptr.captureStackTrace(ret_addr, .alloc); |
| 611 | if (config.resize_stack_traces) |
| 612 | entry.value_ptr.captureStackTrace(ret_addr, .alloc); |
| 602 | 613 | |
| 603 | 614 | // Update the key of the hash map if the memory was relocated. |
| 604 | 615 | if (resized_ptr != old_mem.ptr) { |
| ... | ... | @@ -791,12 +802,16 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { |
| 791 | 802 | new_len: usize, |
| 792 | 803 | return_address: usize, |
| 793 | 804 | ) bool { |
| 794 | | _ = context; |
| 795 | | _ = memory; |
| 796 | | _ = alignment; |
| 797 | | _ = new_len; |
| 798 | | _ = return_address; |
| 799 | | return false; |
| 805 | const self: *Self = @ptrCast(@alignCast(context)); |
| 806 | self.mutex.lock(); |
| 807 | defer self.mutex.unlock(); |
| 808 | |
| 809 | const size_class_index: usize = @max(@bitSizeOf(usize) - @clz(memory.len - 1), @intFromEnum(alignment)); |
| 810 | if (size_class_index >= self.buckets.len) { |
| 811 | return self.resizeLarge(memory, alignment, new_len, return_address, false) != null; |
| 812 | } else { |
| 813 | return resizeSmall(self, memory, alignment, new_len, return_address, size_class_index); |
| 814 | } |
| 800 | 815 | } |
| 801 | 816 | |
| 802 | 817 | fn remap( |
| ... | ... | @@ -806,12 +821,16 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { |
| 806 | 821 | new_len: usize, |
| 807 | 822 | return_address: usize, |
| 808 | 823 | ) ?[*]u8 { |
| 809 | | _ = context; |
| 810 | | _ = memory; |
| 811 | | _ = alignment; |
| 812 | | _ = new_len; |
| 813 | | _ = return_address; |
| 814 | | return null; |
| 824 | const self: *Self = @ptrCast(@alignCast(context)); |
| 825 | self.mutex.lock(); |
| 826 | defer self.mutex.unlock(); |
| 827 | |
| 828 | const size_class_index: usize = @max(@bitSizeOf(usize) - @clz(memory.len - 1), @intFromEnum(alignment)); |
| 829 | if (size_class_index >= self.buckets.len) { |
| 830 | return self.resizeLarge(memory, alignment, new_len, return_address, true); |
| 831 | } else { |
| 832 | return if (resizeSmall(self, memory, alignment, new_len, return_address, size_class_index)) memory.ptr else null; |
| 833 | } |
| 815 | 834 | } |
| 816 | 835 | |
| 817 | 836 | fn free( |
| ... | ... | @@ -894,8 +913,10 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { |
| 894 | 913 | self.total_requested_bytes -= old_memory.len; |
| 895 | 914 | } |
| 896 | 915 | |
| 897 | | // Capture stack trace to be the "first free", in case a double free happens. |
| 898 | | bucket.captureStackTrace(return_address, slot_count, slot_index, .free); |
| 916 | if (config.stack_trace_frames > 0) { |
| 917 | // Capture stack trace to be the "first free", in case a double free happens. |
| 918 | bucket.captureStackTrace(return_address, slot_count, slot_index, .free); |
| 919 | } |
| 899 | 920 | |
| 900 | 921 | used_byte.* &= ~(@as(u8, 1) << used_bit_index); |
| 901 | 922 | if (config.safety) { |
| ... | ... | @@ -915,6 +936,91 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { |
| 915 | 936 | log.info("small free {d} bytes at {*}", .{ old_memory.len, old_memory.ptr }); |
| 916 | 937 | } |
| 917 | 938 | } |
| 939 | |
| 940 | fn resizeSmall( |
| 941 | self: *Self, |
| 942 | memory: []u8, |
| 943 | alignment: mem.Alignment, |
| 944 | new_len: usize, |
| 945 | return_address: usize, |
| 946 | size_class_index: usize, |
| 947 | ) bool { |
| 948 | const new_size_class_index: usize = @max(@bitSizeOf(usize) - @clz(new_len - 1), @intFromEnum(alignment)); |
| 949 | if (!config.safety) return new_size_class_index == size_class_index; |
| 950 | const slot_count = slot_counts[size_class_index]; |
| 951 | const memory_addr = @intFromPtr(memory.ptr); |
| 952 | const page_addr = memory_addr & ~(page_size - 1); |
| 953 | const bucket: *BucketHeader = .fromPage(page_addr, slot_count); |
| 954 | if (bucket.canary != config.canary) @panic("Invalid free"); |
| 955 | const page_offset = memory_addr - page_addr; |
| 956 | const size_class = @as(usize, 1) << @as(u6, @intCast(size_class_index)); |
| 957 | const slot_index: SlotIndex = @intCast(page_offset / size_class); |
| 958 | const used_byte_index = slot_index / 8; |
| 959 | const used_bit_index: u3 = @intCast(slot_index % 8); |
| 960 | const used_byte = bucket.usedBits(used_byte_index); |
| 961 | const is_used = @as(u1, @truncate(used_byte.* >> used_bit_index)) != 0; |
| 962 | if (!is_used) { |
| 963 | reportDoubleFree( |
| 964 | return_address, |
| 965 | bucketStackTrace(bucket, slot_count, slot_index, .alloc), |
| 966 | bucketStackTrace(bucket, slot_count, slot_index, .free), |
| 967 | ); |
| 968 | // Recoverable since this is a free. |
| 969 | return false; |
| 970 | } |
| 971 | |
| 972 | // Definitely an in-use small alloc now. |
| 973 | const requested_size = bucket.requestedSizes(slot_count)[slot_index]; |
| 974 | if (requested_size == 0) @panic("Invalid free"); |
| 975 | const slot_alignment = bucket.log2PtrAligns(slot_count)[slot_index]; |
| 976 | if (memory.len != requested_size or alignment != slot_alignment) { |
| 977 | var addresses: [stack_n]usize = [1]usize{0} ** stack_n; |
| 978 | var free_stack_trace: StackTrace = .{ |
| 979 | .instruction_addresses = &addresses, |
| 980 | .index = 0, |
| 981 | }; |
| 982 | std.debug.captureStackTrace(return_address, &free_stack_trace); |
| 983 | if (memory.len != requested_size) { |
| 984 | log.err("Allocation size {d} bytes does not match free size {d}. Allocation: {} Free: {}", .{ |
| 985 | requested_size, |
| 986 | memory.len, |
| 987 | bucketStackTrace(bucket, slot_count, slot_index, .alloc), |
| 988 | free_stack_trace, |
| 989 | }); |
| 990 | } |
| 991 | if (alignment != slot_alignment) { |
| 992 | log.err("Allocation alignment {d} does not match free alignment {d}. Allocation: {} Free: {}", .{ |
| 993 | slot_alignment.toByteUnits(), |
| 994 | alignment.toByteUnits(), |
| 995 | bucketStackTrace(bucket, slot_count, slot_index, .alloc), |
| 996 | free_stack_trace, |
| 997 | }); |
| 998 | } |
| 999 | } |
| 1000 | |
| 1001 | if (new_size_class_index != size_class_index) return false; |
| 1002 | |
| 1003 | const prev_req_bytes = self.total_requested_bytes; |
| 1004 | if (config.enable_memory_limit) { |
| 1005 | const new_req_bytes = prev_req_bytes - memory.len + new_len; |
| 1006 | if (new_req_bytes > prev_req_bytes and new_req_bytes > self.requested_memory_limit) { |
| 1007 | return false; |
| 1008 | } |
| 1009 | self.total_requested_bytes = new_req_bytes; |
| 1010 | } |
| 1011 | |
| 1012 | if (memory.len > new_len) @memset(memory[new_len..], undefined); |
| 1013 | if (config.verbose_log) |
| 1014 | log.info("small resize {d} bytes at {*} to {d}", .{ memory.len, memory.ptr, new_len }); |
| 1015 | |
| 1016 | if (config.safety) |
| 1017 | bucket.requestedSizes(slot_count)[slot_index] = @intCast(new_len); |
| 1018 | |
| 1019 | if (config.resize_stack_traces) |
| 1020 | bucket.captureStackTrace(return_address, slot_count, slot_index, .alloc); |
| 1021 | |
| 1022 | return true; |
| 1023 | } |
| 918 | 1024 | }; |
| 919 | 1025 | } |
| 920 | 1026 | |
| ... | ... | @@ -1023,12 +1129,8 @@ test "shrink" { |
| 1023 | 1129 | try std.testing.expect(b == 0x11); |
| 1024 | 1130 | } |
| 1025 | 1131 | |
| 1026 | | try std.testing.expect(allocator.resize(slice, 16)); |
| 1027 | | slice = slice[0..16]; |
| 1028 | | |
| 1029 | | for (slice) |b| { |
| 1030 | | try std.testing.expect(b == 0x11); |
| 1031 | | } |
| 1132 | // Does not cross size class boundaries when shrinking. |
| 1133 | try std.testing.expect(!allocator.resize(slice, 16)); |
| 1032 | 1134 | } |
| 1033 | 1135 | |
| 1034 | 1136 | test "large object - grow" { |
| ... | ... | @@ -1212,14 +1314,14 @@ test "realloc large object to larger alignment" { |
| 1212 | 1314 | try std.testing.expect(slice[16] == 0x34); |
| 1213 | 1315 | } |
| 1214 | 1316 | |
| 1215 | | test "large object shrinks to small but allocation fails during shrink" { |
| 1317 | test "large object rejects shrinking to small" { |
| 1216 | 1318 | if (builtin.target.isWasm()) { |
| 1217 | 1319 | // Not expected to pass on targets that do not have memory mapping. |
| 1218 | 1320 | return error.SkipZigTest; |
| 1219 | 1321 | } |
| 1220 | 1322 | |
| 1221 | 1323 | var failing_allocator = std.testing.FailingAllocator.init(std.heap.page_allocator, .{ .fail_index = 3 }); |
| 1222 | | var gpa = GeneralPurposeAllocator(.{}){ .backing_allocator = failing_allocator.allocator() }; |
| 1324 | var gpa: GeneralPurposeAllocator(.{}) = .{ .backing_allocator = failing_allocator.allocator() }; |
| 1223 | 1325 | defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); |
| 1224 | 1326 | const allocator = gpa.allocator(); |
| 1225 | 1327 | |
| ... | ... | @@ -1228,10 +1330,7 @@ test "large object shrinks to small but allocation fails during shrink" { |
| 1228 | 1330 | slice[0] = 0x12; |
| 1229 | 1331 | slice[3] = 0x34; |
| 1230 | 1332 | |
| 1231 | | // Next allocation will fail in the backing allocator of the GeneralPurposeAllocator |
| 1232 | | |
| 1233 | | try std.testing.expect(allocator.resize(slice, 4)); |
| 1234 | | slice = slice[0..4]; |
| 1333 | try std.testing.expect(!allocator.resize(slice, 4)); |
| 1235 | 1334 | try std.testing.expect(slice[0] == 0x12); |
| 1236 | 1335 | try std.testing.expect(slice[3] == 0x34); |
| 1237 | 1336 | } |
| ... | ... | @@ -1274,17 +1373,16 @@ test "setting a memory cap" { |
| 1274 | 1373 | allocator.free(exact); |
| 1275 | 1374 | } |
| 1276 | 1375 | |
| 1277 | | test "bug 9995 fix, large allocs count requested size not backing size" { |
| 1278 | | // with AtLeast, buffer likely to be larger than requested, especially when shrinking |
| 1279 | | var gpa = GeneralPurposeAllocator(.{ .enable_memory_limit = true }){}; |
| 1376 | test "large allocations count requested size not backing size" { |
| 1377 | var gpa: GeneralPurposeAllocator(.{ .enable_memory_limit = true }) = .{}; |
| 1280 | 1378 | const allocator = gpa.allocator(); |
| 1281 | 1379 | |
| 1282 | 1380 | var buf = try allocator.alignedAlloc(u8, 1, page_size + 1); |
| 1283 | | try std.testing.expect(gpa.total_requested_bytes == page_size + 1); |
| 1381 | try std.testing.expectEqual(page_size + 1, gpa.total_requested_bytes); |
| 1284 | 1382 | buf = try allocator.realloc(buf, 1); |
| 1285 | | try std.testing.expect(gpa.total_requested_bytes == 1); |
| 1383 | try std.testing.expectEqual(1, gpa.total_requested_bytes); |
| 1286 | 1384 | buf = try allocator.realloc(buf, 2); |
| 1287 | | try std.testing.expect(gpa.total_requested_bytes == 2); |
| 1385 | try std.testing.expectEqual(2, gpa.total_requested_bytes); |
| 1288 | 1386 | } |
| 1289 | 1387 | |
| 1290 | 1388 | test "retain metadata and never unmap" { |
Andrew Kelley <andrew@ziglang.org> 2025-02-05 00:18:43-08:00
lib