| author |  Andrew Kelley <andrew@ziglang.org> 2025-02-05 19:18:22-08:00 |
| committer | Andrew Kelley <andrew@ziglang.org> 2025-02-06 14:23:23-08:00 |
| log | cd99ab32294a3c22f09615c93d611593a2887cc3 |
| tree | 7251e1608aeae310cf6b249077c4d790b3e6794e |
| parent | 5e9b8c38d360a254bf951674f4b39ea7a602c515 |
std.heap: rename GeneralPurposeAllocator to DebugAllocator
3 files changed, 1412 insertions(+), 1414 deletions(-)
lib/std/heap.zig+8-3
| ... | @@ -9,15 +9,20 @@ const Allocator = std.mem.Allocator; | ... | @@ -9,15 +9,20 @@ const Allocator = std.mem.Allocator; |
| 9 | const windows = std.os.windows; | 9 | const windows = std.os.windows; |
| 10 | 10 | ||
| 11 | pub const ArenaAllocator = @import("heap/arena_allocator.zig").ArenaAllocator; | 11 | pub const ArenaAllocator = @import("heap/arena_allocator.zig").ArenaAllocator; |
| 12 | pub const GeneralPurposeAllocatorConfig = @import("heap/general_purpose_allocator.zig").Config; | ||
| 13 | pub const GeneralPurposeAllocator = @import("heap/general_purpose_allocator.zig").GeneralPurposeAllocator; | ||
| 14 | pub const Check = @import("heap/general_purpose_allocator.zig").Check; | ||
| 15 | pub const WasmAllocator = @import("heap/WasmAllocator.zig"); | 12 | pub const WasmAllocator = @import("heap/WasmAllocator.zig"); |
| 16 | pub const PageAllocator = @import("heap/PageAllocator.zig"); | 13 | pub const PageAllocator = @import("heap/PageAllocator.zig"); |
| 17 | pub const ThreadSafeAllocator = @import("heap/ThreadSafeAllocator.zig"); | 14 | pub const ThreadSafeAllocator = @import("heap/ThreadSafeAllocator.zig"); |
| 18 | pub const SbrkAllocator = @import("heap/sbrk_allocator.zig").SbrkAllocator; | 15 | pub const SbrkAllocator = @import("heap/sbrk_allocator.zig").SbrkAllocator; |
| 19 | pub const FixedBufferAllocator = @import("heap/FixedBufferAllocator.zig"); | 16 | pub const FixedBufferAllocator = @import("heap/FixedBufferAllocator.zig"); |
| 20 | 17 | ||
| 18 | pub const DebugAllocatorConfig = @import("heap/debug_allocator.zig").Config; | ||
| 19 | pub const DebugAllocator = @import("heap/debug_allocator.zig").DebugAllocator; | ||
| 20 | pub const Check = enum { ok, leak }; | ||
| 21 | /// Deprecated; to be removed after 0.15.0 is tagged. | ||
| 22 | pub const GeneralPurposeAllocatorConfig = DebugAllocatorConfig; | ||
| 23 | /// Deprecated; to be removed after 0.15.0 is tagged. | ||
| 24 | pub const GeneralPurposeAllocator = DebugAllocator; | ||
| 25 | |||
| 21 | const memory_pool = @import("heap/memory_pool.zig"); | 26 | const memory_pool = @import("heap/memory_pool.zig"); |
| 22 | pub const MemoryPool = memory_pool.MemoryPool; | 27 | pub const MemoryPool = memory_pool.MemoryPool; |
| 23 | pub const MemoryPoolAligned = memory_pool.MemoryPoolAligned; | 28 | pub const MemoryPoolAligned = memory_pool.MemoryPoolAligned; |
lib/std/heap/debug_allocator.zig created+1404
| ... | @@ -0,0 +1,1404 @@ | ||
| 1 | //! An allocator that is intended to be used in Debug mode. | ||
| 2 | //! | ||
| 3 | //! ## Features | ||
| 4 | //! | ||
| 5 | //! * Captures stack traces on allocation, free, and optionally resize. | ||
| 6 | //! * Double free detection, which prints all three traces (first alloc, first | ||
| 7 | //! free, second free). | ||
| 8 | //! * Leak detection, with stack traces. | ||
| 9 | //! * Never reuses memory addresses, making it easier for Zig to detect branch | ||
| 10 | //! on undefined values in case of dangling pointers. This relies on | ||
| 11 | //! the backing allocator to also not reuse addresses. | ||
| 12 | //! * Uses a minimum backing allocation size to avoid operating system errors | ||
| 13 | //! from having too many active memory mappings. | ||
| 14 | //! * When a page of memory is no longer needed, give it back to resident | ||
| 15 | //! memory as soon as possible, so that it causes page faults when used. | ||
| 16 | //! * Cross platform. Operates based on a backing allocator which makes it work | ||
| 17 | //! everywhere, even freestanding. | ||
| 18 | //! * Compile-time configuration. | ||
| 19 | //! | ||
| 20 | //! These features require the allocator to be quite slow and wasteful. For | ||
| 21 | //! example, when allocating a single byte, the efficiency is less than 1%; | ||
| 22 | //! it requires more than 100 bytes of overhead to manage the allocation for | ||
| 23 | //! one byte. The efficiency gets better with larger allocations. | ||
| 24 | //! | ||
| 25 | //! ## Basic Design | ||
| 26 | //! | ||
| 27 | //! Allocations are divided into two categories, small and large. | ||
| 28 | //! | ||
| 29 | //! Small allocations are divided into buckets based on `page_size`: | ||
| 30 | //! | ||
| 31 | //! ``` | ||
| 32 | //! index obj_size | ||
| 33 | //! 0 1 | ||
| 34 | //! 1 2 | ||
| 35 | //! 2 4 | ||
| 36 | //! 3 8 | ||
| 37 | //! 4 16 | ||
| 38 | //! 5 32 | ||
| 39 | //! 6 64 | ||
| 40 | //! 7 128 | ||
| 41 | //! 8 256 | ||
| 42 | //! 9 512 | ||
| 43 | //! 10 1024 | ||
| 44 | //! 11 2048 | ||
| 45 | //! ... | ||
| 46 | //! ``` | ||
| 47 | //! | ||
| 48 | //! This goes on for `small_bucket_count` indexes. | ||
| 49 | //! | ||
| 50 | //! Allocations are grouped into an object size based on max(len, alignment), | ||
| 51 | //! rounded up to the next power of two. | ||
| 52 | //! | ||
| 53 | //! The main allocator state has an array of all the "current" buckets for each | ||
| 54 | //! size class. Each slot in the array can be null, meaning the bucket for that | ||
| 55 | //! size class is not allocated. When the first object is allocated for a given | ||
| 56 | //! size class, it makes one `page_size` allocation from the backing allocator. | ||
| 57 | //! This allocation is divided into "slots" - one per allocated object, leaving | ||
| 58 | //! room for the allocation metadata (starting with `BucketHeader`), which is | ||
| 59 | //! located at the very end of the "page". | ||
| 60 | //! | ||
| 61 | //! The allocation metadata includes "used bits" - 1 bit per slot representing | ||
| 62 | //! whether the slot is used. Allocations always take the next available slot | ||
| 63 | //! from the current bucket, setting the corresponding used bit, as well as | ||
| 64 | //! incrementing `allocated_count`. | ||
| 65 | //! | ||
| 66 | //! Frees recover the allocation metadata based on the address, length, and | ||
| 67 | //! alignment, relying on the backing allocation's large alignment, combined | ||
| 68 | //! with the fact that allocations are never moved from small to large, or vice | ||
| 69 | //! versa. | ||
| 70 | //! | ||
| 71 | //! When a bucket is full, a new one is allocated, containing a pointer to the | ||
| 72 | //! previous one. This singly-linked list is iterated during leak detection. | ||
| 73 | //! | ||
| 74 | //! Resizing and remapping work the same on small allocations: if the size | ||
| 75 | //! class would not change, then the operation succeeds, and the address is | ||
| 76 | //! unchanged. Otherwise, the request is rejected. | ||
| 77 | //! | ||
| 78 | //! Large objects are allocated directly using the backing allocator. Metadata | ||
| 79 | //! is stored separately in a `std.HashMap` using the backing allocator. | ||
| 80 | //! | ||
| 81 | //! Resizing and remapping are forwarded directly to the backing allocator, | ||
| 82 | //! except where such operations would change the category from large to small. | ||
| 83 | |||
| 84 | const std = @import("std"); | ||
| 85 | const builtin = @import("builtin"); | ||
| 86 | const log = std.log.scoped(.gpa); | ||
| 87 | const math = std.math; | ||
| 88 | const assert = std.debug.assert; | ||
| 89 | const mem = std.mem; | ||
| 90 | const Allocator = std.mem.Allocator; | ||
| 91 | const StackTrace = std.builtin.StackTrace; | ||
| 92 | |||
| 93 | const page_size: usize = @max(std.heap.page_size_max, switch (builtin.os.tag) { | ||
| 94 | .windows => 64 * 1024, // Makes `std.heap.PageAllocator` take the happy path. | ||
| 95 | .wasi => 64 * 1024, // Max alignment supported by `std.heap.WasmAllocator`. | ||
| 96 | else => 128 * 1024, // Avoids too many active mappings when `page_size_max` is low. | ||
| 97 | }); | ||
| 98 | const page_align: mem.Alignment = .fromByteUnits(page_size); | ||
| 99 | |||
| 100 | /// Integer type for pointing to slots in a small allocation | ||
| 101 | const SlotIndex = std.meta.Int(.unsigned, math.log2(page_size) + 1); | ||
| 102 | const Log2USize = std.math.Log2Int(usize); | ||
| 103 | |||
| 104 | const default_sys_stack_trace_frames: usize = if (std.debug.sys_can_stack_trace) 6 else 0; | ||
| 105 | const default_stack_trace_frames: usize = switch (builtin.mode) { | ||
| 106 | .Debug => default_sys_stack_trace_frames, | ||
| 107 | else => 0, | ||
| 108 | }; | ||
| 109 | |||
| 110 | pub const Config = struct { | ||
| 111 | /// Number of stack frames to capture. | ||
| 112 | stack_trace_frames: usize = default_stack_trace_frames, | ||
| 113 | |||
| 114 | /// If true, the allocator will have two fields: | ||
| 115 | /// * `total_requested_bytes` which tracks the total allocated bytes of memory requested. | ||
| 116 | /// * `requested_memory_limit` which causes allocations to return `error.OutOfMemory` | ||
| 117 | /// when the `total_requested_bytes` exceeds this limit. | ||
| 118 | /// If false, these fields will be `void`. | ||
| 119 | enable_memory_limit: bool = false, | ||
| 120 | |||
| 121 | /// Whether to enable safety checks. | ||
| 122 | safety: bool = std.debug.runtime_safety, | ||
| 123 | |||
| 124 | /// Whether the allocator may be used simultaneously from multiple threads. | ||
| 125 | thread_safe: bool = !builtin.single_threaded, | ||
| 126 | |||
| 127 | /// What type of mutex you'd like to use, for thread safety. | ||
| 128 | /// when specified, the mutex type must have the same shape as `std.Thread.Mutex` and | ||
| 129 | /// `DummyMutex`, and have no required fields. Specifying this field causes | ||
| 130 | /// the `thread_safe` field to be ignored. | ||
| 131 | /// | ||
| 132 | /// when null (default): | ||
| 133 | /// * the mutex type defaults to `std.Thread.Mutex` when thread_safe is enabled. | ||
| 134 | /// * the mutex type defaults to `DummyMutex` otherwise. | ||
| 135 | MutexType: ?type = null, | ||
| 136 | |||
| 137 | /// This is a temporary debugging trick you can use to turn segfaults into more helpful | ||
| 138 | /// logged error messages with stack trace details. The downside is that every allocation | ||
| 139 | /// will be leaked, unless used with retain_metadata! | ||
| 140 | never_unmap: bool = false, | ||
| 141 | |||
| 142 | /// This is a temporary debugging aid that retains metadata about allocations indefinitely. | ||
| 143 | /// This allows a greater range of double frees to be reported. All metadata is freed when | ||
| 144 | /// deinit is called. When used with never_unmap, deliberately leaked memory is also freed | ||
| 145 | /// during deinit. Currently should be used with never_unmap to avoid segfaults. | ||
| 146 | /// TODO https://github.com/ziglang/zig/issues/4298 will allow use without never_unmap | ||
| 147 | retain_metadata: bool = false, | ||
| 148 | |||
| 149 | /// Enables emitting info messages with the size and address of every allocation. | ||
| 150 | verbose_log: bool = false, | ||
| 151 | |||
| 152 | /// Tell whether the backing allocator returns already-zeroed memory. | ||
| 153 | backing_allocator_zeroes: bool = true, | ||
| 154 | |||
| 155 | /// When resizing an allocation, refresh the stack trace with the resize | ||
| 156 | /// callsite. Comes with a performance penalty. | ||
| 157 | resize_stack_traces: bool = false, | ||
| 158 | |||
| 159 | /// Magic value that distinguishes allocations owned by this allocator from | ||
| 160 | /// other regions of memory. | ||
| 161 | canary: usize = @truncate(0x9232a6ff85dff10f), | ||
| 162 | }; | ||
| 163 | |||
| 164 | /// Default initialization of this struct is deprecated; use `.init` instead. | ||
| 165 | pub fn DebugAllocator(comptime config: Config) type { | ||
| 166 | return struct { | ||
| 167 | backing_allocator: Allocator = std.heap.page_allocator, | ||
| 168 | /// Tracks the active bucket, which is the one that has free slots in it. | ||
| 169 | buckets: [small_bucket_count]?*BucketHeader = [1]?*BucketHeader{null} ** small_bucket_count, | ||
| 170 | large_allocations: LargeAllocTable = .empty, | ||
| 171 | total_requested_bytes: @TypeOf(total_requested_bytes_init) = total_requested_bytes_init, | ||
| 172 | requested_memory_limit: @TypeOf(requested_memory_limit_init) = requested_memory_limit_init, | ||
| 173 | mutex: @TypeOf(mutex_init) = mutex_init, | ||
| 174 | |||
| 175 | const Self = @This(); | ||
| 176 | |||
| 177 | pub const init: Self = .{}; | ||
| 178 | |||
| 179 | /// These can be derived from size_class_index but the calculation is nontrivial. | ||
| 180 | const slot_counts: [small_bucket_count]SlotIndex = init: { | ||
| 181 | @setEvalBranchQuota(10000); | ||
| 182 | var result: [small_bucket_count]SlotIndex = undefined; | ||
| 183 | for (&result, 0..) |*elem, i| elem.* = calculateSlotCount(i); | ||
| 184 | break :init result; | ||
| 185 | }; | ||
| 186 | |||
| 187 | const total_requested_bytes_init = if (config.enable_memory_limit) @as(usize, 0) else {}; | ||
| 188 | const requested_memory_limit_init = if (config.enable_memory_limit) @as(usize, math.maxInt(usize)) else {}; | ||
| 189 | |||
| 190 | const mutex_init = if (config.MutexType) |T| | ||
| 191 | T{} | ||
| 192 | else if (config.thread_safe) | ||
| 193 | std.Thread.Mutex{} | ||
| 194 | else | ||
| 195 | DummyMutex{}; | ||
| 196 | |||
| 197 | const DummyMutex = struct { | ||
| 198 | inline fn lock(_: *DummyMutex) void {} | ||
| 199 | inline fn unlock(_: *DummyMutex) void {} | ||
| 200 | }; | ||
| 201 | |||
| 202 | const stack_n = config.stack_trace_frames; | ||
| 203 | const one_trace_size = @sizeOf(usize) * stack_n; | ||
| 204 | const traces_per_slot = 2; | ||
| 205 | |||
| 206 | pub const Error = mem.Allocator.Error; | ||
| 207 | |||
| 208 | /// Avoids creating buckets that would only be able to store a small | ||
| 209 | /// number of slots. Value of 1 means 2 is the minimum slot count. | ||
| 210 | const minimum_slots_per_bucket_log2 = 1; | ||
| 211 | const small_bucket_count = math.log2(page_size) - minimum_slots_per_bucket_log2; | ||
| 212 | const largest_bucket_object_size = 1 << (small_bucket_count - 1); | ||
| 213 | const LargestSizeClassInt = std.math.IntFittingRange(0, largest_bucket_object_size); | ||
| 214 | |||
| 215 | const bucketCompare = struct { | ||
| 216 | fn compare(a: *BucketHeader, b: *BucketHeader) std.math.Order { | ||
| 217 | return std.math.order(@intFromPtr(a.page), @intFromPtr(b.page)); | ||
| 218 | } | ||
| 219 | }.compare; | ||
| 220 | |||
| 221 | const LargeAlloc = struct { | ||
| 222 | bytes: []u8, | ||
| 223 | requested_size: if (config.enable_memory_limit) usize else void, | ||
| 224 | stack_addresses: [trace_n][stack_n]usize, | ||
| 225 | freed: if (config.retain_metadata) bool else void, | ||
| 226 | alignment: if (config.never_unmap and config.retain_metadata) mem.Alignment else void, | ||
| 227 | |||
| 228 | const trace_n = if (config.retain_metadata) traces_per_slot else 1; | ||
| 229 | |||
| 230 | fn dumpStackTrace(self: *LargeAlloc, trace_kind: TraceKind) void { | ||
| 231 | std.debug.dumpStackTrace(self.getStackTrace(trace_kind)); | ||
| 232 | } | ||
| 233 | |||
| 234 | fn getStackTrace(self: *LargeAlloc, trace_kind: TraceKind) std.builtin.StackTrace { | ||
| 235 | assert(@intFromEnum(trace_kind) < trace_n); | ||
| 236 | const stack_addresses = &self.stack_addresses[@intFromEnum(trace_kind)]; | ||
| 237 | var len: usize = 0; | ||
| 238 | while (len < stack_n and stack_addresses[len] != 0) { | ||
| 239 | len += 1; | ||
| 240 | } | ||
| 241 | return .{ | ||
| 242 | .instruction_addresses = stack_addresses, | ||
| 243 | .index = len, | ||
| 244 | }; | ||
| 245 | } | ||
| 246 | |||
| 247 | fn captureStackTrace(self: *LargeAlloc, ret_addr: usize, trace_kind: TraceKind) void { | ||
| 248 | assert(@intFromEnum(trace_kind) < trace_n); | ||
| 249 | const stack_addresses = &self.stack_addresses[@intFromEnum(trace_kind)]; | ||
| 250 | collectStackTrace(ret_addr, stack_addresses); | ||
| 251 | } | ||
| 252 | }; | ||
| 253 | const LargeAllocTable = std.AutoHashMapUnmanaged(usize, LargeAlloc); | ||
| 254 | |||
| 255 | /// Bucket: In memory, in order: | ||
| 256 | /// * BucketHeader | ||
| 257 | /// * bucket_used_bits: [N]usize, // 1 bit for every slot | ||
| 258 | /// -- below only exists when config.safety is true -- | ||
| 259 | /// * requested_sizes: [N]LargestSizeClassInt // 1 int for every slot | ||
| 260 | /// * log2_ptr_aligns: [N]u8 // 1 byte for every slot | ||
| 261 | /// -- above only exists when config.safety is true -- | ||
| 262 | /// * stack_trace_addresses: [N]usize, // traces_per_slot for every allocation | ||
| 263 | const BucketHeader = struct { | ||
| 264 | allocated_count: SlotIndex, | ||
| 265 | freed_count: SlotIndex, | ||
| 266 | prev: ?*BucketHeader, | ||
| 267 | canary: usize = config.canary, | ||
| 268 | |||
| 269 | fn fromPage(page_addr: usize, slot_count: usize) *BucketHeader { | ||
| 270 | const unaligned = page_addr + page_size - bucketSize(slot_count); | ||
| 271 | return @ptrFromInt(unaligned & ~(@as(usize, @alignOf(BucketHeader)) - 1)); | ||
| 272 | } | ||
| 273 | |||
| 274 | fn usedBits(bucket: *BucketHeader, index: usize) *usize { | ||
| 275 | const ptr: [*]u8 = @ptrCast(bucket); | ||
| 276 | const bits: [*]usize = @alignCast(@ptrCast(ptr + @sizeOf(BucketHeader))); | ||
| 277 | return &bits[index]; | ||
| 278 | } | ||
| 279 | |||
| 280 | fn requestedSizes(bucket: *BucketHeader, slot_count: usize) []LargestSizeClassInt { | ||
| 281 | if (!config.safety) @compileError("requested size is only stored when safety is enabled"); | ||
| 282 | const start_ptr = @as([*]u8, @ptrCast(bucket)) + bucketRequestedSizesStart(slot_count); | ||
| 283 | const sizes = @as([*]LargestSizeClassInt, @ptrCast(@alignCast(start_ptr))); | ||
| 284 | return sizes[0..slot_count]; | ||
| 285 | } | ||
| 286 | |||
| 287 | fn log2PtrAligns(bucket: *BucketHeader, slot_count: usize) []mem.Alignment { | ||
| 288 | if (!config.safety) @compileError("requested size is only stored when safety is enabled"); | ||
| 289 | const aligns_ptr = @as([*]u8, @ptrCast(bucket)) + bucketAlignsStart(slot_count); | ||
| 290 | return @ptrCast(aligns_ptr[0..slot_count]); | ||
| 291 | } | ||
| 292 | |||
| 293 | fn stackTracePtr( | ||
| 294 | bucket: *BucketHeader, | ||
| 295 | slot_count: usize, | ||
| 296 | slot_index: SlotIndex, | ||
| 297 | trace_kind: TraceKind, | ||
| 298 | ) *[stack_n]usize { | ||
| 299 | const start_ptr = @as([*]u8, @ptrCast(bucket)) + bucketStackFramesStart(slot_count); | ||
| 300 | const addr = start_ptr + one_trace_size * traces_per_slot * slot_index + | ||
| 301 | @intFromEnum(trace_kind) * @as(usize, one_trace_size); | ||
| 302 | return @ptrCast(@alignCast(addr)); | ||
| 303 | } | ||
| 304 | |||
| 305 | fn captureStackTrace( | ||
| 306 | bucket: *BucketHeader, | ||
| 307 | ret_addr: usize, | ||
| 308 | slot_count: usize, | ||
| 309 | slot_index: SlotIndex, | ||
| 310 | trace_kind: TraceKind, | ||
| 311 | ) void { | ||
| 312 | // Initialize them to 0. When determining the count we must look | ||
| 313 | // for non zero addresses. | ||
| 314 | const stack_addresses = bucket.stackTracePtr(slot_count, slot_index, trace_kind); | ||
| 315 | collectStackTrace(ret_addr, stack_addresses); | ||
| 316 | } | ||
| 317 | }; | ||
| 318 | |||
| 319 | pub fn allocator(self: *Self) Allocator { | ||
| 320 | return .{ | ||
| 321 | .ptr = self, | ||
| 322 | .vtable = &.{ | ||
| 323 | .alloc = alloc, | ||
| 324 | .resize = resize, | ||
| 325 | .remap = remap, | ||
| 326 | .free = free, | ||
| 327 | }, | ||
| 328 | }; | ||
| 329 | } | ||
| 330 | |||
| 331 | fn bucketStackTrace( | ||
| 332 | bucket: *BucketHeader, | ||
| 333 | slot_count: usize, | ||
| 334 | slot_index: SlotIndex, | ||
| 335 | trace_kind: TraceKind, | ||
| 336 | ) StackTrace { | ||
| 337 | const stack_addresses = bucket.stackTracePtr(slot_count, slot_index, trace_kind); | ||
| 338 | var len: usize = 0; | ||
| 339 | while (len < stack_n and stack_addresses[len] != 0) { | ||
| 340 | len += 1; | ||
| 341 | } | ||
| 342 | return .{ | ||
| 343 | .instruction_addresses = stack_addresses, | ||
| 344 | .index = len, | ||
| 345 | }; | ||
| 346 | } | ||
| 347 | |||
| 348 | fn bucketRequestedSizesStart(slot_count: usize) usize { | ||
| 349 | if (!config.safety) @compileError("requested sizes are not stored unless safety is enabled"); | ||
| 350 | return mem.alignForward( | ||
| 351 | usize, | ||
| 352 | @sizeOf(BucketHeader) + usedBitsSize(slot_count), | ||
| 353 | @alignOf(LargestSizeClassInt), | ||
| 354 | ); | ||
| 355 | } | ||
| 356 | |||
| 357 | fn bucketAlignsStart(slot_count: usize) usize { | ||
| 358 | if (!config.safety) @compileError("requested sizes are not stored unless safety is enabled"); | ||
| 359 | return bucketRequestedSizesStart(slot_count) + (@sizeOf(LargestSizeClassInt) * slot_count); | ||
| 360 | } | ||
| 361 | |||
| 362 | fn bucketStackFramesStart(slot_count: usize) usize { | ||
| 363 | const unaligned_start = if (config.safety) | ||
| 364 | bucketAlignsStart(slot_count) + slot_count | ||
| 365 | else | ||
| 366 | @sizeOf(BucketHeader) + usedBitsSize(slot_count); | ||
| 367 | return mem.alignForward(usize, unaligned_start, @alignOf(usize)); | ||
| 368 | } | ||
| 369 | |||
| 370 | fn bucketSize(slot_count: usize) usize { | ||
| 371 | return bucketStackFramesStart(slot_count) + one_trace_size * traces_per_slot * slot_count; | ||
| 372 | } | ||
| 373 | |||
| 374 | /// This is executed only at compile-time to prepopulate a lookup table. | ||
| 375 | fn calculateSlotCount(size_class_index: usize) SlotIndex { | ||
| 376 | const size_class = @as(usize, 1) << @as(Log2USize, @intCast(size_class_index)); | ||
| 377 | var lower: usize = 1 << minimum_slots_per_bucket_log2; | ||
| 378 | var upper: usize = (page_size - bucketSize(lower)) / size_class; | ||
| 379 | while (upper > lower) { | ||
| 380 | const proposed: usize = lower + (upper - lower) / 2; | ||
| 381 | if (proposed == lower) return lower; | ||
| 382 | const slots_end = proposed * size_class; | ||
| 383 | const header_begin = mem.alignForward(usize, slots_end, @alignOf(BucketHeader)); | ||
| 384 | const end = header_begin + bucketSize(proposed); | ||
| 385 | if (end > page_size) { | ||
| 386 | upper = proposed - 1; | ||
| 387 | } else { | ||
| 388 | lower = proposed; | ||
| 389 | } | ||
| 390 | } | ||
| 391 | const slots_end = lower * size_class; | ||
| 392 | const header_begin = mem.alignForward(usize, slots_end, @alignOf(BucketHeader)); | ||
| 393 | const end = header_begin + bucketSize(lower); | ||
| 394 | assert(end <= page_size); | ||
| 395 | return lower; | ||
| 396 | } | ||
| 397 | |||
| 398 | fn usedBitsCount(slot_count: usize) usize { | ||
| 399 | return (slot_count + (@bitSizeOf(usize) - 1)) / @bitSizeOf(usize); | ||
| 400 | } | ||
| 401 | |||
| 402 | fn usedBitsSize(slot_count: usize) usize { | ||
| 403 | return usedBitsCount(slot_count) * @sizeOf(usize); | ||
| 404 | } | ||
| 405 | |||
| 406 | fn detectLeaksInBucket(bucket: *BucketHeader, size_class_index: usize, used_bits_count: usize) bool { | ||
| 407 | const size_class = @as(usize, 1) << @as(Log2USize, @intCast(size_class_index)); | ||
| 408 | const slot_count = slot_counts[size_class_index]; | ||
| 409 | var leaks = false; | ||
| 410 | for (0..used_bits_count) |used_bits_byte| { | ||
| 411 | const used_int = bucket.usedBits(used_bits_byte).*; | ||
| 412 | if (used_int != 0) { | ||
| 413 | for (0..@bitSizeOf(usize)) |bit_index_usize| { | ||
| 414 | const bit_index: Log2USize = @intCast(bit_index_usize); | ||
| 415 | const is_used = @as(u1, @truncate(used_int >> bit_index)) != 0; | ||
| 416 | if (is_used) { | ||
| 417 | const slot_index: SlotIndex = @intCast(used_bits_byte * @bitSizeOf(usize) + bit_index); | ||
| 418 | const stack_trace = bucketStackTrace(bucket, slot_count, slot_index, .alloc); | ||
| 419 | const page_addr = @intFromPtr(bucket) & ~(page_size - 1); | ||
| 420 | const addr = page_addr + slot_index * size_class; | ||
| 421 | log.err("memory address 0x{x} leaked: {}", .{ addr, stack_trace }); | ||
| 422 | leaks = true; | ||
| 423 | } | ||
| 424 | } | ||
| 425 | } | ||
| 426 | } | ||
| 427 | return leaks; | ||
| 428 | } | ||
| 429 | |||
| 430 | /// Emits log messages for leaks and then returns whether there were any leaks. | ||
| 431 | pub fn detectLeaks(self: *Self) bool { | ||
| 432 | var leaks = false; | ||
| 433 | |||
| 434 | for (self.buckets, 0..) |init_optional_bucket, size_class_index| { | ||
| 435 | var optional_bucket = init_optional_bucket; | ||
| 436 | const slot_count = slot_counts[size_class_index]; | ||
| 437 | const used_bits_count = usedBitsCount(slot_count); | ||
| 438 | while (optional_bucket) |bucket| { | ||
| 439 | leaks = detectLeaksInBucket(bucket, size_class_index, used_bits_count) or leaks; | ||
| 440 | optional_bucket = bucket.prev; | ||
| 441 | } | ||
| 442 | } | ||
| 443 | |||
| 444 | var it = self.large_allocations.valueIterator(); | ||
| 445 | while (it.next()) |large_alloc| { | ||
| 446 | if (config.retain_metadata and large_alloc.freed) continue; | ||
| 447 | const stack_trace = large_alloc.getStackTrace(.alloc); | ||
| 448 | log.err("memory address 0x{x} leaked: {}", .{ | ||
| 449 | @intFromPtr(large_alloc.bytes.ptr), stack_trace, | ||
| 450 | }); | ||
| 451 | leaks = true; | ||
| 452 | } | ||
| 453 | return leaks; | ||
| 454 | } | ||
| 455 | |||
| 456 | fn freeRetainedMetadata(self: *Self) void { | ||
| 457 | comptime assert(config.retain_metadata); | ||
| 458 | if (config.never_unmap) { | ||
| 459 | // free large allocations that were intentionally leaked by never_unmap | ||
| 460 | var it = self.large_allocations.iterator(); | ||
| 461 | while (it.next()) |large| { | ||
| 462 | if (large.value_ptr.freed) { | ||
| 463 | self.backing_allocator.rawFree(large.value_ptr.bytes, large.value_ptr.alignment, @returnAddress()); | ||
| 464 | } | ||
| 465 | } | ||
| 466 | } | ||
| 467 | } | ||
| 468 | |||
| 469 | pub fn flushRetainedMetadata(self: *Self) void { | ||
| 470 | comptime assert(config.retain_metadata); | ||
| 471 | self.freeRetainedMetadata(); | ||
| 472 | // also remove entries from large_allocations | ||
| 473 | var it = self.large_allocations.iterator(); | ||
| 474 | while (it.next()) |large| { | ||
| 475 | if (large.value_ptr.freed) { | ||
| 476 | _ = self.large_allocations.remove(@intFromPtr(large.value_ptr.bytes.ptr)); | ||
| 477 | } | ||
| 478 | } | ||
| 479 | } | ||
| 480 | |||
| 481 | /// Returns `std.heap.Check.leak` if there were leaks; `std.heap.Check.ok` otherwise. | ||
| 482 | pub fn deinit(self: *Self) std.heap.Check { | ||
| 483 | const leaks = if (config.safety) self.detectLeaks() else false; | ||
| 484 | if (config.retain_metadata) self.freeRetainedMetadata(); | ||
| 485 | self.large_allocations.deinit(self.backing_allocator); | ||
| 486 | self.* = undefined; | ||
| 487 | return if (leaks) .leak else .ok; | ||
| 488 | } | ||
| 489 | |||
| 490 | fn collectStackTrace(first_trace_addr: usize, addresses: *[stack_n]usize) void { | ||
| 491 | if (stack_n == 0) return; | ||
| 492 | @memset(addresses, 0); | ||
| 493 | var stack_trace: StackTrace = .{ | ||
| 494 | .instruction_addresses = addresses, | ||
| 495 | .index = 0, | ||
| 496 | }; | ||
| 497 | std.debug.captureStackTrace(first_trace_addr, &stack_trace); | ||
| 498 | } | ||
| 499 | |||
| 500 | fn reportDoubleFree(ret_addr: usize, alloc_stack_trace: StackTrace, free_stack_trace: StackTrace) void { | ||
| 501 | var addresses: [stack_n]usize = @splat(0); | ||
| 502 | var second_free_stack_trace: StackTrace = .{ | ||
| 503 | .instruction_addresses = &addresses, | ||
| 504 | .index = 0, | ||
| 505 | }; | ||
| 506 | std.debug.captureStackTrace(ret_addr, &second_free_stack_trace); | ||
| 507 | log.err("Double free detected. Allocation: {} First free: {} Second free: {}", .{ | ||
| 508 | alloc_stack_trace, free_stack_trace, second_free_stack_trace, | ||
| 509 | }); | ||
| 510 | } | ||
| 511 | |||
| 512 | /// This function assumes the object is in the large object storage regardless | ||
| 513 | /// of the parameters. | ||
| 514 | fn resizeLarge( | ||
| 515 | self: *Self, | ||
| 516 | old_mem: []u8, | ||
| 517 | alignment: mem.Alignment, | ||
| 518 | new_size: usize, | ||
| 519 | ret_addr: usize, | ||
| 520 | may_move: bool, | ||
| 521 | ) ?[*]u8 { | ||
| 522 | if (config.retain_metadata and may_move) { | ||
| 523 | // Before looking up the entry (since this could invalidate | ||
| 524 | // it), we must reserve space for the new entry in case the | ||
| 525 | // allocation is relocated. | ||
| 526 | self.large_allocations.ensureUnusedCapacity(self.backing_allocator, 1) catch return null; | ||
| 527 | } | ||
| 528 | |||
| 529 | const entry = self.large_allocations.getEntry(@intFromPtr(old_mem.ptr)) orelse { | ||
| 530 | if (config.safety) { | ||
| 531 | @panic("Invalid free"); | ||
| 532 | } else { | ||
| 533 | unreachable; | ||
| 534 | } | ||
| 535 | }; | ||
| 536 | |||
| 537 | if (config.retain_metadata and entry.value_ptr.freed) { | ||
| 538 | if (config.safety) { | ||
| 539 | reportDoubleFree(ret_addr, entry.value_ptr.getStackTrace(.alloc), entry.value_ptr.getStackTrace(.free)); | ||
| 540 | @panic("Unrecoverable double free"); | ||
| 541 | } else { | ||
| 542 | unreachable; | ||
| 543 | } | ||
| 544 | } | ||
| 545 | |||
| 546 | if (config.safety and old_mem.len != entry.value_ptr.bytes.len) { | ||
| 547 | var addresses: [stack_n]usize = [1]usize{0} ** stack_n; | ||
| 548 | var free_stack_trace: StackTrace = .{ | ||
| 549 | .instruction_addresses = &addresses, | ||
| 550 | .index = 0, | ||
| 551 | }; | ||
| 552 | std.debug.captureStackTrace(ret_addr, &free_stack_trace); | ||
| 553 | log.err("Allocation size {d} bytes does not match free size {d}. Allocation: {} Free: {}", .{ | ||
| 554 | entry.value_ptr.bytes.len, | ||
| 555 | old_mem.len, | ||
| 556 | entry.value_ptr.getStackTrace(.alloc), | ||
| 557 | free_stack_trace, | ||
| 558 | }); | ||
| 559 | } | ||
| 560 | |||
| 561 | // If this would move the allocation into a small size class, | ||
| 562 | // refuse the request, because it would require creating small | ||
| 563 | // allocation metadata. | ||
| 564 | const new_size_class_index: usize = @max(@bitSizeOf(usize) - @clz(new_size - 1), @intFromEnum(alignment)); | ||
| 565 | if (new_size_class_index < self.buckets.len) return null; | ||
| 566 | |||
| 567 | // Do memory limit accounting with requested sizes rather than what | ||
| 568 | // backing_allocator returns because if we want to return | ||
| 569 | // error.OutOfMemory, we have to leave allocation untouched, and | ||
| 570 | // that is impossible to guarantee after calling | ||
| 571 | // backing_allocator.rawResize. | ||
| 572 | const prev_req_bytes = self.total_requested_bytes; | ||
| 573 | if (config.enable_memory_limit) { | ||
| 574 | const new_req_bytes = prev_req_bytes + new_size - entry.value_ptr.requested_size; | ||
| 575 | if (new_req_bytes > prev_req_bytes and new_req_bytes > self.requested_memory_limit) { | ||
| 576 | return null; | ||
| 577 | } | ||
| 578 | self.total_requested_bytes = new_req_bytes; | ||
| 579 | } | ||
| 580 | |||
| 581 | const opt_resized_ptr = if (may_move) | ||
| 582 | self.backing_allocator.rawRemap(old_mem, alignment, new_size, ret_addr) | ||
| 583 | else if (self.backing_allocator.rawResize(old_mem, alignment, new_size, ret_addr)) | ||
| 584 | old_mem.ptr | ||
| 585 | else | ||
| 586 | null; | ||
| 587 | |||
| 588 | const resized_ptr = opt_resized_ptr orelse { | ||
| 589 | if (config.enable_memory_limit) { | ||
| 590 | self.total_requested_bytes = prev_req_bytes; | ||
| 591 | } | ||
| 592 | return null; | ||
| 593 | }; | ||
| 594 | |||
| 595 | if (config.enable_memory_limit) { | ||
| 596 | entry.value_ptr.requested_size = new_size; | ||
| 597 | } | ||
| 598 | |||
| 599 | if (config.verbose_log) { | ||
| 600 | log.info("large resize {d} bytes at {*} to {d} at {*}", .{ | ||
| 601 | old_mem.len, old_mem.ptr, new_size, resized_ptr, | ||
| 602 | }); | ||
| 603 | } | ||
| 604 | entry.value_ptr.bytes = resized_ptr[0..new_size]; | ||
| 605 | if (config.resize_stack_traces) | ||
| 606 | entry.value_ptr.captureStackTrace(ret_addr, .alloc); | ||
| 607 | |||
| 608 | // Update the key of the hash map if the memory was relocated. | ||
| 609 | if (resized_ptr != old_mem.ptr) { | ||
| 610 | const large_alloc = entry.value_ptr.*; | ||
| 611 | if (config.retain_metadata) { | ||
| 612 | entry.value_ptr.freed = true; | ||
| 613 | entry.value_ptr.captureStackTrace(ret_addr, .free); | ||
| 614 | } else { | ||
| 615 | self.large_allocations.removeByPtr(entry.key_ptr); | ||
| 616 | } | ||
| 617 | |||
| 618 | const gop = self.large_allocations.getOrPutAssumeCapacity(@intFromPtr(resized_ptr)); | ||
| 619 | if (config.retain_metadata and !config.never_unmap) { | ||
| 620 | // Backing allocator may be reusing memory that we're retaining metadata for | ||
| 621 | assert(!gop.found_existing or gop.value_ptr.freed); | ||
| 622 | } else { | ||
| 623 | assert(!gop.found_existing); // This would mean the kernel double-mapped pages. | ||
| 624 | } | ||
| 625 | gop.value_ptr.* = large_alloc; | ||
| 626 | } | ||
| 627 | |||
| 628 | return resized_ptr; | ||
| 629 | } | ||
| 630 | |||
| 631 | /// This function assumes the object is in the large object storage regardless | ||
| 632 | /// of the parameters. | ||
| 633 | fn freeLarge( | ||
| 634 | self: *Self, | ||
| 635 | old_mem: []u8, | ||
| 636 | alignment: mem.Alignment, | ||
| 637 | ret_addr: usize, | ||
| 638 | ) void { | ||
| 639 | const entry = self.large_allocations.getEntry(@intFromPtr(old_mem.ptr)) orelse { | ||
| 640 | if (config.safety) { | ||
| 641 | @panic("Invalid free"); | ||
| 642 | } else { | ||
| 643 | unreachable; | ||
| 644 | } | ||
| 645 | }; | ||
| 646 | |||
| 647 | if (config.retain_metadata and entry.value_ptr.freed) { | ||
| 648 | if (config.safety) { | ||
| 649 | reportDoubleFree(ret_addr, entry.value_ptr.getStackTrace(.alloc), entry.value_ptr.getStackTrace(.free)); | ||
| 650 | return; | ||
| 651 | } else { | ||
| 652 | unreachable; | ||
| 653 | } | ||
| 654 | } | ||
| 655 | |||
| 656 | if (config.safety and old_mem.len != entry.value_ptr.bytes.len) { | ||
| 657 | var addresses: [stack_n]usize = [1]usize{0} ** stack_n; | ||
| 658 | var free_stack_trace = StackTrace{ | ||
| 659 | .instruction_addresses = &addresses, | ||
| 660 | .index = 0, | ||
| 661 | }; | ||
| 662 | std.debug.captureStackTrace(ret_addr, &free_stack_trace); | ||
| 663 | log.err("Allocation size {d} bytes does not match free size {d}. Allocation: {} Free: {}", .{ | ||
| 664 | entry.value_ptr.bytes.len, | ||
| 665 | old_mem.len, | ||
| 666 | entry.value_ptr.getStackTrace(.alloc), | ||
| 667 | free_stack_trace, | ||
| 668 | }); | ||
| 669 | } | ||
| 670 | |||
| 671 | if (!config.never_unmap) { | ||
| 672 | self.backing_allocator.rawFree(old_mem, alignment, ret_addr); | ||
| 673 | } | ||
| 674 | |||
| 675 | if (config.enable_memory_limit) { | ||
| 676 | self.total_requested_bytes -= entry.value_ptr.requested_size; | ||
| 677 | } | ||
| 678 | |||
| 679 | if (config.verbose_log) { | ||
| 680 | log.info("large free {d} bytes at {*}", .{ old_mem.len, old_mem.ptr }); | ||
| 681 | } | ||
| 682 | |||
| 683 | if (!config.retain_metadata) { | ||
| 684 | assert(self.large_allocations.remove(@intFromPtr(old_mem.ptr))); | ||
| 685 | } else { | ||
| 686 | entry.value_ptr.freed = true; | ||
| 687 | entry.value_ptr.captureStackTrace(ret_addr, .free); | ||
| 688 | } | ||
| 689 | } | ||
| 690 | |||
| 691 | fn alloc(context: *anyopaque, len: usize, alignment: mem.Alignment, ret_addr: usize) ?[*]u8 { | ||
| 692 | const self: *Self = @ptrCast(@alignCast(context)); | ||
| 693 | self.mutex.lock(); | ||
| 694 | defer self.mutex.unlock(); | ||
| 695 | |||
| 696 | if (config.enable_memory_limit) { | ||
| 697 | const new_req_bytes = self.total_requested_bytes + len; | ||
| 698 | if (new_req_bytes > self.requested_memory_limit) return null; | ||
| 699 | self.total_requested_bytes = new_req_bytes; | ||
| 700 | } | ||
| 701 | |||
| 702 | const size_class_index: usize = @max(@bitSizeOf(usize) - @clz(len - 1), @intFromEnum(alignment)); | ||
| 703 | if (size_class_index >= self.buckets.len) { | ||
| 704 | @branchHint(.unlikely); | ||
| 705 | self.large_allocations.ensureUnusedCapacity(self.backing_allocator, 1) catch return null; | ||
| 706 | const ptr = self.backing_allocator.rawAlloc(len, alignment, ret_addr) orelse return null; | ||
| 707 | const slice = ptr[0..len]; | ||
| 708 | |||
| 709 | const gop = self.large_allocations.getOrPutAssumeCapacity(@intFromPtr(slice.ptr)); | ||
| 710 | if (config.retain_metadata and !config.never_unmap) { | ||
| 711 | // Backing allocator may be reusing memory that we're retaining metadata for | ||
| 712 | assert(!gop.found_existing or gop.value_ptr.freed); | ||
| 713 | } else { | ||
| 714 | assert(!gop.found_existing); // This would mean the kernel double-mapped pages. | ||
| 715 | } | ||
| 716 | gop.value_ptr.bytes = slice; | ||
| 717 | if (config.enable_memory_limit) | ||
| 718 | gop.value_ptr.requested_size = len; | ||
| 719 | gop.value_ptr.captureStackTrace(ret_addr, .alloc); | ||
| 720 | if (config.retain_metadata) { | ||
| 721 | gop.value_ptr.freed = false; | ||
| 722 | if (config.never_unmap) { | ||
| 723 | gop.value_ptr.alignment = alignment; | ||
| 724 | } | ||
| 725 | } | ||
| 726 | |||
| 727 | if (config.verbose_log) { | ||
| 728 | log.info("large alloc {d} bytes at {*}", .{ slice.len, slice.ptr }); | ||
| 729 | } | ||
| 730 | return slice.ptr; | ||
| 731 | } | ||
| 732 | |||
| 733 | const slot_count = slot_counts[size_class_index]; | ||
| 734 | |||
| 735 | if (self.buckets[size_class_index]) |bucket| { | ||
| 736 | @branchHint(.likely); | ||
| 737 | const slot_index = bucket.allocated_count; | ||
| 738 | if (slot_index < slot_count) { | ||
| 739 | @branchHint(.likely); | ||
| 740 | bucket.allocated_count = slot_index + 1; | ||
| 741 | const used_bits_byte = bucket.usedBits(slot_index / @bitSizeOf(usize)); | ||
| 742 | const used_bit_index: Log2USize = @intCast(slot_index % @bitSizeOf(usize)); | ||
| 743 | used_bits_byte.* |= (@as(usize, 1) << used_bit_index); | ||
| 744 | const size_class = @as(usize, 1) << @as(Log2USize, @intCast(size_class_index)); | ||
| 745 | if (config.stack_trace_frames > 0) { | ||
| 746 | bucket.captureStackTrace(ret_addr, slot_count, slot_index, .alloc); | ||
| 747 | } | ||
| 748 | if (config.safety) { | ||
| 749 | bucket.requestedSizes(slot_count)[slot_index] = @intCast(len); | ||
| 750 | bucket.log2PtrAligns(slot_count)[slot_index] = alignment; | ||
| 751 | } | ||
| 752 | const page_addr = @intFromPtr(bucket) & ~(page_size - 1); | ||
| 753 | const addr = page_addr + slot_index * size_class; | ||
| 754 | if (config.verbose_log) { | ||
| 755 | log.info("small alloc {d} bytes at 0x{x}", .{ len, addr }); | ||
| 756 | } | ||
| 757 | return @ptrFromInt(addr); | ||
| 758 | } | ||
| 759 | } | ||
| 760 | |||
| 761 | const page = self.backing_allocator.rawAlloc(page_size, page_align, @returnAddress()) orelse | ||
| 762 | return null; | ||
| 763 | const bucket: *BucketHeader = .fromPage(@intFromPtr(page), slot_count); | ||
| 764 | bucket.* = .{ | ||
| 765 | .allocated_count = 1, | ||
| 766 | .freed_count = 0, | ||
| 767 | .prev = self.buckets[size_class_index], | ||
| 768 | }; | ||
| 769 | self.buckets[size_class_index] = bucket; | ||
| 770 | |||
| 771 | if (!config.backing_allocator_zeroes) { | ||
| 772 | @memset(@as([*]usize, @as(*[1]usize, bucket.usedBits(0)))[0..usedBitsCount(slot_count)], 0); | ||
| 773 | if (config.safety) @memset(bucket.requestedSizes(slot_count), 0); | ||
| 774 | } | ||
| 775 | |||
| 776 | bucket.usedBits(0).* = 0b1; | ||
| 777 | |||
| 778 | if (config.stack_trace_frames > 0) { | ||
| 779 | bucket.captureStackTrace(ret_addr, slot_count, 0, .alloc); | ||
| 780 | } | ||
| 781 | |||
| 782 | if (config.safety) { | ||
| 783 | bucket.requestedSizes(slot_count)[0] = @intCast(len); | ||
| 784 | bucket.log2PtrAligns(slot_count)[0] = alignment; | ||
| 785 | } | ||
| 786 | |||
| 787 | if (config.verbose_log) { | ||
| 788 | log.info("small alloc {d} bytes at 0x{x}", .{ len, @intFromPtr(page) }); | ||
| 789 | } | ||
| 790 | |||
| 791 | return page; | ||
| 792 | } | ||
| 793 | |||
| 794 | fn resize( | ||
| 795 | context: *anyopaque, | ||
| 796 | memory: []u8, | ||
| 797 | alignment: mem.Alignment, | ||
| 798 | new_len: usize, | ||
| 799 | return_address: usize, | ||
| 800 | ) bool { | ||
| 801 | const self: *Self = @ptrCast(@alignCast(context)); | ||
| 802 | self.mutex.lock(); | ||
| 803 | defer self.mutex.unlock(); | ||
| 804 | |||
| 805 | const size_class_index: usize = @max(@bitSizeOf(usize) - @clz(memory.len - 1), @intFromEnum(alignment)); | ||
| 806 | if (size_class_index >= self.buckets.len) { | ||
| 807 | return self.resizeLarge(memory, alignment, new_len, return_address, false) != null; | ||
| 808 | } else { | ||
| 809 | return resizeSmall(self, memory, alignment, new_len, return_address, size_class_index); | ||
| 810 | } | ||
| 811 | } | ||
| 812 | |||
| 813 | fn remap( | ||
| 814 | context: *anyopaque, | ||
| 815 | memory: []u8, | ||
| 816 | alignment: mem.Alignment, | ||
| 817 | new_len: usize, | ||
| 818 | return_address: usize, | ||
| 819 | ) ?[*]u8 { | ||
| 820 | const self: *Self = @ptrCast(@alignCast(context)); | ||
| 821 | self.mutex.lock(); | ||
| 822 | defer self.mutex.unlock(); | ||
| 823 | |||
| 824 | const size_class_index: usize = @max(@bitSizeOf(usize) - @clz(memory.len - 1), @intFromEnum(alignment)); | ||
| 825 | if (size_class_index >= self.buckets.len) { | ||
| 826 | return self.resizeLarge(memory, alignment, new_len, return_address, true); | ||
| 827 | } else { | ||
| 828 | return if (resizeSmall(self, memory, alignment, new_len, return_address, size_class_index)) memory.ptr else null; | ||
| 829 | } | ||
| 830 | } | ||
| 831 | |||
| 832 | fn free( | ||
| 833 | context: *anyopaque, | ||
| 834 | old_memory: []u8, | ||
| 835 | alignment: mem.Alignment, | ||
| 836 | return_address: usize, | ||
| 837 | ) void { | ||
| 838 | const self: *Self = @ptrCast(@alignCast(context)); | ||
| 839 | self.mutex.lock(); | ||
| 840 | defer self.mutex.unlock(); | ||
| 841 | |||
| 842 | assert(old_memory.len != 0); | ||
| 843 | |||
| 844 | const size_class_index: usize = @max(@bitSizeOf(usize) - @clz(old_memory.len - 1), @intFromEnum(alignment)); | ||
| 845 | if (size_class_index >= self.buckets.len) { | ||
| 846 | @branchHint(.unlikely); | ||
| 847 | self.freeLarge(old_memory, alignment, return_address); | ||
| 848 | return; | ||
| 849 | } | ||
| 850 | |||
| 851 | const slot_count = slot_counts[size_class_index]; | ||
| 852 | const freed_addr = @intFromPtr(old_memory.ptr); | ||
| 853 | const page_addr = freed_addr & ~(page_size - 1); | ||
| 854 | const bucket: *BucketHeader = .fromPage(page_addr, slot_count); | ||
| 855 | if (bucket.canary != config.canary) @panic("Invalid free"); | ||
| 856 | const page_offset = freed_addr - page_addr; | ||
| 857 | const size_class = @as(usize, 1) << @as(Log2USize, @intCast(size_class_index)); | ||
| 858 | const slot_index: SlotIndex = @intCast(page_offset / size_class); | ||
| 859 | const used_byte_index = slot_index / @bitSizeOf(usize); | ||
| 860 | const used_bit_index: Log2USize = @intCast(slot_index % @bitSizeOf(usize)); | ||
| 861 | const used_byte = bucket.usedBits(used_byte_index); | ||
| 862 | const is_used = @as(u1, @truncate(used_byte.* >> used_bit_index)) != 0; | ||
| 863 | if (!is_used) { | ||
| 864 | if (config.safety) { | ||
| 865 | reportDoubleFree( | ||
| 866 | return_address, | ||
| 867 | bucketStackTrace(bucket, slot_count, slot_index, .alloc), | ||
| 868 | bucketStackTrace(bucket, slot_count, slot_index, .free), | ||
| 869 | ); | ||
| 870 | // Recoverable since this is a free. | ||
| 871 | return; | ||
| 872 | } else { | ||
| 873 | unreachable; | ||
| 874 | } | ||
| 875 | } | ||
| 876 | |||
| 877 | // Definitely an in-use small alloc now. | ||
| 878 | if (config.safety) { | ||
| 879 | const requested_size = bucket.requestedSizes(slot_count)[slot_index]; | ||
| 880 | if (requested_size == 0) @panic("Invalid free"); | ||
| 881 | const slot_alignment = bucket.log2PtrAligns(slot_count)[slot_index]; | ||
| 882 | if (old_memory.len != requested_size or alignment != slot_alignment) { | ||
| 883 | var addresses: [stack_n]usize = [1]usize{0} ** stack_n; | ||
| 884 | var free_stack_trace: StackTrace = .{ | ||
| 885 | .instruction_addresses = &addresses, | ||
| 886 | .index = 0, | ||
| 887 | }; | ||
| 888 | std.debug.captureStackTrace(return_address, &free_stack_trace); | ||
| 889 | if (old_memory.len != requested_size) { | ||
| 890 | log.err("Allocation size {d} bytes does not match free size {d}. Allocation: {} Free: {}", .{ | ||
| 891 | requested_size, | ||
| 892 | old_memory.len, | ||
| 893 | bucketStackTrace(bucket, slot_count, slot_index, .alloc), | ||
| 894 | free_stack_trace, | ||
| 895 | }); | ||
| 896 | } | ||
| 897 | if (alignment != slot_alignment) { | ||
| 898 | log.err("Allocation alignment {d} does not match free alignment {d}. Allocation: {} Free: {}", .{ | ||
| 899 | slot_alignment.toByteUnits(), | ||
| 900 | alignment.toByteUnits(), | ||
| 901 | bucketStackTrace(bucket, slot_count, slot_index, .alloc), | ||
| 902 | free_stack_trace, | ||
| 903 | }); | ||
| 904 | } | ||
| 905 | } | ||
| 906 | } | ||
| 907 | |||
| 908 | if (config.enable_memory_limit) { | ||
| 909 | self.total_requested_bytes -= old_memory.len; | ||
| 910 | } | ||
| 911 | |||
| 912 | if (config.stack_trace_frames > 0) { | ||
| 913 | // Capture stack trace to be the "first free", in case a double free happens. | ||
| 914 | bucket.captureStackTrace(return_address, slot_count, slot_index, .free); | ||
| 915 | } | ||
| 916 | |||
| 917 | used_byte.* &= ~(@as(usize, 1) << used_bit_index); | ||
| 918 | if (config.safety) { | ||
| 919 | bucket.requestedSizes(slot_count)[slot_index] = 0; | ||
| 920 | } | ||
| 921 | bucket.freed_count += 1; | ||
| 922 | if (bucket.freed_count == bucket.allocated_count) { | ||
| 923 | if (self.buckets[size_class_index] == bucket) { | ||
| 924 | self.buckets[size_class_index] = null; | ||
| 925 | } | ||
| 926 | if (!config.never_unmap) { | ||
| 927 | const page: [*]align(page_size) u8 = @ptrFromInt(page_addr); | ||
| 928 | self.backing_allocator.rawFree(page[0..page_size], page_align, @returnAddress()); | ||
| 929 | } | ||
| 930 | } | ||
| 931 | if (config.verbose_log) { | ||
| 932 | log.info("small free {d} bytes at {*}", .{ old_memory.len, old_memory.ptr }); | ||
| 933 | } | ||
| 934 | } | ||
| 935 | |||
| 936 | fn resizeSmall( | ||
| 937 | self: *Self, | ||
| 938 | memory: []u8, | ||
| 939 | alignment: mem.Alignment, | ||
| 940 | new_len: usize, | ||
| 941 | return_address: usize, | ||
| 942 | size_class_index: usize, | ||
| 943 | ) bool { | ||
| 944 | const new_size_class_index: usize = @max(@bitSizeOf(usize) - @clz(new_len - 1), @intFromEnum(alignment)); | ||
| 945 | if (!config.safety) return new_size_class_index == size_class_index; | ||
| 946 | const slot_count = slot_counts[size_class_index]; | ||
| 947 | const memory_addr = @intFromPtr(memory.ptr); | ||
| 948 | const page_addr = memory_addr & ~(page_size - 1); | ||
| 949 | const bucket: *BucketHeader = .fromPage(page_addr, slot_count); | ||
| 950 | if (bucket.canary != config.canary) @panic("Invalid free"); | ||
| 951 | const page_offset = memory_addr - page_addr; | ||
| 952 | const size_class = @as(usize, 1) << @as(Log2USize, @intCast(size_class_index)); | ||
| 953 | const slot_index: SlotIndex = @intCast(page_offset / size_class); | ||
| 954 | const used_byte_index = slot_index / @bitSizeOf(usize); | ||
| 955 | const used_bit_index: Log2USize = @intCast(slot_index % @bitSizeOf(usize)); | ||
| 956 | const used_byte = bucket.usedBits(used_byte_index); | ||
| 957 | const is_used = @as(u1, @truncate(used_byte.* >> used_bit_index)) != 0; | ||
| 958 | if (!is_used) { | ||
| 959 | reportDoubleFree( | ||
| 960 | return_address, | ||
| 961 | bucketStackTrace(bucket, slot_count, slot_index, .alloc), | ||
| 962 | bucketStackTrace(bucket, slot_count, slot_index, .free), | ||
| 963 | ); | ||
| 964 | // Recoverable since this is a free. | ||
| 965 | return false; | ||
| 966 | } | ||
| 967 | |||
| 968 | // Definitely an in-use small alloc now. | ||
| 969 | const requested_size = bucket.requestedSizes(slot_count)[slot_index]; | ||
| 970 | if (requested_size == 0) @panic("Invalid free"); | ||
| 971 | const slot_alignment = bucket.log2PtrAligns(slot_count)[slot_index]; | ||
| 972 | if (memory.len != requested_size or alignment != slot_alignment) { | ||
| 973 | var addresses: [stack_n]usize = [1]usize{0} ** stack_n; | ||
| 974 | var free_stack_trace: StackTrace = .{ | ||
| 975 | .instruction_addresses = &addresses, | ||
| 976 | .index = 0, | ||
| 977 | }; | ||
| 978 | std.debug.captureStackTrace(return_address, &free_stack_trace); | ||
| 979 | if (memory.len != requested_size) { | ||
| 980 | log.err("Allocation size {d} bytes does not match free size {d}. Allocation: {} Free: {}", .{ | ||
| 981 | requested_size, | ||
| 982 | memory.len, | ||
| 983 | bucketStackTrace(bucket, slot_count, slot_index, .alloc), | ||
| 984 | free_stack_trace, | ||
| 985 | }); | ||
| 986 | } | ||
| 987 | if (alignment != slot_alignment) { | ||
| 988 | log.err("Allocation alignment {d} does not match free alignment {d}. Allocation: {} Free: {}", .{ | ||
| 989 | slot_alignment.toByteUnits(), | ||
| 990 | alignment.toByteUnits(), | ||
| 991 | bucketStackTrace(bucket, slot_count, slot_index, .alloc), | ||
| 992 | free_stack_trace, | ||
| 993 | }); | ||
| 994 | } | ||
| 995 | } | ||
| 996 | |||
| 997 | if (new_size_class_index != size_class_index) return false; | ||
| 998 | |||
| 999 | const prev_req_bytes = self.total_requested_bytes; | ||
| 1000 | if (config.enable_memory_limit) { | ||
| 1001 | const new_req_bytes = prev_req_bytes - memory.len + new_len; | ||
| 1002 | if (new_req_bytes > prev_req_bytes and new_req_bytes > self.requested_memory_limit) { | ||
| 1003 | return false; | ||
| 1004 | } | ||
| 1005 | self.total_requested_bytes = new_req_bytes; | ||
| 1006 | } | ||
| 1007 | |||
| 1008 | if (memory.len > new_len) @memset(memory[new_len..], undefined); | ||
| 1009 | if (config.verbose_log) | ||
| 1010 | log.info("small resize {d} bytes at {*} to {d}", .{ memory.len, memory.ptr, new_len }); | ||
| 1011 | |||
| 1012 | if (config.safety) | ||
| 1013 | bucket.requestedSizes(slot_count)[slot_index] = @intCast(new_len); | ||
| 1014 | |||
| 1015 | if (config.resize_stack_traces) | ||
| 1016 | bucket.captureStackTrace(return_address, slot_count, slot_index, .alloc); | ||
| 1017 | |||
| 1018 | return true; | ||
| 1019 | } | ||
| 1020 | }; | ||
| 1021 | } | ||
| 1022 | |||
| 1023 | const TraceKind = enum { | ||
| 1024 | alloc, | ||
| 1025 | free, | ||
| 1026 | }; | ||
| 1027 | |||
| 1028 | const test_config = Config{}; | ||
| 1029 | |||
| 1030 | test "small allocations - free in same order" { | ||
| 1031 | var gpa = DebugAllocator(test_config){}; | ||
| 1032 | defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); | ||
| 1033 | const allocator = gpa.allocator(); | ||
| 1034 | |||
| 1035 | var list = std.ArrayList(*u64).init(std.testing.allocator); | ||
| 1036 | defer list.deinit(); | ||
| 1037 | |||
| 1038 | var i: usize = 0; | ||
| 1039 | while (i < 513) : (i += 1) { | ||
| 1040 | const ptr = try allocator.create(u64); | ||
| 1041 | try list.append(ptr); | ||
| 1042 | } | ||
| 1043 | |||
| 1044 | for (list.items) |ptr| { | ||
| 1045 | allocator.destroy(ptr); | ||
| 1046 | } | ||
| 1047 | } | ||
| 1048 | |||
| 1049 | test "small allocations - free in reverse order" { | ||
| 1050 | var gpa = DebugAllocator(test_config){}; | ||
| 1051 | defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); | ||
| 1052 | const allocator = gpa.allocator(); | ||
| 1053 | |||
| 1054 | var list = std.ArrayList(*u64).init(std.testing.allocator); | ||
| 1055 | defer list.deinit(); | ||
| 1056 | |||
| 1057 | var i: usize = 0; | ||
| 1058 | while (i < 513) : (i += 1) { | ||
| 1059 | const ptr = try allocator.create(u64); | ||
| 1060 | try list.append(ptr); | ||
| 1061 | } | ||
| 1062 | |||
| 1063 | while (list.popOrNull()) |ptr| { | ||
| 1064 | allocator.destroy(ptr); | ||
| 1065 | } | ||
| 1066 | } | ||
| 1067 | |||
| 1068 | test "large allocations" { | ||
| 1069 | var gpa = DebugAllocator(test_config){}; | ||
| 1070 | defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); | ||
| 1071 | const allocator = gpa.allocator(); | ||
| 1072 | |||
| 1073 | const ptr1 = try allocator.alloc(u64, 42768); | ||
| 1074 | const ptr2 = try allocator.alloc(u64, 52768); | ||
| 1075 | allocator.free(ptr1); | ||
| 1076 | const ptr3 = try allocator.alloc(u64, 62768); | ||
| 1077 | allocator.free(ptr3); | ||
| 1078 | allocator.free(ptr2); | ||
| 1079 | } | ||
| 1080 | |||
| 1081 | test "very large allocation" { | ||
| 1082 | var gpa = DebugAllocator(test_config){}; | ||
| 1083 | defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); | ||
| 1084 | const allocator = gpa.allocator(); | ||
| 1085 | |||
| 1086 | try std.testing.expectError(error.OutOfMemory, allocator.alloc(u8, math.maxInt(usize))); | ||
| 1087 | } | ||
| 1088 | |||
| 1089 | test "realloc" { | ||
| 1090 | var gpa = DebugAllocator(test_config){}; | ||
| 1091 | defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); | ||
| 1092 | const allocator = gpa.allocator(); | ||
| 1093 | |||
| 1094 | var slice = try allocator.alignedAlloc(u8, @alignOf(u32), 1); | ||
| 1095 | defer allocator.free(slice); | ||
| 1096 | slice[0] = 0x12; | ||
| 1097 | |||
| 1098 | // This reallocation should keep its pointer address. | ||
| 1099 | const old_slice = slice; | ||
| 1100 | slice = try allocator.realloc(slice, 2); | ||
| 1101 | try std.testing.expect(old_slice.ptr == slice.ptr); | ||
| 1102 | try std.testing.expect(slice[0] == 0x12); | ||
| 1103 | slice[1] = 0x34; | ||
| 1104 | |||
| 1105 | // This requires upgrading to a larger size class | ||
| 1106 | slice = try allocator.realloc(slice, 17); | ||
| 1107 | try std.testing.expect(slice[0] == 0x12); | ||
| 1108 | try std.testing.expect(slice[1] == 0x34); | ||
| 1109 | } | ||
| 1110 | |||
| 1111 | test "shrink" { | ||
| 1112 | var gpa: DebugAllocator(test_config) = .{}; | ||
| 1113 | defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); | ||
| 1114 | const allocator = gpa.allocator(); | ||
| 1115 | |||
| 1116 | var slice = try allocator.alloc(u8, 20); | ||
| 1117 | defer allocator.free(slice); | ||
| 1118 | |||
| 1119 | @memset(slice, 0x11); | ||
| 1120 | |||
| 1121 | try std.testing.expect(allocator.resize(slice, 17)); | ||
| 1122 | slice = slice[0..17]; | ||
| 1123 | |||
| 1124 | for (slice) |b| { | ||
| 1125 | try std.testing.expect(b == 0x11); | ||
| 1126 | } | ||
| 1127 | |||
| 1128 | // Does not cross size class boundaries when shrinking. | ||
| 1129 | try std.testing.expect(!allocator.resize(slice, 16)); | ||
| 1130 | } | ||
| 1131 | |||
| 1132 | test "large object - grow" { | ||
| 1133 | if (builtin.target.isWasm()) { | ||
| 1134 | // Not expected to pass on targets that do not have memory mapping. | ||
| 1135 | return error.SkipZigTest; | ||
| 1136 | } | ||
| 1137 | var gpa: DebugAllocator(test_config) = .{}; | ||
| 1138 | defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); | ||
| 1139 | const allocator = gpa.allocator(); | ||
| 1140 | |||
| 1141 | var slice1 = try allocator.alloc(u8, page_size * 2 - 20); | ||
| 1142 | defer allocator.free(slice1); | ||
| 1143 | |||
| 1144 | const old = slice1; | ||
| 1145 | slice1 = try allocator.realloc(slice1, page_size * 2 - 10); | ||
| 1146 | try std.testing.expect(slice1.ptr == old.ptr); | ||
| 1147 | |||
| 1148 | slice1 = try allocator.realloc(slice1, page_size * 2); | ||
| 1149 | try std.testing.expect(slice1.ptr == old.ptr); | ||
| 1150 | |||
| 1151 | slice1 = try allocator.realloc(slice1, page_size * 2 + 1); | ||
| 1152 | } | ||
| 1153 | |||
| 1154 | test "realloc small object to large object" { | ||
| 1155 | var gpa = DebugAllocator(test_config){}; | ||
| 1156 | defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); | ||
| 1157 | const allocator = gpa.allocator(); | ||
| 1158 | |||
| 1159 | var slice = try allocator.alloc(u8, 70); | ||
| 1160 | defer allocator.free(slice); | ||
| 1161 | slice[0] = 0x12; | ||
| 1162 | slice[60] = 0x34; | ||
| 1163 | |||
| 1164 | // This requires upgrading to a large object | ||
| 1165 | const large_object_size = page_size * 2 + 50; | ||
| 1166 | slice = try allocator.realloc(slice, large_object_size); | ||
| 1167 | try std.testing.expect(slice[0] == 0x12); | ||
| 1168 | try std.testing.expect(slice[60] == 0x34); | ||
| 1169 | } | ||
| 1170 | |||
| 1171 | test "shrink large object to large object" { | ||
| 1172 | var gpa: DebugAllocator(test_config) = .{}; | ||
| 1173 | defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); | ||
| 1174 | const allocator = gpa.allocator(); | ||
| 1175 | |||
| 1176 | var slice = try allocator.alloc(u8, page_size * 2 + 50); | ||
| 1177 | defer allocator.free(slice); | ||
| 1178 | slice[0] = 0x12; | ||
| 1179 | slice[60] = 0x34; | ||
| 1180 | |||
| 1181 | if (!allocator.resize(slice, page_size * 2 + 1)) return; | ||
| 1182 | slice = slice.ptr[0 .. page_size * 2 + 1]; | ||
| 1183 | try std.testing.expect(slice[0] == 0x12); | ||
| 1184 | try std.testing.expect(slice[60] == 0x34); | ||
| 1185 | |||
| 1186 | try std.testing.expect(allocator.resize(slice, page_size * 2 + 1)); | ||
| 1187 | slice = slice[0 .. page_size * 2 + 1]; | ||
| 1188 | try std.testing.expect(slice[0] == 0x12); | ||
| 1189 | try std.testing.expect(slice[60] == 0x34); | ||
| 1190 | |||
| 1191 | slice = try allocator.realloc(slice, page_size * 2); | ||
| 1192 | try std.testing.expect(slice[0] == 0x12); | ||
| 1193 | try std.testing.expect(slice[60] == 0x34); | ||
| 1194 | } | ||
| 1195 | |||
| 1196 | test "shrink large object to large object with larger alignment" { | ||
| 1197 | if (!builtin.link_libc and builtin.os.tag == .wasi) return error.SkipZigTest; // https://github.com/ziglang/zig/issues/22731 | ||
| 1198 | |||
| 1199 | var gpa = DebugAllocator(test_config){}; | ||
| 1200 | defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); | ||
| 1201 | const allocator = gpa.allocator(); | ||
| 1202 | |||
| 1203 | var debug_buffer: [1000]u8 = undefined; | ||
| 1204 | var fba = std.heap.FixedBufferAllocator.init(&debug_buffer); | ||
| 1205 | const debug_allocator = fba.allocator(); | ||
| 1206 | |||
| 1207 | const alloc_size = page_size * 2 + 50; | ||
| 1208 | var slice = try allocator.alignedAlloc(u8, 16, alloc_size); | ||
| 1209 | defer allocator.free(slice); | ||
| 1210 | |||
| 1211 | const big_alignment: usize = switch (builtin.os.tag) { | ||
| 1212 | .windows => page_size * 32, // Windows aligns to 64K. | ||
| 1213 | else => page_size * 2, | ||
| 1214 | }; | ||
| 1215 | // This loop allocates until we find a page that is not aligned to the big | ||
| 1216 | // alignment. Then we shrink the allocation after the loop, but increase the | ||
| 1217 | // alignment to the higher one, that we know will force it to realloc. | ||
| 1218 | var stuff_to_free = std.ArrayList([]align(16) u8).init(debug_allocator); | ||
| 1219 | while (mem.isAligned(@intFromPtr(slice.ptr), big_alignment)) { | ||
| 1220 | try stuff_to_free.append(slice); | ||
| 1221 | slice = try allocator.alignedAlloc(u8, 16, alloc_size); | ||
| 1222 | } | ||
| 1223 | while (stuff_to_free.popOrNull()) |item| { | ||
| 1224 | allocator.free(item); | ||
| 1225 | } | ||
| 1226 | slice[0] = 0x12; | ||
| 1227 | slice[60] = 0x34; | ||
| 1228 | |||
| 1229 | slice = try allocator.reallocAdvanced(slice, big_alignment, alloc_size / 2); | ||
| 1230 | try std.testing.expect(slice[0] == 0x12); | ||
| 1231 | try std.testing.expect(slice[60] == 0x34); | ||
| 1232 | } | ||
| 1233 | |||
| 1234 | test "realloc large object to small object" { | ||
| 1235 | var gpa = DebugAllocator(test_config){}; | ||
| 1236 | defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); | ||
| 1237 | const allocator = gpa.allocator(); | ||
| 1238 | |||
| 1239 | var slice = try allocator.alloc(u8, page_size * 2 + 50); | ||
| 1240 | defer allocator.free(slice); | ||
| 1241 | slice[0] = 0x12; | ||
| 1242 | slice[16] = 0x34; | ||
| 1243 | |||
| 1244 | slice = try allocator.realloc(slice, 19); | ||
| 1245 | try std.testing.expect(slice[0] == 0x12); | ||
| 1246 | try std.testing.expect(slice[16] == 0x34); | ||
| 1247 | } | ||
| 1248 | |||
| 1249 | test "overridable mutexes" { | ||
| 1250 | var gpa = DebugAllocator(.{ .MutexType = std.Thread.Mutex }){ | ||
| 1251 | .backing_allocator = std.testing.allocator, | ||
| 1252 | .mutex = std.Thread.Mutex{}, | ||
| 1253 | }; | ||
| 1254 | defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); | ||
| 1255 | const allocator = gpa.allocator(); | ||
| 1256 | |||
| 1257 | const ptr = try allocator.create(i32); | ||
| 1258 | defer allocator.destroy(ptr); | ||
| 1259 | } | ||
| 1260 | |||
| 1261 | test "non-page-allocator backing allocator" { | ||
| 1262 | var gpa: DebugAllocator(.{ | ||
| 1263 | .backing_allocator_zeroes = false, | ||
| 1264 | }) = .{ | ||
| 1265 | .backing_allocator = std.testing.allocator, | ||
| 1266 | }; | ||
| 1267 | defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); | ||
| 1268 | const allocator = gpa.allocator(); | ||
| 1269 | |||
| 1270 | const ptr = try allocator.create(i32); | ||
| 1271 | defer allocator.destroy(ptr); | ||
| 1272 | } | ||
| 1273 | |||
| 1274 | test "realloc large object to larger alignment" { | ||
| 1275 | if (!builtin.link_libc and builtin.os.tag == .wasi) return error.SkipZigTest; // https://github.com/ziglang/zig/issues/22731 | ||
| 1276 | |||
| 1277 | var gpa = DebugAllocator(test_config){}; | ||
| 1278 | defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); | ||
| 1279 | const allocator = gpa.allocator(); | ||
| 1280 | |||
| 1281 | var debug_buffer: [1000]u8 = undefined; | ||
| 1282 | var fba = std.heap.FixedBufferAllocator.init(&debug_buffer); | ||
| 1283 | const debug_allocator = fba.allocator(); | ||
| 1284 | |||
| 1285 | var slice = try allocator.alignedAlloc(u8, 16, page_size * 2 + 50); | ||
| 1286 | defer allocator.free(slice); | ||
| 1287 | |||
| 1288 | const big_alignment: usize = switch (builtin.os.tag) { | ||
| 1289 | .windows => page_size * 32, // Windows aligns to 64K. | ||
| 1290 | else => page_size * 2, | ||
| 1291 | }; | ||
| 1292 | // This loop allocates until we find a page that is not aligned to the big alignment. | ||
| 1293 | var stuff_to_free = std.ArrayList([]align(16) u8).init(debug_allocator); | ||
| 1294 | while (mem.isAligned(@intFromPtr(slice.ptr), big_alignment)) { | ||
| 1295 | try stuff_to_free.append(slice); | ||
| 1296 | slice = try allocator.alignedAlloc(u8, 16, page_size * 2 + 50); | ||
| 1297 | } | ||
| 1298 | while (stuff_to_free.popOrNull()) |item| { | ||
| 1299 | allocator.free(item); | ||
| 1300 | } | ||
| 1301 | slice[0] = 0x12; | ||
| 1302 | slice[16] = 0x34; | ||
| 1303 | |||
| 1304 | slice = try allocator.reallocAdvanced(slice, 32, page_size * 2 + 100); | ||
| 1305 | try std.testing.expect(slice[0] == 0x12); | ||
| 1306 | try std.testing.expect(slice[16] == 0x34); | ||
| 1307 | |||
| 1308 | slice = try allocator.reallocAdvanced(slice, 32, page_size * 2 + 25); | ||
| 1309 | try std.testing.expect(slice[0] == 0x12); | ||
| 1310 | try std.testing.expect(slice[16] == 0x34); | ||
| 1311 | |||
| 1312 | slice = try allocator.reallocAdvanced(slice, big_alignment, page_size * 2 + 100); | ||
| 1313 | try std.testing.expect(slice[0] == 0x12); | ||
| 1314 | try std.testing.expect(slice[16] == 0x34); | ||
| 1315 | } | ||
| 1316 | |||
| 1317 | test "large object rejects shrinking to small" { | ||
| 1318 | if (builtin.target.isWasm()) { | ||
| 1319 | // Not expected to pass on targets that do not have memory mapping. | ||
| 1320 | return error.SkipZigTest; | ||
| 1321 | } | ||
| 1322 | |||
| 1323 | var failing_allocator = std.testing.FailingAllocator.init(std.heap.page_allocator, .{ .fail_index = 3 }); | ||
| 1324 | var gpa: DebugAllocator(.{}) = .{ | ||
| 1325 | .backing_allocator = failing_allocator.allocator(), | ||
| 1326 | }; | ||
| 1327 | defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); | ||
| 1328 | const allocator = gpa.allocator(); | ||
| 1329 | |||
| 1330 | var slice = try allocator.alloc(u8, page_size * 2 + 50); | ||
| 1331 | defer allocator.free(slice); | ||
| 1332 | slice[0] = 0x12; | ||
| 1333 | slice[3] = 0x34; | ||
| 1334 | |||
| 1335 | try std.testing.expect(!allocator.resize(slice, 4)); | ||
| 1336 | try std.testing.expect(slice[0] == 0x12); | ||
| 1337 | try std.testing.expect(slice[3] == 0x34); | ||
| 1338 | } | ||
| 1339 | |||
| 1340 | test "objects of size 1024 and 2048" { | ||
| 1341 | var gpa = DebugAllocator(test_config){}; | ||
| 1342 | defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); | ||
| 1343 | const allocator = gpa.allocator(); | ||
| 1344 | |||
| 1345 | const slice = try allocator.alloc(u8, 1025); | ||
| 1346 | const slice2 = try allocator.alloc(u8, 3000); | ||
| 1347 | |||
| 1348 | allocator.free(slice); | ||
| 1349 | allocator.free(slice2); | ||
| 1350 | } | ||
| 1351 | |||
| 1352 | test "setting a memory cap" { | ||
| 1353 | var gpa = DebugAllocator(.{ .enable_memory_limit = true }){}; | ||
| 1354 | defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); | ||
| 1355 | const allocator = gpa.allocator(); | ||
| 1356 | |||
| 1357 | gpa.requested_memory_limit = 1010; | ||
| 1358 | |||
| 1359 | const small = try allocator.create(i32); | ||
| 1360 | try std.testing.expect(gpa.total_requested_bytes == 4); | ||
| 1361 | |||
| 1362 | const big = try allocator.alloc(u8, 1000); | ||
| 1363 | try std.testing.expect(gpa.total_requested_bytes == 1004); | ||
| 1364 | |||
| 1365 | try std.testing.expectError(error.OutOfMemory, allocator.create(u64)); | ||
| 1366 | |||
| 1367 | allocator.destroy(small); | ||
| 1368 | try std.testing.expect(gpa.total_requested_bytes == 1000); | ||
| 1369 | |||
| 1370 | allocator.free(big); | ||
| 1371 | try std.testing.expect(gpa.total_requested_bytes == 0); | ||
| 1372 | |||
| 1373 | const exact = try allocator.alloc(u8, 1010); | ||
| 1374 | try std.testing.expect(gpa.total_requested_bytes == 1010); | ||
| 1375 | allocator.free(exact); | ||
| 1376 | } | ||
| 1377 | |||
| 1378 | test "large allocations count requested size not backing size" { | ||
| 1379 | var gpa: DebugAllocator(.{ .enable_memory_limit = true }) = .{}; | ||
| 1380 | const allocator = gpa.allocator(); | ||
| 1381 | |||
| 1382 | var buf = try allocator.alignedAlloc(u8, 1, page_size + 1); | ||
| 1383 | try std.testing.expectEqual(page_size + 1, gpa.total_requested_bytes); | ||
| 1384 | buf = try allocator.realloc(buf, 1); | ||
| 1385 | try std.testing.expectEqual(1, gpa.total_requested_bytes); | ||
| 1386 | buf = try allocator.realloc(buf, 2); | ||
| 1387 | try std.testing.expectEqual(2, gpa.total_requested_bytes); | ||
| 1388 | } | ||
| 1389 | |||
| 1390 | test "retain metadata and never unmap" { | ||
| 1391 | var gpa = std.heap.DebugAllocator(.{ | ||
| 1392 | .safety = true, | ||
| 1393 | .never_unmap = true, | ||
| 1394 | .retain_metadata = true, | ||
| 1395 | }){}; | ||
| 1396 | defer std.debug.assert(gpa.deinit() == .ok); | ||
| 1397 | const allocator = gpa.allocator(); | ||
| 1398 | |||
| 1399 | const alloc = try allocator.alloc(u8, 8); | ||
| 1400 | allocator.free(alloc); | ||
| 1401 | |||
| 1402 | const alloc2 = try allocator.alloc(u8, 8); | ||
| 1403 | allocator.free(alloc2); | ||
| 1404 | } | ||
lib/std/heap/general_purpose_allocator.zig deleted-1411
| ... | @@ -1,1411 +0,0 @@ | ||
| 1 | //! # General Purpose Allocator | ||
| 2 | //! | ||
| 3 | //! ## Design Priorities | ||
| 4 | //! | ||
| 5 | //! ### `OptimizationMode.debug` and `OptimizationMode.release_safe`: | ||
| 6 | //! | ||
| 7 | //! * Detect double free, and emit stack trace of: | ||
| 8 | //! - Where it was first allocated | ||
| 9 | //! - Where it was freed the first time | ||
| 10 | //! - Where it was freed the second time | ||
| 11 | //! | ||
| 12 | //! * Detect leaks and emit stack trace of: | ||
| 13 | //! - Where it was allocated | ||
| 14 | //! | ||
| 15 | //! * When a page of memory is no longer needed, give it back to resident memory | ||
| 16 | //! as soon as possible, so that it causes page faults when used. | ||
| 17 | //! | ||
| 18 | //! * Do not re-use memory slots, so that memory safety is upheld. For small | ||
| 19 | //! allocations, this is handled here; for larger ones it is handled in the | ||
| 20 | //! backing allocator (by default `std.heap.page_allocator`). | ||
| 21 | //! | ||
| 22 | //! * Make pointer math errors unlikely to harm memory from | ||
| 23 | //! unrelated allocations. | ||
| 24 | //! | ||
| 25 | //! * It's OK for these mechanisms to cost some extra overhead bytes. | ||
| 26 | //! | ||
| 27 | //! * It's OK for performance cost for these mechanisms. | ||
| 28 | //! | ||
| 29 | //! * Rogue memory writes should not harm the allocator's state. | ||
| 30 | //! | ||
| 31 | //! * Cross platform. Operates based on a backing allocator which makes it work | ||
| 32 | //! everywhere, even freestanding. | ||
| 33 | //! | ||
| 34 | //! * Compile-time configuration. | ||
| 35 | //! | ||
| 36 | //! ### `OptimizationMode.release_fast` (note: not much work has gone into this use case yet): | ||
| 37 | //! | ||
| 38 | //! * Low fragmentation is primary concern | ||
| 39 | //! * Performance of worst-case latency is secondary concern | ||
| 40 | //! * Performance of average-case latency is next | ||
| 41 | //! * Finally, having freed memory unmapped, and pointer math errors unlikely to | ||
| 42 | //! harm memory from unrelated allocations are nice-to-haves. | ||
| 43 | //! | ||
| 44 | //! ### `OptimizationMode.release_small` (note: not much work has gone into this use case yet): | ||
| 45 | //! | ||
| 46 | //! * Small binary code size of the executable is the primary concern. | ||
| 47 | //! * Next, defer to the `.release_fast` priority list. | ||
| 48 | //! | ||
| 49 | //! ## Basic Design: | ||
| 50 | //! | ||
| 51 | //! Small allocations are divided into buckets: | ||
| 52 | //! | ||
| 53 | //! ``` | ||
| 54 | //! index obj_size | ||
| 55 | //! 0 1 | ||
| 56 | //! 1 2 | ||
| 57 | //! 2 4 | ||
| 58 | //! 3 8 | ||
| 59 | //! 4 16 | ||
| 60 | //! 5 32 | ||
| 61 | //! 6 64 | ||
| 62 | //! 7 128 | ||
| 63 | //! 8 256 | ||
| 64 | //! 9 512 | ||
| 65 | //! 10 1024 | ||
| 66 | //! 11 2048 | ||
| 67 | //! ``` | ||
| 68 | //! | ||
| 69 | //! The main allocator state has an array of all the "current" buckets for each | ||
| 70 | //! size class. Each slot in the array can be null, meaning the bucket for that | ||
| 71 | //! size class is not allocated. When the first object is allocated for a given | ||
| 72 | //! size class, it allocates 1 page of memory from the OS. This page is | ||
| 73 | //! divided into "slots" - one per allocated object. Along with the page of memory | ||
| 74 | //! for object slots, as many pages as necessary are allocated to store the | ||
| 75 | //! BucketHeader, followed by "used bits", and two stack traces for each slot | ||
| 76 | //! (allocation trace and free trace). | ||
| 77 | //! | ||
| 78 | //! The "used bits" are 1 bit per slot representing whether the slot is used. | ||
| 79 | //! Allocations use the data to iterate to find a free slot. Frees assert that the | ||
| 80 | //! corresponding bit is 1 and set it to 0. | ||
| 81 | //! | ||
| 82 | //! Buckets have prev and next pointers. When there is only one bucket for a given | ||
| 83 | //! size class, both prev and next point to itself. When all slots of a bucket are | ||
| 84 | //! used, a new bucket is allocated, and enters the doubly linked list. The main | ||
| 85 | //! allocator state tracks the "current" bucket for each size class. Leak detection | ||
| 86 | //! currently only checks the current bucket. | ||
| 87 | //! | ||
| 88 | //! Resizing detects if the size class is unchanged or smaller, in which case the same | ||
| 89 | //! pointer is returned unmodified. If a larger size class is required, | ||
| 90 | //! `error.OutOfMemory` is returned. | ||
| 91 | //! | ||
| 92 | //! Large objects are allocated directly using the backing allocator and their metadata is stored | ||
| 93 | //! in a `std.HashMap` using the backing allocator. | ||
| 94 | |||
| 95 | const std = @import("std"); | ||
| 96 | const builtin = @import("builtin"); | ||
| 97 | const log = std.log.scoped(.gpa); | ||
| 98 | const math = std.math; | ||
| 99 | const assert = std.debug.assert; | ||
| 100 | const mem = std.mem; | ||
| 101 | const Allocator = std.mem.Allocator; | ||
| 102 | const StackTrace = std.builtin.StackTrace; | ||
| 103 | |||
| 104 | const page_size: usize = @max(std.heap.page_size_max, switch (builtin.os.tag) { | ||
| 105 | .windows => 64 * 1024, // Makes `std.heap.PageAllocator` take the happy path. | ||
| 106 | .wasi => 64 * 1024, // Max alignment supported by `std.heap.WasmAllocator`. | ||
| 107 | else => 128 * 1024, // Avoids too many active mappings when `page_size_max` is low. | ||
| 108 | }); | ||
| 109 | const page_align: mem.Alignment = .fromByteUnits(page_size); | ||
| 110 | |||
| 111 | /// Integer type for pointing to slots in a small allocation | ||
| 112 | const SlotIndex = std.meta.Int(.unsigned, math.log2(page_size) + 1); | ||
| 113 | const Log2USize = std.math.Log2Int(usize); | ||
| 114 | |||
| 115 | const default_sys_stack_trace_frames: usize = if (std.debug.sys_can_stack_trace) 6 else 0; | ||
| 116 | const default_stack_trace_frames: usize = switch (builtin.mode) { | ||
| 117 | .Debug => default_sys_stack_trace_frames, | ||
| 118 | else => 0, | ||
| 119 | }; | ||
| 120 | |||
| 121 | pub const Config = struct { | ||
| 122 | /// Number of stack frames to capture. | ||
| 123 | stack_trace_frames: usize = default_stack_trace_frames, | ||
| 124 | |||
| 125 | /// If true, the allocator will have two fields: | ||
| 126 | /// * `total_requested_bytes` which tracks the total allocated bytes of memory requested. | ||
| 127 | /// * `requested_memory_limit` which causes allocations to return `error.OutOfMemory` | ||
| 128 | /// when the `total_requested_bytes` exceeds this limit. | ||
| 129 | /// If false, these fields will be `void`. | ||
| 130 | enable_memory_limit: bool = false, | ||
| 131 | |||
| 132 | /// Whether to enable safety checks. | ||
| 133 | safety: bool = std.debug.runtime_safety, | ||
| 134 | |||
| 135 | /// Whether the allocator may be used simultaneously from multiple threads. | ||
| 136 | thread_safe: bool = !builtin.single_threaded, | ||
| 137 | |||
| 138 | /// What type of mutex you'd like to use, for thread safety. | ||
| 139 | /// when specified, the mutex type must have the same shape as `std.Thread.Mutex` and | ||
| 140 | /// `DummyMutex`, and have no required fields. Specifying this field causes | ||
| 141 | /// the `thread_safe` field to be ignored. | ||
| 142 | /// | ||
| 143 | /// when null (default): | ||
| 144 | /// * the mutex type defaults to `std.Thread.Mutex` when thread_safe is enabled. | ||
| 145 | /// * the mutex type defaults to `DummyMutex` otherwise. | ||
| 146 | MutexType: ?type = null, | ||
| 147 | |||
| 148 | /// This is a temporary debugging trick you can use to turn segfaults into more helpful | ||
| 149 | /// logged error messages with stack trace details. The downside is that every allocation | ||
| 150 | /// will be leaked, unless used with retain_metadata! | ||
| 151 | never_unmap: bool = false, | ||
| 152 | |||
| 153 | /// This is a temporary debugging aid that retains metadata about allocations indefinitely. | ||
| 154 | /// This allows a greater range of double frees to be reported. All metadata is freed when | ||
| 155 | /// deinit is called. When used with never_unmap, deliberately leaked memory is also freed | ||
| 156 | /// during deinit. Currently should be used with never_unmap to avoid segfaults. | ||
| 157 | /// TODO https://github.com/ziglang/zig/issues/4298 will allow use without never_unmap | ||
| 158 | retain_metadata: bool = false, | ||
| 159 | |||
| 160 | /// Enables emitting info messages with the size and address of every allocation. | ||
| 161 | verbose_log: bool = false, | ||
| 162 | |||
| 163 | /// Tell whether the backing allocator returns already-zeroed memory. | ||
| 164 | backing_allocator_zeroes: bool = true, | ||
| 165 | |||
| 166 | /// When resizing an allocation, refresh the stack trace with the resize | ||
| 167 | /// callsite. Comes with a performance penalty. | ||
| 168 | resize_stack_traces: bool = false, | ||
| 169 | |||
| 170 | /// Magic value that distinguishes allocations owned by this allocator from | ||
| 171 | /// other regions of memory. | ||
| 172 | canary: usize = @truncate(0x9232a6ff85dff10f), | ||
| 173 | }; | ||
| 174 | |||
| 175 | pub const Check = enum { ok, leak }; | ||
| 176 | |||
| 177 | /// Default initialization of this struct is deprecated; use `.init` instead. | ||
| 178 | pub fn GeneralPurposeAllocator(comptime config: Config) type { | ||
| 179 | return struct { | ||
| 180 | backing_allocator: Allocator = std.heap.page_allocator, | ||
| 181 | /// Tracks the active bucket, which is the one that has free slots in it. | ||
| 182 | buckets: [small_bucket_count]?*BucketHeader = [1]?*BucketHeader{null} ** small_bucket_count, | ||
| 183 | large_allocations: LargeAllocTable = .empty, | ||
| 184 | total_requested_bytes: @TypeOf(total_requested_bytes_init) = total_requested_bytes_init, | ||
| 185 | requested_memory_limit: @TypeOf(requested_memory_limit_init) = requested_memory_limit_init, | ||
| 186 | mutex: @TypeOf(mutex_init) = mutex_init, | ||
| 187 | |||
| 188 | const Self = @This(); | ||
| 189 | |||
| 190 | pub const init: Self = .{}; | ||
| 191 | |||
| 192 | /// These can be derived from size_class_index but the calculation is nontrivial. | ||
| 193 | const slot_counts: [small_bucket_count]SlotIndex = init: { | ||
| 194 | @setEvalBranchQuota(10000); | ||
| 195 | var result: [small_bucket_count]SlotIndex = undefined; | ||
| 196 | for (&result, 0..) |*elem, i| elem.* = calculateSlotCount(i); | ||
| 197 | break :init result; | ||
| 198 | }; | ||
| 199 | |||
| 200 | const total_requested_bytes_init = if (config.enable_memory_limit) @as(usize, 0) else {}; | ||
| 201 | const requested_memory_limit_init = if (config.enable_memory_limit) @as(usize, math.maxInt(usize)) else {}; | ||
| 202 | |||
| 203 | const mutex_init = if (config.MutexType) |T| | ||
| 204 | T{} | ||
| 205 | else if (config.thread_safe) | ||
| 206 | std.Thread.Mutex{} | ||
| 207 | else | ||
| 208 | DummyMutex{}; | ||
| 209 | |||
| 210 | const DummyMutex = struct { | ||
| 211 | inline fn lock(_: *DummyMutex) void {} | ||
| 212 | inline fn unlock(_: *DummyMutex) void {} | ||
| 213 | }; | ||
| 214 | |||
| 215 | const stack_n = config.stack_trace_frames; | ||
| 216 | const one_trace_size = @sizeOf(usize) * stack_n; | ||
| 217 | const traces_per_slot = 2; | ||
| 218 | |||
| 219 | pub const Error = mem.Allocator.Error; | ||
| 220 | |||
| 221 | /// Avoids creating buckets that would only be able to store a small | ||
| 222 | /// number of slots. Value of 1 means 2 is the minimum slot count. | ||
| 223 | const minimum_slots_per_bucket_log2 = 1; | ||
| 224 | const small_bucket_count = math.log2(page_size) - minimum_slots_per_bucket_log2; | ||
| 225 | const largest_bucket_object_size = 1 << (small_bucket_count - 1); | ||
| 226 | const LargestSizeClassInt = std.math.IntFittingRange(0, largest_bucket_object_size); | ||
| 227 | |||
| 228 | const bucketCompare = struct { | ||
| 229 | fn compare(a: *BucketHeader, b: *BucketHeader) std.math.Order { | ||
| 230 | return std.math.order(@intFromPtr(a.page), @intFromPtr(b.page)); | ||
| 231 | } | ||
| 232 | }.compare; | ||
| 233 | |||
| 234 | const LargeAlloc = struct { | ||
| 235 | bytes: []u8, | ||
| 236 | requested_size: if (config.enable_memory_limit) usize else void, | ||
| 237 | stack_addresses: [trace_n][stack_n]usize, | ||
| 238 | freed: if (config.retain_metadata) bool else void, | ||
| 239 | alignment: if (config.never_unmap and config.retain_metadata) mem.Alignment else void, | ||
| 240 | |||
| 241 | const trace_n = if (config.retain_metadata) traces_per_slot else 1; | ||
| 242 | |||
| 243 | fn dumpStackTrace(self: *LargeAlloc, trace_kind: TraceKind) void { | ||
| 244 | std.debug.dumpStackTrace(self.getStackTrace(trace_kind)); | ||
| 245 | } | ||
| 246 | |||
| 247 | fn getStackTrace(self: *LargeAlloc, trace_kind: TraceKind) std.builtin.StackTrace { | ||
| 248 | assert(@intFromEnum(trace_kind) < trace_n); | ||
| 249 | const stack_addresses = &self.stack_addresses[@intFromEnum(trace_kind)]; | ||
| 250 | var len: usize = 0; | ||
| 251 | while (len < stack_n and stack_addresses[len] != 0) { | ||
| 252 | len += 1; | ||
| 253 | } | ||
| 254 | return .{ | ||
| 255 | .instruction_addresses = stack_addresses, | ||
| 256 | .index = len, | ||
| 257 | }; | ||
| 258 | } | ||
| 259 | |||
| 260 | fn captureStackTrace(self: *LargeAlloc, ret_addr: usize, trace_kind: TraceKind) void { | ||
| 261 | assert(@intFromEnum(trace_kind) < trace_n); | ||
| 262 | const stack_addresses = &self.stack_addresses[@intFromEnum(trace_kind)]; | ||
| 263 | collectStackTrace(ret_addr, stack_addresses); | ||
| 264 | } | ||
| 265 | }; | ||
| 266 | const LargeAllocTable = std.AutoHashMapUnmanaged(usize, LargeAlloc); | ||
| 267 | |||
| 268 | /// Bucket: In memory, in order: | ||
| 269 | /// * BucketHeader | ||
| 270 | /// * bucket_used_bits: [N]usize, // 1 bit for every slot | ||
| 271 | /// -- below only exists when config.safety is true -- | ||
| 272 | /// * requested_sizes: [N]LargestSizeClassInt // 1 int for every slot | ||
| 273 | /// * log2_ptr_aligns: [N]u8 // 1 byte for every slot | ||
| 274 | /// -- above only exists when config.safety is true -- | ||
| 275 | /// * stack_trace_addresses: [N]usize, // traces_per_slot for every allocation | ||
| 276 | const BucketHeader = struct { | ||
| 277 | allocated_count: SlotIndex, | ||
| 278 | freed_count: SlotIndex, | ||
| 279 | prev: ?*BucketHeader, | ||
| 280 | canary: usize = config.canary, | ||
| 281 | |||
| 282 | fn fromPage(page_addr: usize, slot_count: usize) *BucketHeader { | ||
| 283 | const unaligned = page_addr + page_size - bucketSize(slot_count); | ||
| 284 | return @ptrFromInt(unaligned & ~(@as(usize, @alignOf(BucketHeader)) - 1)); | ||
| 285 | } | ||
| 286 | |||
| 287 | fn usedBits(bucket: *BucketHeader, index: usize) *usize { | ||
| 288 | const ptr: [*]u8 = @ptrCast(bucket); | ||
| 289 | const bits: [*]usize = @alignCast(@ptrCast(ptr + @sizeOf(BucketHeader))); | ||
| 290 | return &bits[index]; | ||
| 291 | } | ||
| 292 | |||
| 293 | fn requestedSizes(bucket: *BucketHeader, slot_count: usize) []LargestSizeClassInt { | ||
| 294 | if (!config.safety) @compileError("requested size is only stored when safety is enabled"); | ||
| 295 | const start_ptr = @as([*]u8, @ptrCast(bucket)) + bucketRequestedSizesStart(slot_count); | ||
| 296 | const sizes = @as([*]LargestSizeClassInt, @ptrCast(@alignCast(start_ptr))); | ||
| 297 | return sizes[0..slot_count]; | ||
| 298 | } | ||
| 299 | |||
| 300 | fn log2PtrAligns(bucket: *BucketHeader, slot_count: usize) []mem.Alignment { | ||
| 301 | if (!config.safety) @compileError("requested size is only stored when safety is enabled"); | ||
| 302 | const aligns_ptr = @as([*]u8, @ptrCast(bucket)) + bucketAlignsStart(slot_count); | ||
| 303 | return @ptrCast(aligns_ptr[0..slot_count]); | ||
| 304 | } | ||
| 305 | |||
| 306 | fn stackTracePtr( | ||
| 307 | bucket: *BucketHeader, | ||
| 308 | slot_count: usize, | ||
| 309 | slot_index: SlotIndex, | ||
| 310 | trace_kind: TraceKind, | ||
| 311 | ) *[stack_n]usize { | ||
| 312 | const start_ptr = @as([*]u8, @ptrCast(bucket)) + bucketStackFramesStart(slot_count); | ||
| 313 | const addr = start_ptr + one_trace_size * traces_per_slot * slot_index + | ||
| 314 | @intFromEnum(trace_kind) * @as(usize, one_trace_size); | ||
| 315 | return @ptrCast(@alignCast(addr)); | ||
| 316 | } | ||
| 317 | |||
| 318 | fn captureStackTrace( | ||
| 319 | bucket: *BucketHeader, | ||
| 320 | ret_addr: usize, | ||
| 321 | slot_count: usize, | ||
| 322 | slot_index: SlotIndex, | ||
| 323 | trace_kind: TraceKind, | ||
| 324 | ) void { | ||
| 325 | // Initialize them to 0. When determining the count we must look | ||
| 326 | // for non zero addresses. | ||
| 327 | const stack_addresses = bucket.stackTracePtr(slot_count, slot_index, trace_kind); | ||
| 328 | collectStackTrace(ret_addr, stack_addresses); | ||
| 329 | } | ||
| 330 | }; | ||
| 331 | |||
| 332 | pub fn allocator(self: *Self) Allocator { | ||
| 333 | return .{ | ||
| 334 | .ptr = self, | ||
| 335 | .vtable = &.{ | ||
| 336 | .alloc = alloc, | ||
| 337 | .resize = resize, | ||
| 338 | .remap = remap, | ||
| 339 | .free = free, | ||
| 340 | }, | ||
| 341 | }; | ||
| 342 | } | ||
| 343 | |||
| 344 | fn bucketStackTrace( | ||
| 345 | bucket: *BucketHeader, | ||
| 346 | slot_count: usize, | ||
| 347 | slot_index: SlotIndex, | ||
| 348 | trace_kind: TraceKind, | ||
| 349 | ) StackTrace { | ||
| 350 | const stack_addresses = bucket.stackTracePtr(slot_count, slot_index, trace_kind); | ||
| 351 | var len: usize = 0; | ||
| 352 | while (len < stack_n and stack_addresses[len] != 0) { | ||
| 353 | len += 1; | ||
| 354 | } | ||
| 355 | return .{ | ||
| 356 | .instruction_addresses = stack_addresses, | ||
| 357 | .index = len, | ||
| 358 | }; | ||
| 359 | } | ||
| 360 | |||
| 361 | fn bucketRequestedSizesStart(slot_count: usize) usize { | ||
| 362 | if (!config.safety) @compileError("requested sizes are not stored unless safety is enabled"); | ||
| 363 | return mem.alignForward( | ||
| 364 | usize, | ||
| 365 | @sizeOf(BucketHeader) + usedBitsSize(slot_count), | ||
| 366 | @alignOf(LargestSizeClassInt), | ||
| 367 | ); | ||
| 368 | } | ||
| 369 | |||
| 370 | fn bucketAlignsStart(slot_count: usize) usize { | ||
| 371 | if (!config.safety) @compileError("requested sizes are not stored unless safety is enabled"); | ||
| 372 | return bucketRequestedSizesStart(slot_count) + (@sizeOf(LargestSizeClassInt) * slot_count); | ||
| 373 | } | ||
| 374 | |||
| 375 | fn bucketStackFramesStart(slot_count: usize) usize { | ||
| 376 | const unaligned_start = if (config.safety) | ||
| 377 | bucketAlignsStart(slot_count) + slot_count | ||
| 378 | else | ||
| 379 | @sizeOf(BucketHeader) + usedBitsSize(slot_count); | ||
| 380 | return mem.alignForward(usize, unaligned_start, @alignOf(usize)); | ||
| 381 | } | ||
| 382 | |||
| 383 | fn bucketSize(slot_count: usize) usize { | ||
| 384 | return bucketStackFramesStart(slot_count) + one_trace_size * traces_per_slot * slot_count; | ||
| 385 | } | ||
| 386 | |||
| 387 | /// This is executed only at compile-time to prepopulate a lookup table. | ||
| 388 | fn calculateSlotCount(size_class_index: usize) SlotIndex { | ||
| 389 | const size_class = @as(usize, 1) << @as(Log2USize, @intCast(size_class_index)); | ||
| 390 | var lower: usize = 1 << minimum_slots_per_bucket_log2; | ||
| 391 | var upper: usize = (page_size - bucketSize(lower)) / size_class; | ||
| 392 | while (upper > lower) { | ||
| 393 | const proposed: usize = lower + (upper - lower) / 2; | ||
| 394 | if (proposed == lower) return lower; | ||
| 395 | const slots_end = proposed * size_class; | ||
| 396 | const header_begin = mem.alignForward(usize, slots_end, @alignOf(BucketHeader)); | ||
| 397 | const end = header_begin + bucketSize(proposed); | ||
| 398 | if (end > page_size) { | ||
| 399 | upper = proposed - 1; | ||
| 400 | } else { | ||
| 401 | lower = proposed; | ||
| 402 | } | ||
| 403 | } | ||
| 404 | const slots_end = lower * size_class; | ||
| 405 | const header_begin = mem.alignForward(usize, slots_end, @alignOf(BucketHeader)); | ||
| 406 | const end = header_begin + bucketSize(lower); | ||
| 407 | assert(end <= page_size); | ||
| 408 | return lower; | ||
| 409 | } | ||
| 410 | |||
| 411 | fn usedBitsCount(slot_count: usize) usize { | ||
| 412 | return (slot_count + (@bitSizeOf(usize) - 1)) / @bitSizeOf(usize); | ||
| 413 | } | ||
| 414 | |||
| 415 | fn usedBitsSize(slot_count: usize) usize { | ||
| 416 | return usedBitsCount(slot_count) * @sizeOf(usize); | ||
| 417 | } | ||
| 418 | |||
| 419 | fn detectLeaksInBucket(bucket: *BucketHeader, size_class_index: usize, used_bits_count: usize) bool { | ||
| 420 | const size_class = @as(usize, 1) << @as(Log2USize, @intCast(size_class_index)); | ||
| 421 | const slot_count = slot_counts[size_class_index]; | ||
| 422 | var leaks = false; | ||
| 423 | for (0..used_bits_count) |used_bits_byte| { | ||
| 424 | const used_int = bucket.usedBits(used_bits_byte).*; | ||
| 425 | if (used_int != 0) { | ||
| 426 | for (0..@bitSizeOf(usize)) |bit_index_usize| { | ||
| 427 | const bit_index: Log2USize = @intCast(bit_index_usize); | ||
| 428 | const is_used = @as(u1, @truncate(used_int >> bit_index)) != 0; | ||
| 429 | if (is_used) { | ||
| 430 | const slot_index: SlotIndex = @intCast(used_bits_byte * @bitSizeOf(usize) + bit_index); | ||
| 431 | const stack_trace = bucketStackTrace(bucket, slot_count, slot_index, .alloc); | ||
| 432 | const page_addr = @intFromPtr(bucket) & ~(page_size - 1); | ||
| 433 | const addr = page_addr + slot_index * size_class; | ||
| 434 | log.err("memory address 0x{x} leaked: {}", .{ addr, stack_trace }); | ||
| 435 | leaks = true; | ||
| 436 | } | ||
| 437 | } | ||
| 438 | } | ||
| 439 | } | ||
| 440 | return leaks; | ||
| 441 | } | ||
| 442 | |||
| 443 | /// Emits log messages for leaks and then returns whether there were any leaks. | ||
| 444 | pub fn detectLeaks(self: *Self) bool { | ||
| 445 | var leaks = false; | ||
| 446 | |||
| 447 | for (self.buckets, 0..) |init_optional_bucket, size_class_index| { | ||
| 448 | var optional_bucket = init_optional_bucket; | ||
| 449 | const slot_count = slot_counts[size_class_index]; | ||
| 450 | const used_bits_count = usedBitsCount(slot_count); | ||
| 451 | while (optional_bucket) |bucket| { | ||
| 452 | leaks = detectLeaksInBucket(bucket, size_class_index, used_bits_count) or leaks; | ||
| 453 | optional_bucket = bucket.prev; | ||
| 454 | } | ||
| 455 | } | ||
| 456 | |||
| 457 | var it = self.large_allocations.valueIterator(); | ||
| 458 | while (it.next()) |large_alloc| { | ||
| 459 | if (config.retain_metadata and large_alloc.freed) continue; | ||
| 460 | const stack_trace = large_alloc.getStackTrace(.alloc); | ||
| 461 | log.err("memory address 0x{x} leaked: {}", .{ | ||
| 462 | @intFromPtr(large_alloc.bytes.ptr), stack_trace, | ||
| 463 | }); | ||
| 464 | leaks = true; | ||
| 465 | } | ||
| 466 | return leaks; | ||
| 467 | } | ||
| 468 | |||
| 469 | fn freeRetainedMetadata(self: *Self) void { | ||
| 470 | comptime assert(config.retain_metadata); | ||
| 471 | if (config.never_unmap) { | ||
| 472 | // free large allocations that were intentionally leaked by never_unmap | ||
| 473 | var it = self.large_allocations.iterator(); | ||
| 474 | while (it.next()) |large| { | ||
| 475 | if (large.value_ptr.freed) { | ||
| 476 | self.backing_allocator.rawFree(large.value_ptr.bytes, large.value_ptr.alignment, @returnAddress()); | ||
| 477 | } | ||
| 478 | } | ||
| 479 | } | ||
| 480 | } | ||
| 481 | |||
| 482 | pub fn flushRetainedMetadata(self: *Self) void { | ||
| 483 | comptime assert(config.retain_metadata); | ||
| 484 | self.freeRetainedMetadata(); | ||
| 485 | // also remove entries from large_allocations | ||
| 486 | var it = self.large_allocations.iterator(); | ||
| 487 | while (it.next()) |large| { | ||
| 488 | if (large.value_ptr.freed) { | ||
| 489 | _ = self.large_allocations.remove(@intFromPtr(large.value_ptr.bytes.ptr)); | ||
| 490 | } | ||
| 491 | } | ||
| 492 | } | ||
| 493 | |||
| 494 | /// Returns `Check.leak` if there were leaks; `Check.ok` otherwise. | ||
| 495 | pub fn deinit(self: *Self) Check { | ||
| 496 | const leaks = if (config.safety) self.detectLeaks() else false; | ||
| 497 | if (config.retain_metadata) self.freeRetainedMetadata(); | ||
| 498 | self.large_allocations.deinit(self.backing_allocator); | ||
| 499 | self.* = undefined; | ||
| 500 | return if (leaks) .leak else .ok; | ||
| 501 | } | ||
| 502 | |||
| 503 | fn collectStackTrace(first_trace_addr: usize, addresses: *[stack_n]usize) void { | ||
| 504 | if (stack_n == 0) return; | ||
| 505 | @memset(addresses, 0); | ||
| 506 | var stack_trace: StackTrace = .{ | ||
| 507 | .instruction_addresses = addresses, | ||
| 508 | .index = 0, | ||
| 509 | }; | ||
| 510 | std.debug.captureStackTrace(first_trace_addr, &stack_trace); | ||
| 511 | } | ||
| 512 | |||
| 513 | fn reportDoubleFree(ret_addr: usize, alloc_stack_trace: StackTrace, free_stack_trace: StackTrace) void { | ||
| 514 | var addresses: [stack_n]usize = @splat(0); | ||
| 515 | var second_free_stack_trace: StackTrace = .{ | ||
| 516 | .instruction_addresses = &addresses, | ||
| 517 | .index = 0, | ||
| 518 | }; | ||
| 519 | std.debug.captureStackTrace(ret_addr, &second_free_stack_trace); | ||
| 520 | log.err("Double free detected. Allocation: {} First free: {} Second free: {}", .{ | ||
| 521 | alloc_stack_trace, free_stack_trace, second_free_stack_trace, | ||
| 522 | }); | ||
| 523 | } | ||
| 524 | |||
| 525 | /// This function assumes the object is in the large object storage regardless | ||
| 526 | /// of the parameters. | ||
| 527 | fn resizeLarge( | ||
| 528 | self: *Self, | ||
| 529 | old_mem: []u8, | ||
| 530 | alignment: mem.Alignment, | ||
| 531 | new_size: usize, | ||
| 532 | ret_addr: usize, | ||
| 533 | may_move: bool, | ||
| 534 | ) ?[*]u8 { | ||
| 535 | if (config.retain_metadata and may_move) { | ||
| 536 | // Before looking up the entry (since this could invalidate | ||
| 537 | // it), we must reserve space for the new entry in case the | ||
| 538 | // allocation is relocated. | ||
| 539 | self.large_allocations.ensureUnusedCapacity(self.backing_allocator, 1) catch return null; | ||
| 540 | } | ||
| 541 | |||
| 542 | const entry = self.large_allocations.getEntry(@intFromPtr(old_mem.ptr)) orelse { | ||
| 543 | if (config.safety) { | ||
| 544 | @panic("Invalid free"); | ||
| 545 | } else { | ||
| 546 | unreachable; | ||
| 547 | } | ||
| 548 | }; | ||
| 549 | |||
| 550 | if (config.retain_metadata and entry.value_ptr.freed) { | ||
| 551 | if (config.safety) { | ||
| 552 | reportDoubleFree(ret_addr, entry.value_ptr.getStackTrace(.alloc), entry.value_ptr.getStackTrace(.free)); | ||
| 553 | @panic("Unrecoverable double free"); | ||
| 554 | } else { | ||
| 555 | unreachable; | ||
| 556 | } | ||
| 557 | } | ||
| 558 | |||
| 559 | if (config.safety and old_mem.len != entry.value_ptr.bytes.len) { | ||
| 560 | var addresses: [stack_n]usize = [1]usize{0} ** stack_n; | ||
| 561 | var free_stack_trace: StackTrace = .{ | ||
| 562 | .instruction_addresses = &addresses, | ||
| 563 | .index = 0, | ||
| 564 | }; | ||
| 565 | std.debug.captureStackTrace(ret_addr, &free_stack_trace); | ||
| 566 | log.err("Allocation size {d} bytes does not match free size {d}. Allocation: {} Free: {}", .{ | ||
| 567 | entry.value_ptr.bytes.len, | ||
| 568 | old_mem.len, | ||
| 569 | entry.value_ptr.getStackTrace(.alloc), | ||
| 570 | free_stack_trace, | ||
| 571 | }); | ||
| 572 | } | ||
| 573 | |||
| 574 | // If this would move the allocation into a small size class, | ||
| 575 | // refuse the request, because it would require creating small | ||
| 576 | // allocation metadata. | ||
| 577 | const new_size_class_index: usize = @max(@bitSizeOf(usize) - @clz(new_size - 1), @intFromEnum(alignment)); | ||
| 578 | if (new_size_class_index < self.buckets.len) return null; | ||
| 579 | |||
| 580 | // Do memory limit accounting with requested sizes rather than what | ||
| 581 | // backing_allocator returns because if we want to return | ||
| 582 | // error.OutOfMemory, we have to leave allocation untouched, and | ||
| 583 | // that is impossible to guarantee after calling | ||
| 584 | // backing_allocator.rawResize. | ||
| 585 | const prev_req_bytes = self.total_requested_bytes; | ||
| 586 | if (config.enable_memory_limit) { | ||
| 587 | const new_req_bytes = prev_req_bytes + new_size - entry.value_ptr.requested_size; | ||
| 588 | if (new_req_bytes > prev_req_bytes and new_req_bytes > self.requested_memory_limit) { | ||
| 589 | return null; | ||
| 590 | } | ||
| 591 | self.total_requested_bytes = new_req_bytes; | ||
| 592 | } | ||
| 593 | |||
| 594 | const opt_resized_ptr = if (may_move) | ||
| 595 | self.backing_allocator.rawRemap(old_mem, alignment, new_size, ret_addr) | ||
| 596 | else if (self.backing_allocator.rawResize(old_mem, alignment, new_size, ret_addr)) | ||
| 597 | old_mem.ptr | ||
| 598 | else | ||
| 599 | null; | ||
| 600 | |||
| 601 | const resized_ptr = opt_resized_ptr orelse { | ||
| 602 | if (config.enable_memory_limit) { | ||
| 603 | self.total_requested_bytes = prev_req_bytes; | ||
| 604 | } | ||
| 605 | return null; | ||
| 606 | }; | ||
| 607 | |||
| 608 | if (config.enable_memory_limit) { | ||
| 609 | entry.value_ptr.requested_size = new_size; | ||
| 610 | } | ||
| 611 | |||
| 612 | if (config.verbose_log) { | ||
| 613 | log.info("large resize {d} bytes at {*} to {d} at {*}", .{ | ||
| 614 | old_mem.len, old_mem.ptr, new_size, resized_ptr, | ||
| 615 | }); | ||
| 616 | } | ||
| 617 | entry.value_ptr.bytes = resized_ptr[0..new_size]; | ||
| 618 | if (config.resize_stack_traces) | ||
| 619 | entry.value_ptr.captureStackTrace(ret_addr, .alloc); | ||
| 620 | |||
| 621 | // Update the key of the hash map if the memory was relocated. | ||
| 622 | if (resized_ptr != old_mem.ptr) { | ||
| 623 | const large_alloc = entry.value_ptr.*; | ||
| 624 | if (config.retain_metadata) { | ||
| 625 | entry.value_ptr.freed = true; | ||
| 626 | entry.value_ptr.captureStackTrace(ret_addr, .free); | ||
| 627 | } else { | ||
| 628 | self.large_allocations.removeByPtr(entry.key_ptr); | ||
| 629 | } | ||
| 630 | |||
| 631 | const gop = self.large_allocations.getOrPutAssumeCapacity(@intFromPtr(resized_ptr)); | ||
| 632 | if (config.retain_metadata and !config.never_unmap) { | ||
| 633 | // Backing allocator may be reusing memory that we're retaining metadata for | ||
| 634 | assert(!gop.found_existing or gop.value_ptr.freed); | ||
| 635 | } else { | ||
| 636 | assert(!gop.found_existing); // This would mean the kernel double-mapped pages. | ||
| 637 | } | ||
| 638 | gop.value_ptr.* = large_alloc; | ||
| 639 | } | ||
| 640 | |||
| 641 | return resized_ptr; | ||
| 642 | } | ||
| 643 | |||
| 644 | /// This function assumes the object is in the large object storage regardless | ||
| 645 | /// of the parameters. | ||
| 646 | fn freeLarge( | ||
| 647 | self: *Self, | ||
| 648 | old_mem: []u8, | ||
| 649 | alignment: mem.Alignment, | ||
| 650 | ret_addr: usize, | ||
| 651 | ) void { | ||
| 652 | const entry = self.large_allocations.getEntry(@intFromPtr(old_mem.ptr)) orelse { | ||
| 653 | if (config.safety) { | ||
| 654 | @panic("Invalid free"); | ||
| 655 | } else { | ||
| 656 | unreachable; | ||
| 657 | } | ||
| 658 | }; | ||
| 659 | |||
| 660 | if (config.retain_metadata and entry.value_ptr.freed) { | ||
| 661 | if (config.safety) { | ||
| 662 | reportDoubleFree(ret_addr, entry.value_ptr.getStackTrace(.alloc), entry.value_ptr.getStackTrace(.free)); | ||
| 663 | return; | ||
| 664 | } else { | ||
| 665 | unreachable; | ||
| 666 | } | ||
| 667 | } | ||
| 668 | |||
| 669 | if (config.safety and old_mem.len != entry.value_ptr.bytes.len) { | ||
| 670 | var addresses: [stack_n]usize = [1]usize{0} ** stack_n; | ||
| 671 | var free_stack_trace = StackTrace{ | ||
| 672 | .instruction_addresses = &addresses, | ||
| 673 | .index = 0, | ||
| 674 | }; | ||
| 675 | std.debug.captureStackTrace(ret_addr, &free_stack_trace); | ||
| 676 | log.err("Allocation size {d} bytes does not match free size {d}. Allocation: {} Free: {}", .{ | ||
| 677 | entry.value_ptr.bytes.len, | ||
| 678 | old_mem.len, | ||
| 679 | entry.value_ptr.getStackTrace(.alloc), | ||
| 680 | free_stack_trace, | ||
| 681 | }); | ||
| 682 | } | ||
| 683 | |||
| 684 | if (!config.never_unmap) { | ||
| 685 | self.backing_allocator.rawFree(old_mem, alignment, ret_addr); | ||
| 686 | } | ||
| 687 | |||
| 688 | if (config.enable_memory_limit) { | ||
| 689 | self.total_requested_bytes -= entry.value_ptr.requested_size; | ||
| 690 | } | ||
| 691 | |||
| 692 | if (config.verbose_log) { | ||
| 693 | log.info("large free {d} bytes at {*}", .{ old_mem.len, old_mem.ptr }); | ||
| 694 | } | ||
| 695 | |||
| 696 | if (!config.retain_metadata) { | ||
| 697 | assert(self.large_allocations.remove(@intFromPtr(old_mem.ptr))); | ||
| 698 | } else { | ||
| 699 | entry.value_ptr.freed = true; | ||
| 700 | entry.value_ptr.captureStackTrace(ret_addr, .free); | ||
| 701 | } | ||
| 702 | } | ||
| 703 | |||
| 704 | fn alloc(context: *anyopaque, len: usize, alignment: mem.Alignment, ret_addr: usize) ?[*]u8 { | ||
| 705 | const self: *Self = @ptrCast(@alignCast(context)); | ||
| 706 | self.mutex.lock(); | ||
| 707 | defer self.mutex.unlock(); | ||
| 708 | |||
| 709 | if (config.enable_memory_limit) { | ||
| 710 | const new_req_bytes = self.total_requested_bytes + len; | ||
| 711 | if (new_req_bytes > self.requested_memory_limit) return null; | ||
| 712 | self.total_requested_bytes = new_req_bytes; | ||
| 713 | } | ||
| 714 | |||
| 715 | const size_class_index: usize = @max(@bitSizeOf(usize) - @clz(len - 1), @intFromEnum(alignment)); | ||
| 716 | if (size_class_index >= self.buckets.len) { | ||
| 717 | @branchHint(.unlikely); | ||
| 718 | self.large_allocations.ensureUnusedCapacity(self.backing_allocator, 1) catch return null; | ||
| 719 | const ptr = self.backing_allocator.rawAlloc(len, alignment, ret_addr) orelse return null; | ||
| 720 | const slice = ptr[0..len]; | ||
| 721 | |||
| 722 | const gop = self.large_allocations.getOrPutAssumeCapacity(@intFromPtr(slice.ptr)); | ||
| 723 | if (config.retain_metadata and !config.never_unmap) { | ||
| 724 | // Backing allocator may be reusing memory that we're retaining metadata for | ||
| 725 | assert(!gop.found_existing or gop.value_ptr.freed); | ||
| 726 | } else { | ||
| 727 | assert(!gop.found_existing); // This would mean the kernel double-mapped pages. | ||
| 728 | } | ||
| 729 | gop.value_ptr.bytes = slice; | ||
| 730 | if (config.enable_memory_limit) | ||
| 731 | gop.value_ptr.requested_size = len; | ||
| 732 | gop.value_ptr.captureStackTrace(ret_addr, .alloc); | ||
| 733 | if (config.retain_metadata) { | ||
| 734 | gop.value_ptr.freed = false; | ||
| 735 | if (config.never_unmap) { | ||
| 736 | gop.value_ptr.alignment = alignment; | ||
| 737 | } | ||
| 738 | } | ||
| 739 | |||
| 740 | if (config.verbose_log) { | ||
| 741 | log.info("large alloc {d} bytes at {*}", .{ slice.len, slice.ptr }); | ||
| 742 | } | ||
| 743 | return slice.ptr; | ||
| 744 | } | ||
| 745 | |||
| 746 | const slot_count = slot_counts[size_class_index]; | ||
| 747 | |||
| 748 | if (self.buckets[size_class_index]) |bucket| { | ||
| 749 | @branchHint(.likely); | ||
| 750 | const slot_index = bucket.allocated_count; | ||
| 751 | if (slot_index < slot_count) { | ||
| 752 | @branchHint(.likely); | ||
| 753 | bucket.allocated_count = slot_index + 1; | ||
| 754 | const used_bits_byte = bucket.usedBits(slot_index / @bitSizeOf(usize)); | ||
| 755 | const used_bit_index: Log2USize = @intCast(slot_index % @bitSizeOf(usize)); | ||
| 756 | used_bits_byte.* |= (@as(usize, 1) << used_bit_index); | ||
| 757 | const size_class = @as(usize, 1) << @as(Log2USize, @intCast(size_class_index)); | ||
| 758 | if (config.stack_trace_frames > 0) { | ||
| 759 | bucket.captureStackTrace(ret_addr, slot_count, slot_index, .alloc); | ||
| 760 | } | ||
| 761 | if (config.safety) { | ||
| 762 | bucket.requestedSizes(slot_count)[slot_index] = @intCast(len); | ||
| 763 | bucket.log2PtrAligns(slot_count)[slot_index] = alignment; | ||
| 764 | } | ||
| 765 | const page_addr = @intFromPtr(bucket) & ~(page_size - 1); | ||
| 766 | const addr = page_addr + slot_index * size_class; | ||
| 767 | if (config.verbose_log) { | ||
| 768 | log.info("small alloc {d} bytes at 0x{x}", .{ len, addr }); | ||
| 769 | } | ||
| 770 | return @ptrFromInt(addr); | ||
| 771 | } | ||
| 772 | } | ||
| 773 | |||
| 774 | const page = self.backing_allocator.rawAlloc(page_size, page_align, @returnAddress()) orelse | ||
| 775 | return null; | ||
| 776 | const bucket: *BucketHeader = .fromPage(@intFromPtr(page), slot_count); | ||
| 777 | bucket.* = .{ | ||
| 778 | .allocated_count = 1, | ||
| 779 | .freed_count = 0, | ||
| 780 | .prev = self.buckets[size_class_index], | ||
| 781 | }; | ||
| 782 | self.buckets[size_class_index] = bucket; | ||
| 783 | |||
| 784 | if (!config.backing_allocator_zeroes) { | ||
| 785 | @memset(@as([*]usize, @as(*[1]usize, bucket.usedBits(0)))[0..usedBitsCount(slot_count)], 0); | ||
| 786 | if (config.safety) @memset(bucket.requestedSizes(slot_count), 0); | ||
| 787 | } | ||
| 788 | |||
| 789 | bucket.usedBits(0).* = 0b1; | ||
| 790 | |||
| 791 | if (config.stack_trace_frames > 0) { | ||
| 792 | bucket.captureStackTrace(ret_addr, slot_count, 0, .alloc); | ||
| 793 | } | ||
| 794 | |||
| 795 | if (config.safety) { | ||
| 796 | bucket.requestedSizes(slot_count)[0] = @intCast(len); | ||
| 797 | bucket.log2PtrAligns(slot_count)[0] = alignment; | ||
| 798 | } | ||
| 799 | |||
| 800 | if (config.verbose_log) { | ||
| 801 | log.info("small alloc {d} bytes at 0x{x}", .{ len, @intFromPtr(page) }); | ||
| 802 | } | ||
| 803 | |||
| 804 | return page; | ||
| 805 | } | ||
| 806 | |||
| 807 | fn resize( | ||
| 808 | context: *anyopaque, | ||
| 809 | memory: []u8, | ||
| 810 | alignment: mem.Alignment, | ||
| 811 | new_len: usize, | ||
| 812 | return_address: usize, | ||
| 813 | ) bool { | ||
| 814 | const self: *Self = @ptrCast(@alignCast(context)); | ||
| 815 | self.mutex.lock(); | ||
| 816 | defer self.mutex.unlock(); | ||
| 817 | |||
| 818 | const size_class_index: usize = @max(@bitSizeOf(usize) - @clz(memory.len - 1), @intFromEnum(alignment)); | ||
| 819 | if (size_class_index >= self.buckets.len) { | ||
| 820 | return self.resizeLarge(memory, alignment, new_len, return_address, false) != null; | ||
| 821 | } else { | ||
| 822 | return resizeSmall(self, memory, alignment, new_len, return_address, size_class_index); | ||
| 823 | } | ||
| 824 | } | ||
| 825 | |||
| 826 | fn remap( | ||
| 827 | context: *anyopaque, | ||
| 828 | memory: []u8, | ||
| 829 | alignment: mem.Alignment, | ||
| 830 | new_len: usize, | ||
| 831 | return_address: usize, | ||
| 832 | ) ?[*]u8 { | ||
| 833 | const self: *Self = @ptrCast(@alignCast(context)); | ||
| 834 | self.mutex.lock(); | ||
| 835 | defer self.mutex.unlock(); | ||
| 836 | |||
| 837 | const size_class_index: usize = @max(@bitSizeOf(usize) - @clz(memory.len - 1), @intFromEnum(alignment)); | ||
| 838 | if (size_class_index >= self.buckets.len) { | ||
| 839 | return self.resizeLarge(memory, alignment, new_len, return_address, true); | ||
| 840 | } else { | ||
| 841 | return if (resizeSmall(self, memory, alignment, new_len, return_address, size_class_index)) memory.ptr else null; | ||
| 842 | } | ||
| 843 | } | ||
| 844 | |||
| 845 | fn free( | ||
| 846 | context: *anyopaque, | ||
| 847 | old_memory: []u8, | ||
| 848 | alignment: mem.Alignment, | ||
| 849 | return_address: usize, | ||
| 850 | ) void { | ||
| 851 | const self: *Self = @ptrCast(@alignCast(context)); | ||
| 852 | self.mutex.lock(); | ||
| 853 | defer self.mutex.unlock(); | ||
| 854 | |||
| 855 | assert(old_memory.len != 0); | ||
| 856 | |||
| 857 | const size_class_index: usize = @max(@bitSizeOf(usize) - @clz(old_memory.len - 1), @intFromEnum(alignment)); | ||
| 858 | if (size_class_index >= self.buckets.len) { | ||
| 859 | @branchHint(.unlikely); | ||
| 860 | self.freeLarge(old_memory, alignment, return_address); | ||
| 861 | return; | ||
| 862 | } | ||
| 863 | |||
| 864 | const slot_count = slot_counts[size_class_index]; | ||
| 865 | const freed_addr = @intFromPtr(old_memory.ptr); | ||
| 866 | const page_addr = freed_addr & ~(page_size - 1); | ||
| 867 | const bucket: *BucketHeader = .fromPage(page_addr, slot_count); | ||
| 868 | if (bucket.canary != config.canary) @panic("Invalid free"); | ||
| 869 | const page_offset = freed_addr - page_addr; | ||
| 870 | const size_class = @as(usize, 1) << @as(Log2USize, @intCast(size_class_index)); | ||
| 871 | const slot_index: SlotIndex = @intCast(page_offset / size_class); | ||
| 872 | const used_byte_index = slot_index / @bitSizeOf(usize); | ||
| 873 | const used_bit_index: Log2USize = @intCast(slot_index % @bitSizeOf(usize)); | ||
| 874 | const used_byte = bucket.usedBits(used_byte_index); | ||
| 875 | const is_used = @as(u1, @truncate(used_byte.* >> used_bit_index)) != 0; | ||
| 876 | if (!is_used) { | ||
| 877 | if (config.safety) { | ||
| 878 | reportDoubleFree( | ||
| 879 | return_address, | ||
| 880 | bucketStackTrace(bucket, slot_count, slot_index, .alloc), | ||
| 881 | bucketStackTrace(bucket, slot_count, slot_index, .free), | ||
| 882 | ); | ||
| 883 | // Recoverable since this is a free. | ||
| 884 | return; | ||
| 885 | } else { | ||
| 886 | unreachable; | ||
| 887 | } | ||
| 888 | } | ||
| 889 | |||
| 890 | // Definitely an in-use small alloc now. | ||
| 891 | if (config.safety) { | ||
| 892 | const requested_size = bucket.requestedSizes(slot_count)[slot_index]; | ||
| 893 | if (requested_size == 0) @panic("Invalid free"); | ||
| 894 | const slot_alignment = bucket.log2PtrAligns(slot_count)[slot_index]; | ||
| 895 | if (old_memory.len != requested_size or alignment != slot_alignment) { | ||
| 896 | var addresses: [stack_n]usize = [1]usize{0} ** stack_n; | ||
| 897 | var free_stack_trace: StackTrace = .{ | ||
| 898 | .instruction_addresses = &addresses, | ||
| 899 | .index = 0, | ||
| 900 | }; | ||
| 901 | std.debug.captureStackTrace(return_address, &free_stack_trace); | ||
| 902 | if (old_memory.len != requested_size) { | ||
| 903 | log.err("Allocation size {d} bytes does not match free size {d}. Allocation: {} Free: {}", .{ | ||
| 904 | requested_size, | ||
| 905 | old_memory.len, | ||
| 906 | bucketStackTrace(bucket, slot_count, slot_index, .alloc), | ||
| 907 | free_stack_trace, | ||
| 908 | }); | ||
| 909 | } | ||
| 910 | if (alignment != slot_alignment) { | ||
| 911 | log.err("Allocation alignment {d} does not match free alignment {d}. Allocation: {} Free: {}", .{ | ||
| 912 | slot_alignment.toByteUnits(), | ||
| 913 | alignment.toByteUnits(), | ||
| 914 | bucketStackTrace(bucket, slot_count, slot_index, .alloc), | ||
| 915 | free_stack_trace, | ||
| 916 | }); | ||
| 917 | } | ||
| 918 | } | ||
| 919 | } | ||
| 920 | |||
| 921 | if (config.enable_memory_limit) { | ||
| 922 | self.total_requested_bytes -= old_memory.len; | ||
| 923 | } | ||
| 924 | |||
| 925 | if (config.stack_trace_frames > 0) { | ||
| 926 | // Capture stack trace to be the "first free", in case a double free happens. | ||
| 927 | bucket.captureStackTrace(return_address, slot_count, slot_index, .free); | ||
| 928 | } | ||
| 929 | |||
| 930 | used_byte.* &= ~(@as(usize, 1) << used_bit_index); | ||
| 931 | if (config.safety) { | ||
| 932 | bucket.requestedSizes(slot_count)[slot_index] = 0; | ||
| 933 | } | ||
| 934 | bucket.freed_count += 1; | ||
| 935 | if (bucket.freed_count == bucket.allocated_count) { | ||
| 936 | if (self.buckets[size_class_index] == bucket) { | ||
| 937 | self.buckets[size_class_index] = null; | ||
| 938 | } | ||
| 939 | if (!config.never_unmap) { | ||
| 940 | const page: [*]align(page_size) u8 = @ptrFromInt(page_addr); | ||
| 941 | self.backing_allocator.rawFree(page[0..page_size], page_align, @returnAddress()); | ||
| 942 | } | ||
| 943 | } | ||
| 944 | if (config.verbose_log) { | ||
| 945 | log.info("small free {d} bytes at {*}", .{ old_memory.len, old_memory.ptr }); | ||
| 946 | } | ||
| 947 | } | ||
| 948 | |||
| 949 | fn resizeSmall( | ||
| 950 | self: *Self, | ||
| 951 | memory: []u8, | ||
| 952 | alignment: mem.Alignment, | ||
| 953 | new_len: usize, | ||
| 954 | return_address: usize, | ||
| 955 | size_class_index: usize, | ||
| 956 | ) bool { | ||
| 957 | const new_size_class_index: usize = @max(@bitSizeOf(usize) - @clz(new_len - 1), @intFromEnum(alignment)); | ||
| 958 | if (!config.safety) return new_size_class_index == size_class_index; | ||
| 959 | const slot_count = slot_counts[size_class_index]; | ||
| 960 | const memory_addr = @intFromPtr(memory.ptr); | ||
| 961 | const page_addr = memory_addr & ~(page_size - 1); | ||
| 962 | const bucket: *BucketHeader = .fromPage(page_addr, slot_count); | ||
| 963 | if (bucket.canary != config.canary) @panic("Invalid free"); | ||
| 964 | const page_offset = memory_addr - page_addr; | ||
| 965 | const size_class = @as(usize, 1) << @as(Log2USize, @intCast(size_class_index)); | ||
| 966 | const slot_index: SlotIndex = @intCast(page_offset / size_class); | ||
| 967 | const used_byte_index = slot_index / @bitSizeOf(usize); | ||
| 968 | const used_bit_index: Log2USize = @intCast(slot_index % @bitSizeOf(usize)); | ||
| 969 | const used_byte = bucket.usedBits(used_byte_index); | ||
| 970 | const is_used = @as(u1, @truncate(used_byte.* >> used_bit_index)) != 0; | ||
| 971 | if (!is_used) { | ||
| 972 | reportDoubleFree( | ||
| 973 | return_address, | ||
| 974 | bucketStackTrace(bucket, slot_count, slot_index, .alloc), | ||
| 975 | bucketStackTrace(bucket, slot_count, slot_index, .free), | ||
| 976 | ); | ||
| 977 | // Recoverable since this is a free. | ||
| 978 | return false; | ||
| 979 | } | ||
| 980 | |||
| 981 | // Definitely an in-use small alloc now. | ||
| 982 | const requested_size = bucket.requestedSizes(slot_count)[slot_index]; | ||
| 983 | if (requested_size == 0) @panic("Invalid free"); | ||
| 984 | const slot_alignment = bucket.log2PtrAligns(slot_count)[slot_index]; | ||
| 985 | if (memory.len != requested_size or alignment != slot_alignment) { | ||
| 986 | var addresses: [stack_n]usize = [1]usize{0} ** stack_n; | ||
| 987 | var free_stack_trace: StackTrace = .{ | ||
| 988 | .instruction_addresses = &addresses, | ||
| 989 | .index = 0, | ||
| 990 | }; | ||
| 991 | std.debug.captureStackTrace(return_address, &free_stack_trace); | ||
| 992 | if (memory.len != requested_size) { | ||
| 993 | log.err("Allocation size {d} bytes does not match free size {d}. Allocation: {} Free: {}", .{ | ||
| 994 | requested_size, | ||
| 995 | memory.len, | ||
| 996 | bucketStackTrace(bucket, slot_count, slot_index, .alloc), | ||
| 997 | free_stack_trace, | ||
| 998 | }); | ||
| 999 | } | ||
| 1000 | if (alignment != slot_alignment) { | ||
| 1001 | log.err("Allocation alignment {d} does not match free alignment {d}. Allocation: {} Free: {}", .{ | ||
| 1002 | slot_alignment.toByteUnits(), | ||
| 1003 | alignment.toByteUnits(), | ||
| 1004 | bucketStackTrace(bucket, slot_count, slot_index, .alloc), | ||
| 1005 | free_stack_trace, | ||
| 1006 | }); | ||
| 1007 | } | ||
| 1008 | } | ||
| 1009 | |||
| 1010 | if (new_size_class_index != size_class_index) return false; | ||
| 1011 | |||
| 1012 | const prev_req_bytes = self.total_requested_bytes; | ||
| 1013 | if (config.enable_memory_limit) { | ||
| 1014 | const new_req_bytes = prev_req_bytes - memory.len + new_len; | ||
| 1015 | if (new_req_bytes > prev_req_bytes and new_req_bytes > self.requested_memory_limit) { | ||
| 1016 | return false; | ||
| 1017 | } | ||
| 1018 | self.total_requested_bytes = new_req_bytes; | ||
| 1019 | } | ||
| 1020 | |||
| 1021 | if (memory.len > new_len) @memset(memory[new_len..], undefined); | ||
| 1022 | if (config.verbose_log) | ||
| 1023 | log.info("small resize {d} bytes at {*} to {d}", .{ memory.len, memory.ptr, new_len }); | ||
| 1024 | |||
| 1025 | if (config.safety) | ||
| 1026 | bucket.requestedSizes(slot_count)[slot_index] = @intCast(new_len); | ||
| 1027 | |||
| 1028 | if (config.resize_stack_traces) | ||
| 1029 | bucket.captureStackTrace(return_address, slot_count, slot_index, .alloc); | ||
| 1030 | |||
| 1031 | return true; | ||
| 1032 | } | ||
| 1033 | }; | ||
| 1034 | } | ||
| 1035 | |||
| 1036 | const TraceKind = enum { | ||
| 1037 | alloc, | ||
| 1038 | free, | ||
| 1039 | }; | ||
| 1040 | |||
| 1041 | const test_config = Config{}; | ||
| 1042 | |||
| 1043 | test "small allocations - free in same order" { | ||
| 1044 | var gpa = GeneralPurposeAllocator(test_config){}; | ||
| 1045 | defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); | ||
| 1046 | const allocator = gpa.allocator(); | ||
| 1047 | |||
| 1048 | var list = std.ArrayList(*u64).init(std.testing.allocator); | ||
| 1049 | defer list.deinit(); | ||
| 1050 | |||
| 1051 | var i: usize = 0; | ||
| 1052 | while (i < 513) : (i += 1) { | ||
| 1053 | const ptr = try allocator.create(u64); | ||
| 1054 | try list.append(ptr); | ||
| 1055 | } | ||
| 1056 | |||
| 1057 | for (list.items) |ptr| { | ||
| 1058 | allocator.destroy(ptr); | ||
| 1059 | } | ||
| 1060 | } | ||
| 1061 | |||
| 1062 | test "small allocations - free in reverse order" { | ||
| 1063 | var gpa = GeneralPurposeAllocator(test_config){}; | ||
| 1064 | defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); | ||
| 1065 | const allocator = gpa.allocator(); | ||
| 1066 | |||
| 1067 | var list = std.ArrayList(*u64).init(std.testing.allocator); | ||
| 1068 | defer list.deinit(); | ||
| 1069 | |||
| 1070 | var i: usize = 0; | ||
| 1071 | while (i < 513) : (i += 1) { | ||
| 1072 | const ptr = try allocator.create(u64); | ||
| 1073 | try list.append(ptr); | ||
| 1074 | } | ||
| 1075 | |||
| 1076 | while (list.popOrNull()) |ptr| { | ||
| 1077 | allocator.destroy(ptr); | ||
| 1078 | } | ||
| 1079 | } | ||
| 1080 | |||
| 1081 | test "large allocations" { | ||
| 1082 | var gpa = GeneralPurposeAllocator(test_config){}; | ||
| 1083 | defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); | ||
| 1084 | const allocator = gpa.allocator(); | ||
| 1085 | |||
| 1086 | const ptr1 = try allocator.alloc(u64, 42768); | ||
| 1087 | const ptr2 = try allocator.alloc(u64, 52768); | ||
| 1088 | allocator.free(ptr1); | ||
| 1089 | const ptr3 = try allocator.alloc(u64, 62768); | ||
| 1090 | allocator.free(ptr3); | ||
| 1091 | allocator.free(ptr2); | ||
| 1092 | } | ||
| 1093 | |||
| 1094 | test "very large allocation" { | ||
| 1095 | var gpa = GeneralPurposeAllocator(test_config){}; | ||
| 1096 | defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); | ||
| 1097 | const allocator = gpa.allocator(); | ||
| 1098 | |||
| 1099 | try std.testing.expectError(error.OutOfMemory, allocator.alloc(u8, math.maxInt(usize))); | ||
| 1100 | } | ||
| 1101 | |||
| 1102 | test "realloc" { | ||
| 1103 | var gpa = GeneralPurposeAllocator(test_config){}; | ||
| 1104 | defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); | ||
| 1105 | const allocator = gpa.allocator(); | ||
| 1106 | |||
| 1107 | var slice = try allocator.alignedAlloc(u8, @alignOf(u32), 1); | ||
| 1108 | defer allocator.free(slice); | ||
| 1109 | slice[0] = 0x12; | ||
| 1110 | |||
| 1111 | // This reallocation should keep its pointer address. | ||
| 1112 | const old_slice = slice; | ||
| 1113 | slice = try allocator.realloc(slice, 2); | ||
| 1114 | try std.testing.expect(old_slice.ptr == slice.ptr); | ||
| 1115 | try std.testing.expect(slice[0] == 0x12); | ||
| 1116 | slice[1] = 0x34; | ||
| 1117 | |||
| 1118 | // This requires upgrading to a larger size class | ||
| 1119 | slice = try allocator.realloc(slice, 17); | ||
| 1120 | try std.testing.expect(slice[0] == 0x12); | ||
| 1121 | try std.testing.expect(slice[1] == 0x34); | ||
| 1122 | } | ||
| 1123 | |||
| 1124 | test "shrink" { | ||
| 1125 | var gpa: GeneralPurposeAllocator(test_config) = .{}; | ||
| 1126 | defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); | ||
| 1127 | const allocator = gpa.allocator(); | ||
| 1128 | |||
| 1129 | var slice = try allocator.alloc(u8, 20); | ||
| 1130 | defer allocator.free(slice); | ||
| 1131 | |||
| 1132 | @memset(slice, 0x11); | ||
| 1133 | |||
| 1134 | try std.testing.expect(allocator.resize(slice, 17)); | ||
| 1135 | slice = slice[0..17]; | ||
| 1136 | |||
| 1137 | for (slice) |b| { | ||
| 1138 | try std.testing.expect(b == 0x11); | ||
| 1139 | } | ||
| 1140 | |||
| 1141 | // Does not cross size class boundaries when shrinking. | ||
| 1142 | try std.testing.expect(!allocator.resize(slice, 16)); | ||
| 1143 | } | ||
| 1144 | |||
| 1145 | test "large object - grow" { | ||
| 1146 | if (builtin.target.isWasm()) { | ||
| 1147 | // Not expected to pass on targets that do not have memory mapping. | ||
| 1148 | return error.SkipZigTest; | ||
| 1149 | } | ||
| 1150 | var gpa: GeneralPurposeAllocator(test_config) = .{}; | ||
| 1151 | defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); | ||
| 1152 | const allocator = gpa.allocator(); | ||
| 1153 | |||
| 1154 | var slice1 = try allocator.alloc(u8, page_size * 2 - 20); | ||
| 1155 | defer allocator.free(slice1); | ||
| 1156 | |||
| 1157 | const old = slice1; | ||
| 1158 | slice1 = try allocator.realloc(slice1, page_size * 2 - 10); | ||
| 1159 | try std.testing.expect(slice1.ptr == old.ptr); | ||
| 1160 | |||
| 1161 | slice1 = try allocator.realloc(slice1, page_size * 2); | ||
| 1162 | try std.testing.expect(slice1.ptr == old.ptr); | ||
| 1163 | |||
| 1164 | slice1 = try allocator.realloc(slice1, page_size * 2 + 1); | ||
| 1165 | } | ||
| 1166 | |||
| 1167 | test "realloc small object to large object" { | ||
| 1168 | var gpa = GeneralPurposeAllocator(test_config){}; | ||
| 1169 | defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); | ||
| 1170 | const allocator = gpa.allocator(); | ||
| 1171 | |||
| 1172 | var slice = try allocator.alloc(u8, 70); | ||
| 1173 | defer allocator.free(slice); | ||
| 1174 | slice[0] = 0x12; | ||
| 1175 | slice[60] = 0x34; | ||
| 1176 | |||
| 1177 | // This requires upgrading to a large object | ||
| 1178 | const large_object_size = page_size * 2 + 50; | ||
| 1179 | slice = try allocator.realloc(slice, large_object_size); | ||
| 1180 | try std.testing.expect(slice[0] == 0x12); | ||
| 1181 | try std.testing.expect(slice[60] == 0x34); | ||
| 1182 | } | ||
| 1183 | |||
| 1184 | test "shrink large object to large object" { | ||
| 1185 | var gpa: GeneralPurposeAllocator(test_config) = .{}; | ||
| 1186 | defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); | ||
| 1187 | const allocator = gpa.allocator(); | ||
| 1188 | |||
| 1189 | var slice = try allocator.alloc(u8, page_size * 2 + 50); | ||
| 1190 | defer allocator.free(slice); | ||
| 1191 | slice[0] = 0x12; | ||
| 1192 | slice[60] = 0x34; | ||
| 1193 | |||
| 1194 | if (!allocator.resize(slice, page_size * 2 + 1)) return; | ||
| 1195 | slice = slice.ptr[0 .. page_size * 2 + 1]; | ||
| 1196 | try std.testing.expect(slice[0] == 0x12); | ||
| 1197 | try std.testing.expect(slice[60] == 0x34); | ||
| 1198 | |||
| 1199 | try std.testing.expect(allocator.resize(slice, page_size * 2 + 1)); | ||
| 1200 | slice = slice[0 .. page_size * 2 + 1]; | ||
| 1201 | try std.testing.expect(slice[0] == 0x12); | ||
| 1202 | try std.testing.expect(slice[60] == 0x34); | ||
| 1203 | |||
| 1204 | slice = try allocator.realloc(slice, page_size * 2); | ||
| 1205 | try std.testing.expect(slice[0] == 0x12); | ||
| 1206 | try std.testing.expect(slice[60] == 0x34); | ||
| 1207 | } | ||
| 1208 | |||
| 1209 | test "shrink large object to large object with larger alignment" { | ||
| 1210 | if (!builtin.link_libc and builtin.os.tag == .wasi) return error.SkipZigTest; // https://github.com/ziglang/zig/issues/22731 | ||
| 1211 | |||
| 1212 | var gpa = GeneralPurposeAllocator(test_config){}; | ||
| 1213 | defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); | ||
| 1214 | const allocator = gpa.allocator(); | ||
| 1215 | |||
| 1216 | var debug_buffer: [1000]u8 = undefined; | ||
| 1217 | var fba = std.heap.FixedBufferAllocator.init(&debug_buffer); | ||
| 1218 | const debug_allocator = fba.allocator(); | ||
| 1219 | |||
| 1220 | const alloc_size = page_size * 2 + 50; | ||
| 1221 | var slice = try allocator.alignedAlloc(u8, 16, alloc_size); | ||
| 1222 | defer allocator.free(slice); | ||
| 1223 | |||
| 1224 | const big_alignment: usize = switch (builtin.os.tag) { | ||
| 1225 | .windows => page_size * 32, // Windows aligns to 64K. | ||
| 1226 | else => page_size * 2, | ||
| 1227 | }; | ||
| 1228 | // This loop allocates until we find a page that is not aligned to the big | ||
| 1229 | // alignment. Then we shrink the allocation after the loop, but increase the | ||
| 1230 | // alignment to the higher one, that we know will force it to realloc. | ||
| 1231 | var stuff_to_free = std.ArrayList([]align(16) u8).init(debug_allocator); | ||
| 1232 | while (mem.isAligned(@intFromPtr(slice.ptr), big_alignment)) { | ||
| 1233 | try stuff_to_free.append(slice); | ||
| 1234 | slice = try allocator.alignedAlloc(u8, 16, alloc_size); | ||
| 1235 | } | ||
| 1236 | while (stuff_to_free.popOrNull()) |item| { | ||
| 1237 | allocator.free(item); | ||
| 1238 | } | ||
| 1239 | slice[0] = 0x12; | ||
| 1240 | slice[60] = 0x34; | ||
| 1241 | |||
| 1242 | slice = try allocator.reallocAdvanced(slice, big_alignment, alloc_size / 2); | ||
| 1243 | try std.testing.expect(slice[0] == 0x12); | ||
| 1244 | try std.testing.expect(slice[60] == 0x34); | ||
| 1245 | } | ||
| 1246 | |||
| 1247 | test "realloc large object to small object" { | ||
| 1248 | var gpa = GeneralPurposeAllocator(test_config){}; | ||
| 1249 | defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); | ||
| 1250 | const allocator = gpa.allocator(); | ||
| 1251 | |||
| 1252 | var slice = try allocator.alloc(u8, page_size * 2 + 50); | ||
| 1253 | defer allocator.free(slice); | ||
| 1254 | slice[0] = 0x12; | ||
| 1255 | slice[16] = 0x34; | ||
| 1256 | |||
| 1257 | slice = try allocator.realloc(slice, 19); | ||
| 1258 | try std.testing.expect(slice[0] == 0x12); | ||
| 1259 | try std.testing.expect(slice[16] == 0x34); | ||
| 1260 | } | ||
| 1261 | |||
| 1262 | test "overridable mutexes" { | ||
| 1263 | var gpa = GeneralPurposeAllocator(.{ .MutexType = std.Thread.Mutex }){ | ||
| 1264 | .backing_allocator = std.testing.allocator, | ||
| 1265 | .mutex = std.Thread.Mutex{}, | ||
| 1266 | }; | ||
| 1267 | defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); | ||
| 1268 | const allocator = gpa.allocator(); | ||
| 1269 | |||
| 1270 | const ptr = try allocator.create(i32); | ||
| 1271 | defer allocator.destroy(ptr); | ||
| 1272 | } | ||
| 1273 | |||
| 1274 | test "non-page-allocator backing allocator" { | ||
| 1275 | var gpa = GeneralPurposeAllocator(.{}){ .backing_allocator = std.testing.allocator }; | ||
| 1276 | defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); | ||
| 1277 | const allocator = gpa.allocator(); | ||
| 1278 | |||
| 1279 | const ptr = try allocator.create(i32); | ||
| 1280 | defer allocator.destroy(ptr); | ||
| 1281 | } | ||
| 1282 | |||
| 1283 | test "realloc large object to larger alignment" { | ||
| 1284 | if (!builtin.link_libc and builtin.os.tag == .wasi) return error.SkipZigTest; // https://github.com/ziglang/zig/issues/22731 | ||
| 1285 | |||
| 1286 | var gpa = GeneralPurposeAllocator(test_config){}; | ||
| 1287 | defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); | ||
| 1288 | const allocator = gpa.allocator(); | ||
| 1289 | |||
| 1290 | var debug_buffer: [1000]u8 = undefined; | ||
| 1291 | var fba = std.heap.FixedBufferAllocator.init(&debug_buffer); | ||
| 1292 | const debug_allocator = fba.allocator(); | ||
| 1293 | |||
| 1294 | var slice = try allocator.alignedAlloc(u8, 16, page_size * 2 + 50); | ||
| 1295 | defer allocator.free(slice); | ||
| 1296 | |||
| 1297 | const big_alignment: usize = switch (builtin.os.tag) { | ||
| 1298 | .windows => page_size * 32, // Windows aligns to 64K. | ||
| 1299 | else => page_size * 2, | ||
| 1300 | }; | ||
| 1301 | // This loop allocates until we find a page that is not aligned to the big alignment. | ||
| 1302 | var stuff_to_free = std.ArrayList([]align(16) u8).init(debug_allocator); | ||
| 1303 | while (mem.isAligned(@intFromPtr(slice.ptr), big_alignment)) { | ||
| 1304 | try stuff_to_free.append(slice); | ||
| 1305 | slice = try allocator.alignedAlloc(u8, 16, page_size * 2 + 50); | ||
| 1306 | } | ||
| 1307 | while (stuff_to_free.popOrNull()) |item| { | ||
| 1308 | allocator.free(item); | ||
| 1309 | } | ||
| 1310 | slice[0] = 0x12; | ||
| 1311 | slice[16] = 0x34; | ||
| 1312 | |||
| 1313 | slice = try allocator.reallocAdvanced(slice, 32, page_size * 2 + 100); | ||
| 1314 | try std.testing.expect(slice[0] == 0x12); | ||
| 1315 | try std.testing.expect(slice[16] == 0x34); | ||
| 1316 | |||
| 1317 | slice = try allocator.reallocAdvanced(slice, 32, page_size * 2 + 25); | ||
| 1318 | try std.testing.expect(slice[0] == 0x12); | ||
| 1319 | try std.testing.expect(slice[16] == 0x34); | ||
| 1320 | |||
| 1321 | slice = try allocator.reallocAdvanced(slice, big_alignment, page_size * 2 + 100); | ||
| 1322 | try std.testing.expect(slice[0] == 0x12); | ||
| 1323 | try std.testing.expect(slice[16] == 0x34); | ||
| 1324 | } | ||
| 1325 | |||
| 1326 | test "large object rejects shrinking to small" { | ||
| 1327 | if (builtin.target.isWasm()) { | ||
| 1328 | // Not expected to pass on targets that do not have memory mapping. | ||
| 1329 | return error.SkipZigTest; | ||
| 1330 | } | ||
| 1331 | |||
| 1332 | var failing_allocator = std.testing.FailingAllocator.init(std.heap.page_allocator, .{ .fail_index = 3 }); | ||
| 1333 | var gpa: GeneralPurposeAllocator(.{}) = .{ .backing_allocator = failing_allocator.allocator() }; | ||
| 1334 | defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); | ||
| 1335 | const allocator = gpa.allocator(); | ||
| 1336 | |||
| 1337 | var slice = try allocator.alloc(u8, page_size * 2 + 50); | ||
| 1338 | defer allocator.free(slice); | ||
| 1339 | slice[0] = 0x12; | ||
| 1340 | slice[3] = 0x34; | ||
| 1341 | |||
| 1342 | try std.testing.expect(!allocator.resize(slice, 4)); | ||
| 1343 | try std.testing.expect(slice[0] == 0x12); | ||
| 1344 | try std.testing.expect(slice[3] == 0x34); | ||
| 1345 | } | ||
| 1346 | |||
| 1347 | test "objects of size 1024 and 2048" { | ||
| 1348 | var gpa = GeneralPurposeAllocator(test_config){}; | ||
| 1349 | defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); | ||
| 1350 | const allocator = gpa.allocator(); | ||
| 1351 | |||
| 1352 | const slice = try allocator.alloc(u8, 1025); | ||
| 1353 | const slice2 = try allocator.alloc(u8, 3000); | ||
| 1354 | |||
| 1355 | allocator.free(slice); | ||
| 1356 | allocator.free(slice2); | ||
| 1357 | } | ||
| 1358 | |||
| 1359 | test "setting a memory cap" { | ||
| 1360 | var gpa = GeneralPurposeAllocator(.{ .enable_memory_limit = true }){}; | ||
| 1361 | defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); | ||
| 1362 | const allocator = gpa.allocator(); | ||
| 1363 | |||
| 1364 | gpa.requested_memory_limit = 1010; | ||
| 1365 | |||
| 1366 | const small = try allocator.create(i32); | ||
| 1367 | try std.testing.expect(gpa.total_requested_bytes == 4); | ||
| 1368 | |||
| 1369 | const big = try allocator.alloc(u8, 1000); | ||
| 1370 | try std.testing.expect(gpa.total_requested_bytes == 1004); | ||
| 1371 | |||
| 1372 | try std.testing.expectError(error.OutOfMemory, allocator.create(u64)); | ||
| 1373 | |||
| 1374 | allocator.destroy(small); | ||
| 1375 | try std.testing.expect(gpa.total_requested_bytes == 1000); | ||
| 1376 | |||
| 1377 | allocator.free(big); | ||
| 1378 | try std.testing.expect(gpa.total_requested_bytes == 0); | ||
| 1379 | |||
| 1380 | const exact = try allocator.alloc(u8, 1010); | ||
| 1381 | try std.testing.expect(gpa.total_requested_bytes == 1010); | ||
| 1382 | allocator.free(exact); | ||
| 1383 | } | ||
| 1384 | |||
| 1385 | test "large allocations count requested size not backing size" { | ||
| 1386 | var gpa: GeneralPurposeAllocator(.{ .enable_memory_limit = true }) = .{}; | ||
| 1387 | const allocator = gpa.allocator(); | ||
| 1388 | |||
| 1389 | var buf = try allocator.alignedAlloc(u8, 1, page_size + 1); | ||
| 1390 | try std.testing.expectEqual(page_size + 1, gpa.total_requested_bytes); | ||
| 1391 | buf = try allocator.realloc(buf, 1); | ||
| 1392 | try std.testing.expectEqual(1, gpa.total_requested_bytes); | ||
| 1393 | buf = try allocator.realloc(buf, 2); | ||
| 1394 | try std.testing.expectEqual(2, gpa.total_requested_bytes); | ||
| 1395 | } | ||
| 1396 | |||
| 1397 | test "retain metadata and never unmap" { | ||
| 1398 | var gpa = std.heap.GeneralPurposeAllocator(.{ | ||
| 1399 | .safety = true, | ||
| 1400 | .never_unmap = true, | ||
| 1401 | .retain_metadata = true, | ||
| 1402 | }){}; | ||
| 1403 | defer std.debug.assert(gpa.deinit() == .ok); | ||
| 1404 | const allocator = gpa.allocator(); | ||
| 1405 | |||
| 1406 | const alloc = try allocator.alloc(u8, 8); | ||
| 1407 | allocator.free(alloc); | ||
| 1408 | |||
| 1409 | const alloc2 = try allocator.alloc(u8, 8); | ||
| 1410 | allocator.free(alloc2); | ||
| 1411 | } | ||