From a9423234a6a4eb5bad25e12829b5a1a09ba9ca96 Mon Sep 17 00:00:00 2001 From: Kendall Condon Date: Mon, 20 Apr 2026 17:41:38 -0400 Subject: [PATCH 1/4] add SafeAllocator MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements a thread-safe allocator with the following guarantees: * `deinit` reports all leaks and frees all backing memory. * All allocation mismatches result in either a panic or segmentation fault. * Allocations from other `SafeAllocator` instances cause a panic (if `Options.canary` differ). * Double frees and operation (resize, remap, and free) races panic or segmentation fault. Given the backing allocator does not reuse memory, it does not reuse memory either and * Most writes after free will segmentation fault or are eventually detected and panic. `std.heap.DebugAllocator` has been deprecated (I have also deprecated `std.heap.Check` since this was its last usage and returning a `usize` leak count is a much cleaner approach). - General Design Every allocation is trailed by an `AllocFooter` which contains metadata for the allocation and stack traces. It is protected by a checksum to catch corruption from allocation overwrites and report canary mismatches. An allocation's memory has a minimum alignment of `AllocFooter` so that the footer is at a fixed offset determined from the allocation size. An allocation's memory is stored either: * Inside linearly-filled buckets for small allocations. * Inside an allocation directly from the backing allocator. To track allocations, each thread maintains a table of backing allocations. The table may be modified by other threads in the case of a producer-consumer operation, so the table is a linked list only expanded by creating new segments. Each thread maintains a linked list of free entries, which may contain entries from other threads' tables. In the case of producer-consumer operations, acquire/release ordering is assumed to be provided externally. This is also assumed by all other thread-safe allocators that reuse memory as otherwise there would be data races on reuse of allocated memory. - Fuzz Tests Two fuzz tests have also been added for the allocator. They check that there is no memory reuse, that returned memory is writable, and that it is not overwritten. The multi-threaded fuzz test spawns a number of worker threads which are used for all the test runs. I have run these tests extensively under TSAN. - Performance Measurements Building the standard library tests with a RelaseSafe compiler build and `-Ddebug-allocator`: ``` Benchmark 1 (3 runs): ./master-out/bin/zig test --zig-lib-dir lib lib/std/std.zig -femit-bin=test --test-no-exec measurement mean ± σ min … max outliers delta wall_time 29.4s ± 157ms 29.2s … 29.5s 0 ( 0%) 0% peak_rss 2.24GB ± 3.49MB 2.23GB … 2.24GB 0 ( 0%) 0% cpu_cycles 143G ± 999M 142G … 144G 0 ( 0%) 0% instructions 268G ± 5.22M 268G … 268G 0 ( 0%) 0% cache_references 13.1G ± 88.8M 13.0G … 13.2G 0 ( 0%) 0% cache_misses 2.38G ± 30.7M 2.35G … 2.41G 0 ( 0%) 0% branch_misses 634M ± 6.22M 629M … 641M 0 ( 0%) 0% Benchmark 2 (3 runs): ./branch-out/bin/zig test --zig-lib-dir lib lib/std/std.zig -femit-bin=test --test-no-exec measurement mean ± σ min … max outliers delta wall_time 22.1s ± 88.6ms 22.0s … 22.2s 0 ( 0%) ⚡- 24.7% ± 1.0% peak_rss 1.11GB ± 799KB 1.11GB … 1.11GB 0 ( 0%) ⚡- 50.3% ± 0.3% cpu_cycles 136G ± 480M 136G … 137G 0 ( 0%) ⚡- 4.4% ± 1.2% instructions 273G ± 2.07M 273G … 273G 0 ( 0%) 💩+ 1.6% ± 0.0% cache_references 12.3G ± 71.3M 12.2G … 12.4G 0 ( 0%) ⚡- 6.0% ± 1.4% cache_misses 2.02G ± 11.5M 2.01G … 2.03G 0 ( 0%) ⚡- 14.9% ± 2.2% branch_misses 569M ± 2.65M 567M … 572M 0 ( 0%) ⚡- 10.2% ± 1.7% ``` --- lib/std/heap.zig | 16 + lib/std/heap/SafeAllocator.zig | 2180 ++++++++++++++++++++++++++++++++ lib/std/mem/Allocator.zig | 2 + 3 files changed, 2198 insertions(+) create mode 100644 lib/std/heap/SafeAllocator.zig diff --git a/lib/std/heap.zig b/lib/std/heap.zig index 283b01cc3700a5fdab01cf7615df575579d91ed3..401026a6fdb3e0f56b12305b2ef3104b9da54377 100644 --- a/lib/std/heap.zig +++ b/lib/std/heap.zig @@ -11,14 +11,18 @@ const Alignment = std.mem.Alignment; pub const ArenaAllocator = @import("heap/ArenaAllocator.zig"); pub const SmpAllocator = @import("heap/SmpAllocator.zig"); +pub const SafeAllocator = @import("heap/SafeAllocator.zig"); pub const FixedBufferAllocator = @import("heap/FixedBufferAllocator.zig"); pub const BufferFirstAllocator = @import("heap/BufferFirstAllocator.zig"); pub const PageAllocator = @import("heap/PageAllocator.zig"); pub const WasmAllocator = if (builtin.single_threaded) BrkAllocator else @compileError("unimplemented"); pub const BrkAllocator = @import("heap/BrkAllocator.zig"); +/// Deprecated; use `SafeAllocator.Options`. pub const DebugAllocatorConfig = @import("heap/debug_allocator.zig").Config; +/// Deprecated; use `SafeAllocator`. pub const DebugAllocator = @import("heap/debug_allocator.zig").DebugAllocator; +/// Deprecated. pub const Check = enum { ok, leak }; /// A memory pool that can allocate objects of a single type very quickly. @@ -378,6 +382,17 @@ test smp_allocator { try testAllocatorAlignedShrink(smp_allocator); } +test SafeAllocator { + var instance: SafeAllocator = .init(page_allocator, .{}); + defer _ = instance.deinit(); + const allocator = instance.allocator(); + + try testAllocator(allocator); + try testAllocatorAligned(allocator); + try testAllocatorLargeAlignment(allocator); + try testAllocatorAlignedShrink(allocator); +} + test PageAllocator { const allocator = page_allocator; try testAllocator(allocator); @@ -880,6 +895,7 @@ test { _ = @import("heap/memory_pool.zig"); _ = ArenaAllocator; _ = DebugAllocator(.{}); + _ = SafeAllocator; _ = FixedBufferAllocator; _ = BufferFirstAllocator; if (builtin.single_threaded) { diff --git a/lib/std/heap/SafeAllocator.zig b/lib/std/heap/SafeAllocator.zig new file mode 100644 index 0000000000000000000000000000000000000000..fa566a21f9a229ef65dff1db525282ef45c4399c --- /dev/null +++ b/lib/std/heap/SafeAllocator.zig @@ -0,0 +1,2180 @@ +//! Provides the following guarantees: +//! * `deinit` reports all leaks and frees all backing memory. +//! * All allocation mismatches result in either a panic or segmentation fault. +//! * Allocations from other `SafeAllocator` instances cause a panic (if `Options.canary` differ). +//! * Double frees and operation (resize, remap, and free) races panic or segmentation fault. +//! +//! Given the backing allocator does not reuse memory, this does not reuse memory either and +//! * Most writes after free will segmentation fault or are eventually detected and panic. +//! +//! Thread-safe + +// General Design: +// +// Every allocation is trailed by an `AllocFooter` which contains metadata for the allocation and +// stack traces. It is protected by a checksum to catch corruption from allocation overwrites and +// report canary mismatches. An allocation's memory has a minimum alignment of `AllocFooter` so +// that the footer is at a fixed offset determined from the allocation size. An allocation's memory +// is stored either: +// * Inside linearly-filled buckets for small allocations. +// * Inside an allocation directly from the backing allocator. +// +// To track allocations, each thread maintains a table of backing allocations. The table may be +// modified by other threads in the case of a producer-consumer operation, so the table is a linked +// list only expanded by creating new segments. Each thread maintains a linked list of free +// entries, which may contain entries from other threads' tables. +// +// In the case of producer-consumer operations, acquire/release ordering is assumed to be provided +// externally. This is also assumed by all other thread-safe allocators that reuse memory as +// otherwise there would be data races on reuse of allocated memory. + +const std = @import("../std.zig"); +const math = std.math; +const mem = std.mem; +const Alignment = mem.Alignment; +const assert = std.debug.assert; +const panic = std.debug.panic; + +const SafeAllocator = @This(); +const scoped_log = std.log.scoped(.SafeAllocator); + +pub const Options = struct { + const is_debug = @import("builtin").mode == .Debug; + const page_size_log2 = @max(math.log2_int(usize, std.heap.page_size_max), 8); + + stack_trace_frames: usize = if (is_debug and std.debug.sys_can_stack_trace) 7 else 0, + check_write_after_free: bool = is_debug, + /// A unique value used to check that allocations created by other + /// `SafeAllocator` instances are not passed to this one. + canary: u32 = 0x85dff10f, + + /// Controls the block size and alignment of allocation buckets. + /// + /// Changing this is useful to save memory if the backing allocator offers better granuality, + /// or if the backing allocator has a limit on active allocations, however decreasing this + /// can harm performance. + /// + /// Asserted to be >= 8 + bucket_size_log2: u5 = @max(page_size_log2, 13), + /// Controls the block size of internal metadata. + /// + /// Changing this is useful to save memory if the backing allocator offers better granuality, + /// or if the backing allocator has a limit on active allocations, however decreasing this + /// can harm performance. + /// + /// Asserted to be >= 8 + block_size_log2: u5 = page_size_log2, +}; + +var n_threads: usize = 0; +threadlocal var thread_index: usize = 0; + +backing: mem.Allocator, +// Needs to be a fixed size so the max `n_threads` value is agreed upon by all instances. +threads: [128]Thread, + +bucket_size_log2: u5, +block_size_log2: u5, +/// In `usize`s +stack_trace_size: usize, +/// In `usize`s +allocs_entry_count: usize, +large_alloc_threshold: usize, + +canary: u32, +check_write_after_free: bool, + +fn bucketSize(s: *SafeAllocator) u32 { + return @as(u32, 1) << s.bucket_size_log2; +} + +fn bucketMask(s: *SafeAllocator) u32 { + return s.bucketSize() - 1; +} + +const Thread = struct { + /// Avoid false sharing. + _: void align(std.atomic.cache_line) = {}, + + mutex: std.atomic.Mutex, + fill_bucket: ?*Bucket, + free_entry: ?*Allocs.Entry, + allocs_next: usize, + allocs_first: ?*Allocs, +}; + +/// Trailed by `[allocs_entry_count]Entry` +const Allocs = extern struct { + next: ?*Allocs, + + comptime { + assert(@alignOf(@This()) == @alignOf(usize)); + assert(@sizeOf(@This()) == @sizeOf(usize)); + } + + fn usizes(a: *Allocs, s: *SafeAllocator) []usize { + return @as([*]usize, @ptrCast(a))[0 .. 1 + s.allocs_entry_count]; + } + + fn entries(a: *Allocs, s: *SafeAllocator) []Entry { + return @as([*]Entry, @ptrCast(a))[1..][0..s.allocs_entry_count]; + } + + const Entry = packed struct(usize) { + kind: Kind, + ptr_high: @Int(.unsigned, @bitSizeOf(usize) - 2), + + const Kind = enum(u2) { free, bucket, large_alloc }; + + comptime { + assert(@alignOf(Entry) >= 4); + assert(@alignOf(Bucket) >= 4); + assert(@alignOf(AllocFooter) >= 4); + } + + fn fromFree(ptr: ?*Entry) Entry { + return .{ + .ptr_high = @intCast(@intFromPtr(ptr) >> 2), + .kind = .free, + }; + } + + fn fromBucket(ptr: *Bucket) Entry { + return .{ + .ptr_high = @intCast(@intFromPtr(ptr) >> 2), + .kind = .bucket, + }; + } + + fn fromLargeAlloc(ptr: *AllocFooter) Entry { + return .{ + .ptr_high = @intCast(@intFromPtr(ptr) >> 2), + .kind = .large_alloc, + }; + } + + fn toFree(ent: Entry) ?*Entry { + assert(ent.kind == .free); + return @ptrFromInt(@as(usize, ent.ptr_high) << 2); + } + + fn toBucket(ent: Entry) *Bucket { + assert(ent.kind == .bucket); + return @ptrFromInt(@as(usize, ent.ptr_high) << 2); + } + + fn toLargeAlloc(ent: Entry) *AllocFooter { + assert(ent.kind == .large_alloc); + return @ptrFromInt(@as(usize, ent.ptr_high) << 2); + } + }; +}; + +/// This struct contains the header for a bucket. It is always part of a larger +/// allocations of length and alignment `bucketSize()`. +/// +/// All allocations inside buckets have a minimum of 8-byte alignment (including length) +/// so that allocations with 8-byte alignment or less do not need to store the location +/// of the previous footer since it is directly before it. This property is used by non- +/// extended footers to omit the offset of the previous footer. +const Bucket = struct { + entry: *Allocs.Entry, + /// Accesed atomically with `.acquire` / `.release` ordering to + /// provide memory ordering for allocation footers, **expect for + /// the `modify` field**. This needs `.acquire` fenced every time + /// footer data is updated (`AllocCount.fenceAcqRel`). + alloc_count: AllocCount, + /// Accesed atomically with `.monotonic` ordering. Alternatively, + /// this is also synchronized by `alloc_count`. + fill: Fill, + + /// So that `@sizeOf(Bucket)` is the start of first allocation if it is 8-byte aligned or less. + _: void align(8) = {}, + comptime { + assert(@alignOf(@This()) >= 8); + } + + const AllocCount = packed struct(u32) { + n: u31, + /// If `true`, this bucket cannot be freed yet. + filling: bool, + + fn fenceAcqRel(a: *AllocCount) void { + _ = @atomicRmw(AllocCount, a, .Or, .{ .n = 0, .filling = false }, .acq_rel); + } + }; + + const Fill = packed struct(u32) { + at: u31, + last_is_extended: bool, + }; + + fn of(s: *SafeAllocator, ptr: [*]u8) *Bucket { + const size_log2 = s.bucket_size_log2; + return @ptrFromInt(@intFromPtr(ptr) >> @intCast(size_log2) << @intCast(size_log2)); + } + + fn fillAt(s: *SafeAllocator, ptr: [*]const u8) u32 { + return @intCast(@intFromPtr(ptr) & s.bucketMask()); + } + + fn bytes(b: *Bucket, s: *SafeAllocator) []u8 { + assert(@intFromPtr(b) & s.bucketMask() == 0); + return @as([*]u8, @ptrCast(b))[0..s.bucketSize()]; + } + + fn lastAlloc(b: *Bucket, s: *SafeAllocator, fill: Fill) ?*AllocFooter { + return b.allocFooterBefore(s, fill.at, fill.last_is_extended); + } + + fn allocFooterBefore(b: *Bucket, s: *SafeAllocator, at: u32, is_extended: bool) ?*AllocFooter { + if (at - @sizeOf(Bucket) == 0) return null; + const off = at - AllocFooter.lenBucket(s, is_extended); + assert(off >= @sizeOf(Bucket)); + return @ptrCast(@alignCast(b.bytes(s)[off..])); + } + + /// Checks that no writes after frees were performed. + /// + /// Assumes `b.alloc_count` has been loaded with `.acquire` ordering. + fn check(b: *Bucket, s: *SafeAllocator) void { + var footer = b.lastAlloc(s, b.fill).?; + while (true) { + const modify = @atomicLoad( + AllocFooter.Modify, + &footer.modify, + // The only possible value should be `.freed` since `b.alloc_count` + // has been loaded with `.acquire`. However, another thread may be trying to + // modify the allocation after it is freed and so the other thread is going + // to panic even if this thread still sees `.freed`. + .unordered, + ).storedXor(&footer.modify); + if (modify != .freed or footer.actualChecksum(s) != footer.checksum ^ s.canary) { + panic("corrupted footer metadata in bucket at *{x}", .{@intFromPtr(&footer)}); + } + + s.checkFreed(footer); + footer = footer.bucketPrev(b, s) orelse break; + } + } +}; + +/// Trails the allocation, which has the following advantages: +/// * For buckets, the footer of the last allocation is always at the current fill. +/// * Aligning the allocation is simpler and wastes less space. +/// * Allocation overwrites are more likely to be caught by the footer getting corrupted. +/// For bucket allocs, this is trailed by `[2][stack_trace_size]usize`. +/// For large allocs, this is trailed by `[1][stack_trace_size]usize`. +const AllocFooter = struct { + /// Hash of `data` with the seed as the hash of its address so that memcpys of allocation + /// metadata are detected or are at least caught across runs. + /// + /// This stored value is xored with the canary value so that canary mismatches are detected. + checksum: u32, + /// Accesed atomically with `.monotonic` ordering to catch operation races. + /// + /// This stored value is xored with the hash of its address so that memcpys of allocation + /// metadata are detected or are at least caught across runs. + modify: Modify, + data: Data, + + /// `8`: minimum alignment for `Bucket` allocations + /// `@alignOf(usize)`: so that the offset of trailing data is at `@sizeOf(@This())` + _: void align(@max(8, @alignOf(usize))) = {}, + + comptime { + assert(@alignOf(@This()) >= @max(8, @alignOf(usize))); + } + + const Data = packed struct(u16) { + len: Len, + /// Low bits of the alignment. + /// + /// For non-extended headers, this is the entire alignment. The location of the previous + /// header is directly before this allocation since footers in `Bucket` are gauraunteed to + /// have at least 8-byte alignment. + alignment: u2, + /// Used only for bucket allocations. + prev_extended: bool, + + const Len = enum(u13) { + _, + + /// This footer is trailed (before the traces) by `Extended`. + /// The high bits of the alignment are encoded as the offset from `extended_start`. + /// + /// This may be set even if `Extended` is not strictly necesary + /// as a result of resizes and remaps. + const extended_start: u13 = math.maxInt(u13) - ((@bitSizeOf(usize) - 1) >> 2); + }; + }; + + const Extended = struct { + len: usize, + container: Container, + + const Container = union { + bucket_prev: ?*AllocFooter, + large_entry: *Allocs.Entry, + }; + + comptime { + // Exactly `usize` so this is directly after the regular footer + // and so that traces start directly after `@sizeOf(@This())`. + assert(@alignOf(@This()) == @alignOf(usize)); + } + }; + + const Modify = enum(u16) { + // Random non-linear enum values to decrease the chance of undetected corruption. + none = 0x2962, + resized = 0x0030, + remaped = 0x9068, + freeing = 0x7f3d, + freed = 0xb98b, + _, + + fn setNone(m: *Modify) void { + _ = @atomicRmw(Modify, m, .Xchg, .storedXor(.none, m), .monotonic); + } + + fn opName(m: Modify) []const u8 { + return switch (m) { + .resized => "resize", + .remaped => "remap", + .freeing => "free", + _, .none, .freed => unreachable, + }; + } + + fn stateName(m: Modify) []const u8 { + return switch (m) { + .resized => "after resize", + .remaped => "after remap", + .freeing => "during free", + .freed => "after free", + _, .none => unreachable, + }; + } + + fn storedXor(m: Modify, ptr: *Modify) Modify { + const addr_hash: u16 = @truncate(std.hash.int(@intFromPtr(ptr))); + return @enumFromInt(@intFromEnum(m) ^ addr_hash); + } + }; + + fn isExtended(f: *AllocFooter) bool { + return @intFromEnum(f.data.len) >= Data.Len.extended_start; + } + + fn extended(f: *AllocFooter) *Extended { + assert(f.isExtended()); + return @ptrFromInt(@intFromPtr(f) + @sizeOf(AllocFooter)); + } + + fn userMemory(f: *AllocFooter) []u8 { + const memory_addr = @intFromPtr(f) - allocOffset(f.userLen()); + assert(f.userAlign().check(memory_addr)); + const memory_ptr: [*]u8 = @ptrFromInt(memory_addr); + return memory_ptr[0..f.userLen()]; + } + + fn userLen(f: *AllocFooter) usize { + const len_int = @intFromEnum(f.data.len); + return if (len_int < Data.Len.extended_start) len_int else f.extended().len; + } + + fn userAlign(f: *AllocFooter) Alignment { + const high = (@intFromEnum(f.data.len) -| Data.Len.extended_start) << 2; + return @enumFromInt(high | f.data.alignment); + } + + fn bucketPrev(f: *AllocFooter, b: *Bucket, s: *SafeAllocator) ?*AllocFooter { + if (f.isExtended()) return f.extended().container.bucket_prev; + return b.allocFooterBefore(s, Bucket.fillAt(s, f.userMemory().ptr), f.data.prev_extended); + } + + fn tracesPtr(f: *AllocFooter) [*]usize { + const off_footer = @divExact(@sizeOf(AllocFooter), @sizeOf(usize)); + const off_extended = @as(usize, @divExact(@sizeOf(Extended), @sizeOf(usize))) * + @intFromBool(f.isExtended()); + return @as([*]usize, @ptrCast(f))[off_footer + off_extended ..]; + } + + fn allocTrace(f: *AllocFooter, s: *SafeAllocator) []usize { + return f.tracesPtr()[0..s.stack_trace_size]; + } + + fn freeTrace(f: *AllocFooter, s: *SafeAllocator) []usize { + const trace_size = s.stack_trace_size; + return f.tracesPtr()[trace_size..][0..trace_size]; + } + + fn actualChecksum(f: *AllocFooter, s: *SafeAllocator) u32 { + if (f.isExtended()) { + const len = f.extended().len; + const addr: usize = if (s.isLarge(len, f.userAlign())) + @intFromPtr(f.extended().container.large_entry) + else + @intFromPtr(f.extended().container.bucket_prev); + + const len_bytes: [@sizeOf(usize)]u8 = @bitCast(len); + const container: [@sizeOf(usize)]u8 = @bitCast(addr); + const regular_bytes: [2]u8 = @bitCast(f.data); + const data_bytes = len_bytes ++ container ++ regular_bytes; + + return @truncate(std.hash.Wyhash.hash(@truncate(@intFromPtr(f)), &data_bytes)); + } + return @truncate(std.hash.int(@as(u16, @bitCast(f.data)) ^ @intFromPtr(f))); + } + + fn allocOffset(len: usize) usize { + return Alignment.of(AllocFooter).forward(len); + } + + fn allocAlign(a: Alignment) Alignment { + return a.max(.of(AllocFooter)); + } + + /// Assumes the footer is in a bucket allocation; all + /// large allocations require an extended header. + fn requiresExtended(len: usize, alignment: Alignment) bool { + return len >= Data.Len.extended_start or @intFromEnum(alignment) > math.maxInt(u2); + } + + fn lenBucket(s: *SafeAllocator, is_extended: bool) usize { + return Alignment.forward(.@"8", @sizeOf(AllocFooter) + + @as(usize, @sizeOf(Extended)) * @intFromBool(is_extended) + + s.stack_trace_size * @sizeOf(usize) * 2); + } + + fn lenLarge(s: *SafeAllocator) usize { + return @sizeOf(AllocFooter) + @sizeOf(Extended) + s.stack_trace_size * @sizeOf(usize); + } + + fn allocLenBucket(s: *SafeAllocator, len: usize, is_extended: bool) usize { + return allocOffset(len) + lenBucket(s, is_extended); + } + + fn allocLenLarge(s: *SafeAllocator, len: usize) usize { + return allocOffset(len) + lenLarge(s); + } + + fn allocOffsetOrOom(len: usize) error{OutOfMemory}!usize { + return alignForwardOrOom(.of(AllocFooter), len); + } + + fn allocLenBucketOrOom( + s: *SafeAllocator, + len: usize, + is_extended: bool, + ) error{OutOfMemory}!usize { + return addOrOom(try allocOffsetOrOom(len), lenBucket(s, is_extended)); + } + + fn of(user_memory: []u8) *AllocFooter { + // Avoid panicing now if `memory.ptr` is not correctly aligned since a more + // useful panic will be provided later by a mismatch or invalid footer. + const aligned_start = Alignment.backward(.of(AllocFooter), @intFromPtr(user_memory.ptr)); + return @ptrFromInt(aligned_start + allocOffset(user_memory.len)); + } + + fn startModify(f: *AllocFooter, m: Modify, s: *SafeAllocator, mem_fmt: FormatMemory) void { + const prev = @atomicRmw( + Modify, + &f.modify, + .Xchg, + .storedXor(m, &f.modify), + .monotonic, + ).storedXor(&f.modify); + + if (prev != .none) { + @branchHint(.cold); + const op_name = m.opName(); + switch (prev) { + .none => unreachable, + .resized, .remaped => panic( + \\{s} {s} of {f} + \\alloc: {f} + \\{s}: + // (panic stack trace) + , .{ + op_name, + prev.stateName(), + mem_fmt, + // The stack trace may have been overwritten, but at least give it a try + formatStackTrace(f.allocTrace(s)), + op_name, + }), + .freeing, .freed => { + if (prev == .freeing) { + // Wait for trace to become available + const complete: Modify = .storedXor(.freed, &f.modify); + while (@atomicLoad(Modify, &f.modify, .monotonic) != complete) {} + const b: *Bucket = .of(s, @ptrCast(f)); + b.alloc_count.fenceAcqRel(); + } + if (m == .freeing) { + panic( + \\double free of {f} + \\alloc: {f} + \\first free: {f} + \\second free: + // (panic stack trace) + , .{ + mem_fmt, + formatStackTrace(f.allocTrace(s)), + formatStackTrace(f.freeTrace(s)), + }); + } else { + panic( + \\{s} {s} of {f} + \\alloc: {f} + \\free: {f} + \\{s}: + // (panic stack trace) + , .{ + op_name, + prev.stateName(), + mem_fmt, + formatStackTrace(f.allocTrace(s)), + formatStackTrace(f.freeTrace(s)), + op_name, + }); + } + }, + _ => panic( + "{s} of invalid memory {f} or corrupted metadata", + .{ m.opName(), mem_fmt }, + ), + } + comptime unreachable; + } + + const expected_checksum = f.actualChecksum(s); + if (f.checksum ^ s.canary != expected_checksum) { + @branchHint(.cold); + const other_canary = f.checksum ^ expected_checksum; + panic( + "{s} of invalid memory {f}, corrupted metadata, or foreign allocation from canary 0x{x}", + .{ m.opName(), mem_fmt, other_canary }, + ); + } + + if (f.userLen() != mem_fmt.memory.len or f.userAlign() != mem_fmt.alignment) { + const op_name = m.opName(); + panic( + \\{s} of {f} mismatches allocation of {f} + \\alloc: {f} + \\{s}: + // (panic stack trace) + , .{ op_name, mem_fmt, FormatMemory{ + .memory = f.userMemory(), + .alignment = f.userAlign(), + }, formatStackTrace(f.allocTrace(s)), op_name }); + } + } + + /// It is the caller's responsibility to `.acquire` fence the respective `Bucket.alloc_count`. + fn populate( + memory: []align(@alignOf(AllocFooter)) u8, + len: usize, + alignment: Alignment, + ra: usize, + /// `true` for large allocations + is_extended: bool, + /// `false` for large allocations + prev_extended: bool, + container: Extended.Container, + s: *SafeAllocator, + ) *AllocFooter { + const footer: *AllocFooter = @ptrCast(@alignCast(memory[allocOffset(len)..].ptr)); + + if (!is_extended) { + footer.data = .{ + .len = @enumFromInt(len), + .alignment = @intCast(@intFromEnum(alignment)), + .prev_extended = prev_extended, + }; + assert(!footer.isExtended()); + } else { + footer.data = .{ + .len = @enumFromInt(Data.Len.extended_start + (@intFromEnum(alignment) >> 2)), + .alignment = @truncate(@intFromEnum(alignment)), + .prev_extended = prev_extended, + }; + assert(footer.isExtended()); + footer.extended().* = .{ + .len = len, + .container = container, + }; + } + + captureStackTrace(footer.allocTrace(s), ra); + footer.checksum = footer.actualChecksum(s) ^ s.canary; + footer.modify.setNone(); + + return footer; + } +}; + +pub fn init( + /// Must be thread-safe for this allocator to be thread-safe + backing: mem.Allocator, + options: Options, +) SafeAllocator { + assert(options.block_size_log2 >= 8); + assert(options.bucket_size_log2 >= 8); + + const allocs_entry_count = (@as(usize, 1) << options.block_size_log2) / @sizeOf(usize); + return .{ + .backing = backing, + .threads = @splat(.{ + .mutex = .unlocked, + .fill_bucket = null, + .free_entry = null, + .allocs_next = allocs_entry_count, + .allocs_first = null, + }), + + .bucket_size_log2 = options.bucket_size_log2, + .block_size_log2 = options.block_size_log2, + .stack_trace_size = options.stack_trace_frames + + @intFromBool(options.stack_trace_frames != 0), + .allocs_entry_count = allocs_entry_count, + .large_alloc_threshold = (@as(usize, 1) << options.bucket_size_log2) * 3 / 4, + + .canary = options.canary, + .check_write_after_free = options.check_write_after_free, + }; +} + +/// Returns the number of leaks +pub fn deinit(s: *SafeAllocator) usize { + return s.deinitLog(true); +} + +/// Same as `deinit`, expect if `log` is `false`, it will not log leaks. +pub fn deinitLog(s: *SafeAllocator, log: bool) usize { + var leaks: usize = 0; + const thread_count = @atomicRmw(usize, &n_threads, .Or, 0, .monotonic); + for (s.threads[0..@max(1, thread_count)]) |*t| { + assert(t.mutex == .unlocked); // use of allocator during `deinit` + + var maybe_allocs = t.allocs_first; + var n_entries = t.allocs_next; + while (maybe_allocs) |allocs| { + for (allocs.entries(s)[0..n_entries]) |*ent| { + switch (ent.kind) { + .free => { + @branchHint(.likely); + }, + .bucket => leaks += s.deinitLeakedBucket(ent.toBucket(), log), + .large_alloc => { + leaks += 1; + s.deinitLargeAlloc(ent.toLargeAlloc(), log); + }, + } + } + maybe_allocs = allocs.next; + n_entries = s.allocs_entry_count; + s.backing.rawFree(@ptrCast(allocs.usizes(s)), .of(usize), 0); + } + } + return leaks; +} + +/// Returns the true count of leaks +fn deinitLeakedBucket(s: *SafeAllocator, b: *Bucket, log: bool) usize { + var leaks: usize = 0; + + const expected = @atomicLoad(Bucket.AllocCount, &b.alloc_count, .acquire); + if (expected.n == 0) assert(expected.filling); + + var footer = b.lastAlloc(s, b.fill).?; + while (true) { + const modify = @atomicRmw( + AllocFooter.Modify, + &footer.modify, + .Xchg, + undefined, + .monotonic, + ).storedXor(&footer.modify); + + const bad_modify = modify != .none and modify != .freed; + if (bad_modify or footer.actualChecksum(s) != footer.checksum ^ s.canary) { + panic("corrupted footer metadata in bucket at *{x}", .{@intFromPtr(&footer)}); + } + + switch (modify) { + .none => { + leaks += 1; + if (log) scoped_log.err("leaked {f} allocated at: {f}", .{ FormatMemory{ + .memory = footer.userMemory(), + .alignment = footer.userAlign(), + }, formatStackTrace(footer.allocTrace(s)) }); + }, + .freed => s.checkFreed(footer), + else => unreachable, + } + + footer = footer.bucketPrev(b, s) orelse break; + } + s.backing.rawFree(b.bytes(s), @enumFromInt(s.bucket_size_log2), 0); + + assert(leaks == expected.n); + return leaks; +} + +fn deinitLargeAlloc(s: *SafeAllocator, footer: *AllocFooter, log: bool) void { + const modify = footer.modify.storedXor(&footer.modify); + if (modify != .none or footer.checksum ^ s.canary != footer.actualChecksum(s)) { + panic("corrupted footer metadata at *{x}", .{@intFromPtr(&footer)}); + } + + const memory = footer.userMemory(); + if (log) scoped_log.err("leaked {f} allocated at {f}", .{ FormatMemory{ + .memory = memory, + .alignment = footer.userAlign(), + }, formatStackTrace(footer.allocTrace(s)) }); + + s.backing.rawFree( + memory.ptr[0..AllocFooter.allocLenLarge(s, memory.len)], + AllocFooter.allocAlign(footer.userAlign()), + 0, + ); +} + +/// Returned allocator is thread-safe +pub fn allocator(s: *SafeAllocator) mem.Allocator { + return .{ .ptr = s, .vtable = &vtable }; +} + +fn acquireThread(s: *SafeAllocator) *Thread { + while (true) { + const t = &s.threads[thread_index]; + if (t.mutex.tryLock()) { + @branchHint(.likely); + return t; + } + + var max = @atomicLoad(usize, &n_threads, .unordered); + if (max == 0) { + @branchHint(.unlikely); + max = @min(std.Thread.getCpuCount() catch s.threads.len, s.threads.len); + max = @cmpxchgStrong(usize, &n_threads, 0, max, .monotonic, .monotonic) orelse max; + } + + thread_index += 1; + // thread_index may be greater than max if the zero is returned by getCpuCount + thread_index *= @intFromBool(thread_index < max); + } +} + +fn alignForwardOrOom(a: Alignment, addr: usize) error{OutOfMemory}!usize { + const x = a.toByteUnits() - 1; + return try addOrOom(addr, x) & ~x; +} + +fn addOrOom(a: usize, b: usize) error{OutOfMemory}!usize { + return math.add(usize, a, b) catch error.OutOfMemory; +} + +fn isLarge(s: *SafeAllocator, len: usize, alignment: Alignment) bool { + const max_align_waste = alignment.toByteUnits() - 1; + const max_use = max_align_waste + AllocFooter.allocLenBucket(s, len, true); + return max_use >= s.large_alloc_threshold; +} + +fn isLargeOrOom(s: *SafeAllocator, len: usize, alignment: Alignment) error{OutOfMemory}!bool { + const max_align_waste = alignment.toByteUnits() - 1; + const max_use = try addOrOom(max_align_waste, try AllocFooter.allocLenBucketOrOom(s, len, true)); + return max_use >= s.large_alloc_threshold; +} + +fn newAllocEntry(s: *SafeAllocator, t: *Thread, ra: usize) error{OutOfMemory}!*Allocs.Entry { + if (t.free_entry) |ent| { + @branchHint(.likely); + t.free_entry = ent.toFree(); + return ent; + } + + if (s.allocs_entry_count - t.allocs_next != 0) { + @branchHint(.likely); + const ent = &t.allocs_first.?.entries(s)[t.allocs_next]; + t.allocs_next += 1; + return ent; + } + + const new_segment: *Allocs = @ptrCast(@alignCast(s.backing.rawAlloc( + (1 + s.allocs_entry_count) * @sizeOf(usize), + .of(usize), + ra, + ) orelse return error.OutOfMemory)); + new_segment.next = t.allocs_first; + t.allocs_first = new_segment; + t.allocs_next = 1; + return &new_segment.entries(s)[0]; +} + +fn freeAllocEntry(t: *Thread, ent: *Allocs.Entry) void { + ent.* = .fromFree(t.free_entry); + t.free_entry = ent; +} + +fn overwriteFreed(s: *SafeAllocator, bytes: []u8) void { + if (!s.check_write_after_free) return; + // 0x55 is used so that undefined writes of 0xaa are still caught. Another option would be a + // stream of random bytes seeded by the address, however that makes debugging reads after frees + // more difficult and has a performance penalty and so is not worth catching slightly more + // writes after frees. + @memset(bytes, 0x55); +} + +/// Returns the first address of a write after free +fn checkFreed(s: *SafeAllocator, footer: *AllocFooter) void { + if (!s.check_write_after_free) return; + const memory = footer.userMemory(); + for (memory) |*b| if (b.* != 0x55) { + panic( + \\write after free at *{x} + \\original alloc of {f}: {f} + \\free: {f} + \\stack trace: + // (panic stack trace) + , .{ + @intFromPtr(b), + FormatMemory{ .memory = memory, .alignment = footer.userAlign() }, + formatStackTrace(footer.allocTrace(s)), + formatStackTrace(footer.freeTrace(s)), + }); + }; +} + +const FormatMemory = struct { + memory: []const u8, + alignment: Alignment, + + pub fn format(m: FormatMemory, w: *std.Io.Writer) std.Io.Writer.Error!void { + return w.print( + "[addr: {x}, len: {} (0x{x}) align: {}]", + .{ @intFromPtr(m.memory.ptr), m.memory.len, m.memory.len, m.alignment.toByteUnits() }, + ); + } +}; + +/// The first element stores the length of the stack trace including skipped frames. +/// The remaining elements store the return addresses. +fn captureStackTrace(trace_buf: []usize, ra: usize) void { + if (trace_buf.len == 0) return; + + if (ra == 0) { // No return address provided + @branchHint(.unlikely); + trace_buf[0] = 0; + return; + } + + const t = std.debug.captureCurrentStackTrace(.{ .first_address = ra }, trace_buf[1..]); + const skipped = @intFromEnum(t.skipped) * + @intFromBool(t.return_addresses.len == trace_buf[1..].len); + trace_buf[0] = t.return_addresses.len +| skipped; +} + +fn formatStackTrace(trace_buf: []usize) std.debug.FormatStackTrace { + return .{ + .stack_trace = if (trace_buf.len != 0) trace: { + const frames = trace_buf[0]; + const addrs = trace_buf[1..]; + break :trace .{ + .return_addresses = addrs[0..@min(frames, addrs.len)], + .skipped = switch (frames) { + else => @enumFromInt(frames -| addrs.len), + 0, math.maxInt(usize) => .unknown, + }, + }; + } else .{ + .return_addresses = &.{}, + .skipped = .unknown, + }, + .terminal_mode = std.log.terminalMode(), + }; +} + +/// If this fails, future allocations to the bucket are illegal +fn allocBucket( + s: *SafeAllocator, + t: *Thread, + b: *Bucket, + len: usize, + alignment: Alignment, + ra: usize, +) ?[*]u8 { + const fill = &b.fill; + const is_extended = AllocFooter.requiresExtended(len, alignment); + const alloc_len: u32 = @intCast(AllocFooter.allocLenBucket(s, len, is_extended)); + const alloc_align = AllocFooter.allocAlign(alignment); + + var prev_fill = @atomicLoad(Bucket.Fill, fill, .monotonic); + var start: u32 = undefined; + var end: u32 = undefined; + while (true) { + start = @intCast(alloc_align.forward(prev_fill.at)); + end = start + alloc_len; + + if (end > s.bucketSize()) { + @branchHint(.unlikely); + + const prev_count = @atomicRmw( + Bucket.AllocCount, + &b.alloc_count, + .Sub, + .{ .filling = true, .n = 0 }, + .acq_rel, + ); + assert(prev_count.filling); + + if (prev_count.n == 0) { + @branchHint(.unlikely); + freeAllocEntry(t, b.entry); + b.check(s); + s.backing.rawFree(b.bytes(s), @enumFromInt(s.bucket_size_log2), ra); + } + + return null; + } + + prev_fill = @cmpxchgWeak( + Bucket.Fill, + fill, + prev_fill, + .{ .at = @intCast(end), .last_is_extended = is_extended }, + .monotonic, + .monotonic, + ) orelse { + @branchHint(.likely); + break; + }; + // b.fill was changed during a resize (or a sporadic cmpxchgWeak failure) + } + + const memory = b.bytes(s)[start..end]; + _ = AllocFooter.populate( + @alignCast(memory), + len, + alignment, + ra, + is_extended, + prev_fill.last_is_extended, + .{ .bucket_prev = b.lastAlloc(s, prev_fill) }, + s, + ); + + assert(@atomicRmw( + Bucket.AllocCount, + &b.alloc_count, + .Add, + .{ .filling = false, .n = 1 }, + .acq_rel, + ).filling); + + return memory.ptr; +} + +fn growingResizeBucket( + s: *SafeAllocator, + f: *AllocFooter, + memory: []const u8, + alignment: Alignment, + new_len: usize, + ra: usize, +) bool { + assert(new_len >= memory.len); + return s.advanceBucketAlloc(f, Bucket.fillAt(s, memory.ptr), false, alignment, new_len, ra); +} + +fn advanceBucketAlloc( + s: *SafeAllocator, + old: *AllocFooter, + new_start: u32, + start_moved: bool, + alignment: Alignment, + new_len: usize, + ra: usize, +) bool { + assert(AllocFooter.allocAlign(alignment).check(new_start)); + const b: *Bucket = .of(s, @ptrCast(old)); + + const old_is_extended = old.isExtended(); + const old_footer_len = AllocFooter.lenBucket(s, old_is_extended); + const old_fill: u32 = @intCast(Bucket.fillAt(s, @ptrCast(old)) + old_footer_len); + + const new_is_extended = old_is_extended or start_moved or + AllocFooter.requiresExtended(new_len, alignment); + const new_footer_len = AllocFooter.lenBucket(s, new_is_extended); + const new_fill: u32 = @intCast(new_start + AllocFooter.allocOffset(new_len) + new_footer_len); + + assert(old_fill <= new_fill); + if (new_fill > s.bucketSize()) { + return false; + } + + if (old_fill == new_fill or @cmpxchgStrong( + Bucket.Fill, + &b.fill, + .{ .last_is_extended = old_is_extended, .at = @intCast(old_fill) }, + .{ .last_is_extended = new_is_extended, .at = @intCast(new_fill) }, + .monotonic, + .monotonic, + ) != null) { + return false; + } + + _ = AllocFooter.populate( + @alignCast(b.bytes(s)[new_start..new_fill]), + new_len, + alignment, + ra, + new_is_extended, + old.data.prev_extended, + .{ .bucket_prev = old.bucketPrev(b, s) }, + s, + ); + b.alloc_count.fenceAcqRel(); + return true; +} + +const vtable: mem.Allocator.VTable = .{ + .alloc = alloc, + .free = free, + .resize = resize, + .remap = remap, +}; + +fn alloc(ctx: *anyopaque, len: usize, alignment: Alignment, ra: usize) ?[*]u8 { + assert(len != 0); + + const s: *SafeAllocator = @ptrCast(@alignCast(ctx)); + const t = s.acquireThread(); + defer t.mutex.unlock(); + + if (s.isLargeOrOom(len, alignment) catch return null) { + @branchHint(.unlikely); + + const entry = s.newAllocEntry(t, ra) catch return null; + const alloc_len = AllocFooter.allocLenLarge(s, len); + const alloc_align = AllocFooter.allocAlign(alignment); + const alloc_ptr = s.backing.rawAlloc(alloc_len, alloc_align, ra) orelse { + freeAllocEntry(t, entry); + return null; + }; + + const footer = AllocFooter.populate( + @alignCast(alloc_ptr[0..alloc_len]), + len, + alignment, + ra, + true, + false, + .{ .large_entry = entry }, + s, + ); + entry.* = .fromLargeAlloc(footer); + + return alloc_ptr; + } + + if (t.fill_bucket) |bucket| { + @branchHint(.likely); + if (s.allocBucket(t, bucket, len, alignment, ra)) |ptr| { + @branchHint(.likely); + return ptr; + } + } + t.fill_bucket = null; // In case of OOM below, this bucket will still be unusable for future + // allocations. + + const entry = s.newAllocEntry(t, ra) catch return null; + const bucket: *Bucket = @ptrCast(@alignCast(s.backing.rawAlloc( + s.bucketSize(), + @enumFromInt(s.bucket_size_log2), + ra, + ) orelse { + freeAllocEntry(t, entry); + return null; + })); + bucket.* = .{ + .entry = entry, + // No atomic stores necessary because this thread is the + // first to atomically update these below in allocBucket. + .alloc_count = .{ .filling = true, .n = 0 }, + .fill = .{ .at = @sizeOf(Bucket), .last_is_extended = false }, + }; + entry.* = .fromBucket(bucket); + + t.fill_bucket = bucket; + return s.allocBucket(t, bucket, len, alignment, ra); +} + +fn free(ctx: *anyopaque, memory: []u8, alignment: Alignment, ra: usize) void { + const s: *SafeAllocator = @ptrCast(@alignCast(ctx)); + const f: *AllocFooter = .of(memory); + f.startModify(.freeing, s, .{ .memory = memory, .alignment = alignment }); + + if (s.isLarge(memory.len, alignment)) { + @branchHint(.unlikely); + + const t = s.acquireThread(); + freeAllocEntry(t, f.extended().container.large_entry); + t.mutex.unlock(); + s.backing.rawFree( + memory.ptr[0..AllocFooter.allocLenLarge(s, memory.len)], + AllocFooter.allocAlign(alignment), + ra, + ); + return; + } + + const b: *Bucket = .of(s, memory.ptr); + s.overwriteFreed(memory); + captureStackTrace(f.freeTrace(s), ra); + + // Fence the alloc count before setting `f.modify` to `.freed`. + // This way, if another thread is waiting for the trace to become + // available, it will not be racing with us to see this `.release`. + // + // The below alloc count update can not be moved up here instead + // since that would allow another thread to see the `.freeing` state. + b.alloc_count.fenceAcqRel(); + + // If this result is different than .freeing, then some other thread + // is in the process of panicing. So, just ignore it. (This is also + // the reasoning for several other places.) + _ = @atomicRmw( + AllocFooter.Modify, + &f.modify, + .Xchg, + .storedXor(.freed, &f.modify), + .monotonic, + ); + + const prev_count = @atomicRmw( + Bucket.AllocCount, + &b.alloc_count, + .Sub, + .{ .filling = false, .n = 1 }, + .acq_rel, + ); + + if (prev_count.n - 1 == 0 and !prev_count.filling) { + @branchHint(.unlikely); + const t = s.acquireThread(); + freeAllocEntry(t, b.entry); + t.mutex.unlock(); + b.check(s); + s.backing.rawFree(b.bytes(s), @enumFromInt(s.bucket_size_log2), ra); + } +} + +fn resize(ctx: *anyopaque, memory: []u8, alignment: Alignment, new_len: usize, ra: usize) bool { + assert(new_len != 0); + + const s: *SafeAllocator = @ptrCast(@alignCast(ctx)); + const f: *AllocFooter = .of(memory); + f.startModify(.resized, s, .{ .memory = memory, .alignment = alignment }); + + // Check that the allocation is not moving between a bucket and large allocation. This is + // done after the above so that it is still checked that valid memory is passed and there + // is no double modify. + const from_large_alloc = s.isLarge(memory.len, alignment); + const to_large_alloc = s.isLargeOrOom(new_len, alignment) catch { + f.modify.setNone(); + return false; + }; + if (from_large_alloc != to_large_alloc) { + @branchHint(.unlikely); + f.modify.setNone(); + return false; + } + + if (from_large_alloc) { + @branchHint(.unlikely); + + const entry = f.extended().container.large_entry; + const new_alloc_len = AllocFooter.allocLenLarge(s, new_len); + if (!s.backing.rawResize( + memory.ptr[0..AllocFooter.allocLenLarge(s, memory.len)], + AllocFooter.allocAlign(alignment), + new_alloc_len, + ra, + )) { + f.modify.setNone(); + return false; + } + + const new_footer = AllocFooter.populate( + @alignCast(memory.ptr[0..new_alloc_len]), + new_len, + alignment, + ra, + true, + false, + .{ .large_entry = entry }, + s, + ); + assert(entry.kind == .large_alloc); + entry.* = .fromLargeAlloc(new_footer); + return true; + } + + if (new_len < memory.len) { + // Resize shrinks are disallowed in all cases since the linked list would be broken. Even + // if this footer is the final one, the fill value would need decreased which would allow + // memory to be reused. + f.modify.setNone(); + return false; + } + + if (s.growingResizeBucket(f, memory, alignment, new_len, ra)) { + @branchHint(.likely); + return true; + } else { + f.modify.setNone(); + return false; + } +} + +fn remap(ctx: *anyopaque, memory: []u8, alignment: Alignment, new_len: usize, ra: usize) ?[*]u8 { + assert(new_len != 0); + + const s: *SafeAllocator = @ptrCast(@alignCast(ctx)); + const f: *AllocFooter = .of(memory); + f.startModify(.remaped, s, .{ .memory = memory, .alignment = alignment }); + + // Check that the allocation is not moving between a bucket and large allocation. This is + // done after the above so that it is still checked that valid memory is passed and there + // is no double modify. + const from_large_alloc = s.isLarge(memory.len, alignment); + const to_large_alloc = s.isLargeOrOom(new_len, alignment) catch { + f.modify.setNone(); + return null; + }; + if (from_large_alloc != to_large_alloc) { + @branchHint(.unlikely); + f.modify.setNone(); + return null; + } + + if (from_large_alloc) { + @branchHint(.unlikely); + + const entry = f.extended().container.large_entry; + const new_alloc_len = AllocFooter.allocLenLarge(s, new_len); + const new_memory = s.backing.rawRemap( + memory.ptr[0..AllocFooter.allocLenLarge(s, memory.len)], + AllocFooter.allocAlign(alignment), + new_alloc_len, + ra, + ) orelse { + f.modify.setNone(); + return null; + }; + + const new_footer = AllocFooter.populate( + @alignCast(new_memory[0..new_alloc_len]), + new_len, + alignment, + ra, + true, + false, + .{ .large_entry = entry }, + s, + ); + assert(entry.kind == .large_alloc); + entry.* = .fromLargeAlloc(new_footer); + return new_memory; + } + + if (new_len < memory.len) { + // Move the allocation forward to avoid bucket reuse + + const fixed_start = Bucket.fillAt(s, @ptrCast(f)) - AllocFooter.allocOffset(new_len); + const moved_start = alignment.forward(fixed_start); + if (moved_start != fixed_start or !f.isExtended()) { + @branchHint(.unlikely); + // For `moved_start != fixed_start`: the footer needs moved forward as well to + // maintain the correct allocOffset. + // + // For `!f.isExtended()`: since the memory will no longer be directly after the + // previous footer, the footer needs promoted to an extended one to encode the + // location of the previous footer. + if (!s.advanceBucketAlloc(f, @intCast(moved_start), true, alignment, new_len, ra)) { + @branchHint(.unlikely); + f.modify.setNone(); + return null; + } + const new_memory = Bucket.bytes(.of(s, @ptrCast(f)), s)[moved_start..][0..new_len]; + @memmove(new_memory, memory[0..new_memory.len]); + return new_memory.ptr; + } + + // The footer can be modified in place + const b: *Bucket = .of(s, @ptrCast(f)); + f.extended().len = new_len; + f.checksum = f.actualChecksum(s) ^ s.canary; + captureStackTrace(f.allocTrace(s), ra); + + f.modify.setNone(); + b.alloc_count.fenceAcqRel(); + + const new_memory = f.userMemory(); + @memmove(new_memory, memory[0..new_memory.len]); + return new_memory.ptr; + } + + if (s.growingResizeBucket(f, memory, alignment, new_len, ra)) { + @branchHint(.likely); + return memory.ptr; + } else { + f.modify.setNone(); + return null; + } +} + +const Smith = std.testing.Smith; + +/// Shared between single-threaded and multi-threaded fuzzing. +const fuzz_probs = struct { + const alignment: []const Smith.Weight = &.{ + .rangeAtMost(Alignment, .@"1", .@"16", 32), // ~75% + .rangeAtMost(Alignment, .@"16", @enumFromInt(@bitSizeOf(usize) - 1), 1), + .value(Alignment, @enumFromInt(@bitSizeOf(usize) - 1), 32), // More likely overflow cases + }; + + const eos: []const Smith.Weight = &.{ + // Very high false weight so that expanding allocation tables, OOM cases, + // and multi-threaded consumer-producer cases get tested thoroughly. + .value(bool, false, 255), + .value(bool, true, 1), + }; + + fn generateOptions(smith: *Smith) Options { + @disableInstrumentation(); + + const size_log2_weights: []const Smith.Weight = &.{ + .value(u5, 8, 1024), // 8x odds of below + .rangeAtMost(u5, 8, 16, 16), + .rangeAtMost(u5, 17, 31, 1), // 1/32 odds of above since these just OOM with the fixed buffer + }; + return .{ + .stack_trace_frames = smith.valueWeighted(u16, &.{ + .value(u16, 0, 1 << 18), // 4x - stack traces have no tested properties except I.B. + .rangeAtMost(u16, 0, math.maxInt(u16), 1), + }), + // If set, it is aimed to allocate much fewer bytes since freeing becomes O(n). + // Without this, it is O(1) since mem.Allocator is bypassed so there is no memsets + // of the data. + .check_write_after_free = smith.valueWeighted(bool, &.{ + .value(bool, false, 31), + .value(bool, false, 1), + }), + .canary = smith.value(u32), + + .block_size_log2 = smith.valueWeighted(u5, size_log2_weights), + .bucket_size_log2 = smith.valueWeighted(u5, size_log2_weights), + }; + } + + const Op = enum(u8) { alloc, free, resize, remap }; + fn generateOp(smith: *Smith, any_allocs: bool) Op { + @disableInstrumentation(); + return if (any_allocs) smith.valueWeighted(Op, &.{ + .rangeAtMost(Op, .alloc, .free, 4), + .rangeAtMost(Op, .resize, .remap, 1), + }) else .alloc; + } + + fn generateSplat(smith: *Smith) ?u8 { + @disableInstrumentation(); + + // Same rationale for `check_write_after_free` + const n = smith.valueWeighted(u16, &.{ + .value(u16, 256, 256 * 31), + .rangeAtMost(u16, 0, 255, 1), + }); + return if (n == 256) null else @intCast(n); + } + + fn generateLen(smith: *Smith, will_memset: bool) usize { + @disableInstrumentation(); + + // 1 << 24 indicates to generate an unweighted usize. + // 1 << 25 indicates to provide a value relative to the maximum usize. + const len = smith.valueWeightedWithHash( + u32, + if (!will_memset) comptime &.{ + // zig fmt: off + .rangeLessThan(u32, 1 , 1 << 6 , 1 << 15), // 2^21 - 2^4 times below so 16x odds + .rangeLessThan(u32, 1 << 6, 1 << 17, 1 ), // 2^17 - 2^4 times below so 16x odds + .value (u32, 1 << 24, 1 << 12), // 2^12 + .value (u32, 1 << 25, 1 << 12), // 2^12 + // zig fmt: on + } else comptime &.{ + // zig fmt: off + .rangeLessThan(u32, 1 , 1 << 6, 1 << 17), // 2^23 - 2^6 times below so 64x odds + .rangeLessThan(u32, 1 << 6, 1 << 17, 1 ), // 2^17 - 2^6 times below so 64x odds + .value (u32, 1 << 24, 1 << 10), // 2^10 + .value (u32, 1 << 25, 1 << 10), // 2^10 + // zig fmt: on + }, + // Give the fuzzer different hashes when the weights used differ + // so that it does not reuse values from other probabilities. + if (!will_memset) 0x38a74424 else 0xec581ff0, + ); + + if (len == 1 << 24) return @max(1, smith.value(usize)); + if (len == 1 << 25) return @as(usize, math.maxInt(usize)) - smith.value(u16); + return len; + } + + fn checkSplat(splat: ?u8, bytes: []const u8) void { + @disableInstrumentation(); + + const byte = splat orelse return; + for (bytes) |*b| if (b.* != byte) { + panic("SafeAllocator corrupted allocation data at *{x}", .{@intFromPtr(b)}); + }; + } +}; + +test "fuzz single threaded" { + // This single threaded fuzz test has the following advantages: + // * Higher throughput and deterministic, which helps the fuzzer. + // * Easier debugging of single-threaded reproducable bugs. + const testing_buf = try std.testing.allocator.alloc(u8, 65536); + defer std.testing.allocator.free(testing_buf); + const backing_buf = try std.testing.allocator.alloc(u8, 1 << 17); + defer std.testing.allocator.free(backing_buf); + try std.testing.fuzz(FuzzSingleThreadedContext{ + .testing_buf = testing_buf, + .backing_buf = backing_buf, + }, fuzzSingleThreaded, .{}); +} + +const FuzzSingleThreadedContext = struct { + testing_buf: []u8, + backing_buf: []u8, +}; + +/// Guarantees memory will not be reused. +const FuzzSingleThreadedAllocator = struct { + gpa: mem.Allocator, + smith: *std.testing.Smith, + + buf: []u8, + fill: usize, + allocs: std.MultiArrayList(AllocInfo), + + const AllocInfo = struct { + ptr: [*]u8, + len: usize, + alignment: Alignment, + }; + + fn allocator(f: *FuzzSingleThreadedAllocator) mem.Allocator { + @disableInstrumentation(); + return .{ .ptr = f, .vtable = &.{ + .alloc = FuzzSingleThreadedAllocator.alloc, + .free = FuzzSingleThreadedAllocator.free, + .resize = FuzzSingleThreadedAllocator.resize, + .remap = FuzzSingleThreadedAllocator.remap, + } }; + } + + fn alloc(ctx: *anyopaque, len: usize, alignment: Alignment, _: usize) ?[*]u8 { + @disableInstrumentation(); + + const f: *FuzzSingleThreadedAllocator = @ptrCast(@alignCast(ctx)); + f.allocs.ensureUnusedCapacity(f.gpa, 1) catch return null; + + const ptr = f.allocInner(len, alignment) orelse return null; + f.allocs.appendAssumeCapacity(.{ + .ptr = ptr, + .len = len, + .alignment = alignment, + }); + return ptr; + } + + fn allocInner(f: *FuzzSingleThreadedAllocator, len: usize, alignment: Alignment) ?[*]u8 { + @disableInstrumentation(); + + const start_addr = alignment.forward(@intFromPtr(f.buf[f.fill..].ptr)); + const start = @as([*]u8, @ptrFromInt(start_addr)) - f.buf.ptr; + if (start +| len > f.buf.len or f.smith.boolWeighted(31, 1)) return null; + f.fill = start + len; + return f.buf[start..][0..len].ptr; + } + + fn allocIndex(f: *FuzzSingleThreadedAllocator, memory: []u8, alignment: Alignment) usize { + @disableInstrumentation(); + + const allocs_slice = f.allocs.slice(); + const i = mem.indexOfScalar([*]u8, allocs_slice.items(.ptr), memory.ptr) orelse panic( + "invalid SafeAllocator free of {f}", + .{FormatMemory{ .memory = memory, .alignment = alignment }}, + ); + const expected_len = allocs_slice.items(.len)[i]; + const expected_align = allocs_slice.items(.alignment)[i]; + if (memory.len != expected_len or allocs_slice.items(.alignment)[i] != expected_align) { + panic("SafeAllocator free {f} mismatches alloc {f}", .{ + FormatMemory{ .memory = memory, .alignment = alignment }, + FormatMemory{ .memory = memory.ptr[0..expected_len], .alignment = expected_align }, + }); + } + return i; + } + + fn free(ctx: *anyopaque, memory: []u8, alignment: Alignment, _: usize) void { + @disableInstrumentation(); + + const f: *FuzzSingleThreadedAllocator = @ptrCast(@alignCast(ctx)); + f.allocs.swapRemove(f.allocIndex(memory, alignment)); + } + + fn resize(ctx: *anyopaque, memory: []u8, alignment: Alignment, new_len: usize, _: usize) bool { + @disableInstrumentation(); + + const f: *FuzzSingleThreadedAllocator = @ptrCast(@alignCast(ctx)); + const i = f.allocIndex(memory, alignment); + + const start = memory.ptr - f.buf.ptr; + const old_end = start + memory.len; + const new_end = start +| new_len; + if (new_end > f.buf.len or f.smith.value(bool)) { + return false; + } + + if (new_len <= memory.len) { + // The fill is not decreased so memory is not reused. + } else if (f.fill == old_end) { + f.fill = new_end; + } else { + return false; + } + f.allocs.items(.len)[i] = new_len; + return true; + } + + fn remap(ctx: *anyopaque, memory: []u8, alignment: Alignment, new_len: usize, _: usize) ?[*]u8 { + @disableInstrumentation(); + + const f: *FuzzSingleThreadedAllocator = @ptrCast(@alignCast(ctx)); + if (f.smith.value(bool)) { + const resized = FuzzSingleThreadedAllocator.resize( + ctx, + memory, + alignment, + new_len, + undefined, + ); + return if (resized) memory.ptr else null; + } + + const i = f.allocIndex(memory, alignment); + if (f.smith.value(bool)) return null; + + const new_ptr = f.allocInner(new_len, alignment) orelse return null; + const copy_len = @min(memory.len, new_len); + @memcpy(new_ptr[0..copy_len], memory[0..copy_len]); + + f.allocs.set(i, .{ + .ptr = new_ptr, + .len = new_len, + .alignment = alignment, + }); + return new_ptr; + } +}; + +fn fuzzSingleThreaded(ctx: FuzzSingleThreadedContext, smith: *Smith) !void { + @disableInstrumentation(); + + var gpa_instance: std.heap.FixedBufferAllocator = .init(ctx.testing_buf); + const gpa = gpa_instance.allocator(); + var backing_gpa_instance: FuzzSingleThreadedAllocator = .{ + .gpa = gpa, + .smith = smith, + + .buf = ctx.backing_buf, + .fill = 0, + .allocs = .empty, + }; + const backing_gpa = backing_gpa_instance.allocator(); + + const options = fuzz_probs.generateOptions(smith); + var s: SafeAllocator = .init(backing_gpa, options); + const no_ra: usize = 0; + + var allocs: std.MultiArrayList(struct { + memory: []u8, + alignment: Alignment, + splat: ?u8, + }) = .empty; + var used_memory: std.ArrayList(struct { + start: usize, + end: usize, + }) = .empty; + + while (!smith.eosWeighted(fuzz_probs.eos)) { + const op = fuzz_probs.generateOp(smith, allocs.len != 0); + const new_mem: []const u8, const old_mem: ?[]const u8 = new_alloc: switch (op) { + .alloc => { + used_memory.ensureUnusedCapacity(gpa, 1) catch break; + allocs.ensureUnusedCapacity(gpa, 1) catch break; + + const splat = fuzz_probs.generateSplat(smith); + const will_memset = options.check_write_after_free or splat != null; + const len = fuzz_probs.generateLen(smith, will_memset); + const alignment = smith.valueWeighted(Alignment, fuzz_probs.alignment); + + const ptr = alloc(&s, len, alignment, no_ra) orelse continue; + if (!alignment.check(@intFromPtr(ptr))) @panic("bad returned alignment"); + const memory = ptr[0..len]; + if (splat) |b| @memset(memory, b); + + allocs.appendAssumeCapacity(.{ + .memory = memory, + .alignment = alignment, + .splat = splat, + }); + break :new_alloc .{ memory, null }; + }, + .free => { + const i = smith.valueRangeLessThan(u32, 0, @intCast(allocs.len)); + const alloc_info = allocs.get(i); + allocs.swapRemove(i); + + fuzz_probs.checkSplat(alloc_info.splat, alloc_info.memory); + free(&s, alloc_info.memory, alloc_info.alignment, no_ra); + continue; + }, + .resize => { + used_memory.ensureUnusedCapacity(gpa, 1) catch break; + const i = smith.valueRangeLessThan(u32, 0, @intCast(allocs.len)); + const allocs_slice = allocs.slice(); + + const prev_alloc = allocs_slice.get(i); + const old_len = prev_alloc.memory.len; + + const alloc_memory = &allocs_slice.items(.memory)[i]; + const splat = prev_alloc.splat; + const will_memset = options.check_write_after_free or splat != null; + + const new_len = fuzz_probs.generateLen(smith, will_memset); + if (!resize(&s, prev_alloc.memory, prev_alloc.alignment, new_len, no_ra)) { + fuzz_probs.checkSplat(prev_alloc.splat, prev_alloc.memory); + continue; + } + alloc_memory.len = new_len; + + fuzz_probs.checkSplat(prev_alloc.splat, alloc_memory.*[0..@min(old_len, new_len)]); + if (splat) |b| @memset(alloc_memory.*[@min(old_len, new_len)..], b); + + break :new_alloc .{ alloc_memory.*, prev_alloc.memory }; + }, + .remap => { + used_memory.ensureUnusedCapacity(gpa, 1) catch break; + const i = smith.valueRangeLessThan(u32, 0, @intCast(allocs.len)); + const allocs_slice = allocs.slice(); + + const prev_alloc = allocs_slice.get(i); + const old_len = prev_alloc.memory.len; + + const alloc_memory = &allocs_slice.items(.memory)[i]; + const alignment = prev_alloc.alignment; + const splat = prev_alloc.splat; + const will_memset = options.check_write_after_free or splat != null; + + const new_len = fuzz_probs.generateLen(smith, will_memset); + const new_ptr = remap( + &s, + prev_alloc.memory, + prev_alloc.alignment, + new_len, + no_ra, + ) orelse { + fuzz_probs.checkSplat(prev_alloc.splat, prev_alloc.memory); + continue; + }; + alloc_memory.* = new_ptr[0..new_len]; + + if (!alignment.check(@intFromPtr(new_ptr))) @panic("bad returned alignment"); + fuzz_probs.checkSplat(prev_alloc.splat, alloc_memory.*[0..@min(old_len, new_len)]); + if (splat) |b| @memset(alloc_memory.*[@min(old_len, new_len)..], b); + + break :new_alloc .{ alloc_memory.*, prev_alloc.memory }; + }, + }; + + const new_start = @intFromPtr(new_mem.ptr); + const new_end = new_start + new_mem.len; + const old_start = if (old_mem) |old| @intFromPtr(old.ptr) else 0; + const old_end = new_start + if (old_mem) |old| old.len else 0; + for (used_memory.items) |used| { + if (old_start <= used.end and used.start <= old_end) { + continue; + } + if (new_start <= used.end and used.start <= new_end) { + panic( + "memory reuse between [addr: {x}, len: {}] and new [addr: {x}, len: {}]", + .{ used.start, used.end, new_start, new_end }, + ); + } + } + used_memory.appendAssumeCapacity(.{ .start = new_start, .end = new_end }); + } + + try std.testing.expectEqual(allocs.len, s.deinitLog(false)); + const leaks_slice = backing_gpa_instance.allocs.slice(); + for (0..leaks_slice.len) |i| { + const leak = leaks_slice.get(i); + std.log.err("SafeAllocator leaked {f}", .{FormatMemory{ + .memory = leak.ptr[0..leak.len], + .alignment = leak.alignment, + }}); + } + try std.testing.expectEqual(0, leaks_slice.len); // no leaks +} + +test "fuzz multi threaded" { + if (@import("builtin").single_threaded) return error.SkipZigTest; + + const testing_buf = try std.testing.allocator.alloc(u8, 65536); + defer std.testing.allocator.free(testing_buf); + const backing_buf = try std.testing.allocator.alloc(u8, 1 << 17); + defer std.testing.allocator.free(backing_buf); + + // `std.testing` instances are overwritten during `std.testing.fuzz` so + // it is necessary to use our own io and gpa instances. + var threaded_io: std.Io.Threaded = .init(std.heap.smp_allocator, .{}); + defer threaded_io.deinit(); + const io = threaded_io.io(); + + var ops: FuzzMultiThreadedContext.ThreadOps = undefined; + ops.run = .{ .n = false }; + var group: std.Io.Group = .init; + defer group.cancel(io); + for (0..FuzzMultiThreadedContext.n_threads) |_| { + try group.concurrent(io, fuzzMultiThreadedWorker, .{ io, &ops }); + } + + try std.testing.fuzz(FuzzMultiThreadedContext{ + .testing_buf = testing_buf, + .backing_buf = backing_buf, + + .io = io, + .ops = &ops, + }, fuzzMultiThreaded, .{}); +} + +const FuzzMultiThreadedContext = struct { + testing_buf: []u8, + backing_buf: []u8, + + io: std.Io, + ops: *ThreadOps, + + const n_threads = 4; + + const ThreadOps = struct { + /// Switches between two values for each time a run starts. + run: Run, + /// While this can be calculated as `n_threads - (i -| ops.items.len)`, + /// this also serves as `.release` synchronization for each thread. + running: u32, + + instance: SafeAllocator, + i: usize, + items: []Op, + + const Run = packed struct(u32) { + n: bool, + pad: u31 = 0, + + fn wait(ptr: *Run, val: Run, io: std.Io) error{Canceled}!void { + assert(val.pad == 0); + while (true) { + // This cannot load a previous value since this thread previously loaded the + // latest value. + const prev = @atomicLoad(Run, ptr, .acquire); + assert(prev.pad == 0); + if (prev == val) break; + + try io.futexWait(Run, ptr, prev); + } + } + + fn next(r: Run) Run { + assert(r.pad == 0); + return .{ .n = !r.n }; + } + }; + + const Op = union(fuzz_probs.Op) { + alloc: struct { + len: usize, + alignment: Alignment, + + splat: ?u8, + /// Not embeded directly in the struct as a workaround for tsan since a + /// switch directly on `Op` loads the entire value non-atomically. + result: *MemoryDependency, + }, + free: struct { + memory: *MemoryDependency, + alignment: Alignment, + + splat: ?u8, + }, + resize: Realloc, + remap: Realloc, + + const Realloc = struct { + memory: *MemoryDependency, + alignment: Alignment, + new_len: usize, + + splat: ?u8, + /// Not embeded directly in the struct as a workaround for tsan since a + /// switch directly on `Op` loads the entire value non-atomically. + result: *MemoryDependency, + }; + + const MemoryDependency = struct { + ready: std.Io.Event, + /// Null if the memory failed to be allocated + memory: ?[]u8, + + const init: MemoryDependency = .{ + .ready = .unset, + .memory = undefined, + }; + + fn get(dep: *MemoryDependency, io: std.Io) ?[]u8 { + dep.ready.waitUncancelable(io); + return dep.memory; + } + }; + }; + }; +}; + +/// Guarantees memory will not be reused. +const FuzzMultiThreadedAllocator = struct { + gpa: mem.Allocator, + + fill: usize, + active_allocs: usize, + fail_i: usize, + fixed_remap_i: usize, + + // The below are assumed to be externally synchronized + // i.e. each thread has an acquire fence before **first** using the allocator + buf: []u8, + fails: []const bool, + fixed_remaps: []const bool, + + fn allocator(f: *FuzzMultiThreadedAllocator) mem.Allocator { + @disableInstrumentation(); + return .{ .ptr = f, .vtable = &.{ + .alloc = FuzzMultiThreadedAllocator.alloc, + .free = FuzzMultiThreadedAllocator.free, + .resize = FuzzMultiThreadedAllocator.resize, + .remap = FuzzMultiThreadedAllocator.remap, + } }; + } + + fn maybeFail(f: *FuzzMultiThreadedAllocator) bool { + @disableInstrumentation(); + const i = @atomicRmw(usize, &f.fail_i, .Add, 1, .monotonic); + return i < f.fails.len and f.fails[i]; + } + + fn maybeFixedRemap(f: *FuzzMultiThreadedAllocator) bool { + @disableInstrumentation(); + const i = @atomicRmw(usize, &f.fixed_remap_i, .Add, 1, .monotonic); + return i < f.fixed_remaps.len and f.fixed_remaps[i]; + } + + fn alloc(ctx: *anyopaque, len: usize, alignment: Alignment, _: usize) ?[*]u8 { + @disableInstrumentation(); + + const f: *FuzzMultiThreadedAllocator = @ptrCast(@alignCast(ctx)); + const memory = f.allocInner(len, alignment) orelse return null; + _ = @atomicRmw(usize, &f.active_allocs, .Add, 1, .monotonic); + return memory; + } + + fn allocInner(f: *FuzzMultiThreadedAllocator, len: usize, alignment: Alignment) ?[*]u8 { + var prev_fill = @atomicLoad(usize, &f.fill, .monotonic); + var start: usize = undefined; + while (true) { + const start_addr = alignment.forward(@intFromPtr(f.buf[prev_fill..].ptr)); + start = @as([*]u8, @ptrFromInt(start_addr)) - f.buf.ptr; + if (start +| len > f.buf.len or f.maybeFail()) return null; + prev_fill = @cmpxchgStrong( + usize, + &f.fill, + prev_fill, + start + len, + .monotonic, + .monotonic, + ) orelse { + @branchHint(.likely); + break; + }; + } + return f.buf[start..][0..len].ptr; + } + + fn free(ctx: *anyopaque, _: []u8, _: Alignment, _: usize) void { + @disableInstrumentation(); + + const f: *FuzzMultiThreadedAllocator = @ptrCast(@alignCast(ctx)); + assert(@atomicRmw(usize, &f.active_allocs, .Sub, 1, .monotonic) != 0); + } + + fn resize(ctx: *anyopaque, memory: []u8, _: Alignment, new_len: usize, _: usize) bool { + @disableInstrumentation(); + + const f: *FuzzMultiThreadedAllocator = @ptrCast(@alignCast(ctx)); + const start = memory.ptr - f.buf.ptr; + const old_end = start + memory.len; + const new_end = start +| new_len; + if (new_end > f.buf.len or f.maybeFail()) { + return false; + } + + if (new_len <= memory.len) { + // The fill is not decreased so memory is not reused. + return true; + } + + return @cmpxchgStrong(usize, &f.fill, old_end, new_end, .monotonic, .monotonic) == null; + } + + fn remap(ctx: *anyopaque, memory: []u8, alignment: Alignment, new_len: usize, _: usize) ?[*]u8 { + @disableInstrumentation(); + + if (maybeFixedRemap(@ptrCast(@alignCast(ctx)))) { + const resized = FuzzMultiThreadedAllocator.resize( + ctx, + memory, + alignment, + new_len, + undefined, + ); + return if (resized) memory.ptr else null; + } + + const f: *FuzzMultiThreadedAllocator = @ptrCast(@alignCast(ctx)); + const new_ptr = f.allocInner(new_len, alignment) orelse return null; + const copy_len = @min(memory.len, new_len); + @memcpy(new_ptr[0..copy_len], memory[0..copy_len]); + return new_ptr; + } +}; + +fn fuzzMultiThreaded(ctx: FuzzMultiThreadedContext, smith: *Smith) !void { + @disableInstrumentation(); + + var gpa_instance: std.heap.FixedBufferAllocator = .init(ctx.testing_buf); + const gpa = gpa_instance.allocator(); + + var op_count: u32 = 0; + while (!smith.eosWeighted(fuzz_probs.eos)) op_count += 1; + const Op = FuzzMultiThreadedContext.ThreadOps.Op; + const ops = gpa.alloc(Op, op_count) catch return error.SkipZigTest; + const op_results = gpa.alloc(Op.MemoryDependency, op_count) catch return error.SkipZigTest; + @memset(op_results, .init); + + const allocs = gpa.alloc(struct { + memory: *FuzzMultiThreadedContext.ThreadOps.Op.MemoryDependency, + alignment: Alignment, + splat: ?u8, + }, op_count) catch return error.SkipZigTest; + var allocs_n: u32 = 0; + var expected_remaps: usize = 0; + + const options = fuzz_probs.generateOptions(smith); + for (ops, op_results) |*op, *result| switch (fuzz_probs.generateOp(smith, allocs_n != 0)) { + .alloc => { + const splat = fuzz_probs.generateSplat(smith); + const will_memset = options.check_write_after_free or splat != null; + op.* = .{ .alloc = .{ + .len = fuzz_probs.generateLen(smith, will_memset), + .alignment = smith.valueWeighted(Alignment, fuzz_probs.alignment), + + .splat = splat, + .result = result, + } }; + allocs[allocs_n] = .{ + .memory = result, + .alignment = op.alloc.alignment, + .splat = splat, + }; + allocs_n += 1; + }, + .free => { + const i = smith.valueRangeLessThan(u32, 0, allocs_n); + op.* = .{ .free = .{ + .memory = allocs[i].memory, + .alignment = allocs[i].alignment, + + .splat = allocs[i].splat, + } }; + + allocs_n -= 1; + allocs[i] = allocs[allocs_n]; + }, + .resize, .remap => |kind| { + op.* = switch (kind) { + .remap => .{ .remap = undefined }, + .resize => .{ .resize = undefined }, + else => unreachable, + }; + const realloc = switch (kind) { + .remap => &op.remap, + .resize => &op.resize, + else => unreachable, + }; + expected_remaps += @intFromBool(kind == .remap); + + const i = smith.valueRangeLessThan(u32, 0, allocs_n); + realloc.* = .{ + .memory = allocs[i].memory, + .alignment = allocs[i].alignment, + .new_len = fuzz_probs.generateLen(smith, options.check_write_after_free), + + .splat = allocs[i].splat, + .result = result, + }; + allocs[i].memory = result; + }, + }; + + const fails: []bool = gpa.alloc(bool, ops.len * 2 + smith.value(u8)) catch &.{}; + const fixed_remaps: []bool = gpa.alloc(bool, expected_remaps + smith.value(u8)) catch &.{}; + for (fails) |*f| f.* = smith.boolWeighted(31, 1); + for (fixed_remaps) |*f| f.* = smith.value(bool); + var backing_gpa_instance: FuzzMultiThreadedAllocator = .{ + .gpa = gpa, + + .fill = 0, + .active_allocs = 0, + .fail_i = 0, + .fixed_remap_i = 0, + + .buf = ctx.backing_buf, + .fails = fails, + .fixed_remaps = fixed_remaps, + }; + const backing_gpa = backing_gpa_instance.allocator(); + + ctx.ops.instance = .init(backing_gpa, options); + ctx.ops.i = 0; + ctx.ops.items = ops; + + ctx.ops.running = FuzzMultiThreadedContext.n_threads; + // Loading `ctx.ops.run` non-atomically is fine since this is the only thread that writes to it. + @atomicStore(FuzzMultiThreadedContext.ThreadOps.Run, &ctx.ops.run, ctx.ops.run.next(), .release); + ctx.io.futexWake(FuzzMultiThreadedContext.ThreadOps.Run, &ctx.ops.run, math.maxInt(u32)); + while (true) { + const prev_running = @atomicLoad(u32, &ctx.ops.running, .acquire); + if (prev_running == 0) break; + ctx.io.futexWaitUncancelable(u32, &ctx.ops.running, prev_running); + } + + var expected_allocs = allocs_n; + for (allocs[0..allocs_n]) |a| { + expected_allocs -= @intFromBool(a.memory.memory == null); + } + try std.testing.expectEqual(expected_allocs, ctx.ops.instance.deinitLog(false)); + try std.testing.expectEqual(0, backing_gpa_instance.active_allocs); // no leaks +} + +fn fuzzMultiThreadedWorker( + io: std.Io, + ops: *FuzzMultiThreadedContext.ThreadOps, +) error{Canceled}!void { + const no_ra: usize = 0; + var next_run: FuzzMultiThreadedContext.ThreadOps.Run = .{ .n = true }; + while (true) { + try ops.run.wait(next_run, io); + next_run = .next(next_run); + + while (true) { + const i = @atomicRmw(usize, &ops.i, .Add, 1, .monotonic); + if (i >= ops.items.len) { + // `.acq_rel` is necessary since acquire loads only synchronize with the thread + // which the read value was written from, not all previous writer threads. + const prev_rem = @atomicRmw(u32, &ops.running, .Sub, 1, .acq_rel); + if (prev_rem - 1 == 0) { + io.futexWake(u32, &ops.running, 1); + } + break; + } + + switch (ops.items[i]) { + .alloc => |call| { + const alloc_ptr = alloc(&ops.instance, call.len, call.alignment, no_ra); + if (alloc_ptr) |memory_ptr| { + const memory = memory_ptr[0..call.len]; + if (call.splat) |b| @memset(memory, b); + call.result.memory = memory; + } else { + call.result.memory = null; + } + call.result.ready.set(io); + }, + .free => |call| { + const memory = call.memory.get(io) orelse continue; + fuzz_probs.checkSplat(call.splat, memory); + free(&ops.instance, memory, call.alignment, no_ra); + }, + .resize, .remap => |call, kind| { + const memory = call.memory.get(io) orelse { + call.result.memory = null; + call.result.ready.set(io); + continue; + }; + const new_memory: []u8 = switch (kind) { + .remap => if (remap( + &ops.instance, + memory, + call.alignment, + call.new_len, + no_ra, + )) |new_ptr| new_ptr[0..call.new_len] else memory, + .resize => if (resize( + &ops.instance, + memory, + call.alignment, + call.new_len, + no_ra, + )) memory.ptr[0..call.new_len] else memory, + else => unreachable, + }; + + const old_len = memory.len; + const new_len = new_memory.len; + fuzz_probs.checkSplat(call.splat, new_memory[0..@min(old_len, new_len)]); + if (call.splat) |b| @memset(new_memory[@min(old_len, new_len)..], b); + + call.result.memory = new_memory; + call.result.ready.set(io); + }, + } + } + } +} diff --git a/lib/std/mem/Allocator.zig b/lib/std/mem/Allocator.zig index ec3426146cc0b93f1ea9b7e4a364cc61ab6d641f..e348e02921c1e345f1a2451072e9c25eea96cc05 100644 --- a/lib/std/mem/Allocator.zig +++ b/lib/std/mem/Allocator.zig @@ -23,6 +23,8 @@ pub const VTable = struct { /// Return a pointer to `len` bytes with specified `alignment`, or return /// `null` indicating the allocation failed. /// + /// `new_len` must be greater than zero. + /// /// `ret_addr` is optionally provided as the first return address of the /// allocation call stack. If the value is `0` it means no return address /// has been provided. -- 2.54.0 From e1ce81eb546e10ae505b411b6882266d1834fce1 Mon Sep 17 00:00:00 2001 From: Kendall Condon Date: Mon, 20 Apr 2026 17:43:49 -0400 Subject: [PATCH 2/4] link.Elf: fix large abi aligned globals --- src/link/Elf/ZigObject.zig | 2 +- src/link/Elf2.zig | 2 +- test/behavior/align.zig | 9 +++++++++ 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/link/Elf/ZigObject.zig b/src/link/Elf/ZigObject.zig index 4832a46accb4c33170019f1be354fa208baec830..da3b4678f282eccdd36b9f8e480e88391ad00f42 100644 --- a/src/link/Elf/ZigObject.zig +++ b/src/link/Elf/ZigObject.zig @@ -1386,7 +1386,7 @@ fn updateNavCode( .none => switch (mod.optimize_mode) { .Debug, .ReleaseSafe, .ReleaseFast => target_util.defaultFunctionAlignment(target), .ReleaseSmall => target_util.minFunctionAlignment(target), - }, + }.maxStrict(Type.fromInterned(nav.resolved.?.type).abiAlignment(zcu)), else => |a| a.maxStrict(target_util.minFunctionAlignment(target)), }; diff --git a/src/link/Elf2.zig b/src/link/Elf2.zig index 46a5106046785aa1f1c9b0de8617d3b20b9ecebc..f7e7aa69fd375f57dfbf56ec0f4379e9e0f3bd0d 100644 --- a/src/link/Elf2.zig +++ b/src/link/Elf2.zig @@ -2863,7 +2863,7 @@ fn navMapIndex(elf: *Elf, zcu: *Zcu, nav_index: InternPool.Nav.Index) !Node.NavM .ReleaseFast, => target_util.defaultFunctionAlignment(target), .ReleaseSmall => min, - }, + }.maxStrict(Type.fromInterned(nav.resolved.?.type).abiAlignment(zcu)), }; }, else => switch (nav.resolved.?.@"align") { diff --git a/test/behavior/align.zig b/test/behavior/align.zig index 0bc7195d3999f9a2cc04e9923a960bec79317b74..3ade0f6164d08e6e57c7f33ecbe9fed48b232b83 100644 --- a/test/behavior/align.zig +++ b/test/behavior/align.zig @@ -15,6 +15,15 @@ test "global variable alignment" { } } +test "large abi alignment of global" { + const S = struct { + var global: @This() = undefined; + x: u64 align(64), + }; + + try std.testing.expect(@ctz(@intFromPtr(&S.global)) >= 6); +} + test "large alignment of local constant" { if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest; // flaky -- 2.54.0 From c805c7228978df4af34ee0d63948234fb0ab504e Mon Sep 17 00:00:00 2001 From: Kendall Condon Date: Mon, 20 Apr 2026 17:44:52 -0400 Subject: [PATCH 3/4] upgrade most uses of DebugAllocator to SafeAllocator --- build.zig | 2 +- lib/compiler/build_runner.zig | 6 +++--- lib/compiler/test_runner.zig | 38 +++++++++++++++++++++------------ lib/fuzzer.zig | 6 +++--- lib/std/heap/ArenaAllocator.zig | 8 +++---- lib/std/mem.zig | 18 ++++++++++------ lib/std/start.zig | 15 ++++++------- lib/std/testing.zig | 18 +++++----------- src/main.zig | 37 ++++++++++++++------------------ 9 files changed, 74 insertions(+), 74 deletions(-) diff --git a/build.zig b/build.zig index 3c4f56b4ed159167fbf052eeccfea210ecdfcc05..fcbf1a46dc6df7d5dc2aa958488b559f0e891516 100644 --- a/build.zig +++ b/build.zig @@ -183,7 +183,7 @@ pub fn build(b: *std.Build) !void { const tracy_callstack = b.option(bool, "tracy-callstack", "Include callstack information with Tracy data. Does nothing if -Dtracy is not provided") orelse (tracy != null); const tracy_allocation = b.option(bool, "tracy-allocation", "Include allocation information with Tracy data. Does nothing if -Dtracy is not provided") orelse (tracy != null); const tracy_callstack_depth: u32 = b.option(u32, "tracy-callstack-depth", "Declare callstack depth for Tracy data. Does nothing if -Dtracy_callstack is not provided") orelse 10; - const debug_gpa = b.option(bool, "debug-allocator", "Force the compiler to use DebugAllocator") orelse false; + const debug_gpa = b.option(bool, "debug-allocator", "Force the compiler to use SafeAllocator") orelse false; const link_libc = b.option(bool, "force-link-libc", "Force self-hosted compiler to link libc") orelse (enable_llvm or only_c); const sanitize_thread = b.option(bool, "sanitize-thread", "Enable thread-sanitization") orelse false; const strip = b.option(bool, "strip", "Omit debug information"); diff --git a/lib/compiler/build_runner.zig b/lib/compiler/build_runner.zig index 6d8b5cca706e4c59a58a4197a5b7b11ab7a5e068..439e46bb6e82bf93bf53652cc506c129781da740 100644 --- a/lib/compiler/build_runner.zig +++ b/lib/compiler/build_runner.zig @@ -26,9 +26,9 @@ pub const std_options: std.Options = .{ pub fn main(init: process.Init.Minimal) !void { // The build runner is often short-lived, but thanks to `--watch` and `--webui`, that's not // always the case. So, we do need a true gpa for some things. - var debug_gpa_state: std.heap.DebugAllocator(.{}) = .init; - defer _ = debug_gpa_state.deinit(); - const gpa = debug_gpa_state.allocator(); + var safe_gpa_state: std.heap.SafeAllocator = .init(std.heap.page_allocator, .{}); + defer _ = safe_gpa_state.deinit(); + const gpa = safe_gpa_state.allocator(); var threaded: std.Io.Threaded = .init(gpa, .{ .environ = init.environ, diff --git a/lib/compiler/test_runner.zig b/lib/compiler/test_runner.zig index 1359924a9e5003e0eb34538ad09a9c9fb421b37d..69b078a4bcd7a9c1277901d52b47a354fad51437 100644 --- a/lib/compiler/test_runner.zig +++ b/lib/compiler/test_runner.zig @@ -91,8 +91,8 @@ fn mainServer(init: std.process.Init.Minimal) !void { return std.process.exit(0); }, .query_test_metadata => { - testing.allocator_instance = .{}; - defer if (testing.allocator_instance.deinit() == .leak) { + testing.allocator_instance = .init(std.heap.page_allocator, .{}); + defer if (testing.allocator_instance.deinit() != 0) { @panic("internal test runner memory leak"); }; @@ -123,7 +123,10 @@ fn mainServer(init: std.process.Init.Minimal) !void { .run_test => { testing.environ = init.environ; - testing.allocator_instance = .{}; + testing.allocator_instance = .init(std.heap.page_allocator, .{ + .canary = 0xc3a701ba, + .check_write_after_free = true, + }); testing.io_instance = .init(testing.allocator, .{ .argv0 = .init(init.args), .environ = init.environ, @@ -150,8 +153,7 @@ fn mainServer(init: std.process.Init.Minimal) !void { }, }; testing.io_instance.deinit(); - const leak_count = testing.allocator_instance.detectLeaks(); - testing.allocator_instance.deinitWithoutLeakChecks(); + const leak_count = testing.allocator_instance.deinit(); try server.serveTestResults(.{ .index = index, .flags = .{ @@ -173,8 +175,8 @@ fn mainServer(init: std.process.Init.Minimal) !void { // since they are not present. if (!builtin.fuzz) unreachable; - var gpa_instance: std.heap.DebugAllocator(.{}) = .init; - defer if (gpa_instance.deinit() == .leak) { + var gpa_instance: std.heap.SafeAllocator = .init(std.heap.page_allocator, .{}); + defer if (gpa_instance.deinit() != 0) { @panic("internal test runner memory leak"); }; const gpa = gpa_instance.allocator(); @@ -271,14 +273,17 @@ fn mainTerminal(init: std.process.Init.Minimal) void { var leaks: usize = 0; for (test_fn_list, 0..) |test_fn, i| { - testing.allocator_instance = .{}; + testing.allocator_instance = .init(std.heap.page_allocator, .{ + .canary = 0xc3a701ba, + .check_write_after_free = true, + }); testing.io_instance = .init(testing.allocator, .{ .argv0 = .init(init.args), .environ = init.environ, }); defer { testing.io_instance.deinit(); - if (testing.allocator_instance.deinit() == .leak) leaks += 1; + if (testing.allocator_instance.deinit() != 0) leaks += 1; } testing.log_level = .warn; testing.environ = init.environ; @@ -430,8 +435,11 @@ var fuzz_runner: if (builtin.fuzz) struct { error.WriteFailed => panic("failed to write to stdout: {t}", .{stdout_writer.err.?}), }; - testing.allocator_instance = .{}; - defer if (testing.allocator_instance.deinit() == .leak) std.process.exit(1); + testing.allocator_instance = .init(std.heap.page_allocator, .{ + .canary = 0xc3a701ba, + .check_write_after_free = true, + }); + defer if (testing.allocator_instance.deinit() != 0) std.process.exit(1); is_fuzz_test = false; builtin.test_functions[fuzz_runner.indexes[i]].func() catch |err| switch (err) { @@ -554,8 +562,11 @@ pub fn fuzz( fn test_one() callconv(.c) bool { @disableInstrumentation(); - testing.allocator_instance = .{}; - defer if (testing.allocator_instance.deinit() == .leak) std.process.exit(1); + testing.allocator_instance = .init(std.heap.page_allocator, .{ + .canary = 0xcacce5e0, + .check_write_after_free = true, + }); + defer if (testing.allocator_instance.deinit() != 0) std.process.exit(1); log_err_count = 0; testOne(ctx, @constCast(&testing.Smith{ .in = null })) catch |err| switch (err) { error.SkipZigTest => return true, @@ -582,7 +593,6 @@ pub fn fuzz( if (builtin.fuzz) { // Preserve the calling test's allocator state const prev_allocator_state = testing.allocator_instance; - testing.allocator_instance = .{}; defer testing.allocator_instance = prev_allocator_state; global.ctx = context; diff --git a/lib/fuzzer.zig b/lib/fuzzer.zig index edfde79dd14cb27e0621210e3836a93e5bf736ae..37189a2b635f0a8c8774ee3d88d670e01bc37b90 100644 --- a/lib/fuzzer.zig +++ b/lib/fuzzer.zig @@ -39,10 +39,10 @@ fn logOverride( fw.interface.flush() catch panic("failed to write to fuzzer log: {t}", .{fw.err.?}); } -var debug_allocator: std.heap.DebugAllocator(.{}) = .init; +var safe_allocator: std.heap.SafeAllocator = .init(std.heap.page_allocator, .{}); const gpa = switch (builtin.mode) { - .Debug => debug_allocator.allocator(), - .ReleaseFast, .ReleaseSmall, .ReleaseSafe => std.heap.smp_allocator, + .Debug, .ReleaseSafe => safe_allocator.allocator(), + .ReleaseFast, .ReleaseSmall => std.heap.smp_allocator, }; // Seperate from `exec` to allow initialization before `exec` is. diff --git a/lib/std/heap/ArenaAllocator.zig b/lib/std/heap/ArenaAllocator.zig index ad6c937c2346a1b6c4a3b97408d1a29fdf83c56f..bca6bd2fc081e0839a2f5d4ed8f7e645bc59fb34 100644 --- a/lib/std/heap/ArenaAllocator.zig +++ b/lib/std/heap/ArenaAllocator.zig @@ -671,12 +671,10 @@ test "reset while retaining a buffer" { // Create two internal buffers _ = try a.alloc(u8, 1); - _ = try a.alloc(u8, 1000); - try std.testing.expect(arena_allocator.state.used_list != null); - - // Check that we have at least two buffers - try std.testing.expect(arena_allocator.state.used_list.?.next != null); + while (arena_allocator.state.used_list.?.next == null) { + _ = try a.alloc(u8, 1000); + } // This retains the first allocated buffer try std.testing.expect(arena_allocator.reset(.{ .retain_with_limit = 2 })); diff --git a/lib/std/mem.zig b/lib/std/mem.zig index 7aea379efb5bb45e20de370e9f8433bd04d58a41..2226f25ce6e1de85f5962a4660850de64f3863ce 100644 --- a/lib/std/mem.zig +++ b/lib/std/mem.zig @@ -201,9 +201,12 @@ test "Allocator.resize" { defer testing.allocator.free(values); for (values, 0..) |*v, i| v.* = @as(T, @intCast(i)); - if (!testing.allocator.resize(values, values.len + 10)) return error.OutOfMemory; - values = values.ptr[0 .. values.len + 10]; - try testing.expect(values.len == 110); + if (testing.allocator.resize(values, values.len + 10)) { + values = values.ptr[0 .. values.len + 10]; + try testing.expect(values.len == 110); + } else { + // `resize` is not guaranteed to succeed even if there is sufficient memory. + } } const primitiveFloatTypes = .{ @@ -217,9 +220,12 @@ test "Allocator.resize" { defer testing.allocator.free(values); for (values, 0..) |*v, i| v.* = @as(T, @floatFromInt(i)); - if (!testing.allocator.resize(values, values.len + 10)) return error.OutOfMemory; - values = values.ptr[0 .. values.len + 10]; - try testing.expect(values.len == 110); + if (testing.allocator.resize(values, values.len + 10)) { + values = values.ptr[0 .. values.len + 10]; + try testing.expect(values.len == 110); + } else { + // `resize` is not guaranteed to succeed even if there is sufficient memory. + } } } diff --git a/lib/std/start.zig b/lib/std/start.zig index 2537da9800df607c5b871ec6b3b39c0d3c4a9666..9491ef2045ce81f2d46966979c58cdc04833e8ea 100644 --- a/lib/std/start.zig +++ b/lib/std/start.zig @@ -710,12 +710,11 @@ fn mainWithoutEnv(c_argc: c_int, c_argv: [*][*:0]c_char) callconv(.c) c_int { /// General error message for a malformed return type const bad_main_ret = "expected return type of main to be 'void', '!void', 'noreturn', 'u8', or '!u8'"; -const use_debug_allocator = !is_wasm and switch (builtin.mode) { - .Debug => true, - .ReleaseSafe => !builtin.link_libc, // Not ideal, but the best we have for now. +const use_safe_allocator = !is_wasm and switch (builtin.mode) { + .Debug, .ReleaseSafe => true, .ReleaseFast, .ReleaseSmall => !builtin.link_libc and builtin.single_threaded, // Also not ideal. }; -var debug_allocator: std.heap.DebugAllocator(.{}) = .init; +var safe_allocator: std.heap.SafeAllocator = .init(std.heap.page_allocator, .{}); inline fn callMain(args: std.process.Args.Vector, environ: std.process.Environ.Block) u8 { const fn_info = @typeInfo(@TypeOf(root.main)).@"fn"; @@ -725,8 +724,8 @@ inline fn callMain(args: std.process.Args.Vector, environ: std.process.Environ.B .environ = .{ .block = environ }, })); - const gpa = if (use_debug_allocator) - debug_allocator.allocator() + const gpa = if (use_safe_allocator) + safe_allocator.allocator() else if (builtin.link_libc) std.heap.c_allocator else if (is_wasm) @@ -736,8 +735,8 @@ inline fn callMain(args: std.process.Args.Vector, environ: std.process.Environ.B else comptime unreachable; - defer if (use_debug_allocator) { - _ = debug_allocator.deinit(); // Leaks do not affect return code. + defer if (use_safe_allocator) { + _ = safe_allocator.deinit(); // Leaks do not affect return code. }; const arena_backing_allocator = if (is_wasm) gpa else std.heap.page_allocator; diff --git a/lib/std/testing.zig b/lib/std/testing.zig index d262ca0c43a52c5364f405cc623b380c252d3915..e8a214fa2362fc37e7f25ea8ab69ec65e24a1354 100644 --- a/lib/std/testing.zig +++ b/lib/std/testing.zig @@ -17,19 +17,11 @@ var failing_allocator_instance = FailingAllocator.init(base_allocator_instance.a }); var base_allocator_instance = std.heap.FixedBufferAllocator.init(""); -/// This should only be used in temporary test programs. -pub const allocator = allocator_instance.allocator(); -pub var allocator_instance: std.heap.DebugAllocator(.{ - .stack_trace_frames = if (std.debug.sys_can_stack_trace) 10 else 0, - .resize_stack_traces = true, - // A unique value so that when a default-constructed - // DebugAllocator is incorrectly passed to testing allocator, or - // vice versa, panic occurs. - .canary = @truncate(0x2731e675c3a701ba), -}) = b: { - if (!builtin.is_test) @compileError("testing allocator used when not testing"); - break :b .init; -}; +pub var allocator_instance: std.heap.SafeAllocator = undefined; +pub const allocator = if (builtin.is_test) + allocator_instance.allocator() +else + @compileError("not testing"); pub var io_instance: Io.Threaded = undefined; pub const io = if (builtin.is_test) io_instance.io() else @compileError("not testing"); diff --git a/src/main.zig b/src/main.zig index ccf6ce25d50a7cec688d54cd6c7dd7cf086c7b46..16df63526f3654bacfdfbf116d193ec7a0edd2b2 100644 --- a/src/main.zig +++ b/src/main.zig @@ -157,34 +157,29 @@ pub fn log( std.log.defaultLog(level, scope, format, args); } -const use_debug_allocator = build_options.debug_gpa or +const use_safe_allocator = build_options.debug_gpa or (native_os != .wasi and !builtin.link_libc and switch (builtin.mode) { .Debug, .ReleaseSafe => true, .ReleaseFast, .ReleaseSmall => false, }); -const RootAllocator = if (use_debug_allocator) std.heap.DebugAllocator(.{ +// TODO: The `align(@alignOf(std.heap.SafeAllocator))` can be removed the next time zig1.wasm is updated +var safe_allocator: std.heap.SafeAllocator align(@alignOf(std.heap.SafeAllocator)) = .init(std.heap.page_allocator, .{ .stack_trace_frames = build_options.mem_leak_frames, - .thread_safe = switch (build_options.io_mode) { - .threaded => true, - .evented => false, - }, -}) else struct { - pub const init: RootAllocator = .{}; - pub fn allocator(_: RootAllocator) Allocator { - if (native_os == .wasi) return std.heap.wasm_allocator; - if (builtin.link_libc) return std.heap.c_allocator; - return std.heap.smp_allocator; - } - pub fn deinit(_: RootAllocator) std.heap.Check { - return .ok; - } -}; +}); pub fn main(init: std.process.Init.Minimal) anyerror!void { - var root_allocator: RootAllocator = .init; - defer _ = root_allocator.deinit(); - const root_gpa = root_allocator.allocator(); + const root_gpa = if (use_safe_allocator) + safe_allocator.allocator() + else if (native_os == .wasi) + std.heap.wasm_allocator + else if (builtin.link_libc) + std.heap.c_allocator + else + std.heap.smp_allocator; + defer if (use_safe_allocator) { + _ = safe_allocator.deinit(); + }; var io_impl: IoImpl = undefined; switch (build_options.io_mode) { .threaded => io_impl = .init(root_gpa, .{ @@ -197,7 +192,7 @@ pub fn main(init: std.process.Init.Minimal) anyerror!void { .argv0 = .init(init.args), .environ = init.environ, - .backing_allocator_needs_mutex = use_debug_allocator, + .backing_allocator_needs_mutex = false, }), } defer io_impl.deinit(); -- 2.54.0 From 65819004f32dda70b6f35f7226de005155f68a4c Mon Sep 17 00:00:00 2001 From: Kendall Condon Date: Sun, 3 May 2026 20:28:37 -0400 Subject: [PATCH 4/4] workaround surfaced backend bugs self-hosted wasm workaround provided by pavelverigo in #31991 powerpc workarounds provided by alexrp in #31991 --- lib/std/fmt/float.zig | 2 ++ test/behavior/vector.zig | 1 + test/tests.zig | 9 +++++++++ 3 files changed, 12 insertions(+) diff --git a/lib/std/fmt/float.zig b/lib/std/fmt/float.zig index 44a71f90f42c47b608d6db65b1b7a59243680507..2d6dccf25b5b6c7cbdae3506d5aae90ff8899b45 100644 --- a/lib/std/fmt/float.zig +++ b/lib/std/fmt/float.zig @@ -1646,6 +1646,8 @@ test "format f64" { } test "format f80" { + if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; + try check(f80, 0.0, "0e0"); try check(f80, -0.0, "-0e0"); try check(f80, 1.0, "1e0"); diff --git a/test/behavior/vector.zig b/test/behavior/vector.zig index 69998aeb6963234c6d9611d95d70a197b06f0a86..a01fc0a218dac64e24d7c43637d9dfcae43fe8f3 100644 --- a/test/behavior/vector.zig +++ b/test/behavior/vector.zig @@ -748,6 +748,7 @@ test "vector reduce operation" { if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest; if (builtin.zig_backend == .stage2_c and builtin.cpu.arch.isArm()) return error.SkipZigTest; if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest; + if (builtin.zig_backend == .stage2_llvm and builtin.cpu.arch.isPowerPC()) return error.SkipZigTest; // https://github.com/llvm/llvm-project/issues/195562 const S = struct { fn testReduce(comptime op: std.builtin.ReduceOp, x: anytype, expected: anytype) !void { diff --git a/test/tests.zig b/test/tests.zig index ce65c75c63b4bcd40c45b316270740a88975458f..5ee5c3f3e2b4b7b2a37363d551385ceba8a12ff0 100644 --- a/test/tests.zig +++ b/test/tests.zig @@ -2612,6 +2612,10 @@ fn addOneModuleTest( if (mem.eql(u8, options.name, "compiler-rt") or mem.eql(u8, options.name, "libc")) { these_tests.root_module.stack_protector = false; } + // https://github.com/llvm/llvm-project/issues/195561 + if (target.cpu.arch.isPowerPC()) { + these_tests.root_module.stack_protector = false; + } if (options.build_options) |build_options| { these_tests.root_module.addOptions("build_options", build_options); } @@ -2857,6 +2861,11 @@ pub fn addCAbiTests(b: *std.Build, options: CAbiTestOptions) *Step { .max_rss = options.max_rss, }); + // https://github.com/llvm/llvm-project/issues/195561 + if (target.cpu.arch.isPowerPC()) { + test_step.root_module.stack_protector = false; + } + // This test is intentionally trying to check if the external ABI is // done properly. LTO would be a hindrance to this. test_step.lto = .none; -- 2.54.0