authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-02-07 06:21:51-08:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2025-02-07 06:21:51-08:00
log6a6e72fff820fb641aa1b00700f6835430dae72e
treeea70863e08ba9167cfe954287691cce98716d918
parent8ad0732954df80f0f9a0248525c2bded7e82ba27
parentb8f5cfed457726a77082b7ffe6672b6066c0a66e
signaturebadge-check Signed by PGP key B5690EEEBB952194

Merge pull request #20511 from archbirdplus

runtime page size detection rework GeneralPurposeAllocator to reduce active mapping count Allocator VTable API update

44 files changed, 2981 insertions(+), 2710 deletions(-)

lib/fuzzer.zig+1-1
...@@ -480,7 +480,7 @@ pub const MemoryMappedList = struct {...@@ -480,7 +480,7 @@ pub const MemoryMappedList = struct {
480 /// of this ArrayList in accordance with the respective documentation. In480 /// of this ArrayList in accordance with the respective documentation. In
481 /// all cases, "invalidated" means that the memory has been passed to this481 /// all cases, "invalidated" means that the memory has been passed to this
482 /// allocator's resize or free function.482 /// allocator's resize or free function.
483 items: []align(std.mem.page_size) volatile u8,483 items: []align(std.heap.page_size_min) volatile u8,
484 /// How many bytes this list can hold without allocating additional memory.484 /// How many bytes this list can hold without allocating additional memory.
485 capacity: usize,485 capacity: usize,
486486
lib/std/Build/Fuzz/WebServer.zig+1-1
...@@ -41,7 +41,7 @@ const fuzzer_arch_os_abi = "wasm32-freestanding";...@@ -41,7 +41,7 @@ const fuzzer_arch_os_abi = "wasm32-freestanding";
41const fuzzer_cpu_features = "baseline+atomics+bulk_memory+multivalue+mutable_globals+nontrapping_fptoint+reference_types+sign_ext";41const fuzzer_cpu_features = "baseline+atomics+bulk_memory+multivalue+mutable_globals+nontrapping_fptoint+reference_types+sign_ext";
4242
43const CoverageMap = struct {43const CoverageMap = struct {
44 mapped_memory: []align(std.mem.page_size) const u8,44 mapped_memory: []align(std.heap.page_size_min) const u8,
45 coverage: Coverage,45 coverage: Coverage,
46 source_locations: []Coverage.SourceLocation,46 source_locations: []Coverage.SourceLocation,
47 /// Elements are indexes into `source_locations` pointing to the unit tests that are being fuzz tested.47 /// Elements are indexes into `source_locations` pointing to the unit tests that are being fuzz tested.
lib/std/Thread.zig+3-3
...@@ -769,7 +769,7 @@ const PosixThreadImpl = struct {...@@ -769,7 +769,7 @@ const PosixThreadImpl = struct {
769 // Use the same set of parameters used by the libc-less impl.769 // Use the same set of parameters used by the libc-less impl.
770 const stack_size = @max(config.stack_size, 16 * 1024);770 const stack_size = @max(config.stack_size, 16 * 1024);
771 assert(c.pthread_attr_setstacksize(&attr, stack_size) == .SUCCESS);771 assert(c.pthread_attr_setstacksize(&attr, stack_size) == .SUCCESS);
772 assert(c.pthread_attr_setguardsize(&attr, std.mem.page_size) == .SUCCESS);772 assert(c.pthread_attr_setguardsize(&attr, std.heap.pageSize()) == .SUCCESS);
773773
774 var handle: c.pthread_t = undefined;774 var handle: c.pthread_t = undefined;
775 switch (c.pthread_create(775 switch (c.pthread_create(
...@@ -1155,7 +1155,7 @@ const LinuxThreadImpl = struct {...@@ -1155,7 +1155,7 @@ const LinuxThreadImpl = struct {
1155 completion: Completion = Completion.init(.running),1155 completion: Completion = Completion.init(.running),
1156 child_tid: std.atomic.Value(i32) = std.atomic.Value(i32).init(1),1156 child_tid: std.atomic.Value(i32) = std.atomic.Value(i32).init(1),
1157 parent_tid: i32 = undefined,1157 parent_tid: i32 = undefined,
1158 mapped: []align(std.mem.page_size) u8,1158 mapped: []align(std.heap.page_size_min) u8,
11591159
1160 /// Calls `munmap(mapped.ptr, mapped.len)` then `exit(1)` without touching the stack (which lives in `mapped.ptr`).1160 /// Calls `munmap(mapped.ptr, mapped.len)` then `exit(1)` without touching the stack (which lives in `mapped.ptr`).
1161 /// Ported over from musl libc's pthread detached implementation:1161 /// Ported over from musl libc's pthread detached implementation:
...@@ -1362,7 +1362,7 @@ const LinuxThreadImpl = struct {...@@ -1362,7 +1362,7 @@ const LinuxThreadImpl = struct {
1362 };1362 };
13631363
1364 fn spawn(config: SpawnConfig, comptime f: anytype, args: anytype) !Impl {1364 fn spawn(config: SpawnConfig, comptime f: anytype, args: anytype) !Impl {
1365 const page_size = std.mem.page_size;1365 const page_size = std.heap.pageSize();
1366 const Args = @TypeOf(args);1366 const Args = @TypeOf(args);
1367 const Instance = struct {1367 const Instance = struct {
1368 fn_args: Args,1368 fn_args: Args,
lib/std/array_list.zig+18-19
...@@ -105,21 +105,19 @@ pub fn ArrayListAligned(comptime T: type, comptime alignment: ?u29) type {...@@ -105,21 +105,19 @@ pub fn ArrayListAligned(comptime T: type, comptime alignment: ?u29) type {
105 return result;105 return result;
106 }106 }
107107
108 /// The caller owns the returned memory. Empties this ArrayList,108 /// The caller owns the returned memory. Empties this ArrayList.
109 /// Its capacity is cleared, making deinit() safe but unnecessary to call.109 /// Its capacity is cleared, making `deinit` safe but unnecessary to call.
110 pub fn toOwnedSlice(self: *Self) Allocator.Error!Slice {110 pub fn toOwnedSlice(self: *Self) Allocator.Error!Slice {
111 const allocator = self.allocator;111 const allocator = self.allocator;
112112
113 const old_memory = self.allocatedSlice();113 const old_memory = self.allocatedSlice();
114 if (allocator.resize(old_memory, self.items.len)) {114 if (allocator.remap(old_memory, self.items.len)) |new_items| {
115 const result = self.items;
116 self.* = init(allocator);115 self.* = init(allocator);
117 return result;116 return new_items;
118 }117 }
119118
120 const new_memory = try allocator.alignedAlloc(T, alignment, self.items.len);119 const new_memory = try allocator.alignedAlloc(T, alignment, self.items.len);
121 @memcpy(new_memory, self.items);120 @memcpy(new_memory, self.items);
122 @memset(self.items, undefined);
123 self.clearAndFree();121 self.clearAndFree();
124 return new_memory;122 return new_memory;
125 }123 }
...@@ -185,8 +183,9 @@ pub fn ArrayListAligned(comptime T: type, comptime alignment: ?u29) type {...@@ -185,8 +183,9 @@ pub fn ArrayListAligned(comptime T: type, comptime alignment: ?u29) type {
185 // extra capacity.183 // extra capacity.
186 const new_capacity = growCapacity(self.capacity, new_len);184 const new_capacity = growCapacity(self.capacity, new_len);
187 const old_memory = self.allocatedSlice();185 const old_memory = self.allocatedSlice();
188 if (self.allocator.resize(old_memory, new_capacity)) {186 if (self.allocator.remap(old_memory, new_capacity)) |new_memory| {
189 self.capacity = new_capacity;187 self.items.ptr = new_memory.ptr;
188 self.capacity = new_memory.len;
190 return addManyAtAssumeCapacity(self, index, count);189 return addManyAtAssumeCapacity(self, index, count);
191 }190 }
192191
...@@ -468,8 +467,9 @@ pub fn ArrayListAligned(comptime T: type, comptime alignment: ?u29) type {...@@ -468,8 +467,9 @@ pub fn ArrayListAligned(comptime T: type, comptime alignment: ?u29) type {
468 // the allocator implementation would pointlessly copy our467 // the allocator implementation would pointlessly copy our
469 // extra capacity.468 // extra capacity.
470 const old_memory = self.allocatedSlice();469 const old_memory = self.allocatedSlice();
471 if (self.allocator.resize(old_memory, new_capacity)) {470 if (self.allocator.remap(old_memory, new_capacity)) |new_memory| {
472 self.capacity = new_capacity;471 self.items.ptr = new_memory.ptr;
472 self.capacity = new_memory.len;
473 } else {473 } else {
474 const new_memory = try self.allocator.alignedAlloc(T, alignment, new_capacity);474 const new_memory = try self.allocator.alignedAlloc(T, alignment, new_capacity);
475 @memcpy(new_memory[0..self.items.len], self.items);475 @memcpy(new_memory[0..self.items.len], self.items);
...@@ -707,15 +707,13 @@ pub fn ArrayListAlignedUnmanaged(comptime T: type, comptime alignment: ?u29) typ...@@ -707,15 +707,13 @@ pub fn ArrayListAlignedUnmanaged(comptime T: type, comptime alignment: ?u29) typ
707 /// Its capacity is cleared, making deinit() safe but unnecessary to call.707 /// Its capacity is cleared, making deinit() safe but unnecessary to call.
708 pub fn toOwnedSlice(self: *Self, allocator: Allocator) Allocator.Error!Slice {708 pub fn toOwnedSlice(self: *Self, allocator: Allocator) Allocator.Error!Slice {
709 const old_memory = self.allocatedSlice();709 const old_memory = self.allocatedSlice();
710 if (allocator.resize(old_memory, self.items.len)) {710 if (allocator.remap(old_memory, self.items.len)) |new_items| {
711 const result = self.items;
712 self.* = .empty;711 self.* = .empty;
713 return result;712 return new_items;
714 }713 }
715714
716 const new_memory = try allocator.alignedAlloc(T, alignment, self.items.len);715 const new_memory = try allocator.alignedAlloc(T, alignment, self.items.len);
717 @memcpy(new_memory, self.items);716 @memcpy(new_memory, self.items);
718 @memset(self.items, undefined);
719 self.clearAndFree(allocator);717 self.clearAndFree(allocator);
720 return new_memory;718 return new_memory;
721 }719 }
...@@ -1031,9 +1029,9 @@ pub fn ArrayListAlignedUnmanaged(comptime T: type, comptime alignment: ?u29) typ...@@ -1031,9 +1029,9 @@ pub fn ArrayListAlignedUnmanaged(comptime T: type, comptime alignment: ?u29) typ
1031 }1029 }
10321030
1033 const old_memory = self.allocatedSlice();1031 const old_memory = self.allocatedSlice();
1034 if (allocator.resize(old_memory, new_len)) {1032 if (allocator.remap(old_memory, new_len)) |new_items| {
1035 self.capacity = new_len;1033 self.capacity = new_items.len;
1036 self.items.len = new_len;1034 self.items = new_items;
1037 return;1035 return;
1038 }1036 }
10391037
...@@ -1099,8 +1097,9 @@ pub fn ArrayListAlignedUnmanaged(comptime T: type, comptime alignment: ?u29) typ...@@ -1099,8 +1097,9 @@ pub fn ArrayListAlignedUnmanaged(comptime T: type, comptime alignment: ?u29) typ
1099 // the allocator implementation would pointlessly copy our1097 // the allocator implementation would pointlessly copy our
1100 // extra capacity.1098 // extra capacity.
1101 const old_memory = self.allocatedSlice();1099 const old_memory = self.allocatedSlice();
1102 if (allocator.resize(old_memory, new_capacity)) {1100 if (allocator.remap(old_memory, new_capacity)) |new_memory| {
1103 self.capacity = new_capacity;1101 self.items.ptr = new_memory.ptr;
1102 self.capacity = new_memory.len;
1104 } else {1103 } else {
1105 const new_memory = try allocator.alignedAlloc(T, alignment, new_capacity);1104 const new_memory = try allocator.alignedAlloc(T, alignment, new_capacity);
1106 @memcpy(new_memory[0..self.items.len], self.items);1105 @memcpy(new_memory[0..self.items.len], self.items);
lib/std/c.zig+50-7
...@@ -3,7 +3,7 @@ const builtin = @import("builtin");...@@ -3,7 +3,7 @@ const builtin = @import("builtin");
3const c = @This();3const c = @This();
4const maxInt = std.math.maxInt;4const maxInt = std.math.maxInt;
5const assert = std.debug.assert;5const assert = std.debug.assert;
6const page_size = std.mem.page_size;6const page_size = std.heap.page_size_min;
7const native_abi = builtin.abi;7const native_abi = builtin.abi;
8const native_arch = builtin.cpu.arch;8const native_arch = builtin.cpu.arch;
9const native_os = builtin.os.tag;9const native_os = builtin.os.tag;
...@@ -2227,6 +2227,39 @@ pub const SC = switch (native_os) {...@@ -2227,6 +2227,39 @@ pub const SC = switch (native_os) {
2227 .linux => linux.SC,2227 .linux => linux.SC,
2228 else => void,2228 else => void,
2229};2229};
2230
2231pub const _SC = switch (native_os) {
2232 .driverkit, .ios, .macos, .tvos, .visionos, .watchos => enum(c_int) {
2233 PAGESIZE = 29,
2234 },
2235 .dragonfly => enum(c_int) {
2236 PAGESIZE = 47,
2237 },
2238 .freebsd => enum(c_int) {
2239 PAGESIZE = 47,
2240 },
2241 .fuchsia => enum(c_int) {
2242 PAGESIZE = 30,
2243 },
2244 .haiku => enum(c_int) {
2245 PAGESIZE = 27,
2246 },
2247 .linux => enum(c_int) {
2248 PAGESIZE = 30,
2249 },
2250 .netbsd => enum(c_int) {
2251 PAGESIZE = 28,
2252 },
2253 .openbsd => enum(c_int) {
2254 PAGESIZE = 28,
2255 },
2256 .solaris, .illumos => enum(c_int) {
2257 PAGESIZE = 11,
2258 NPROCESSORS_ONLN = 15,
2259 },
2260 else => void,
2261};
2262
2230pub const SEEK = switch (native_os) {2263pub const SEEK = switch (native_os) {
2231 .linux => linux.SEEK,2264 .linux => linux.SEEK,
2232 .emscripten => emscripten.SEEK,2265 .emscripten => emscripten.SEEK,
...@@ -7834,6 +7867,11 @@ pub const MAP = switch (native_os) {...@@ -7834,6 +7867,11 @@ pub const MAP = switch (native_os) {
7834 else => void,7867 else => void,
7835};7868};
78367869
7870pub const MREMAP = switch (native_os) {
7871 .linux => linux.MREMAP,
7872 else => void,
7873};
7874
7837/// Used by libc to communicate failure. Not actually part of the underlying syscall.7875/// Used by libc to communicate failure. Not actually part of the underlying syscall.
7838pub const MAP_FAILED: *anyopaque = @ptrFromInt(maxInt(usize));7876pub const MAP_FAILED: *anyopaque = @ptrFromInt(maxInt(usize));
78397877
...@@ -9232,7 +9270,7 @@ pub extern "c" fn getpwnam(name: [*:0]const u8) ?*passwd;...@@ -9232,7 +9270,7 @@ pub extern "c" fn getpwnam(name: [*:0]const u8) ?*passwd;
9232pub extern "c" fn getpwuid(uid: uid_t) ?*passwd;9270pub extern "c" fn getpwuid(uid: uid_t) ?*passwd;
9233pub extern "c" fn getrlimit64(resource: rlimit_resource, rlim: *rlimit) c_int;9271pub extern "c" fn getrlimit64(resource: rlimit_resource, rlim: *rlimit) c_int;
9234pub extern "c" fn lseek64(fd: fd_t, offset: i64, whence: c_int) i64;9272pub extern "c" fn lseek64(fd: fd_t, offset: i64, whence: c_int) i64;
9235pub extern "c" fn mmap64(addr: ?*align(std.mem.page_size) anyopaque, len: usize, prot: c_uint, flags: c_uint, fd: fd_t, offset: i64) *anyopaque;9273pub extern "c" fn mmap64(addr: ?*align(page_size) anyopaque, len: usize, prot: c_uint, flags: c_uint, fd: fd_t, offset: i64) *anyopaque;
9236pub extern "c" fn open64(path: [*:0]const u8, oflag: O, ...) c_int;9274pub extern "c" fn open64(path: [*:0]const u8, oflag: O, ...) c_int;
9237pub extern "c" fn openat64(fd: c_int, path: [*:0]const u8, oflag: O, ...) c_int;9275pub extern "c" fn openat64(fd: c_int, path: [*:0]const u8, oflag: O, ...) c_int;
9238pub extern "c" fn pread64(fd: fd_t, buf: [*]u8, nbyte: usize, offset: i64) isize;9276pub extern "c" fn pread64(fd: fd_t, buf: [*]u8, nbyte: usize, offset: i64) isize;
...@@ -9324,13 +9362,13 @@ pub extern "c" fn signalfd(fd: fd_t, mask: *const sigset_t, flags: u32) c_int;...@@ -9324,13 +9362,13 @@ pub extern "c" fn signalfd(fd: fd_t, mask: *const sigset_t, flags: u32) c_int;
93249362
9325pub extern "c" fn prlimit(pid: pid_t, resource: rlimit_resource, new_limit: *const rlimit, old_limit: *rlimit) c_int;9363pub extern "c" fn prlimit(pid: pid_t, resource: rlimit_resource, new_limit: *const rlimit, old_limit: *rlimit) c_int;
9326pub extern "c" fn mincore(9364pub extern "c" fn mincore(
9327 addr: *align(std.mem.page_size) anyopaque,9365 addr: *align(page_size) anyopaque,
9328 length: usize,9366 length: usize,
9329 vec: [*]u8,9367 vec: [*]u8,
9330) c_int;9368) c_int;
93319369
9332pub extern "c" fn madvise(9370pub extern "c" fn madvise(
9333 addr: *align(std.mem.page_size) anyopaque,9371 addr: *align(page_size) anyopaque,
9334 length: usize,9372 length: usize,
9335 advice: u32,9373 advice: u32,
9336) c_int;9374) c_int;
...@@ -9428,6 +9466,10 @@ pub const posix_memalign = switch (native_os) {...@@ -9428,6 +9466,10 @@ pub const posix_memalign = switch (native_os) {
9428 .dragonfly, .netbsd, .freebsd, .solaris, .openbsd, .linux, .macos, .ios, .tvos, .watchos, .visionos => private.posix_memalign,9466 .dragonfly, .netbsd, .freebsd, .solaris, .openbsd, .linux, .macos, .ios, .tvos, .watchos, .visionos => private.posix_memalign,
9429 else => {},9467 else => {},
9430};9468};
9469pub const sysconf = switch (native_os) {
9470 .solaris => solaris.sysconf,
9471 else => private.sysconf,
9472};
94319473
9432pub const sf_hdtr = switch (native_os) {9474pub const sf_hdtr = switch (native_os) {
9433 .freebsd, .macos, .ios, .tvos, .watchos, .visionos => extern struct {9475 .freebsd, .macos, .ios, .tvos, .watchos, .visionos => extern struct {
...@@ -9471,6 +9513,7 @@ pub extern "c" fn write(fd: fd_t, buf: [*]const u8, nbyte: usize) isize;...@@ -9471,6 +9513,7 @@ pub extern "c" fn write(fd: fd_t, buf: [*]const u8, nbyte: usize) isize;
9471pub extern "c" fn pwrite(fd: fd_t, buf: [*]const u8, nbyte: usize, offset: off_t) isize;9513pub extern "c" fn pwrite(fd: fd_t, buf: [*]const u8, nbyte: usize, offset: off_t) isize;
9472pub extern "c" fn mmap(addr: ?*align(page_size) anyopaque, len: usize, prot: c_uint, flags: MAP, fd: fd_t, offset: off_t) *anyopaque;9514pub extern "c" fn mmap(addr: ?*align(page_size) anyopaque, len: usize, prot: c_uint, flags: MAP, fd: fd_t, offset: off_t) *anyopaque;
9473pub extern "c" fn munmap(addr: *align(page_size) const anyopaque, len: usize) c_int;9515pub extern "c" fn munmap(addr: *align(page_size) const anyopaque, len: usize) c_int;
9516pub extern "c" fn mremap(addr: ?*align(page_size) const anyopaque, old_len: usize, new_len: usize, flags: MREMAP, ...) *anyopaque;
9474pub extern "c" fn mprotect(addr: *align(page_size) anyopaque, len: usize, prot: c_uint) c_int;9517pub extern "c" fn mprotect(addr: *align(page_size) anyopaque, len: usize, prot: c_uint) c_int;
9475pub extern "c" fn link(oldpath: [*:0]const u8, newpath: [*:0]const u8) c_int;9518pub extern "c" fn link(oldpath: [*:0]const u8, newpath: [*:0]const u8) c_int;
9476pub extern "c" fn linkat(oldfd: fd_t, oldpath: [*:0]const u8, newfd: fd_t, newpath: [*:0]const u8, flags: c_int) c_int;9519pub extern "c" fn linkat(oldfd: fd_t, oldpath: [*:0]const u8, newfd: fd_t, newpath: [*:0]const u8, flags: c_int) c_int;
...@@ -9823,7 +9866,6 @@ pub const SCM = solaris.SCM;...@@ -9823,7 +9866,6 @@ pub const SCM = solaris.SCM;
9823pub const SETCONTEXT = solaris.SETCONTEXT;9866pub const SETCONTEXT = solaris.SETCONTEXT;
9824pub const SETUSTACK = solaris.GETUSTACK;9867pub const SETUSTACK = solaris.GETUSTACK;
9825pub const SFD = solaris.SFD;9868pub const SFD = solaris.SFD;
9826pub const _SC = solaris._SC;
9827pub const cmsghdr = solaris.cmsghdr;9869pub const cmsghdr = solaris.cmsghdr;
9828pub const ctid_t = solaris.ctid_t;9870pub const ctid_t = solaris.ctid_t;
9829pub const file_obj = solaris.file_obj;9871pub const file_obj = solaris.file_obj;
...@@ -9840,7 +9882,6 @@ pub const priority = solaris.priority;...@@ -9840,7 +9882,6 @@ pub const priority = solaris.priority;
9840pub const procfs = solaris.procfs;9882pub const procfs = solaris.procfs;
9841pub const projid_t = solaris.projid_t;9883pub const projid_t = solaris.projid_t;
9842pub const signalfd_siginfo = solaris.signalfd_siginfo;9884pub const signalfd_siginfo = solaris.signalfd_siginfo;
9843pub const sysconf = solaris.sysconf;
9844pub const taskid_t = solaris.taskid_t;9885pub const taskid_t = solaris.taskid_t;
9845pub const zoneid_t = solaris.zoneid_t;9886pub const zoneid_t = solaris.zoneid_t;
98469887
...@@ -9997,6 +10038,7 @@ pub const host_t = darwin.host_t;...@@ -9997,6 +10038,7 @@ pub const host_t = darwin.host_t;
9997pub const ipc_space_t = darwin.ipc_space_t;10038pub const ipc_space_t = darwin.ipc_space_t;
9998pub const ipc_space_port_t = darwin.ipc_space_port_t;10039pub const ipc_space_port_t = darwin.ipc_space_port_t;
9999pub const kern_return_t = darwin.kern_return_t;10040pub const kern_return_t = darwin.kern_return_t;
10041pub const vm_size_t = darwin.vm_size_t;
10000pub const kevent64 = darwin.kevent64;10042pub const kevent64 = darwin.kevent64;
10001pub const kevent64_s = darwin.kevent64_s;10043pub const kevent64_s = darwin.kevent64_s;
10002pub const mach_absolute_time = darwin.mach_absolute_time;10044pub const mach_absolute_time = darwin.mach_absolute_time;
...@@ -10168,6 +10210,7 @@ const private = struct {...@@ -10168,6 +10210,7 @@ const private = struct {
10168 extern "c" fn socket(domain: c_uint, sock_type: c_uint, protocol: c_uint) c_int;10210 extern "c" fn socket(domain: c_uint, sock_type: c_uint, protocol: c_uint) c_int;
10169 extern "c" fn stat(noalias path: [*:0]const u8, noalias buf: *Stat) c_int;10211 extern "c" fn stat(noalias path: [*:0]const u8, noalias buf: *Stat) c_int;
10170 extern "c" fn sigaltstack(ss: ?*stack_t, old_ss: ?*stack_t) c_int;10212 extern "c" fn sigaltstack(ss: ?*stack_t, old_ss: ?*stack_t) c_int;
10213 extern "c" fn sysconf(sc: c_int) c_long;
1017110214
10172 extern "c" fn pthread_setname_np(thread: pthread_t, name: [*:0]const u8) c_int;10215 extern "c" fn pthread_setname_np(thread: pthread_t, name: [*:0]const u8) c_int;
10173 extern "c" fn getcontext(ucp: *ucontext_t) c_int;10216 extern "c" fn getcontext(ucp: *ucontext_t) c_int;
...@@ -10202,7 +10245,7 @@ const private = struct {...@@ -10202,7 +10245,7 @@ const private = struct {
10202 extern "c" fn __getrusage50(who: c_int, usage: *rusage) c_int;10245 extern "c" fn __getrusage50(who: c_int, usage: *rusage) c_int;
10203 extern "c" fn __gettimeofday50(noalias tv: ?*timeval, noalias tz: ?*timezone) c_int;10246 extern "c" fn __gettimeofday50(noalias tv: ?*timeval, noalias tz: ?*timezone) c_int;
10204 extern "c" fn __libc_thr_yield() c_int;10247 extern "c" fn __libc_thr_yield() c_int;
10205 extern "c" fn __msync13(addr: *align(std.mem.page_size) const anyopaque, len: usize, flags: c_int) c_int;10248 extern "c" fn __msync13(addr: *align(page_size) const anyopaque, len: usize, flags: c_int) c_int;
10206 extern "c" fn __nanosleep50(rqtp: *const timespec, rmtp: ?*timespec) c_int;10249 extern "c" fn __nanosleep50(rqtp: *const timespec, rmtp: ?*timespec) c_int;
10207 extern "c" fn __sigaction14(sig: c_int, noalias act: ?*const Sigaction, noalias oact: ?*Sigaction) c_int;10250 extern "c" fn __sigaction14(sig: c_int, noalias act: ?*const Sigaction, noalias oact: ?*Sigaction) c_int;
10208 extern "c" fn __sigfillset14(set: ?*sigset_t) void;10251 extern "c" fn __sigfillset14(set: ?*sigset_t) void;
lib/std/c/solaris.zig-4
...@@ -154,10 +154,6 @@ pub const AF_SUN = struct {...@@ -154,10 +154,6 @@ pub const AF_SUN = struct {
154 pub const NOPLM = 0x00000004;154 pub const NOPLM = 0x00000004;
155};155};
156156
157pub const _SC = struct {
158 pub const NPROCESSORS_ONLN = 15;
159};
160
161pub const procfs = struct {157pub const procfs = struct {
162 pub const misc_header = extern struct {158 pub const misc_header = extern struct {
163 size: u32,159 size: u32,
lib/std/crypto/tlcsprng.zig+5-5
...@@ -42,7 +42,7 @@ var install_atfork_handler = std.once(struct {...@@ -42,7 +42,7 @@ var install_atfork_handler = std.once(struct {
42 }42 }
43}.do);43}.do);
4444
45threadlocal var wipe_mem: []align(mem.page_size) u8 = &[_]u8{};45threadlocal var wipe_mem: []align(std.heap.page_size_min) u8 = &[_]u8{};
4646
47fn tlsCsprngFill(_: *anyopaque, buffer: []u8) void {47fn tlsCsprngFill(_: *anyopaque, buffer: []u8) void {
48 if (os_has_arc4random) {48 if (os_has_arc4random) {
...@@ -77,7 +77,7 @@ fn tlsCsprngFill(_: *anyopaque, buffer: []u8) void {...@@ -77,7 +77,7 @@ fn tlsCsprngFill(_: *anyopaque, buffer: []u8) void {
77 } else {77 } else {
78 // Use a static thread-local buffer.78 // Use a static thread-local buffer.
79 const S = struct {79 const S = struct {
80 threadlocal var buf: Context align(mem.page_size) = .{80 threadlocal var buf: Context align(std.heap.page_size_min) = .{
81 .init_state = .uninitialized,81 .init_state = .uninitialized,
82 .rng = undefined,82 .rng = undefined,
83 };83 };
...@@ -85,7 +85,7 @@ fn tlsCsprngFill(_: *anyopaque, buffer: []u8) void {...@@ -85,7 +85,7 @@ fn tlsCsprngFill(_: *anyopaque, buffer: []u8) void {
85 wipe_mem = mem.asBytes(&S.buf);85 wipe_mem = mem.asBytes(&S.buf);
86 }86 }
87 }87 }
88 const ctx = @as(*Context, @ptrCast(wipe_mem.ptr));88 const ctx: *Context = @ptrCast(wipe_mem.ptr);
8989
90 switch (ctx.init_state) {90 switch (ctx.init_state) {
91 .uninitialized => {91 .uninitialized => {
...@@ -141,7 +141,7 @@ fn childAtForkHandler() callconv(.c) void {...@@ -141,7 +141,7 @@ fn childAtForkHandler() callconv(.c) void {
141}141}
142142
143fn fillWithCsprng(buffer: []u8) void {143fn fillWithCsprng(buffer: []u8) void {
144 const ctx = @as(*Context, @ptrCast(wipe_mem.ptr));144 const ctx: *Context = @ptrCast(wipe_mem.ptr);
145 return ctx.rng.fill(buffer);145 return ctx.rng.fill(buffer);
146}146}
147147
...@@ -157,7 +157,7 @@ fn initAndFill(buffer: []u8) void {...@@ -157,7 +157,7 @@ fn initAndFill(buffer: []u8) void {
157 // the `std.options.cryptoRandomSeed` function is provided.157 // the `std.options.cryptoRandomSeed` function is provided.
158 std.options.cryptoRandomSeed(&seed);158 std.options.cryptoRandomSeed(&seed);
159159
160 const ctx = @as(*Context, @ptrCast(wipe_mem.ptr));160 const ctx: *Context = @ptrCast(wipe_mem.ptr);
161 ctx.rng = Rng.init(seed);161 ctx.rng = Rng.init(seed);
162 std.crypto.secureZero(u8, &seed);162 std.crypto.secureZero(u8, &seed);
163163
lib/std/debug.zig+8-8
...@@ -1134,7 +1134,7 @@ fn printLineFromFileAnyOs(out_stream: anytype, source_location: SourceLocation)...@@ -1134,7 +1134,7 @@ fn printLineFromFileAnyOs(out_stream: anytype, source_location: SourceLocation)
1134 defer f.close();1134 defer f.close();
1135 // TODO fstat and make sure that the file has the correct size1135 // TODO fstat and make sure that the file has the correct size
11361136
1137 var buf: [mem.page_size]u8 = undefined;1137 var buf: [4096]u8 = undefined;
1138 var amt_read = try f.read(buf[0..]);1138 var amt_read = try f.read(buf[0..]);
1139 const line_start = seek: {1139 const line_start = seek: {
1140 var current_line_start: usize = 0;1140 var current_line_start: usize = 0;
...@@ -1237,7 +1237,7 @@ test printLineFromFileAnyOs {...@@ -1237,7 +1237,7 @@ test printLineFromFileAnyOs {
12371237
1238 const overlap = 10;1238 const overlap = 10;
1239 var writer = file.writer();1239 var writer = file.writer();
1240 try writer.writeByteNTimes('a', mem.page_size - overlap);1240 try writer.writeByteNTimes('a', std.heap.page_size_min - overlap);
1241 try writer.writeByte('\n');1241 try writer.writeByte('\n');
1242 try writer.writeByteNTimes('a', overlap);1242 try writer.writeByteNTimes('a', overlap);
12431243
...@@ -1252,10 +1252,10 @@ test printLineFromFileAnyOs {...@@ -1252,10 +1252,10 @@ test printLineFromFileAnyOs {
1252 defer allocator.free(path);1252 defer allocator.free(path);
12531253
1254 var writer = file.writer();1254 var writer = file.writer();
1255 try writer.writeByteNTimes('a', mem.page_size);1255 try writer.writeByteNTimes('a', std.heap.page_size_max);
12561256
1257 try printLineFromFileAnyOs(output_stream, .{ .file_name = path, .line = 1, .column = 0 });1257 try printLineFromFileAnyOs(output_stream, .{ .file_name = path, .line = 1, .column = 0 });
1258 try expectEqualStrings(("a" ** mem.page_size) ++ "\n", output.items);1258 try expectEqualStrings(("a" ** std.heap.page_size_max) ++ "\n", output.items);
1259 output.clearRetainingCapacity();1259 output.clearRetainingCapacity();
1260 }1260 }
1261 {1261 {
...@@ -1265,18 +1265,18 @@ test printLineFromFileAnyOs {...@@ -1265,18 +1265,18 @@ test printLineFromFileAnyOs {
1265 defer allocator.free(path);1265 defer allocator.free(path);
12661266
1267 var writer = file.writer();1267 var writer = file.writer();
1268 try writer.writeByteNTimes('a', 3 * mem.page_size);1268 try writer.writeByteNTimes('a', 3 * std.heap.page_size_max);
12691269
1270 try expectError(error.EndOfFile, printLineFromFileAnyOs(output_stream, .{ .file_name = path, .line = 2, .column = 0 }));1270 try expectError(error.EndOfFile, printLineFromFileAnyOs(output_stream, .{ .file_name = path, .line = 2, .column = 0 }));
12711271
1272 try printLineFromFileAnyOs(output_stream, .{ .file_name = path, .line = 1, .column = 0 });1272 try printLineFromFileAnyOs(output_stream, .{ .file_name = path, .line = 1, .column = 0 });
1273 try expectEqualStrings(("a" ** (3 * mem.page_size)) ++ "\n", output.items);1273 try expectEqualStrings(("a" ** (3 * std.heap.page_size_max)) ++ "\n", output.items);
1274 output.clearRetainingCapacity();1274 output.clearRetainingCapacity();
12751275
1276 try writer.writeAll("a\na");1276 try writer.writeAll("a\na");
12771277
1278 try printLineFromFileAnyOs(output_stream, .{ .file_name = path, .line = 1, .column = 0 });1278 try printLineFromFileAnyOs(output_stream, .{ .file_name = path, .line = 1, .column = 0 });
1279 try expectEqualStrings(("a" ** (3 * mem.page_size)) ++ "a\n", output.items);1279 try expectEqualStrings(("a" ** (3 * std.heap.page_size_max)) ++ "a\n", output.items);
1280 output.clearRetainingCapacity();1280 output.clearRetainingCapacity();
12811281
1282 try printLineFromFileAnyOs(output_stream, .{ .file_name = path, .line = 2, .column = 0 });1282 try printLineFromFileAnyOs(output_stream, .{ .file_name = path, .line = 2, .column = 0 });
...@@ -1290,7 +1290,7 @@ test printLineFromFileAnyOs {...@@ -1290,7 +1290,7 @@ test printLineFromFileAnyOs {
1290 defer allocator.free(path);1290 defer allocator.free(path);
12911291
1292 var writer = file.writer();1292 var writer = file.writer();
1293 const real_file_start = 3 * mem.page_size;1293 const real_file_start = 3 * std.heap.page_size_min;
1294 try writer.writeByteNTimes('\n', real_file_start);1294 try writer.writeByteNTimes('\n', real_file_start);
1295 try writer.writeAll("abc\ndef");1295 try writer.writeAll("abc\ndef");
12961296
lib/std/debug/Dwarf.zig+5-5
...@@ -2120,8 +2120,8 @@ fn pcRelBase(field_ptr: usize, pc_rel_offset: i64) !usize {...@@ -2120,8 +2120,8 @@ fn pcRelBase(field_ptr: usize, pc_rel_offset: i64) !usize {
2120pub const ElfModule = struct {2120pub const ElfModule = struct {
2121 base_address: usize,2121 base_address: usize,
2122 dwarf: Dwarf,2122 dwarf: Dwarf,
2123 mapped_memory: []align(std.mem.page_size) const u8,2123 mapped_memory: []align(std.heap.page_size_min) const u8,
2124 external_mapped_memory: ?[]align(std.mem.page_size) const u8,2124 external_mapped_memory: ?[]align(std.heap.page_size_min) const u8,
21252125
2126 pub fn deinit(self: *@This(), allocator: Allocator) void {2126 pub fn deinit(self: *@This(), allocator: Allocator) void {
2127 self.dwarf.deinit(allocator);2127 self.dwarf.deinit(allocator);
...@@ -2167,11 +2167,11 @@ pub const ElfModule = struct {...@@ -2167,11 +2167,11 @@ pub const ElfModule = struct {
2167 /// sections from an external file.2167 /// sections from an external file.
2168 pub fn load(2168 pub fn load(
2169 gpa: Allocator,2169 gpa: Allocator,
2170 mapped_mem: []align(std.mem.page_size) const u8,2170 mapped_mem: []align(std.heap.page_size_min) const u8,
2171 build_id: ?[]const u8,2171 build_id: ?[]const u8,
2172 expected_crc: ?u32,2172 expected_crc: ?u32,
2173 parent_sections: *Dwarf.SectionArray,2173 parent_sections: *Dwarf.SectionArray,
2174 parent_mapped_mem: ?[]align(std.mem.page_size) const u8,2174 parent_mapped_mem: ?[]align(std.heap.page_size_min) const u8,
2175 elf_filename: ?[]const u8,2175 elf_filename: ?[]const u8,
2176 ) LoadError!Dwarf.ElfModule {2176 ) LoadError!Dwarf.ElfModule {
2177 if (expected_crc) |crc| if (crc != std.hash.crc.Crc32.hash(mapped_mem)) return error.InvalidDebugInfo;2177 if (expected_crc) |crc| if (crc != std.hash.crc.Crc32.hash(mapped_mem)) return error.InvalidDebugInfo;
...@@ -2423,7 +2423,7 @@ pub const ElfModule = struct {...@@ -2423,7 +2423,7 @@ pub const ElfModule = struct {
2423 build_id: ?[]const u8,2423 build_id: ?[]const u8,
2424 expected_crc: ?u32,2424 expected_crc: ?u32,
2425 parent_sections: *Dwarf.SectionArray,2425 parent_sections: *Dwarf.SectionArray,
2426 parent_mapped_mem: ?[]align(std.mem.page_size) const u8,2426 parent_mapped_mem: ?[]align(std.heap.page_size_min) const u8,
2427 ) LoadError!Dwarf.ElfModule {2427 ) LoadError!Dwarf.ElfModule {
2428 const elf_file = elf_file_path.root_dir.handle.openFile(elf_file_path.sub_path, .{}) catch |err| switch (err) {2428 const elf_file = elf_file_path.root_dir.handle.openFile(elf_file_path.sub_path, .{}) catch |err| switch (err) {
2429 error.FileNotFound => return missing(),2429 error.FileNotFound => return missing(),
lib/std/debug/Info.zig-1
...@@ -10,7 +10,6 @@ const std = @import("../std.zig");...@@ -10,7 +10,6 @@ const std = @import("../std.zig");
10const Allocator = std.mem.Allocator;10const Allocator = std.mem.Allocator;
11const Path = std.Build.Cache.Path;11const Path = std.Build.Cache.Path;
12const Dwarf = std.debug.Dwarf;12const Dwarf = std.debug.Dwarf;
13const page_size = std.mem.page_size;
14const assert = std.debug.assert;13const assert = std.debug.assert;
15const Coverage = std.debug.Coverage;14const Coverage = std.debug.Coverage;
16const SourceLocation = std.debug.Coverage.SourceLocation;15const SourceLocation = std.debug.Coverage.SourceLocation;
lib/std/debug/MemoryAccessor.zig+5-4
...@@ -7,7 +7,7 @@ const native_os = builtin.os.tag;...@@ -7,7 +7,7 @@ const native_os = builtin.os.tag;
7const std = @import("../std.zig");7const std = @import("../std.zig");
8const posix = std.posix;8const posix = std.posix;
9const File = std.fs.File;9const File = std.fs.File;
10const page_size = std.mem.page_size;10const page_size_min = std.heap.page_size_min;
1111
12const MemoryAccessor = @This();12const MemoryAccessor = @This();
1313
...@@ -93,9 +93,10 @@ pub fn isValidMemory(address: usize) bool {...@@ -93,9 +93,10 @@ pub fn isValidMemory(address: usize) bool {
93 // We are unable to determine validity of memory for freestanding targets93 // We are unable to determine validity of memory for freestanding targets
94 if (native_os == .freestanding or native_os == .other or native_os == .uefi) return true;94 if (native_os == .freestanding or native_os == .other or native_os == .uefi) return true;
9595
96 const aligned_address = address & ~@as(usize, @intCast((page_size - 1)));96 const page_size = std.heap.pageSize();
97 const aligned_address = address & ~(page_size - 1);
97 if (aligned_address == 0) return false;98 if (aligned_address == 0) return false;
98 const aligned_memory = @as([*]align(page_size) u8, @ptrFromInt(aligned_address))[0..page_size];99 const aligned_memory = @as([*]align(page_size_min) u8, @ptrFromInt(aligned_address))[0..page_size];
99100
100 if (native_os == .windows) {101 if (native_os == .windows) {
101 const windows = std.os.windows;102 const windows = std.os.windows;
...@@ -104,7 +105,7 @@ pub fn isValidMemory(address: usize) bool {...@@ -104,7 +105,7 @@ pub fn isValidMemory(address: usize) bool {
104105
105 // The only error this function can throw is ERROR_INVALID_PARAMETER.106 // The only error this function can throw is ERROR_INVALID_PARAMETER.
106 // supply an address that invalid i'll be thrown.107 // supply an address that invalid i'll be thrown.
107 const rc = windows.VirtualQuery(aligned_memory, &memory_info, aligned_memory.len) catch {108 const rc = windows.VirtualQuery(@ptrCast(aligned_memory), &memory_info, aligned_memory.len) catch {
108 return false;109 return false;
109 };110 };
110111
lib/std/debug/SelfInfo.zig+3-3
...@@ -504,7 +504,7 @@ pub const Module = switch (native_os) {...@@ -504,7 +504,7 @@ pub const Module = switch (native_os) {
504 .macos, .ios, .watchos, .tvos, .visionos => struct {504 .macos, .ios, .watchos, .tvos, .visionos => struct {
505 base_address: usize,505 base_address: usize,
506 vmaddr_slide: usize,506 vmaddr_slide: usize,
507 mapped_memory: []align(mem.page_size) const u8,507 mapped_memory: []align(std.heap.page_size_min) const u8,
508 symbols: []const MachoSymbol,508 symbols: []const MachoSymbol,
509 strings: [:0]const u8,509 strings: [:0]const u8,
510 ofiles: OFileTable,510 ofiles: OFileTable,
...@@ -1046,7 +1046,7 @@ pub fn readElfDebugInfo(...@@ -1046,7 +1046,7 @@ pub fn readElfDebugInfo(
1046 build_id: ?[]const u8,1046 build_id: ?[]const u8,
1047 expected_crc: ?u32,1047 expected_crc: ?u32,
1048 parent_sections: *Dwarf.SectionArray,1048 parent_sections: *Dwarf.SectionArray,
1049 parent_mapped_mem: ?[]align(mem.page_size) const u8,1049 parent_mapped_mem: ?[]align(std.heap.page_size_min) const u8,
1050) !Dwarf.ElfModule {1050) !Dwarf.ElfModule {
1051 nosuspend {1051 nosuspend {
1052 const elf_file = (if (elf_filename) |filename| blk: {1052 const elf_file = (if (elf_filename) |filename| blk: {
...@@ -1088,7 +1088,7 @@ const MachoSymbol = struct {...@@ -1088,7 +1088,7 @@ const MachoSymbol = struct {
10881088
1089/// Takes ownership of file, even on error.1089/// Takes ownership of file, even on error.
1090/// TODO it's weird to take ownership even on error, rework this code.1090/// TODO it's weird to take ownership even on error, rework this code.
1091fn mapWholeFile(file: File) ![]align(mem.page_size) const u8 {1091fn mapWholeFile(file: File) ![]align(std.heap.page_size_min) const u8 {
1092 nosuspend {1092 nosuspend {
1093 defer file.close();1093 defer file.close();
10941094
lib/std/dynamic_library.zig+7-5
...@@ -143,7 +143,7 @@ pub const ElfDynLib = struct {...@@ -143,7 +143,7 @@ pub const ElfDynLib = struct {
143 hashtab: [*]posix.Elf_Symndx,143 hashtab: [*]posix.Elf_Symndx,
144 versym: ?[*]elf.Versym,144 versym: ?[*]elf.Versym,
145 verdef: ?*elf.Verdef,145 verdef: ?*elf.Verdef,
146 memory: []align(mem.page_size) u8,146 memory: []align(std.heap.page_size_min) u8,
147147
148 pub const Error = ElfDynLibError;148 pub const Error = ElfDynLibError;
149149
...@@ -219,11 +219,13 @@ pub const ElfDynLib = struct {...@@ -219,11 +219,13 @@ pub const ElfDynLib = struct {
219 const stat = try file.stat();219 const stat = try file.stat();
220 const size = std.math.cast(usize, stat.size) orelse return error.FileTooBig;220 const size = std.math.cast(usize, stat.size) orelse return error.FileTooBig;
221221
222 const page_size = std.heap.pageSize();
223
222 // This one is to read the ELF info. We do more mmapping later224 // This one is to read the ELF info. We do more mmapping later
223 // corresponding to the actual LOAD sections.225 // corresponding to the actual LOAD sections.
224 const file_bytes = try posix.mmap(226 const file_bytes = try posix.mmap(
225 null,227 null,
226 mem.alignForward(usize, size, mem.page_size),228 mem.alignForward(usize, size, page_size),
227 posix.PROT.READ,229 posix.PROT.READ,
228 .{ .TYPE = .PRIVATE },230 .{ .TYPE = .PRIVATE },
229 fd,231 fd,
...@@ -284,10 +286,10 @@ pub const ElfDynLib = struct {...@@ -284,10 +286,10 @@ pub const ElfDynLib = struct {
284 elf.PT_LOAD => {286 elf.PT_LOAD => {
285 // The VirtAddr may not be page-aligned; in such case there will be287 // The VirtAddr may not be page-aligned; in such case there will be
286 // extra nonsense mapped before/after the VirtAddr,MemSiz288 // extra nonsense mapped before/after the VirtAddr,MemSiz
287 const aligned_addr = (base + ph.p_vaddr) & ~(@as(usize, mem.page_size) - 1);289 const aligned_addr = (base + ph.p_vaddr) & ~(@as(usize, page_size) - 1);
288 const extra_bytes = (base + ph.p_vaddr) - aligned_addr;290 const extra_bytes = (base + ph.p_vaddr) - aligned_addr;
289 const extended_memsz = mem.alignForward(usize, ph.p_memsz + extra_bytes, mem.page_size);291 const extended_memsz = mem.alignForward(usize, ph.p_memsz + extra_bytes, page_size);
290 const ptr = @as([*]align(mem.page_size) u8, @ptrFromInt(aligned_addr));292 const ptr = @as([*]align(std.heap.page_size_min) u8, @ptrFromInt(aligned_addr));
291 const prot = elfToMmapProt(ph.p_flags);293 const prot = elfToMmapProt(ph.p_flags);
292 if ((ph.p_flags & elf.PF_W) == 0) {294 if ((ph.p_flags & elf.PF_W) == 0) {
293 // If it does not need write access, it can be mapped from the fd.295 // If it does not need write access, it can be mapped from the fd.
lib/std/fifo.zig+1-1
...@@ -91,7 +91,7 @@ pub fn LinearFifo(...@@ -91,7 +91,7 @@ pub fn LinearFifo(
91 mem.copyForwards(T, self.buf[0..self.count], self.buf[self.head..][0..self.count]);91 mem.copyForwards(T, self.buf[0..self.count], self.buf[self.head..][0..self.count]);
92 self.head = 0;92 self.head = 0;
93 } else {93 } else {
94 var tmp: [mem.page_size / 2 / @sizeOf(T)]T = undefined;94 var tmp: [4096 / 2 / @sizeOf(T)]T = undefined;
9595
96 while (self.head != 0) {96 while (self.head != 0) {
97 const n = @min(self.head, tmp.len);97 const n = @min(self.head, tmp.len);
lib/std/hash_map.zig+20
...@@ -413,10 +413,15 @@ pub fn HashMap(...@@ -413,10 +413,15 @@ pub fn HashMap(
413 /// If there is an `Entry` with a matching key, it is deleted from413 /// If there is an `Entry` with a matching key, it is deleted from
414 /// the hash map, and this function returns true. Otherwise this414 /// the hash map, and this function returns true. Otherwise this
415 /// function returns false.415 /// function returns false.
416 ///
417 /// TODO: answer the question in these doc comments, does this
418 /// increase the unused capacity by one?
416 pub fn remove(self: *Self, key: K) bool {419 pub fn remove(self: *Self, key: K) bool {
417 return self.unmanaged.removeContext(key, self.ctx);420 return self.unmanaged.removeContext(key, self.ctx);
418 }421 }
419422
423 /// TODO: answer the question in these doc comments, does this
424 /// increase the unused capacity by one?
420 pub fn removeAdapted(self: *Self, key: anytype, ctx: anytype) bool {425 pub fn removeAdapted(self: *Self, key: anytype, ctx: anytype) bool {
421 return self.unmanaged.removeAdapted(key, ctx);426 return self.unmanaged.removeAdapted(key, ctx);
422 }427 }
...@@ -424,6 +429,9 @@ pub fn HashMap(...@@ -424,6 +429,9 @@ pub fn HashMap(
424 /// Delete the entry with key pointed to by key_ptr from the hash map.429 /// Delete the entry with key pointed to by key_ptr from the hash map.
425 /// key_ptr is assumed to be a valid pointer to a key that is present430 /// key_ptr is assumed to be a valid pointer to a key that is present
426 /// in the hash map.431 /// in the hash map.
432 ///
433 /// TODO: answer the question in these doc comments, does this
434 /// increase the unused capacity by one?
427 pub fn removeByPtr(self: *Self, key_ptr: *K) void {435 pub fn removeByPtr(self: *Self, key_ptr: *K) void {
428 self.unmanaged.removeByPtr(key_ptr);436 self.unmanaged.removeByPtr(key_ptr);
429 }437 }
...@@ -1225,14 +1233,23 @@ pub fn HashMapUnmanaged(...@@ -1225,14 +1233,23 @@ pub fn HashMapUnmanaged(
1225 /// If there is an `Entry` with a matching key, it is deleted from1233 /// If there is an `Entry` with a matching key, it is deleted from
1226 /// the hash map, and this function returns true. Otherwise this1234 /// the hash map, and this function returns true. Otherwise this
1227 /// function returns false.1235 /// function returns false.
1236 ///
1237 /// TODO: answer the question in these doc comments, does this
1238 /// increase the unused capacity by one?
1228 pub fn remove(self: *Self, key: K) bool {1239 pub fn remove(self: *Self, key: K) bool {
1229 if (@sizeOf(Context) != 0)1240 if (@sizeOf(Context) != 0)
1230 @compileError("Cannot infer context " ++ @typeName(Context) ++ ", call removeContext instead.");1241 @compileError("Cannot infer context " ++ @typeName(Context) ++ ", call removeContext instead.");
1231 return self.removeContext(key, undefined);1242 return self.removeContext(key, undefined);
1232 }1243 }
1244
1245 /// TODO: answer the question in these doc comments, does this
1246 /// increase the unused capacity by one?
1233 pub fn removeContext(self: *Self, key: K, ctx: Context) bool {1247 pub fn removeContext(self: *Self, key: K, ctx: Context) bool {
1234 return self.removeAdapted(key, ctx);1248 return self.removeAdapted(key, ctx);
1235 }1249 }
1250
1251 /// TODO: answer the question in these doc comments, does this
1252 /// increase the unused capacity by one?
1236 pub fn removeAdapted(self: *Self, key: anytype, ctx: anytype) bool {1253 pub fn removeAdapted(self: *Self, key: anytype, ctx: anytype) bool {
1237 if (self.getIndex(key, ctx)) |idx| {1254 if (self.getIndex(key, ctx)) |idx| {
1238 self.removeByIndex(idx);1255 self.removeByIndex(idx);
...@@ -1245,6 +1262,9 @@ pub fn HashMapUnmanaged(...@@ -1245,6 +1262,9 @@ pub fn HashMapUnmanaged(
1245 /// Delete the entry with key pointed to by key_ptr from the hash map.1262 /// Delete the entry with key pointed to by key_ptr from the hash map.
1246 /// key_ptr is assumed to be a valid pointer to a key that is present1263 /// key_ptr is assumed to be a valid pointer to a key that is present
1247 /// in the hash map.1264 /// in the hash map.
1265 ///
1266 /// TODO: answer the question in these doc comments, does this
1267 /// increase the unused capacity by one?
1248 pub fn removeByPtr(self: *Self, key_ptr: *K) void {1268 pub fn removeByPtr(self: *Self, key_ptr: *K) void {
1249 // TODO: replace with pointer subtraction once supported by zig1269 // TODO: replace with pointer subtraction once supported by zig
1250 // if @sizeOf(K) == 0 then there is at most one item in the hash1270 // if @sizeOf(K) == 0 then there is at most one item in the hash
lib/std/heap.zig+504-417
...@@ -8,19 +8,20 @@ const c = std.c;...@@ -8,19 +8,20 @@ const c = std.c;
8const Allocator = std.mem.Allocator;8const Allocator = std.mem.Allocator;
9const windows = std.os.windows;9const windows = std.os.windows;
1010
11pub const LoggingAllocator = @import("heap/logging_allocator.zig").LoggingAllocator;
12pub const loggingAllocator = @import("heap/logging_allocator.zig").loggingAllocator;
13pub const ScopedLoggingAllocator = @import("heap/logging_allocator.zig").ScopedLoggingAllocator;
14pub const LogToWriterAllocator = @import("heap/log_to_writer_allocator.zig").LogToWriterAllocator;
15pub const logToWriterAllocator = @import("heap/log_to_writer_allocator.zig").logToWriterAllocator;
16pub const ArenaAllocator = @import("heap/arena_allocator.zig").ArenaAllocator;11pub const ArenaAllocator = @import("heap/arena_allocator.zig").ArenaAllocator;
17pub const GeneralPurposeAllocatorConfig = @import("heap/general_purpose_allocator.zig").Config;
18pub const GeneralPurposeAllocator = @import("heap/general_purpose_allocator.zig").GeneralPurposeAllocator;
19pub const Check = @import("heap/general_purpose_allocator.zig").Check;
20pub const WasmAllocator = @import("heap/WasmAllocator.zig");12pub const WasmAllocator = @import("heap/WasmAllocator.zig");
21pub const PageAllocator = @import("heap/PageAllocator.zig");13pub const PageAllocator = @import("heap/PageAllocator.zig");
22pub const ThreadSafeAllocator = @import("heap/ThreadSafeAllocator.zig");14pub const ThreadSafeAllocator = @import("heap/ThreadSafeAllocator.zig");
23pub const SbrkAllocator = @import("heap/sbrk_allocator.zig").SbrkAllocator;15pub const SbrkAllocator = @import("heap/sbrk_allocator.zig").SbrkAllocator;
16pub const FixedBufferAllocator = @import("heap/FixedBufferAllocator.zig");
17
18pub const DebugAllocatorConfig = @import("heap/debug_allocator.zig").Config;
19pub const DebugAllocator = @import("heap/debug_allocator.zig").DebugAllocator;
20pub const Check = enum { ok, leak };
21/// Deprecated; to be removed after 0.14.0 is tagged.
22pub const GeneralPurposeAllocatorConfig = DebugAllocatorConfig;
23/// Deprecated; to be removed after 0.14.0 is tagged.
24pub const GeneralPurposeAllocator = DebugAllocator;
2425
25const memory_pool = @import("heap/memory_pool.zig");26const memory_pool = @import("heap/memory_pool.zig");
26pub const MemoryPool = memory_pool.MemoryPool;27pub const MemoryPool = memory_pool.MemoryPool;
...@@ -29,7 +30,97 @@ pub const MemoryPoolExtra = memory_pool.MemoryPoolExtra;...@@ -29,7 +30,97 @@ pub const MemoryPoolExtra = memory_pool.MemoryPoolExtra;
29pub const MemoryPoolOptions = memory_pool.Options;30pub const MemoryPoolOptions = memory_pool.Options;
3031
31/// TODO Utilize this on Windows.32/// TODO Utilize this on Windows.
32pub var next_mmap_addr_hint: ?[*]align(mem.page_size) u8 = null;33pub var next_mmap_addr_hint: ?[*]align(page_size_min) u8 = null;
34
35/// comptime-known minimum page size of the target.
36///
37/// All pointers from `mmap` or `VirtualAlloc` are aligned to at least
38/// `page_size_min`, but their actual alignment may be bigger.
39///
40/// This value can be overridden via `std.options.page_size_min`.
41///
42/// On many systems, the actual page size can only be determined at runtime
43/// with `pageSize`.
44pub const page_size_min: usize = std.options.page_size_min orelse (page_size_min_default orelse if (builtin.os.tag == .freestanding or builtin.os.tag == .other)
45 @compileError("freestanding/other page_size_min must provided with std.options.page_size_min")
46else
47 @compileError(@tagName(builtin.cpu.arch) ++ "-" ++ @tagName(builtin.os.tag) ++ " has unknown page_size_min; populate std.options.page_size_min"));
48
49/// comptime-known maximum page size of the target.
50///
51/// Targeting a system with a larger page size may require overriding
52/// `std.options.page_size_max`, as well as providing a corresponding linker
53/// option.
54///
55/// The actual page size can only be determined at runtime with `pageSize`.
56pub const page_size_max: usize = std.options.page_size_max orelse (page_size_max_default orelse if (builtin.os.tag == .freestanding or builtin.os.tag == .other)
57 @compileError("freestanding/other page_size_max must provided with std.options.page_size_max")
58else
59 @compileError(@tagName(builtin.cpu.arch) ++ "-" ++ @tagName(builtin.os.tag) ++ " has unknown page_size_max; populate std.options.page_size_max"));
60
61/// If the page size is comptime-known, return value is comptime.
62/// Otherwise, calls `std.options.queryPageSize` which by default queries the
63/// host operating system at runtime.
64pub inline fn pageSize() usize {
65 if (page_size_min == page_size_max) return page_size_min;
66 return std.options.queryPageSize();
67}
68
69test pageSize {
70 assert(std.math.isPowerOfTwo(pageSize()));
71}
72
73/// The default implementation of `std.options.queryPageSize`.
74/// Asserts that the page size is within `page_size_min` and `page_size_max`
75pub fn defaultQueryPageSize() usize {
76 const global = struct {
77 var cached_result: std.atomic.Value(usize) = .init(0);
78 };
79 var size = global.cached_result.load(.unordered);
80 if (size > 0) return size;
81 size = switch (builtin.os.tag) {
82 .linux => if (builtin.link_libc) @intCast(std.c.sysconf(@intFromEnum(std.c._SC.PAGESIZE))) else std.os.linux.getauxval(std.elf.AT_PAGESZ),
83 .driverkit, .ios, .macos, .tvos, .visionos, .watchos => blk: {
84 const task_port = std.c.mach_task_self();
85 // mach_task_self may fail "if there are any resource failures or other errors".
86 if (task_port == std.c.TASK_NULL)
87 break :blk 0;
88 var info_count = std.c.TASK_VM_INFO_COUNT;
89 var vm_info: std.c.task_vm_info_data_t = undefined;
90 vm_info.page_size = 0;
91 _ = std.c.task_info(
92 task_port,
93 std.c.TASK_VM_INFO,
94 @as(std.c.task_info_t, @ptrCast(&vm_info)),
95 &info_count,
96 );
97 assert(vm_info.page_size != 0);
98 break :blk @intCast(vm_info.page_size);
99 },
100 .windows => blk: {
101 var info: std.os.windows.SYSTEM_INFO = undefined;
102 std.os.windows.kernel32.GetSystemInfo(&info);
103 break :blk info.dwPageSize;
104 },
105 else => if (builtin.link_libc)
106 @intCast(std.c.sysconf(@intFromEnum(std.c._SC.PAGESIZE)))
107 else if (builtin.os.tag == .freestanding or builtin.os.tag == .other)
108 @compileError("unsupported target: freestanding/other")
109 else
110 @compileError("pageSize on " ++ @tagName(builtin.cpu.arch) ++ "-" ++ @tagName(builtin.os.tag) ++ " is not supported without linking libc, using the default implementation"),
111 };
112
113 assert(size >= page_size_min);
114 assert(size <= page_size_max);
115 global.cached_result.store(size, .unordered);
116
117 return size;
118}
119
120test defaultQueryPageSize {
121 if (builtin.cpu.arch.isWasm()) return error.SkipZigTest;
122 assert(std.math.isPowerOfTwo(defaultQueryPageSize()));
123}
33124
34const CAllocator = struct {125const CAllocator = struct {
35 comptime {126 comptime {
...@@ -38,6 +129,13 @@ const CAllocator = struct {...@@ -38,6 +129,13 @@ const CAllocator = struct {
38 }129 }
39 }130 }
40131
132 const vtable: Allocator.VTable = .{
133 .alloc = alloc,
134 .resize = resize,
135 .remap = remap,
136 .free = free,
137 };
138
41 pub const supports_malloc_size = @TypeOf(malloc_size) != void;139 pub const supports_malloc_size = @TypeOf(malloc_size) != void;
42 pub const malloc_size = if (@TypeOf(c.malloc_size) != void)140 pub const malloc_size = if (@TypeOf(c.malloc_size) != void)
43 c.malloc_size141 c.malloc_size
...@@ -53,29 +151,29 @@ const CAllocator = struct {...@@ -53,29 +151,29 @@ const CAllocator = struct {
53 };151 };
54152
55 fn getHeader(ptr: [*]u8) *[*]u8 {153 fn getHeader(ptr: [*]u8) *[*]u8 {
56 return @as(*[*]u8, @ptrFromInt(@intFromPtr(ptr) - @sizeOf(usize)));154 return @alignCast(@ptrCast(ptr - @sizeOf(usize)));
57 }155 }
58156
59 fn alignedAlloc(len: usize, log2_align: u8) ?[*]u8 {157 fn alignedAlloc(len: usize, alignment: mem.Alignment) ?[*]u8 {
60 const alignment = @as(usize, 1) << @as(Allocator.Log2Align, @intCast(log2_align));158 const alignment_bytes = alignment.toByteUnits();
61 if (supports_posix_memalign) {159 if (supports_posix_memalign) {
62 // The posix_memalign only accepts alignment values that are a160 // The posix_memalign only accepts alignment values that are a
63 // multiple of the pointer size161 // multiple of the pointer size
64 const eff_alignment = @max(alignment, @sizeOf(usize));162 const effective_alignment = @max(alignment_bytes, @sizeOf(usize));
65163
66 var aligned_ptr: ?*anyopaque = undefined;164 var aligned_ptr: ?*anyopaque = undefined;
67 if (c.posix_memalign(&aligned_ptr, eff_alignment, len) != 0)165 if (c.posix_memalign(&aligned_ptr, effective_alignment, len) != 0)
68 return null;166 return null;
69167
70 return @as([*]u8, @ptrCast(aligned_ptr));168 return @ptrCast(aligned_ptr);
71 }169 }
72170
73 // Thin wrapper around regular malloc, overallocate to account for171 // Thin wrapper around regular malloc, overallocate to account for
74 // alignment padding and store the original malloc()'ed pointer before172 // alignment padding and store the original malloc()'ed pointer before
75 // the aligned address.173 // the aligned address.
76 const unaligned_ptr = @as([*]u8, @ptrCast(c.malloc(len + alignment - 1 + @sizeOf(usize)) orelse return null));174 const unaligned_ptr = @as([*]u8, @ptrCast(c.malloc(len + alignment_bytes - 1 + @sizeOf(usize)) orelse return null));
77 const unaligned_addr = @intFromPtr(unaligned_ptr);175 const unaligned_addr = @intFromPtr(unaligned_ptr);
78 const aligned_addr = mem.alignForward(usize, unaligned_addr + @sizeOf(usize), alignment);176 const aligned_addr = mem.alignForward(usize, unaligned_addr + @sizeOf(usize), alignment_bytes);
79 const aligned_ptr = unaligned_ptr + (aligned_addr - unaligned_addr);177 const aligned_ptr = unaligned_ptr + (aligned_addr - unaligned_addr);
80 getHeader(aligned_ptr).* = unaligned_ptr;178 getHeader(aligned_ptr).* = unaligned_ptr;
81179
...@@ -104,22 +202,22 @@ const CAllocator = struct {...@@ -104,22 +202,22 @@ const CAllocator = struct {
104 fn alloc(202 fn alloc(
105 _: *anyopaque,203 _: *anyopaque,
106 len: usize,204 len: usize,
107 log2_align: u8,205 alignment: mem.Alignment,
108 return_address: usize,206 return_address: usize,
109 ) ?[*]u8 {207 ) ?[*]u8 {
110 _ = return_address;208 _ = return_address;
111 assert(len > 0);209 assert(len > 0);
112 return alignedAlloc(len, log2_align);210 return alignedAlloc(len, alignment);
113 }211 }
114212
115 fn resize(213 fn resize(
116 _: *anyopaque,214 _: *anyopaque,
117 buf: []u8,215 buf: []u8,
118 log2_buf_align: u8,216 alignment: mem.Alignment,
119 new_len: usize,217 new_len: usize,
120 return_address: usize,218 return_address: usize,
121 ) bool {219 ) bool {
122 _ = log2_buf_align;220 _ = alignment;
123 _ = return_address;221 _ = return_address;
124 if (new_len <= buf.len) {222 if (new_len <= buf.len) {
125 return true;223 return true;
...@@ -133,13 +231,25 @@ const CAllocator = struct {...@@ -133,13 +231,25 @@ const CAllocator = struct {
133 return false;231 return false;
134 }232 }
135233
234 fn remap(
235 context: *anyopaque,
236 memory: []u8,
237 alignment: mem.Alignment,
238 new_len: usize,
239 return_address: usize,
240 ) ?[*]u8 {
241 // realloc would potentially return a new allocation that does not
242 // respect the original alignment.
243 return if (resize(context, memory, alignment, new_len, return_address)) memory.ptr else null;
244 }
245
136 fn free(246 fn free(
137 _: *anyopaque,247 _: *anyopaque,
138 buf: []u8,248 buf: []u8,
139 log2_buf_align: u8,249 alignment: mem.Alignment,
140 return_address: usize,250 return_address: usize,
141 ) void {251 ) void {
142 _ = log2_buf_align;252 _ = alignment;
143 _ = return_address;253 _ = return_address;
144 alignedFree(buf.ptr);254 alignedFree(buf.ptr);
145 }255 }
...@@ -148,78 +258,83 @@ const CAllocator = struct {...@@ -148,78 +258,83 @@ const CAllocator = struct {
148/// Supports the full Allocator interface, including alignment, and exploiting258/// Supports the full Allocator interface, including alignment, and exploiting
149/// `malloc_usable_size` if available. For an allocator that directly calls259/// `malloc_usable_size` if available. For an allocator that directly calls
150/// `malloc`/`free`, see `raw_c_allocator`.260/// `malloc`/`free`, see `raw_c_allocator`.
151pub const c_allocator = Allocator{261pub const c_allocator: Allocator = .{
152 .ptr = undefined,262 .ptr = undefined,
153 .vtable = &c_allocator_vtable,263 .vtable = &CAllocator.vtable,
154};
155const c_allocator_vtable = Allocator.VTable{
156 .alloc = CAllocator.alloc,
157 .resize = CAllocator.resize,
158 .free = CAllocator.free,
159};264};
160265
161/// Asserts allocations are within `@alignOf(std.c.max_align_t)` and directly calls266/// Asserts allocations are within `@alignOf(std.c.max_align_t)` and directly
162/// `malloc`/`free`. Does not attempt to utilize `malloc_usable_size`.267/// calls `malloc`/`free`. Does not attempt to utilize `malloc_usable_size`.
163/// This allocator is safe to use as the backing allocator with268/// This allocator is safe to use as the backing allocator with
164/// `ArenaAllocator` for example and is more optimal in such a case269/// `ArenaAllocator` for example and is more optimal in such a case than
165/// than `c_allocator`.270/// `c_allocator`.
166pub const raw_c_allocator = Allocator{271pub const raw_c_allocator: Allocator = .{
167 .ptr = undefined,272 .ptr = undefined,
168 .vtable = &raw_c_allocator_vtable,273 .vtable = &raw_c_allocator_vtable,
169};274};
170const raw_c_allocator_vtable = Allocator.VTable{275const raw_c_allocator_vtable: Allocator.VTable = .{
171 .alloc = rawCAlloc,276 .alloc = rawCAlloc,
172 .resize = rawCResize,277 .resize = rawCResize,
278 .remap = rawCRemap,
173 .free = rawCFree,279 .free = rawCFree,
174};280};
175281
176fn rawCAlloc(282fn rawCAlloc(
177 _: *anyopaque,283 context: *anyopaque,
178 len: usize,284 len: usize,
179 log2_ptr_align: u8,285 alignment: mem.Alignment,
180 ret_addr: usize,286 return_address: usize,
181) ?[*]u8 {287) ?[*]u8 {
182 _ = ret_addr;288 _ = context;
183 assert(log2_ptr_align <= comptime std.math.log2_int(usize, @alignOf(std.c.max_align_t)));289 _ = return_address;
290 assert(alignment.compare(.lte, comptime .fromByteUnits(@alignOf(std.c.max_align_t))));
184 // Note that this pointer cannot be aligncasted to max_align_t because if291 // Note that this pointer cannot be aligncasted to max_align_t because if
185 // len is < max_align_t then the alignment can be smaller. For example, if292 // len is < max_align_t then the alignment can be smaller. For example, if
186 // max_align_t is 16, but the user requests 8 bytes, there is no built-in293 // max_align_t is 16, but the user requests 8 bytes, there is no built-in
187 // type in C that is size 8 and has 16 byte alignment, so the alignment may294 // type in C that is size 8 and has 16 byte alignment, so the alignment may
188 // be 8 bytes rather than 16. Similarly if only 1 byte is requested, malloc295 // be 8 bytes rather than 16. Similarly if only 1 byte is requested, malloc
189 // is allowed to return a 1-byte aligned pointer.296 // is allowed to return a 1-byte aligned pointer.
190 return @as(?[*]u8, @ptrCast(c.malloc(len)));297 return @ptrCast(c.malloc(len));
191}298}
192299
193fn rawCResize(300fn rawCResize(
194 _: *anyopaque,301 context: *anyopaque,
195 buf: []u8,302 memory: []u8,
196 log2_old_align: u8,303 alignment: mem.Alignment,
197 new_len: usize,304 new_len: usize,
198 ret_addr: usize,305 return_address: usize,
199) bool {306) bool {
200 _ = log2_old_align;307 _ = context;
201 _ = ret_addr;308 _ = memory;
202309 _ = alignment;
203 if (new_len <= buf.len)310 _ = new_len;
204 return true;311 _ = return_address;
205
206 if (CAllocator.supports_malloc_size) {
207 const full_len = CAllocator.malloc_size(buf.ptr);
208 if (new_len <= full_len) return true;
209 }
210
211 return false;312 return false;
212}313}
213314
315fn rawCRemap(
316 context: *anyopaque,
317 memory: []u8,
318 alignment: mem.Alignment,
319 new_len: usize,
320 return_address: usize,
321) ?[*]u8 {
322 _ = context;
323 _ = alignment;
324 _ = return_address;
325 return @ptrCast(c.realloc(memory.ptr, new_len));
326}
327
214fn rawCFree(328fn rawCFree(
215 _: *anyopaque,329 context: *anyopaque,
216 buf: []u8,330 memory: []u8,
217 log2_old_align: u8,331 alignment: mem.Alignment,
218 ret_addr: usize,332 return_address: usize,
219) void {333) void {
220 _ = log2_old_align;334 _ = context;
221 _ = ret_addr;335 _ = alignment;
222 c.free(buf.ptr);336 _ = return_address;
337 c.free(memory.ptr);
223}338}
224339
225/// On operating systems that support memory mapping, this allocator makes a340/// On operating systems that support memory mapping, this allocator makes a
...@@ -253,252 +368,6 @@ pub const wasm_allocator: Allocator = .{...@@ -253,252 +368,6 @@ pub const wasm_allocator: Allocator = .{
253 .vtable = &WasmAllocator.vtable,368 .vtable = &WasmAllocator.vtable,
254};369};
255370
256/// Verifies that the adjusted length will still map to the full length
257pub fn alignPageAllocLen(full_len: usize, len: usize) usize {
258 const aligned_len = mem.alignAllocLen(full_len, len);
259 assert(mem.alignForward(usize, aligned_len, mem.page_size) == full_len);
260 return aligned_len;
261}
262
263pub const HeapAllocator = switch (builtin.os.tag) {
264 .windows => struct {
265 heap_handle: ?HeapHandle,
266
267 const HeapHandle = windows.HANDLE;
268
269 pub fn init() HeapAllocator {
270 return HeapAllocator{
271 .heap_handle = null,
272 };
273 }
274
275 pub fn allocator(self: *HeapAllocator) Allocator {
276 return .{
277 .ptr = self,
278 .vtable = &.{
279 .alloc = alloc,
280 .resize = resize,
281 .free = free,
282 },
283 };
284 }
285
286 pub fn deinit(self: *HeapAllocator) void {
287 if (self.heap_handle) |heap_handle| {
288 windows.HeapDestroy(heap_handle);
289 }
290 }
291
292 fn getRecordPtr(buf: []u8) *align(1) usize {
293 return @as(*align(1) usize, @ptrFromInt(@intFromPtr(buf.ptr) + buf.len));
294 }
295
296 fn alloc(
297 ctx: *anyopaque,
298 n: usize,
299 log2_ptr_align: u8,
300 return_address: usize,
301 ) ?[*]u8 {
302 _ = return_address;
303 const self: *HeapAllocator = @ptrCast(@alignCast(ctx));
304
305 const ptr_align = @as(usize, 1) << @as(Allocator.Log2Align, @intCast(log2_ptr_align));
306 const amt = n + ptr_align - 1 + @sizeOf(usize);
307 const optional_heap_handle = @atomicLoad(?HeapHandle, &self.heap_handle, .seq_cst);
308 const heap_handle = optional_heap_handle orelse blk: {
309 const options = if (builtin.single_threaded) windows.HEAP_NO_SERIALIZE else 0;
310 const hh = windows.kernel32.HeapCreate(options, amt, 0) orelse return null;
311 const other_hh = @cmpxchgStrong(?HeapHandle, &self.heap_handle, null, hh, .seq_cst, .seq_cst) orelse break :blk hh;
312 windows.HeapDestroy(hh);
313 break :blk other_hh.?; // can't be null because of the cmpxchg
314 };
315 const ptr = windows.kernel32.HeapAlloc(heap_handle, 0, amt) orelse return null;
316 const root_addr = @intFromPtr(ptr);
317 const aligned_addr = mem.alignForward(usize, root_addr, ptr_align);
318 const buf = @as([*]u8, @ptrFromInt(aligned_addr))[0..n];
319 getRecordPtr(buf).* = root_addr;
320 return buf.ptr;
321 }
322
323 fn resize(
324 ctx: *anyopaque,
325 buf: []u8,
326 log2_buf_align: u8,
327 new_size: usize,
328 return_address: usize,
329 ) bool {
330 _ = log2_buf_align;
331 _ = return_address;
332 const self: *HeapAllocator = @ptrCast(@alignCast(ctx));
333
334 const root_addr = getRecordPtr(buf).*;
335 const align_offset = @intFromPtr(buf.ptr) - root_addr;
336 const amt = align_offset + new_size + @sizeOf(usize);
337 const new_ptr = windows.kernel32.HeapReAlloc(
338 self.heap_handle.?,
339 windows.HEAP_REALLOC_IN_PLACE_ONLY,
340 @as(*anyopaque, @ptrFromInt(root_addr)),
341 amt,
342 ) orelse return false;
343 assert(new_ptr == @as(*anyopaque, @ptrFromInt(root_addr)));
344 getRecordPtr(buf.ptr[0..new_size]).* = root_addr;
345 return true;
346 }
347
348 fn free(
349 ctx: *anyopaque,
350 buf: []u8,
351 log2_buf_align: u8,
352 return_address: usize,
353 ) void {
354 _ = log2_buf_align;
355 _ = return_address;
356 const self: *HeapAllocator = @ptrCast(@alignCast(ctx));
357 windows.HeapFree(self.heap_handle.?, 0, @as(*anyopaque, @ptrFromInt(getRecordPtr(buf).*)));
358 }
359 },
360 else => @compileError("Unsupported OS"),
361};
362
363fn sliceContainsPtr(container: []u8, ptr: [*]u8) bool {
364 return @intFromPtr(ptr) >= @intFromPtr(container.ptr) and
365 @intFromPtr(ptr) < (@intFromPtr(container.ptr) + container.len);
366}
367
368fn sliceContainsSlice(container: []u8, slice: []u8) bool {
369 return @intFromPtr(slice.ptr) >= @intFromPtr(container.ptr) and
370 (@intFromPtr(slice.ptr) + slice.len) <= (@intFromPtr(container.ptr) + container.len);
371}
372
373pub const FixedBufferAllocator = struct {
374 end_index: usize,
375 buffer: []u8,
376
377 pub fn init(buffer: []u8) FixedBufferAllocator {
378 return FixedBufferAllocator{
379 .buffer = buffer,
380 .end_index = 0,
381 };
382 }
383
384 /// *WARNING* using this at the same time as the interface returned by `threadSafeAllocator` is not thread safe
385 pub fn allocator(self: *FixedBufferAllocator) Allocator {
386 return .{
387 .ptr = self,
388 .vtable = &.{
389 .alloc = alloc,
390 .resize = resize,
391 .free = free,
392 },
393 };
394 }
395
396 /// Provides a lock free thread safe `Allocator` interface to the underlying `FixedBufferAllocator`
397 /// *WARNING* using this at the same time as the interface returned by `allocator` is not thread safe
398 pub fn threadSafeAllocator(self: *FixedBufferAllocator) Allocator {
399 return .{
400 .ptr = self,
401 .vtable = &.{
402 .alloc = threadSafeAlloc,
403 .resize = Allocator.noResize,
404 .free = Allocator.noFree,
405 },
406 };
407 }
408
409 pub fn ownsPtr(self: *FixedBufferAllocator, ptr: [*]u8) bool {
410 return sliceContainsPtr(self.buffer, ptr);
411 }
412
413 pub fn ownsSlice(self: *FixedBufferAllocator, slice: []u8) bool {
414 return sliceContainsSlice(self.buffer, slice);
415 }
416
417 /// NOTE: this will not work in all cases, if the last allocation had an adjusted_index
418 /// then we won't be able to determine what the last allocation was. This is because
419 /// the alignForward operation done in alloc is not reversible.
420 pub fn isLastAllocation(self: *FixedBufferAllocator, buf: []u8) bool {
421 return buf.ptr + buf.len == self.buffer.ptr + self.end_index;
422 }
423
424 fn alloc(ctx: *anyopaque, n: usize, log2_ptr_align: u8, ra: usize) ?[*]u8 {
425 const self: *FixedBufferAllocator = @ptrCast(@alignCast(ctx));
426 _ = ra;
427 const ptr_align = @as(usize, 1) << @as(Allocator.Log2Align, @intCast(log2_ptr_align));
428 const adjust_off = mem.alignPointerOffset(self.buffer.ptr + self.end_index, ptr_align) orelse return null;
429 const adjusted_index = self.end_index + adjust_off;
430 const new_end_index = adjusted_index + n;
431 if (new_end_index > self.buffer.len) return null;
432 self.end_index = new_end_index;
433 return self.buffer.ptr + adjusted_index;
434 }
435
436 fn resize(
437 ctx: *anyopaque,
438 buf: []u8,
439 log2_buf_align: u8,
440 new_size: usize,
441 return_address: usize,
442 ) bool {
443 const self: *FixedBufferAllocator = @ptrCast(@alignCast(ctx));
444 _ = log2_buf_align;
445 _ = return_address;
446 assert(@inComptime() or self.ownsSlice(buf));
447
448 if (!self.isLastAllocation(buf)) {
449 if (new_size > buf.len) return false;
450 return true;
451 }
452
453 if (new_size <= buf.len) {
454 const sub = buf.len - new_size;
455 self.end_index -= sub;
456 return true;
457 }
458
459 const add = new_size - buf.len;
460 if (add + self.end_index > self.buffer.len) return false;
461
462 self.end_index += add;
463 return true;
464 }
465
466 fn free(
467 ctx: *anyopaque,
468 buf: []u8,
469 log2_buf_align: u8,
470 return_address: usize,
471 ) void {
472 const self: *FixedBufferAllocator = @ptrCast(@alignCast(ctx));
473 _ = log2_buf_align;
474 _ = return_address;
475 assert(@inComptime() or self.ownsSlice(buf));
476
477 if (self.isLastAllocation(buf)) {
478 self.end_index -= buf.len;
479 }
480 }
481
482 fn threadSafeAlloc(ctx: *anyopaque, n: usize, log2_ptr_align: u8, ra: usize) ?[*]u8 {
483 const self: *FixedBufferAllocator = @ptrCast(@alignCast(ctx));
484 _ = ra;
485 const ptr_align = @as(usize, 1) << @as(Allocator.Log2Align, @intCast(log2_ptr_align));
486 var end_index = @atomicLoad(usize, &self.end_index, .seq_cst);
487 while (true) {
488 const adjust_off = mem.alignPointerOffset(self.buffer.ptr + end_index, ptr_align) orelse return null;
489 const adjusted_index = end_index + adjust_off;
490 const new_end_index = adjusted_index + n;
491 if (new_end_index > self.buffer.len) return null;
492 end_index = @cmpxchgWeak(usize, &self.end_index, end_index, new_end_index, .seq_cst, .seq_cst) orelse
493 return self.buffer[adjusted_index..new_end_index].ptr;
494 }
495 }
496
497 pub fn reset(self: *FixedBufferAllocator) void {
498 self.end_index = 0;
499 }
500};
501
502/// Returns a `StackFallbackAllocator` allocating using either a371/// Returns a `StackFallbackAllocator` allocating using either a
503/// `FixedBufferAllocator` on an array of size `size` and falling back to372/// `FixedBufferAllocator` on an array of size `size` and falling back to
504/// `fallback_allocator` if that fails.373/// `fallback_allocator` if that fails.
...@@ -537,6 +406,7 @@ pub fn StackFallbackAllocator(comptime size: usize) type {...@@ -537,6 +406,7 @@ pub fn StackFallbackAllocator(comptime size: usize) type {
537 .vtable = &.{406 .vtable = &.{
538 .alloc = alloc,407 .alloc = alloc,
539 .resize = resize,408 .resize = resize,
409 .remap = remap,
540 .free = free,410 .free = free,
541 },411 },
542 };412 };
...@@ -551,40 +421,55 @@ pub fn StackFallbackAllocator(comptime size: usize) type {...@@ -551,40 +421,55 @@ pub fn StackFallbackAllocator(comptime size: usize) type {
551 fn alloc(421 fn alloc(
552 ctx: *anyopaque,422 ctx: *anyopaque,
553 len: usize,423 len: usize,
554 log2_ptr_align: u8,424 alignment: mem.Alignment,
555 ra: usize,425 ra: usize,
556 ) ?[*]u8 {426 ) ?[*]u8 {
557 const self: *Self = @ptrCast(@alignCast(ctx));427 const self: *Self = @ptrCast(@alignCast(ctx));
558 return FixedBufferAllocator.alloc(&self.fixed_buffer_allocator, len, log2_ptr_align, ra) orelse428 return FixedBufferAllocator.alloc(&self.fixed_buffer_allocator, len, alignment, ra) orelse
559 return self.fallback_allocator.rawAlloc(len, log2_ptr_align, ra);429 return self.fallback_allocator.rawAlloc(len, alignment, ra);
560 }430 }
561431
562 fn resize(432 fn resize(
563 ctx: *anyopaque,433 ctx: *anyopaque,
564 buf: []u8,434 buf: []u8,
565 log2_buf_align: u8,435 alignment: mem.Alignment,
566 new_len: usize,436 new_len: usize,
567 ra: usize,437 ra: usize,
568 ) bool {438 ) bool {
569 const self: *Self = @ptrCast(@alignCast(ctx));439 const self: *Self = @ptrCast(@alignCast(ctx));
570 if (self.fixed_buffer_allocator.ownsPtr(buf.ptr)) {440 if (self.fixed_buffer_allocator.ownsPtr(buf.ptr)) {
571 return FixedBufferAllocator.resize(&self.fixed_buffer_allocator, buf, log2_buf_align, new_len, ra);441 return FixedBufferAllocator.resize(&self.fixed_buffer_allocator, buf, alignment, new_len, ra);
572 } else {442 } else {
573 return self.fallback_allocator.rawResize(buf, log2_buf_align, new_len, ra);443 return self.fallback_allocator.rawResize(buf, alignment, new_len, ra);
444 }
445 }
446
447 fn remap(
448 context: *anyopaque,
449 memory: []u8,
450 alignment: mem.Alignment,
451 new_len: usize,
452 return_address: usize,
453 ) ?[*]u8 {
454 const self: *Self = @ptrCast(@alignCast(context));
455 if (self.fixed_buffer_allocator.ownsPtr(memory.ptr)) {
456 return FixedBufferAllocator.remap(&self.fixed_buffer_allocator, memory, alignment, new_len, return_address);
457 } else {
458 return self.fallback_allocator.rawRemap(memory, alignment, new_len, return_address);
574 }459 }
575 }460 }
576461
577 fn free(462 fn free(
578 ctx: *anyopaque,463 ctx: *anyopaque,
579 buf: []u8,464 buf: []u8,
580 log2_buf_align: u8,465 alignment: mem.Alignment,
581 ra: usize,466 ra: usize,
582 ) void {467 ) void {
583 const self: *Self = @ptrCast(@alignCast(ctx));468 const self: *Self = @ptrCast(@alignCast(ctx));
584 if (self.fixed_buffer_allocator.ownsPtr(buf.ptr)) {469 if (self.fixed_buffer_allocator.ownsPtr(buf.ptr)) {
585 return FixedBufferAllocator.free(&self.fixed_buffer_allocator, buf, log2_buf_align, ra);470 return FixedBufferAllocator.free(&self.fixed_buffer_allocator, buf, alignment, ra);
586 } else {471 } else {
587 return self.fallback_allocator.rawFree(buf, log2_buf_align, ra);472 return self.fallback_allocator.rawFree(buf, alignment, ra);
588 }473 }
589 }474 }
590 };475 };
...@@ -605,7 +490,7 @@ test "raw_c_allocator" {...@@ -605,7 +490,7 @@ test "raw_c_allocator" {
605 }490 }
606}491}
607492
608test "PageAllocator" {493test PageAllocator {
609 const allocator = page_allocator;494 const allocator = page_allocator;
610 try testAllocator(allocator);495 try testAllocator(allocator);
611 try testAllocatorAligned(allocator);496 try testAllocatorAligned(allocator);
...@@ -615,35 +500,19 @@ test "PageAllocator" {...@@ -615,35 +500,19 @@ test "PageAllocator" {
615 }500 }
616501
617 if (builtin.os.tag == .windows) {502 if (builtin.os.tag == .windows) {
618 const slice = try allocator.alignedAlloc(u8, mem.page_size, 128);503 const slice = try allocator.alignedAlloc(u8, page_size_min, 128);
619 slice[0] = 0x12;504 slice[0] = 0x12;
620 slice[127] = 0x34;505 slice[127] = 0x34;
621 allocator.free(slice);506 allocator.free(slice);
622 }507 }
623 {508 {
624 var buf = try allocator.alloc(u8, mem.page_size + 1);509 var buf = try allocator.alloc(u8, pageSize() + 1);
625 defer allocator.free(buf);510 defer allocator.free(buf);
626 buf = try allocator.realloc(buf, 1); // shrink past the page boundary511 buf = try allocator.realloc(buf, 1); // shrink past the page boundary
627 }512 }
628}513}
629514
630test "HeapAllocator" {515test ArenaAllocator {
631 if (builtin.os.tag == .windows) {
632 // https://github.com/ziglang/zig/issues/13702
633 if (builtin.cpu.arch == .aarch64) return error.SkipZigTest;
634
635 var heap_allocator = HeapAllocator.init();
636 defer heap_allocator.deinit();
637 const allocator = heap_allocator.allocator();
638
639 try testAllocator(allocator);
640 try testAllocatorAligned(allocator);
641 try testAllocatorLargeAlignment(allocator);
642 try testAllocatorAlignedShrink(allocator);
643 }
644}
645
646test "ArenaAllocator" {
647 var arena_allocator = ArenaAllocator.init(page_allocator);516 var arena_allocator = ArenaAllocator.init(page_allocator);
648 defer arena_allocator.deinit();517 defer arena_allocator.deinit();
649 const allocator = arena_allocator.allocator();518 const allocator = arena_allocator.allocator();
...@@ -654,38 +523,6 @@ test "ArenaAllocator" {...@@ -654,38 +523,6 @@ test "ArenaAllocator" {
654 try testAllocatorAlignedShrink(allocator);523 try testAllocatorAlignedShrink(allocator);
655}524}
656525
657var test_fixed_buffer_allocator_memory: [800000 * @sizeOf(u64)]u8 = undefined;
658test "FixedBufferAllocator" {
659 var fixed_buffer_allocator = mem.validationWrap(FixedBufferAllocator.init(test_fixed_buffer_allocator_memory[0..]));
660 const allocator = fixed_buffer_allocator.allocator();
661
662 try testAllocator(allocator);
663 try testAllocatorAligned(allocator);
664 try testAllocatorLargeAlignment(allocator);
665 try testAllocatorAlignedShrink(allocator);
666}
667
668test "FixedBufferAllocator.reset" {
669 var buf: [8]u8 align(@alignOf(u64)) = undefined;
670 var fba = FixedBufferAllocator.init(buf[0..]);
671 const allocator = fba.allocator();
672
673 const X = 0xeeeeeeeeeeeeeeee;
674 const Y = 0xffffffffffffffff;
675
676 const x = try allocator.create(u64);
677 x.* = X;
678 try testing.expectError(error.OutOfMemory, allocator.create(u64));
679
680 fba.reset();
681 const y = try allocator.create(u64);
682 y.* = Y;
683
684 // we expect Y to have overwritten X.
685 try testing.expect(x.* == y.*);
686 try testing.expect(y.* == Y);
687}
688
689test "StackFallbackAllocator" {526test "StackFallbackAllocator" {
690 {527 {
691 var stack_allocator = stackFallback(4096, std.testing.allocator);528 var stack_allocator = stackFallback(4096, std.testing.allocator);
...@@ -705,46 +542,6 @@ test "StackFallbackAllocator" {...@@ -705,46 +542,6 @@ test "StackFallbackAllocator" {
705 }542 }
706}543}
707544
708test "FixedBufferAllocator Reuse memory on realloc" {
709 var small_fixed_buffer: [10]u8 = undefined;
710 // check if we re-use the memory
711 {
712 var fixed_buffer_allocator = FixedBufferAllocator.init(small_fixed_buffer[0..]);
713 const allocator = fixed_buffer_allocator.allocator();
714
715 const slice0 = try allocator.alloc(u8, 5);
716 try testing.expect(slice0.len == 5);
717 const slice1 = try allocator.realloc(slice0, 10);
718 try testing.expect(slice1.ptr == slice0.ptr);
719 try testing.expect(slice1.len == 10);
720 try testing.expectError(error.OutOfMemory, allocator.realloc(slice1, 11));
721 }
722 // check that we don't re-use the memory if it's not the most recent block
723 {
724 var fixed_buffer_allocator = FixedBufferAllocator.init(small_fixed_buffer[0..]);
725 const allocator = fixed_buffer_allocator.allocator();
726
727 var slice0 = try allocator.alloc(u8, 2);
728 slice0[0] = 1;
729 slice0[1] = 2;
730 const slice1 = try allocator.alloc(u8, 2);
731 const slice2 = try allocator.realloc(slice0, 4);
732 try testing.expect(slice0.ptr != slice2.ptr);
733 try testing.expect(slice1.ptr != slice2.ptr);
734 try testing.expect(slice2[0] == 1);
735 try testing.expect(slice2[1] == 2);
736 }
737}
738
739test "Thread safe FixedBufferAllocator" {
740 var fixed_buffer_allocator = FixedBufferAllocator.init(test_fixed_buffer_allocator_memory[0..]);
741
742 try testAllocator(fixed_buffer_allocator.threadSafeAllocator());
743 try testAllocatorAligned(fixed_buffer_allocator.threadSafeAllocator());
744 try testAllocatorLargeAlignment(fixed_buffer_allocator.threadSafeAllocator());
745 try testAllocatorAlignedShrink(fixed_buffer_allocator.threadSafeAllocator());
746}
747
748/// This one should not try alignments that exceed what C malloc can handle.545/// This one should not try alignments that exceed what C malloc can handle.
749pub fn testAllocator(base_allocator: mem.Allocator) !void {546pub fn testAllocator(base_allocator: mem.Allocator) !void {
750 var validationAllocator = mem.validationWrap(base_allocator);547 var validationAllocator = mem.validationWrap(base_allocator);
...@@ -824,7 +621,7 @@ pub fn testAllocatorLargeAlignment(base_allocator: mem.Allocator) !void {...@@ -824,7 +621,7 @@ pub fn testAllocatorLargeAlignment(base_allocator: mem.Allocator) !void {
824 var validationAllocator = mem.validationWrap(base_allocator);621 var validationAllocator = mem.validationWrap(base_allocator);
825 const allocator = validationAllocator.allocator();622 const allocator = validationAllocator.allocator();
826623
827 const large_align: usize = mem.page_size / 2;624 const large_align: usize = page_size_min / 2;
828625
829 var align_mask: usize = undefined;626 var align_mask: usize = undefined;
830 align_mask = @shlWithOverflow(~@as(usize, 0), @as(Allocator.Log2Align, @ctz(large_align)))[0];627 align_mask = @shlWithOverflow(~@as(usize, 0), @as(Allocator.Log2Align, @ctz(large_align)))[0];
...@@ -857,7 +654,7 @@ pub fn testAllocatorAlignedShrink(base_allocator: mem.Allocator) !void {...@@ -857,7 +654,7 @@ pub fn testAllocatorAlignedShrink(base_allocator: mem.Allocator) !void {
857 var fib = FixedBufferAllocator.init(&debug_buffer);654 var fib = FixedBufferAllocator.init(&debug_buffer);
858 const debug_allocator = fib.allocator();655 const debug_allocator = fib.allocator();
859656
860 const alloc_size = mem.page_size * 2 + 50;657 const alloc_size = pageSize() * 2 + 50;
861 var slice = try allocator.alignedAlloc(u8, 16, alloc_size);658 var slice = try allocator.alignedAlloc(u8, 16, alloc_size);
862 defer allocator.free(slice);659 defer allocator.free(slice);
863660
...@@ -866,7 +663,7 @@ pub fn testAllocatorAlignedShrink(base_allocator: mem.Allocator) !void {...@@ -866,7 +663,7 @@ pub fn testAllocatorAlignedShrink(base_allocator: mem.Allocator) !void {
866 // which is 16 pages, hence the 32. This test may require to increase663 // which is 16 pages, hence the 32. This test may require to increase
867 // the size of the allocations feeding the `allocator` parameter if they664 // the size of the allocations feeding the `allocator` parameter if they
868 // fail, because of this high over-alignment we want to have.665 // fail, because of this high over-alignment we want to have.
869 while (@intFromPtr(slice.ptr) == mem.alignForward(usize, @intFromPtr(slice.ptr), mem.page_size * 32)) {666 while (@intFromPtr(slice.ptr) == mem.alignForward(usize, @intFromPtr(slice.ptr), pageSize() * 32)) {
870 try stuff_to_free.append(slice);667 try stuff_to_free.append(slice);
871 slice = try allocator.alignedAlloc(u8, 16, alloc_size);668 slice = try allocator.alignedAlloc(u8, 16, alloc_size);
872 }669 }
...@@ -881,13 +678,303 @@ pub fn testAllocatorAlignedShrink(base_allocator: mem.Allocator) !void {...@@ -881,13 +678,303 @@ pub fn testAllocatorAlignedShrink(base_allocator: mem.Allocator) !void {
881 try testing.expect(slice[60] == 0x34);678 try testing.expect(slice[60] == 0x34);
882}679}
883680
681const page_size_min_default: ?usize = switch (builtin.os.tag) {
682 .driverkit, .ios, .macos, .tvos, .visionos, .watchos => switch (builtin.cpu.arch) {
683 .x86_64 => 4 << 10,
684 .aarch64 => 16 << 10,
685 else => null,
686 },
687 .windows => switch (builtin.cpu.arch) {
688 // -- <https://devblogs.microsoft.com/oldnewthing/20210510-00/?p=105200>
689 .x86, .x86_64 => 4 << 10,
690 // SuperH => 4 << 10,
691 .mips, .mipsel, .mips64, .mips64el => 4 << 10,
692 .powerpc, .powerpcle, .powerpc64, .powerpc64le => 4 << 10,
693 // DEC Alpha => 8 << 10,
694 // Itanium => 8 << 10,
695 .thumb, .thumbeb, .arm, .armeb, .aarch64, .aarch64_be => 4 << 10,
696 else => null,
697 },
698 .wasi => switch (builtin.cpu.arch) {
699 .wasm32, .wasm64 => 64 << 10,
700 else => null,
701 },
702 // https://github.com/tianocore/edk2/blob/b158dad150bf02879668f72ce306445250838201/MdePkg/Include/Uefi/UefiBaseType.h#L180-L187
703 .uefi => 4 << 10,
704 .freebsd => switch (builtin.cpu.arch) {
705 // FreeBSD/sys/*
706 .x86, .x86_64 => 4 << 10,
707 .thumb, .thumbeb, .arm, .armeb => 4 << 10,
708 .aarch64, .aarch64_be => 4 << 10,
709 .powerpc, .powerpc64, .powerpc64le, .powerpcle => 4 << 10,
710 .riscv32, .riscv64 => 4 << 10,
711 else => null,
712 },
713 .netbsd => switch (builtin.cpu.arch) {
714 // NetBSD/sys/arch/*
715 .x86, .x86_64 => 4 << 10,
716 .thumb, .thumbeb, .arm, .armeb => 4 << 10,
717 .aarch64, .aarch64_be => 4 << 10,
718 .mips, .mipsel, .mips64, .mips64el => 4 << 10,
719 .powerpc, .powerpc64, .powerpc64le, .powerpcle => 4 << 10,
720 .sparc => 4 << 10,
721 .sparc64 => 8 << 10,
722 .riscv32, .riscv64 => 4 << 10,
723 // Sun-2
724 .m68k => 2 << 10,
725 else => null,
726 },
727 .dragonfly => switch (builtin.cpu.arch) {
728 .x86, .x86_64 => 4 << 10,
729 else => null,
730 },
731 .openbsd => switch (builtin.cpu.arch) {
732 // OpenBSD/sys/arch/*
733 .x86, .x86_64 => 4 << 10,
734 .thumb, .thumbeb, .arm, .armeb, .aarch64, .aarch64_be => 4 << 10,
735 .mips64, .mips64el => 4 << 10,
736 .powerpc, .powerpc64, .powerpc64le, .powerpcle => 4 << 10,
737 .riscv64 => 4 << 10,
738 .sparc64 => 8 << 10,
739 else => null,
740 },
741 .solaris, .illumos => switch (builtin.cpu.arch) {
742 // src/uts/*/sys/machparam.h
743 .x86, .x86_64 => 4 << 10,
744 .sparc, .sparc64 => 8 << 10,
745 else => null,
746 },
747 .fuchsia => switch (builtin.cpu.arch) {
748 // fuchsia/kernel/arch/*/include/arch/defines.h
749 .x86_64 => 4 << 10,
750 .aarch64, .aarch64_be => 4 << 10,
751 .riscv64 => 4 << 10,
752 else => null,
753 },
754 // https://github.com/SerenityOS/serenity/blob/62b938b798dc009605b5df8a71145942fc53808b/Kernel/API/POSIX/sys/limits.h#L11-L13
755 .serenity => 4 << 10,
756 .haiku => switch (builtin.cpu.arch) {
757 // haiku/headers/posix/arch/*/limits.h
758 .thumb, .thumbeb, .arm, .armeb => 4 << 10,
759 .aarch64, .aarch64_be => 4 << 10,
760 .m68k => 4 << 10,
761 .mips, .mipsel, .mips64, .mips64el => 4 << 10,
762 .powerpc, .powerpc64, .powerpc64le, .powerpcle => 4 << 10,
763 .riscv64 => 4 << 10,
764 .sparc64 => 8 << 10,
765 .x86, .x86_64 => 4 << 10,
766 else => null,
767 },
768 .hurd => switch (builtin.cpu.arch) {
769 // gnumach/*/include/mach/*/vm_param.h
770 .x86, .x86_64 => 4 << 10,
771 .aarch64 => null,
772 else => null,
773 },
774 .plan9 => switch (builtin.cpu.arch) {
775 // 9front/sys/src/9/*/mem.h
776 .x86, .x86_64 => 4 << 10,
777 .thumb, .thumbeb, .arm, .armeb => 4 << 10,
778 .aarch64, .aarch64_be => 4 << 10,
779 .mips, .mipsel, .mips64, .mips64el => 4 << 10,
780 .powerpc, .powerpcle, .powerpc64, .powerpc64le => 4 << 10,
781 .sparc => 4 << 10,
782 else => null,
783 },
784 .ps3 => switch (builtin.cpu.arch) {
785 // cell/SDK_doc/en/html/C_and_C++_standard_libraries/stdlib.html
786 .powerpc64 => 1 << 20, // 1 MiB
787 else => null,
788 },
789 .ps4 => switch (builtin.cpu.arch) {
790 // https://github.com/ps4dev/ps4sdk/blob/4df9d001b66ae4ec07d9a51b62d1e4c5e270eecc/include/machine/param.h#L95
791 .x86, .x86_64 => 4 << 10,
792 else => null,
793 },
794 .ps5 => switch (builtin.cpu.arch) {
795 // https://github.com/PS5Dev/PS5SDK/blob/a2e03a2a0231a3a3397fa6cd087a01ca6d04f273/include/machine/param.h#L95
796 .x86, .x86_64 => 16 << 10,
797 else => null,
798 },
799 // system/lib/libc/musl/arch/emscripten/bits/limits.h
800 .emscripten => 64 << 10,
801 .linux => switch (builtin.cpu.arch) {
802 // Linux/arch/*/Kconfig
803 .arc => 4 << 10,
804 .thumb, .thumbeb, .arm, .armeb => 4 << 10,
805 .aarch64, .aarch64_be => 4 << 10,
806 .csky => 4 << 10,
807 .hexagon => 4 << 10,
808 .loongarch32, .loongarch64 => 4 << 10,
809 .m68k => 4 << 10,
810 .mips, .mipsel, .mips64, .mips64el => 4 << 10,
811 .powerpc, .powerpc64, .powerpc64le, .powerpcle => 4 << 10,
812 .riscv32, .riscv64 => 4 << 10,
813 .s390x => 4 << 10,
814 .sparc => 4 << 10,
815 .sparc64 => 8 << 10,
816 .x86, .x86_64 => 4 << 10,
817 .xtensa => 4 << 10,
818 else => null,
819 },
820 .freestanding => switch (builtin.cpu.arch) {
821 .wasm32, .wasm64 => 64 << 10,
822 else => null,
823 },
824 else => null,
825};
826
827const page_size_max_default: ?usize = switch (builtin.os.tag) {
828 .driverkit, .ios, .macos, .tvos, .visionos, .watchos => switch (builtin.cpu.arch) {
829 .x86_64 => 4 << 10,
830 .aarch64 => 16 << 10,
831 else => null,
832 },
833 .windows => switch (builtin.cpu.arch) {
834 // -- <https://devblogs.microsoft.com/oldnewthing/20210510-00/?p=105200>
835 .x86, .x86_64 => 4 << 10,
836 // SuperH => 4 << 10,
837 .mips, .mipsel, .mips64, .mips64el => 4 << 10,
838 .powerpc, .powerpcle, .powerpc64, .powerpc64le => 4 << 10,
839 // DEC Alpha => 8 << 10,
840 // Itanium => 8 << 10,
841 .thumb, .thumbeb, .arm, .armeb, .aarch64, .aarch64_be => 4 << 10,
842 else => null,
843 },
844 .wasi => switch (builtin.cpu.arch) {
845 .wasm32, .wasm64 => 64 << 10,
846 else => null,
847 },
848 // https://github.com/tianocore/edk2/blob/b158dad150bf02879668f72ce306445250838201/MdePkg/Include/Uefi/UefiBaseType.h#L180-L187
849 .uefi => 4 << 10,
850 .freebsd => switch (builtin.cpu.arch) {
851 // FreeBSD/sys/*
852 .x86, .x86_64 => 4 << 10,
853 .thumb, .thumbeb, .arm, .armeb => 4 << 10,
854 .aarch64, .aarch64_be => 4 << 10,
855 .powerpc, .powerpc64, .powerpc64le, .powerpcle => 4 << 10,
856 .riscv32, .riscv64 => 4 << 10,
857 else => null,
858 },
859 .netbsd => switch (builtin.cpu.arch) {
860 // NetBSD/sys/arch/*
861 .x86, .x86_64 => 4 << 10,
862 .thumb, .thumbeb, .arm, .armeb => 4 << 10,
863 .aarch64, .aarch64_be => 64 << 10,
864 .mips, .mipsel, .mips64, .mips64el => 16 << 10,
865 .powerpc, .powerpc64, .powerpc64le, .powerpcle => 16 << 10,
866 .sparc => 8 << 10,
867 .sparc64 => 8 << 10,
868 .riscv32, .riscv64 => 4 << 10,
869 .m68k => 8 << 10,
870 else => null,
871 },
872 .dragonfly => switch (builtin.cpu.arch) {
873 .x86, .x86_64 => 4 << 10,
874 else => null,
875 },
876 .openbsd => switch (builtin.cpu.arch) {
877 // OpenBSD/sys/arch/*
878 .x86, .x86_64 => 4 << 10,
879 .thumb, .thumbeb, .arm, .armeb, .aarch64, .aarch64_be => 4 << 10,
880 .mips64, .mips64el => 16 << 10,
881 .powerpc, .powerpc64, .powerpc64le, .powerpcle => 4 << 10,
882 .riscv64 => 4 << 10,
883 .sparc64 => 8 << 10,
884 else => null,
885 },
886 .solaris, .illumos => switch (builtin.cpu.arch) {
887 // src/uts/*/sys/machparam.h
888 .x86, .x86_64 => 4 << 10,
889 .sparc, .sparc64 => 8 << 10,
890 else => null,
891 },
892 .fuchsia => switch (builtin.cpu.arch) {
893 // fuchsia/kernel/arch/*/include/arch/defines.h
894 .x86_64 => 4 << 10,
895 .aarch64, .aarch64_be => 4 << 10,
896 .riscv64 => 4 << 10,
897 else => null,
898 },
899 // https://github.com/SerenityOS/serenity/blob/62b938b798dc009605b5df8a71145942fc53808b/Kernel/API/POSIX/sys/limits.h#L11-L13
900 .serenity => 4 << 10,
901 .haiku => switch (builtin.cpu.arch) {
902 // haiku/headers/posix/arch/*/limits.h
903 .thumb, .thumbeb, .arm, .armeb => 4 << 10,
904 .aarch64, .aarch64_be => 4 << 10,
905 .m68k => 4 << 10,
906 .mips, .mipsel, .mips64, .mips64el => 4 << 10,
907 .powerpc, .powerpc64, .powerpc64le, .powerpcle => 4 << 10,
908 .riscv64 => 4 << 10,
909 .sparc64 => 8 << 10,
910 .x86, .x86_64 => 4 << 10,
911 else => null,
912 },
913 .hurd => switch (builtin.cpu.arch) {
914 // gnumach/*/include/mach/*/vm_param.h
915 .x86, .x86_64 => 4 << 10,
916 .aarch64 => null,
917 else => null,
918 },
919 .plan9 => switch (builtin.cpu.arch) {
920 // 9front/sys/src/9/*/mem.h
921 .x86, .x86_64 => 4 << 10,
922 .thumb, .thumbeb, .arm, .armeb => 4 << 10,
923 .aarch64, .aarch64_be => 64 << 10,
924 .mips, .mipsel, .mips64, .mips64el => 16 << 10,
925 .powerpc, .powerpcle, .powerpc64, .powerpc64le => 4 << 10,
926 .sparc => 4 << 10,
927 else => null,
928 },
929 .ps3 => switch (builtin.cpu.arch) {
930 // cell/SDK_doc/en/html/C_and_C++_standard_libraries/stdlib.html
931 .powerpc64 => 1 << 20, // 1 MiB
932 else => null,
933 },
934 .ps4 => switch (builtin.cpu.arch) {
935 // https://github.com/ps4dev/ps4sdk/blob/4df9d001b66ae4ec07d9a51b62d1e4c5e270eecc/include/machine/param.h#L95
936 .x86, .x86_64 => 4 << 10,
937 else => null,
938 },
939 .ps5 => switch (builtin.cpu.arch) {
940 // https://github.com/PS5Dev/PS5SDK/blob/a2e03a2a0231a3a3397fa6cd087a01ca6d04f273/include/machine/param.h#L95
941 .x86, .x86_64 => 16 << 10,
942 else => null,
943 },
944 // system/lib/libc/musl/arch/emscripten/bits/limits.h
945 .emscripten => 64 << 10,
946 .linux => switch (builtin.cpu.arch) {
947 // Linux/arch/*/Kconfig
948 .arc => 16 << 10,
949 .thumb, .thumbeb, .arm, .armeb => 4 << 10,
950 .aarch64, .aarch64_be => 64 << 10,
951 .csky => 4 << 10,
952 .hexagon => 256 << 10,
953 .loongarch32, .loongarch64 => 64 << 10,
954 .m68k => 8 << 10,
955 .mips, .mipsel, .mips64, .mips64el => 64 << 10,
956 .powerpc, .powerpc64, .powerpc64le, .powerpcle => 256 << 10,
957 .riscv32, .riscv64 => 4 << 10,
958 .s390x => 4 << 10,
959 .sparc => 4 << 10,
960 .sparc64 => 8 << 10,
961 .x86, .x86_64 => 4 << 10,
962 .xtensa => 4 << 10,
963 else => null,
964 },
965 .freestanding => switch (builtin.cpu.arch) {
966 .wasm32, .wasm64 => 64 << 10,
967 else => null,
968 },
969 else => null,
970};
971
884test {972test {
885 _ = LoggingAllocator;
886 _ = LogToWriterAllocator;
887 _ = ScopedLoggingAllocator;
888 _ = @import("heap/memory_pool.zig");973 _ = @import("heap/memory_pool.zig");
889 _ = ArenaAllocator;974 _ = ArenaAllocator;
890 _ = GeneralPurposeAllocator;975 _ = GeneralPurposeAllocator;
976 _ = FixedBufferAllocator;
977 _ = ThreadSafeAllocator;
891 if (builtin.target.isWasm()) {978 if (builtin.target.isWasm()) {
892 _ = WasmAllocator;979 _ = WasmAllocator;
893 }980 }
lib/std/heap/FixedBufferAllocator.zig created+230
...@@ -0,0 +1,230 @@
1const std = @import("../std.zig");
2const Allocator = std.mem.Allocator;
3const assert = std.debug.assert;
4const mem = std.mem;
5
6const FixedBufferAllocator = @This();
7
8end_index: usize,
9buffer: []u8,
10
11pub fn init(buffer: []u8) FixedBufferAllocator {
12 return .{
13 .buffer = buffer,
14 .end_index = 0,
15 };
16}
17
18/// Using this at the same time as the interface returned by `threadSafeAllocator` is not thread safe.
19pub fn allocator(self: *FixedBufferAllocator) Allocator {
20 return .{
21 .ptr = self,
22 .vtable = &.{
23 .alloc = alloc,
24 .resize = resize,
25 .remap = remap,
26 .free = free,
27 },
28 };
29}
30
31/// Provides a lock free thread safe `Allocator` interface to the underlying `FixedBufferAllocator`
32///
33/// Using this at the same time as the interface returned by `allocator` is not thread safe.
34pub fn threadSafeAllocator(self: *FixedBufferAllocator) Allocator {
35 return .{
36 .ptr = self,
37 .vtable = &.{
38 .alloc = threadSafeAlloc,
39 .resize = Allocator.noResize,
40 .remap = Allocator.noRemap,
41 .free = Allocator.noFree,
42 },
43 };
44}
45
46pub fn ownsPtr(self: *FixedBufferAllocator, ptr: [*]u8) bool {
47 return sliceContainsPtr(self.buffer, ptr);
48}
49
50pub fn ownsSlice(self: *FixedBufferAllocator, slice: []u8) bool {
51 return sliceContainsSlice(self.buffer, slice);
52}
53
54/// This has false negatives when the last allocation had an
55/// adjusted_index. In such case we won't be able to determine what the
56/// last allocation was because the alignForward operation done in alloc is
57/// not reversible.
58pub fn isLastAllocation(self: *FixedBufferAllocator, buf: []u8) bool {
59 return buf.ptr + buf.len == self.buffer.ptr + self.end_index;
60}
61
62pub fn alloc(ctx: *anyopaque, n: usize, alignment: mem.Alignment, ra: usize) ?[*]u8 {
63 const self: *FixedBufferAllocator = @ptrCast(@alignCast(ctx));
64 _ = ra;
65 const ptr_align = alignment.toByteUnits();
66 const adjust_off = mem.alignPointerOffset(self.buffer.ptr + self.end_index, ptr_align) orelse return null;
67 const adjusted_index = self.end_index + adjust_off;
68 const new_end_index = adjusted_index + n;
69 if (new_end_index > self.buffer.len) return null;
70 self.end_index = new_end_index;
71 return self.buffer.ptr + adjusted_index;
72}
73
74pub fn resize(
75 ctx: *anyopaque,
76 buf: []u8,
77 alignment: mem.Alignment,
78 new_size: usize,
79 return_address: usize,
80) bool {
81 const self: *FixedBufferAllocator = @ptrCast(@alignCast(ctx));
82 _ = alignment;
83 _ = return_address;
84 assert(@inComptime() or self.ownsSlice(buf));
85
86 if (!self.isLastAllocation(buf)) {
87 if (new_size > buf.len) return false;
88 return true;
89 }
90
91 if (new_size <= buf.len) {
92 const sub = buf.len - new_size;
93 self.end_index -= sub;
94 return true;
95 }
96
97 const add = new_size - buf.len;
98 if (add + self.end_index > self.buffer.len) return false;
99
100 self.end_index += add;
101 return true;
102}
103
104pub fn remap(
105 context: *anyopaque,
106 memory: []u8,
107 alignment: mem.Alignment,
108 new_len: usize,
109 return_address: usize,
110) ?[*]u8 {
111 return if (resize(context, memory, alignment, new_len, return_address)) memory.ptr else null;
112}
113
114pub fn free(
115 ctx: *anyopaque,
116 buf: []u8,
117 alignment: mem.Alignment,
118 return_address: usize,
119) void {
120 const self: *FixedBufferAllocator = @ptrCast(@alignCast(ctx));
121 _ = alignment;
122 _ = return_address;
123 assert(@inComptime() or self.ownsSlice(buf));
124
125 if (self.isLastAllocation(buf)) {
126 self.end_index -= buf.len;
127 }
128}
129
130fn threadSafeAlloc(ctx: *anyopaque, n: usize, alignment: mem.Alignment, ra: usize) ?[*]u8 {
131 const self: *FixedBufferAllocator = @ptrCast(@alignCast(ctx));
132 _ = ra;
133 const ptr_align = alignment.toByteUnits();
134 var end_index = @atomicLoad(usize, &self.end_index, .seq_cst);
135 while (true) {
136 const adjust_off = mem.alignPointerOffset(self.buffer.ptr + end_index, ptr_align) orelse return null;
137 const adjusted_index = end_index + adjust_off;
138 const new_end_index = adjusted_index + n;
139 if (new_end_index > self.buffer.len) return null;
140 end_index = @cmpxchgWeak(usize, &self.end_index, end_index, new_end_index, .seq_cst, .seq_cst) orelse
141 return self.buffer[adjusted_index..new_end_index].ptr;
142 }
143}
144
145pub fn reset(self: *FixedBufferAllocator) void {
146 self.end_index = 0;
147}
148
149fn sliceContainsPtr(container: []u8, ptr: [*]u8) bool {
150 return @intFromPtr(ptr) >= @intFromPtr(container.ptr) and
151 @intFromPtr(ptr) < (@intFromPtr(container.ptr) + container.len);
152}
153
154fn sliceContainsSlice(container: []u8, slice: []u8) bool {
155 return @intFromPtr(slice.ptr) >= @intFromPtr(container.ptr) and
156 (@intFromPtr(slice.ptr) + slice.len) <= (@intFromPtr(container.ptr) + container.len);
157}
158
159var test_fixed_buffer_allocator_memory: [800000 * @sizeOf(u64)]u8 = undefined;
160
161test FixedBufferAllocator {
162 var fixed_buffer_allocator = mem.validationWrap(FixedBufferAllocator.init(test_fixed_buffer_allocator_memory[0..]));
163 const a = fixed_buffer_allocator.allocator();
164
165 try std.heap.testAllocator(a);
166 try std.heap.testAllocatorAligned(a);
167 try std.heap.testAllocatorLargeAlignment(a);
168 try std.heap.testAllocatorAlignedShrink(a);
169}
170
171test reset {
172 var buf: [8]u8 align(@alignOf(u64)) = undefined;
173 var fba = FixedBufferAllocator.init(buf[0..]);
174 const a = fba.allocator();
175
176 const X = 0xeeeeeeeeeeeeeeee;
177 const Y = 0xffffffffffffffff;
178
179 const x = try a.create(u64);
180 x.* = X;
181 try std.testing.expectError(error.OutOfMemory, a.create(u64));
182
183 fba.reset();
184 const y = try a.create(u64);
185 y.* = Y;
186
187 // we expect Y to have overwritten X.
188 try std.testing.expect(x.* == y.*);
189 try std.testing.expect(y.* == Y);
190}
191
192test "reuse memory on realloc" {
193 var small_fixed_buffer: [10]u8 = undefined;
194 // check if we re-use the memory
195 {
196 var fixed_buffer_allocator = FixedBufferAllocator.init(small_fixed_buffer[0..]);
197 const a = fixed_buffer_allocator.allocator();
198
199 const slice0 = try a.alloc(u8, 5);
200 try std.testing.expect(slice0.len == 5);
201 const slice1 = try a.realloc(slice0, 10);
202 try std.testing.expect(slice1.ptr == slice0.ptr);
203 try std.testing.expect(slice1.len == 10);
204 try std.testing.expectError(error.OutOfMemory, a.realloc(slice1, 11));
205 }
206 // check that we don't re-use the memory if it's not the most recent block
207 {
208 var fixed_buffer_allocator = FixedBufferAllocator.init(small_fixed_buffer[0..]);
209 const a = fixed_buffer_allocator.allocator();
210
211 var slice0 = try a.alloc(u8, 2);
212 slice0[0] = 1;
213 slice0[1] = 2;
214 const slice1 = try a.alloc(u8, 2);
215 const slice2 = try a.realloc(slice0, 4);
216 try std.testing.expect(slice0.ptr != slice2.ptr);
217 try std.testing.expect(slice1.ptr != slice2.ptr);
218 try std.testing.expect(slice2[0] == 1);
219 try std.testing.expect(slice2[1] == 2);
220 }
221}
222
223test "thread safe version" {
224 var fixed_buffer_allocator = FixedBufferAllocator.init(test_fixed_buffer_allocator_memory[0..]);
225
226 try std.heap.testAllocator(fixed_buffer_allocator.threadSafeAllocator());
227 try std.heap.testAllocatorAligned(fixed_buffer_allocator.threadSafeAllocator());
228 try std.heap.testAllocatorLargeAlignment(fixed_buffer_allocator.threadSafeAllocator());
229 try std.heap.testAllocatorAlignedShrink(fixed_buffer_allocator.threadSafeAllocator());
230}
lib/std/heap/PageAllocator.zig+128-52
...@@ -7,107 +7,183 @@ const assert = std.debug.assert;...@@ -7,107 +7,183 @@ const assert = std.debug.assert;
7const native_os = builtin.os.tag;7const native_os = builtin.os.tag;
8const windows = std.os.windows;8const windows = std.os.windows;
9const posix = std.posix;9const posix = std.posix;
10const page_size_min = std.heap.page_size_min;
1011
11pub const vtable = Allocator.VTable{12pub const vtable: Allocator.VTable = .{
12 .alloc = alloc,13 .alloc = alloc,
13 .resize = resize,14 .resize = resize,
15 .remap = remap,
14 .free = free,16 .free = free,
15};17};
1618
17fn alloc(_: *anyopaque, n: usize, log2_align: u8, ra: usize) ?[*]u8 {19fn alloc(context: *anyopaque, n: usize, alignment: mem.Alignment, ra: usize) ?[*]u8 {
20 _ = context;
18 _ = ra;21 _ = ra;
19 _ = log2_align;
20 assert(n > 0);22 assert(n > 0);
21 if (n > maxInt(usize) - (mem.page_size - 1)) return null;23
24 const page_size = std.heap.pageSize();
25 if (n >= maxInt(usize) - page_size) return null;
26 const alignment_bytes = alignment.toByteUnits();
2227
23 if (native_os == .windows) {28 if (native_os == .windows) {
29 // According to official documentation, VirtualAlloc aligns to page
30 // boundary, however, empirically it reserves pages on a 64K boundary.
31 // Since it is very likely the requested alignment will be honored,
32 // this logic first tries a call with exactly the size requested,
33 // before falling back to the loop below.
34 // https://devblogs.microsoft.com/oldnewthing/?p=42223
24 const addr = windows.VirtualAlloc(35 const addr = windows.VirtualAlloc(
25 null,36 null,
26
27 // VirtualAlloc will round the length to a multiple of page size.37 // VirtualAlloc will round the length to a multiple of page size.
28 // VirtualAlloc docs: If the lpAddress parameter is NULL, this value is rounded up to the next page boundary38 // "If the lpAddress parameter is NULL, this value is rounded up to
39 // the next page boundary".
29 n,40 n,
30
31 windows.MEM_COMMIT | windows.MEM_RESERVE,41 windows.MEM_COMMIT | windows.MEM_RESERVE,
32 windows.PAGE_READWRITE,42 windows.PAGE_READWRITE,
33 ) catch return null;43 ) catch return null;
34 return @ptrCast(addr);44
45 if (mem.isAligned(@intFromPtr(addr), alignment_bytes))
46 return @ptrCast(addr);
47
48 // Fallback: reserve a range of memory large enough to find a
49 // sufficiently aligned address, then free the entire range and
50 // immediately allocate the desired subset. Another thread may have won
51 // the race to map the target range, in which case a retry is needed.
52 windows.VirtualFree(addr, 0, windows.MEM_RELEASE);
53
54 const overalloc_len = n + alignment_bytes - page_size;
55 const aligned_len = mem.alignForward(usize, n, page_size);
56
57 while (true) {
58 const reserved_addr = windows.VirtualAlloc(
59 null,
60 overalloc_len,
61 windows.MEM_RESERVE,
62 windows.PAGE_NOACCESS,
63 ) catch return null;
64 const aligned_addr = mem.alignForward(usize, @intFromPtr(reserved_addr), alignment_bytes);
65 windows.VirtualFree(reserved_addr, 0, windows.MEM_RELEASE);
66 const ptr = windows.VirtualAlloc(
67 @ptrFromInt(aligned_addr),
68 aligned_len,
69 windows.MEM_COMMIT | windows.MEM_RESERVE,
70 windows.PAGE_READWRITE,
71 ) catch continue;
72 return @ptrCast(ptr);
73 }
35 }74 }
3675
37 const aligned_len = mem.alignForward(usize, n, mem.page_size);76 const aligned_len = mem.alignForward(usize, n, page_size);
77 const max_drop_len = alignment_bytes - @min(alignment_bytes, page_size);
78 const overalloc_len = if (max_drop_len <= aligned_len - n)
79 aligned_len
80 else
81 mem.alignForward(usize, aligned_len + max_drop_len, page_size);
38 const hint = @atomicLoad(@TypeOf(std.heap.next_mmap_addr_hint), &std.heap.next_mmap_addr_hint, .unordered);82 const hint = @atomicLoad(@TypeOf(std.heap.next_mmap_addr_hint), &std.heap.next_mmap_addr_hint, .unordered);
39 const slice = posix.mmap(83 const slice = posix.mmap(
40 hint,84 hint,
41 aligned_len,85 overalloc_len,
42 posix.PROT.READ | posix.PROT.WRITE,86 posix.PROT.READ | posix.PROT.WRITE,
43 .{ .TYPE = .PRIVATE, .ANONYMOUS = true },87 .{ .TYPE = .PRIVATE, .ANONYMOUS = true },
44 -1,88 -1,
45 0,89 0,
46 ) catch return null;90 ) catch return null;
47 assert(mem.isAligned(@intFromPtr(slice.ptr), mem.page_size));91 const result_ptr = mem.alignPointer(slice.ptr, alignment_bytes) orelse return null;
48 const new_hint: [*]align(mem.page_size) u8 = @alignCast(slice.ptr + aligned_len);92 // Unmap the extra bytes that were only requested in order to guarantee
93 // that the range of memory we were provided had a proper alignment in it
94 // somewhere. The extra bytes could be at the beginning, or end, or both.
95 const drop_len = result_ptr - slice.ptr;
96 if (drop_len != 0) posix.munmap(slice[0..drop_len]);
97 const remaining_len = overalloc_len - drop_len;
98 if (remaining_len > aligned_len) posix.munmap(@alignCast(result_ptr[aligned_len..remaining_len]));
99 const new_hint: [*]align(page_size_min) u8 = @alignCast(result_ptr + aligned_len);
49 _ = @cmpxchgStrong(@TypeOf(std.heap.next_mmap_addr_hint), &std.heap.next_mmap_addr_hint, hint, new_hint, .monotonic, .monotonic);100 _ = @cmpxchgStrong(@TypeOf(std.heap.next_mmap_addr_hint), &std.heap.next_mmap_addr_hint, hint, new_hint, .monotonic, .monotonic);
50 return slice.ptr;101 return result_ptr;
51}102}
52103
53fn resize(104fn resize(
54 _: *anyopaque,105 context: *anyopaque,
55 buf_unaligned: []u8,106 memory: []u8,
56 log2_buf_align: u8,107 alignment: mem.Alignment,
57 new_size: usize,108 new_len: usize,
58 return_address: usize,109 return_address: usize,
59) bool {110) bool {
60 _ = log2_buf_align;111 _ = context;
112 _ = alignment;
61 _ = return_address;113 _ = return_address;
62 const new_size_aligned = mem.alignForward(usize, new_size, mem.page_size);114 return realloc(memory, new_len, false) != null;
115}
116
117pub fn remap(
118 context: *anyopaque,
119 memory: []u8,
120 alignment: mem.Alignment,
121 new_len: usize,
122 return_address: usize,
123) ?[*]u8 {
124 _ = context;
125 _ = alignment;
126 _ = return_address;
127 return realloc(memory, new_len, true);
128}
129
130fn free(context: *anyopaque, slice: []u8, alignment: mem.Alignment, return_address: usize) void {
131 _ = context;
132 _ = alignment;
133 _ = return_address;
134
135 if (native_os == .windows) {
136 windows.VirtualFree(slice.ptr, 0, windows.MEM_RELEASE);
137 } else {
138 const buf_aligned_len = mem.alignForward(usize, slice.len, std.heap.pageSize());
139 posix.munmap(@alignCast(slice.ptr[0..buf_aligned_len]));
140 }
141}
142
143fn realloc(uncasted_memory: []u8, new_len: usize, may_move: bool) ?[*]u8 {
144 const memory: []align(std.heap.page_size_min) u8 = @alignCast(uncasted_memory);
145 const page_size = std.heap.pageSize();
146 const new_size_aligned = mem.alignForward(usize, new_len, page_size);
63147
64 if (native_os == .windows) {148 if (native_os == .windows) {
65 if (new_size <= buf_unaligned.len) {149 if (new_len <= memory.len) {
66 const base_addr = @intFromPtr(buf_unaligned.ptr);150 const base_addr = @intFromPtr(memory.ptr);
67 const old_addr_end = base_addr + buf_unaligned.len;151 const old_addr_end = base_addr + memory.len;
68 const new_addr_end = mem.alignForward(usize, base_addr + new_size, mem.page_size);152 const new_addr_end = mem.alignForward(usize, base_addr + new_len, page_size);
69 if (old_addr_end > new_addr_end) {153 if (old_addr_end > new_addr_end) {
70 // For shrinking that is not releasing, we will only154 // For shrinking that is not releasing, we will only decommit
71 // decommit the pages not needed anymore.155 // the pages not needed anymore.
72 windows.VirtualFree(156 windows.VirtualFree(
73 @as(*anyopaque, @ptrFromInt(new_addr_end)),157 @ptrFromInt(new_addr_end),
74 old_addr_end - new_addr_end,158 old_addr_end - new_addr_end,
75 windows.MEM_DECOMMIT,159 windows.MEM_DECOMMIT,
76 );160 );
77 }161 }
78 return true;162 return memory.ptr;
79 }163 }
80 const old_size_aligned = mem.alignForward(usize, buf_unaligned.len, mem.page_size);164 const old_size_aligned = mem.alignForward(usize, memory.len, page_size);
81 if (new_size_aligned <= old_size_aligned) {165 if (new_size_aligned <= old_size_aligned) {
82 return true;166 return memory.ptr;
83 }167 }
84 return false;168 return null;
85 }169 }
86170
87 const buf_aligned_len = mem.alignForward(usize, buf_unaligned.len, mem.page_size);171 const page_aligned_len = mem.alignForward(usize, memory.len, page_size);
88 if (new_size_aligned == buf_aligned_len)172 if (new_size_aligned == page_aligned_len)
89 return true;173 return memory.ptr;
90174
91 if (new_size_aligned < buf_aligned_len) {175 if (posix.MREMAP != void) {
92 const ptr = buf_unaligned.ptr + new_size_aligned;176 // TODO: if the next_mmap_addr_hint is within the remapped range, update it
93 // TODO: if the next_mmap_addr_hint is within the unmapped range, update it177 const new_memory = posix.mremap(memory.ptr, memory.len, new_len, .{ .MAYMOVE = may_move }, null) catch return null;
94 posix.munmap(@alignCast(ptr[0 .. buf_aligned_len - new_size_aligned]));178 return new_memory.ptr;
95 return true;
96 }179 }
97180
98 // TODO: call mremap181 if (new_size_aligned < page_aligned_len) {
99 // TODO: if the next_mmap_addr_hint is within the remapped range, update it182 const ptr = memory.ptr + new_size_aligned;
100 return false;183 // TODO: if the next_mmap_addr_hint is within the unmapped range, update it
101}184 posix.munmap(@alignCast(ptr[0 .. page_aligned_len - new_size_aligned]));
102185 return memory.ptr;
103fn free(_: *anyopaque, slice: []u8, log2_buf_align: u8, return_address: usize) void {
104 _ = log2_buf_align;
105 _ = return_address;
106
107 if (native_os == .windows) {
108 windows.VirtualFree(slice.ptr, 0, windows.MEM_RELEASE);
109 } else {
110 const buf_aligned_len = mem.alignForward(usize, slice.len, mem.page_size);
111 posix.munmap(@alignCast(slice.ptr[0..buf_aligned_len]));
112 }186 }
187
188 return null;
113}189}
lib/std/heap/ThreadSafeAllocator.zig+16-6
...@@ -9,35 +9,45 @@ pub fn allocator(self: *ThreadSafeAllocator) Allocator {...@@ -9,35 +9,45 @@ pub fn allocator(self: *ThreadSafeAllocator) Allocator {
9 .vtable = &.{9 .vtable = &.{
10 .alloc = alloc,10 .alloc = alloc,
11 .resize = resize,11 .resize = resize,
12 .remap = remap,
12 .free = free,13 .free = free,
13 },14 },
14 };15 };
15}16}
1617
17fn alloc(ctx: *anyopaque, n: usize, log2_ptr_align: u8, ra: usize) ?[*]u8 {18fn alloc(ctx: *anyopaque, n: usize, alignment: std.mem.Alignment, ra: usize) ?[*]u8 {
18 const self: *ThreadSafeAllocator = @ptrCast(@alignCast(ctx));19 const self: *ThreadSafeAllocator = @ptrCast(@alignCast(ctx));
19 self.mutex.lock();20 self.mutex.lock();
20 defer self.mutex.unlock();21 defer self.mutex.unlock();
2122
22 return self.child_allocator.rawAlloc(n, log2_ptr_align, ra);23 return self.child_allocator.rawAlloc(n, alignment, ra);
23}24}
2425
25fn resize(ctx: *anyopaque, buf: []u8, log2_buf_align: u8, new_len: usize, ret_addr: usize) bool {26fn resize(ctx: *anyopaque, buf: []u8, alignment: std.mem.Alignment, new_len: usize, ret_addr: usize) bool {
26 const self: *ThreadSafeAllocator = @ptrCast(@alignCast(ctx));27 const self: *ThreadSafeAllocator = @ptrCast(@alignCast(ctx));
2728
28 self.mutex.lock();29 self.mutex.lock();
29 defer self.mutex.unlock();30 defer self.mutex.unlock();
3031
31 return self.child_allocator.rawResize(buf, log2_buf_align, new_len, ret_addr);32 return self.child_allocator.rawResize(buf, alignment, new_len, ret_addr);
32}33}
3334
34fn free(ctx: *anyopaque, buf: []u8, log2_buf_align: u8, ret_addr: usize) void {35fn remap(context: *anyopaque, memory: []u8, alignment: std.mem.Alignment, new_len: usize, return_address: usize) ?[*]u8 {
36 const self: *ThreadSafeAllocator = @ptrCast(@alignCast(context));
37
38 self.mutex.lock();
39 defer self.mutex.unlock();
40
41 return self.child_allocator.rawRemap(memory, alignment, new_len, return_address);
42}
43
44fn free(ctx: *anyopaque, buf: []u8, alignment: std.mem.Alignment, ret_addr: usize) void {
35 const self: *ThreadSafeAllocator = @ptrCast(@alignCast(ctx));45 const self: *ThreadSafeAllocator = @ptrCast(@alignCast(ctx));
3646
37 self.mutex.lock();47 self.mutex.lock();
38 defer self.mutex.unlock();48 defer self.mutex.unlock();
3949
40 return self.child_allocator.rawFree(buf, log2_buf_align, ret_addr);50 return self.child_allocator.rawFree(buf, alignment, ret_addr);
41}51}
4252
43const std = @import("../std.zig");53const std = @import("../std.zig");
lib/std/heap/WasmAllocator.zig+21-11
...@@ -20,6 +20,7 @@ comptime {...@@ -20,6 +20,7 @@ comptime {
20pub const vtable: Allocator.VTable = .{20pub const vtable: Allocator.VTable = .{
21 .alloc = alloc,21 .alloc = alloc,
22 .resize = resize,22 .resize = resize,
23 .remap = remap,
23 .free = free,24 .free = free,
24};25};
2526
...@@ -40,18 +41,17 @@ const size_class_count = math.log2(bigpage_size) - min_class;...@@ -40,18 +41,17 @@ const size_class_count = math.log2(bigpage_size) - min_class;
40/// etc.41/// etc.
41const big_size_class_count = math.log2(bigpage_count);42const big_size_class_count = math.log2(bigpage_count);
4243
43var next_addrs = [1]usize{0} ** size_class_count;44var next_addrs: [size_class_count]usize = @splat(0);
44/// For each size class, points to the freed pointer.45/// For each size class, points to the freed pointer.
45var frees = [1]usize{0} ** size_class_count;46var frees: [size_class_count]usize = @splat(0);
46/// For each big size class, points to the freed pointer.47/// For each big size class, points to the freed pointer.
47var big_frees = [1]usize{0} ** big_size_class_count;48var big_frees: [big_size_class_count]usize = @splat(0);
4849
49fn alloc(ctx: *anyopaque, len: usize, log2_align: u8, return_address: usize) ?[*]u8 {50fn alloc(ctx: *anyopaque, len: usize, alignment: mem.Alignment, return_address: usize) ?[*]u8 {
50 _ = ctx;51 _ = ctx;
51 _ = return_address;52 _ = return_address;
52 // Make room for the freelist next pointer.53 // Make room for the freelist next pointer.
53 const alignment = @as(usize, 1) << @as(Allocator.Log2Align, @intCast(log2_align));54 const actual_len = @max(len +| @sizeOf(usize), alignment.toByteUnits());
54 const actual_len = @max(len +| @sizeOf(usize), alignment);
55 const slot_size = math.ceilPowerOfTwo(usize, actual_len) catch return null;55 const slot_size = math.ceilPowerOfTwo(usize, actual_len) catch return null;
56 const class = math.log2(slot_size) - min_class;56 const class = math.log2(slot_size) - min_class;
57 if (class < size_class_count) {57 if (class < size_class_count) {
...@@ -86,7 +86,7 @@ fn alloc(ctx: *anyopaque, len: usize, log2_align: u8, return_address: usize) ?[*...@@ -86,7 +86,7 @@ fn alloc(ctx: *anyopaque, len: usize, log2_align: u8, return_address: usize) ?[*
86fn resize(86fn resize(
87 ctx: *anyopaque,87 ctx: *anyopaque,
88 buf: []u8,88 buf: []u8,
89 log2_buf_align: u8,89 alignment: mem.Alignment,
90 new_len: usize,90 new_len: usize,
91 return_address: usize,91 return_address: usize,
92) bool {92) bool {
...@@ -94,7 +94,7 @@ fn resize(...@@ -94,7 +94,7 @@ fn resize(
94 _ = return_address;94 _ = return_address;
95 // We don't want to move anything from one size class to another, but we95 // We don't want to move anything from one size class to another, but we
96 // can recover bytes in between powers of two.96 // can recover bytes in between powers of two.
97 const buf_align = @as(usize, 1) << @as(Allocator.Log2Align, @intCast(log2_buf_align));97 const buf_align = alignment.toByteUnits();
98 const old_actual_len = @max(buf.len + @sizeOf(usize), buf_align);98 const old_actual_len = @max(buf.len + @sizeOf(usize), buf_align);
99 const new_actual_len = @max(new_len +| @sizeOf(usize), buf_align);99 const new_actual_len = @max(new_len +| @sizeOf(usize), buf_align);
100 const old_small_slot_size = math.ceilPowerOfTwoAssert(usize, old_actual_len);100 const old_small_slot_size = math.ceilPowerOfTwoAssert(usize, old_actual_len);
...@@ -111,15 +111,25 @@ fn resize(...@@ -111,15 +111,25 @@ fn resize(
111 }111 }
112}112}
113113
114fn remap(
115 context: *anyopaque,
116 memory: []u8,
117 alignment: mem.Alignment,
118 new_len: usize,
119 return_address: usize,
120) ?[*]u8 {
121 return if (resize(context, memory, alignment, new_len, return_address)) memory.ptr else null;
122}
123
114fn free(124fn free(
115 ctx: *anyopaque,125 ctx: *anyopaque,
116 buf: []u8,126 buf: []u8,
117 log2_buf_align: u8,127 alignment: mem.Alignment,
118 return_address: usize,128 return_address: usize,
119) void {129) void {
120 _ = ctx;130 _ = ctx;
121 _ = return_address;131 _ = return_address;
122 const buf_align = @as(usize, 1) << @as(Allocator.Log2Align, @intCast(log2_buf_align));132 const buf_align = alignment.toByteUnits();
123 const actual_len = @max(buf.len + @sizeOf(usize), buf_align);133 const actual_len = @max(buf.len + @sizeOf(usize), buf_align);
124 const slot_size = math.ceilPowerOfTwoAssert(usize, actual_len);134 const slot_size = math.ceilPowerOfTwoAssert(usize, actual_len);
125 const class = math.log2(slot_size) - min_class;135 const class = math.log2(slot_size) - min_class;
...@@ -160,7 +170,7 @@ fn allocBigPages(n: usize) usize {...@@ -160,7 +170,7 @@ fn allocBigPages(n: usize) usize {
160 return @as(usize, @intCast(page_index)) * wasm.page_size;170 return @as(usize, @intCast(page_index)) * wasm.page_size;
161}171}
162172
163const test_ally = Allocator{173const test_ally: Allocator = .{
164 .ptr = undefined,174 .ptr = undefined,
165 .vtable = &vtable,175 .vtable = &vtable,
166};176};
lib/std/heap/arena_allocator.zig+25-17
...@@ -29,12 +29,14 @@ pub const ArenaAllocator = struct {...@@ -29,12 +29,14 @@ pub const ArenaAllocator = struct {
29 .vtable = &.{29 .vtable = &.{
30 .alloc = alloc,30 .alloc = alloc,
31 .resize = resize,31 .resize = resize,
32 .remap = remap,
32 .free = free,33 .free = free,
33 },34 },
34 };35 };
35 }36 }
3637
37 const BufNode = std.SinglyLinkedList(usize).Node;38 const BufNode = std.SinglyLinkedList(usize).Node;
39 const BufNode_alignment: mem.Alignment = .fromByteUnits(@alignOf(BufNode));
3840
39 pub fn init(child_allocator: Allocator) ArenaAllocator {41 pub fn init(child_allocator: Allocator) ArenaAllocator {
40 return (State{}).promote(child_allocator);42 return (State{}).promote(child_allocator);
...@@ -47,9 +49,8 @@ pub const ArenaAllocator = struct {...@@ -47,9 +49,8 @@ pub const ArenaAllocator = struct {
47 while (it) |node| {49 while (it) |node| {
48 // this has to occur before the free because the free frees node50 // this has to occur before the free because the free frees node
49 const next_it = node.next;51 const next_it = node.next;
50 const align_bits = std.math.log2_int(usize, @alignOf(BufNode));
51 const alloc_buf = @as([*]u8, @ptrCast(node))[0..node.data];52 const alloc_buf = @as([*]u8, @ptrCast(node))[0..node.data];
52 self.child_allocator.rawFree(alloc_buf, align_bits, @returnAddress());53 self.child_allocator.rawFree(alloc_buf, BufNode_alignment, @returnAddress());
53 it = next_it;54 it = next_it;
54 }55 }
55 }56 }
...@@ -120,7 +121,6 @@ pub const ArenaAllocator = struct {...@@ -120,7 +121,6 @@ pub const ArenaAllocator = struct {
120 return true;121 return true;
121 }122 }
122 const total_size = requested_capacity + @sizeOf(BufNode);123 const total_size = requested_capacity + @sizeOf(BufNode);
123 const align_bits = std.math.log2_int(usize, @alignOf(BufNode));
124 // Free all nodes except for the last one124 // Free all nodes except for the last one
125 var it = self.state.buffer_list.first;125 var it = self.state.buffer_list.first;
126 const maybe_first_node = while (it) |node| {126 const maybe_first_node = while (it) |node| {
...@@ -129,7 +129,7 @@ pub const ArenaAllocator = struct {...@@ -129,7 +129,7 @@ pub const ArenaAllocator = struct {
129 if (next_it == null)129 if (next_it == null)
130 break node;130 break node;
131 const alloc_buf = @as([*]u8, @ptrCast(node))[0..node.data];131 const alloc_buf = @as([*]u8, @ptrCast(node))[0..node.data];
132 self.child_allocator.rawFree(alloc_buf, align_bits, @returnAddress());132 self.child_allocator.rawFree(alloc_buf, BufNode_alignment, @returnAddress());
133 it = next_it;133 it = next_it;
134 } else null;134 } else null;
135 std.debug.assert(maybe_first_node == null or maybe_first_node.?.next == null);135 std.debug.assert(maybe_first_node == null or maybe_first_node.?.next == null);
...@@ -141,16 +141,16 @@ pub const ArenaAllocator = struct {...@@ -141,16 +141,16 @@ pub const ArenaAllocator = struct {
141 if (first_node.data == total_size)141 if (first_node.data == total_size)
142 return true;142 return true;
143 const first_alloc_buf = @as([*]u8, @ptrCast(first_node))[0..first_node.data];143 const first_alloc_buf = @as([*]u8, @ptrCast(first_node))[0..first_node.data];
144 if (self.child_allocator.rawResize(first_alloc_buf, align_bits, total_size, @returnAddress())) {144 if (self.child_allocator.rawResize(first_alloc_buf, BufNode_alignment, total_size, @returnAddress())) {
145 // successful resize145 // successful resize
146 first_node.data = total_size;146 first_node.data = total_size;
147 } else {147 } else {
148 // manual realloc148 // manual realloc
149 const new_ptr = self.child_allocator.rawAlloc(total_size, align_bits, @returnAddress()) orelse {149 const new_ptr = self.child_allocator.rawAlloc(total_size, BufNode_alignment, @returnAddress()) orelse {
150 // we failed to preheat the arena properly, signal this to the user.150 // we failed to preheat the arena properly, signal this to the user.
151 return false;151 return false;
152 };152 };
153 self.child_allocator.rawFree(first_alloc_buf, align_bits, @returnAddress());153 self.child_allocator.rawFree(first_alloc_buf, BufNode_alignment, @returnAddress());
154 const node: *BufNode = @ptrCast(@alignCast(new_ptr));154 const node: *BufNode = @ptrCast(@alignCast(new_ptr));
155 node.* = .{ .data = total_size };155 node.* = .{ .data = total_size };
156 self.state.buffer_list.first = node;156 self.state.buffer_list.first = node;
...@@ -163,8 +163,7 @@ pub const ArenaAllocator = struct {...@@ -163,8 +163,7 @@ pub const ArenaAllocator = struct {
163 const actual_min_size = minimum_size + (@sizeOf(BufNode) + 16);163 const actual_min_size = minimum_size + (@sizeOf(BufNode) + 16);
164 const big_enough_len = prev_len + actual_min_size;164 const big_enough_len = prev_len + actual_min_size;
165 const len = big_enough_len + big_enough_len / 2;165 const len = big_enough_len + big_enough_len / 2;
166 const log2_align = comptime std.math.log2_int(usize, @alignOf(BufNode));166 const ptr = self.child_allocator.rawAlloc(len, BufNode_alignment, @returnAddress()) orelse
167 const ptr = self.child_allocator.rawAlloc(len, log2_align, @returnAddress()) orelse
168 return null;167 return null;
169 const buf_node: *BufNode = @ptrCast(@alignCast(ptr));168 const buf_node: *BufNode = @ptrCast(@alignCast(ptr));
170 buf_node.* = .{ .data = len };169 buf_node.* = .{ .data = len };
...@@ -173,11 +172,11 @@ pub const ArenaAllocator = struct {...@@ -173,11 +172,11 @@ pub const ArenaAllocator = struct {
173 return buf_node;172 return buf_node;
174 }173 }
175174
176 fn alloc(ctx: *anyopaque, n: usize, log2_ptr_align: u8, ra: usize) ?[*]u8 {175 fn alloc(ctx: *anyopaque, n: usize, alignment: mem.Alignment, ra: usize) ?[*]u8 {
177 const self: *ArenaAllocator = @ptrCast(@alignCast(ctx));176 const self: *ArenaAllocator = @ptrCast(@alignCast(ctx));
178 _ = ra;177 _ = ra;
179178
180 const ptr_align = @as(usize, 1) << @as(Allocator.Log2Align, @intCast(log2_ptr_align));179 const ptr_align = alignment.toByteUnits();
181 var cur_node = if (self.state.buffer_list.first) |first_node|180 var cur_node = if (self.state.buffer_list.first) |first_node|
182 first_node181 first_node
183 else182 else
...@@ -197,8 +196,7 @@ pub const ArenaAllocator = struct {...@@ -197,8 +196,7 @@ pub const ArenaAllocator = struct {
197 }196 }
198197
199 const bigger_buf_size = @sizeOf(BufNode) + new_end_index;198 const bigger_buf_size = @sizeOf(BufNode) + new_end_index;
200 const log2_align = comptime std.math.log2_int(usize, @alignOf(BufNode));199 if (self.child_allocator.rawResize(cur_alloc_buf, BufNode_alignment, bigger_buf_size, @returnAddress())) {
201 if (self.child_allocator.rawResize(cur_alloc_buf, log2_align, bigger_buf_size, @returnAddress())) {
202 cur_node.data = bigger_buf_size;200 cur_node.data = bigger_buf_size;
203 } else {201 } else {
204 // Allocate a new node if that's not possible202 // Allocate a new node if that's not possible
...@@ -207,9 +205,9 @@ pub const ArenaAllocator = struct {...@@ -207,9 +205,9 @@ pub const ArenaAllocator = struct {
207 }205 }
208 }206 }
209207
210 fn resize(ctx: *anyopaque, buf: []u8, log2_buf_align: u8, new_len: usize, ret_addr: usize) bool {208 fn resize(ctx: *anyopaque, buf: []u8, alignment: mem.Alignment, new_len: usize, ret_addr: usize) bool {
211 const self: *ArenaAllocator = @ptrCast(@alignCast(ctx));209 const self: *ArenaAllocator = @ptrCast(@alignCast(ctx));
212 _ = log2_buf_align;210 _ = alignment;
213 _ = ret_addr;211 _ = ret_addr;
214212
215 const cur_node = self.state.buffer_list.first orelse return false;213 const cur_node = self.state.buffer_list.first orelse return false;
...@@ -231,8 +229,18 @@ pub const ArenaAllocator = struct {...@@ -231,8 +229,18 @@ pub const ArenaAllocator = struct {
231 }229 }
232 }230 }
233231
234 fn free(ctx: *anyopaque, buf: []u8, log2_buf_align: u8, ret_addr: usize) void {232 fn remap(
235 _ = log2_buf_align;233 context: *anyopaque,
234 memory: []u8,
235 alignment: mem.Alignment,
236 new_len: usize,
237 return_address: usize,
238 ) ?[*]u8 {
239 return if (resize(context, memory, alignment, new_len, return_address)) memory.ptr else null;
240 }
241
242 fn free(ctx: *anyopaque, buf: []u8, alignment: mem.Alignment, ret_addr: usize) void {
243 _ = alignment;
236 _ = ret_addr;244 _ = ret_addr;
237245
238 const self: *ArenaAllocator = @ptrCast(@alignCast(ctx));246 const self: *ArenaAllocator = @ptrCast(@alignCast(ctx));
lib/std/heap/debug_allocator.zig created+1410
...@@ -0,0 +1,1410 @@
1//! An allocator that is intended to be used in Debug mode.
2//!
3//! ## Features
4//!
5//! * Captures stack traces on allocation, free, and optionally resize.
6//! * Double free detection, which prints all three traces (first alloc, first
7//! free, second free).
8//! * Leak detection, with stack traces.
9//! * Never reuses memory addresses, making it easier for Zig to detect branch
10//! on undefined values in case of dangling pointers. This relies on
11//! the backing allocator to also not reuse addresses.
12//! * Uses a minimum backing allocation size to avoid operating system errors
13//! from having too many active memory mappings.
14//! * When a page of memory is no longer needed, give it back to resident
15//! memory as soon as possible, so that it causes page faults when used.
16//! * Cross platform. Operates based on a backing allocator which makes it work
17//! everywhere, even freestanding.
18//! * Compile-time configuration.
19//!
20//! These features require the allocator to be quite slow and wasteful. For
21//! example, when allocating a single byte, the efficiency is less than 1%;
22//! it requires more than 100 bytes of overhead to manage the allocation for
23//! one byte. The efficiency gets better with larger allocations.
24//!
25//! ## Basic Design
26//!
27//! Allocations are divided into two categories, small and large.
28//!
29//! Small allocations are divided into buckets based on `page_size`:
30//!
31//! ```
32//! index obj_size
33//! 0 1
34//! 1 2
35//! 2 4
36//! 3 8
37//! 4 16
38//! 5 32
39//! 6 64
40//! 7 128
41//! 8 256
42//! 9 512
43//! 10 1024
44//! 11 2048
45//! ...
46//! ```
47//!
48//! This goes on for `small_bucket_count` indexes.
49//!
50//! Allocations are grouped into an object size based on max(len, alignment),
51//! rounded up to the next power of two.
52//!
53//! The main allocator state has an array of all the "current" buckets for each
54//! size class. Each slot in the array can be null, meaning the bucket for that
55//! size class is not allocated. When the first object is allocated for a given
56//! size class, it makes one `page_size` allocation from the backing allocator.
57//! This allocation is divided into "slots" - one per allocated object, leaving
58//! room for the allocation metadata (starting with `BucketHeader`), which is
59//! located at the very end of the "page".
60//!
61//! The allocation metadata includes "used bits" - 1 bit per slot representing
62//! whether the slot is used. Allocations always take the next available slot
63//! from the current bucket, setting the corresponding used bit, as well as
64//! incrementing `allocated_count`.
65//!
66//! Frees recover the allocation metadata based on the address, length, and
67//! alignment, relying on the backing allocation's large alignment, combined
68//! with the fact that allocations are never moved from small to large, or vice
69//! versa.
70//!
71//! When a bucket is full, a new one is allocated, containing a pointer to the
72//! previous one. This singly-linked list is iterated during leak detection.
73//!
74//! Resizing and remapping work the same on small allocations: if the size
75//! class would not change, then the operation succeeds, and the address is
76//! unchanged. Otherwise, the request is rejected.
77//!
78//! Large objects are allocated directly using the backing allocator. Metadata
79//! is stored separately in a `std.HashMap` using the backing allocator.
80//!
81//! Resizing and remapping are forwarded directly to the backing allocator,
82//! except where such operations would change the category from large to small.
83
84const std = @import("std");
85const builtin = @import("builtin");
86const log = std.log.scoped(.gpa);
87const math = std.math;
88const assert = std.debug.assert;
89const mem = std.mem;
90const Allocator = std.mem.Allocator;
91const StackTrace = std.builtin.StackTrace;
92
93const default_page_size: usize = @max(std.heap.page_size_max, switch (builtin.os.tag) {
94 .windows => 64 * 1024, // Makes `std.heap.PageAllocator` take the happy path.
95 .wasi => 64 * 1024, // Max alignment supported by `std.heap.WasmAllocator`.
96 else => 128 * 1024, // Avoids too many active mappings when `page_size_max` is low.
97});
98
99const Log2USize = std.math.Log2Int(usize);
100
101const default_sys_stack_trace_frames: usize = if (std.debug.sys_can_stack_trace) 6 else 0;
102const default_stack_trace_frames: usize = switch (builtin.mode) {
103 .Debug => default_sys_stack_trace_frames,
104 else => 0,
105};
106
107pub const Config = struct {
108 /// Number of stack frames to capture.
109 stack_trace_frames: usize = default_stack_trace_frames,
110
111 /// If true, the allocator will have two fields:
112 /// * `total_requested_bytes` which tracks the total allocated bytes of memory requested.
113 /// * `requested_memory_limit` which causes allocations to return `error.OutOfMemory`
114 /// when the `total_requested_bytes` exceeds this limit.
115 /// If false, these fields will be `void`.
116 enable_memory_limit: bool = false,
117
118 /// Whether to enable safety checks.
119 safety: bool = std.debug.runtime_safety,
120
121 /// Whether the allocator may be used simultaneously from multiple threads.
122 thread_safe: bool = !builtin.single_threaded,
123
124 /// What type of mutex you'd like to use, for thread safety.
125 /// when specified, the mutex type must have the same shape as `std.Thread.Mutex` and
126 /// `DummyMutex`, and have no required fields. Specifying this field causes
127 /// the `thread_safe` field to be ignored.
128 ///
129 /// when null (default):
130 /// * the mutex type defaults to `std.Thread.Mutex` when thread_safe is enabled.
131 /// * the mutex type defaults to `DummyMutex` otherwise.
132 MutexType: ?type = null,
133
134 /// This is a temporary debugging trick you can use to turn segfaults into more helpful
135 /// logged error messages with stack trace details. The downside is that every allocation
136 /// will be leaked, unless used with retain_metadata!
137 never_unmap: bool = false,
138
139 /// This is a temporary debugging aid that retains metadata about allocations indefinitely.
140 /// This allows a greater range of double frees to be reported. All metadata is freed when
141 /// deinit is called. When used with never_unmap, deliberately leaked memory is also freed
142 /// during deinit. Currently should be used with never_unmap to avoid segfaults.
143 /// TODO https://github.com/ziglang/zig/issues/4298 will allow use without never_unmap
144 retain_metadata: bool = false,
145
146 /// Enables emitting info messages with the size and address of every allocation.
147 verbose_log: bool = false,
148
149 /// Tell whether the backing allocator returns already-zeroed memory.
150 backing_allocator_zeroes: bool = true,
151
152 /// When resizing an allocation, refresh the stack trace with the resize
153 /// callsite. Comes with a performance penalty.
154 resize_stack_traces: bool = false,
155
156 /// Magic value that distinguishes allocations owned by this allocator from
157 /// other regions of memory.
158 canary: usize = @truncate(0x9232a6ff85dff10f),
159
160 /// The size of allocations requested from the backing allocator for
161 /// subdividing into slots for small allocations.
162 ///
163 /// Must be a power of two.
164 page_size: usize = default_page_size,
165};
166
167/// Default initialization of this struct is deprecated; use `.init` instead.
168pub fn DebugAllocator(comptime config: Config) type {
169 return struct {
170 backing_allocator: Allocator = std.heap.page_allocator,
171 /// Tracks the active bucket, which is the one that has free slots in it.
172 buckets: [small_bucket_count]?*BucketHeader = [1]?*BucketHeader{null} ** small_bucket_count,
173 large_allocations: LargeAllocTable = .empty,
174 total_requested_bytes: @TypeOf(total_requested_bytes_init) = total_requested_bytes_init,
175 requested_memory_limit: @TypeOf(requested_memory_limit_init) = requested_memory_limit_init,
176 mutex: @TypeOf(mutex_init) = mutex_init,
177
178 const Self = @This();
179
180 pub const init: Self = .{};
181
182 /// These can be derived from size_class_index but the calculation is nontrivial.
183 const slot_counts: [small_bucket_count]SlotIndex = init: {
184 @setEvalBranchQuota(10000);
185 var result: [small_bucket_count]SlotIndex = undefined;
186 for (&result, 0..) |*elem, i| elem.* = calculateSlotCount(i);
187 break :init result;
188 };
189
190 comptime {
191 assert(math.isPowerOfTwo(page_size));
192 }
193
194 const page_size = config.page_size;
195 const page_align: mem.Alignment = .fromByteUnits(page_size);
196 /// Integer type for pointing to slots in a small allocation
197 const SlotIndex = std.meta.Int(.unsigned, math.log2(page_size) + 1);
198
199 const total_requested_bytes_init = if (config.enable_memory_limit) @as(usize, 0) else {};
200 const requested_memory_limit_init = if (config.enable_memory_limit) @as(usize, math.maxInt(usize)) else {};
201
202 const mutex_init = if (config.MutexType) |T|
203 T{}
204 else if (config.thread_safe)
205 std.Thread.Mutex{}
206 else
207 DummyMutex{};
208
209 const DummyMutex = struct {
210 inline fn lock(_: *DummyMutex) void {}
211 inline fn unlock(_: *DummyMutex) void {}
212 };
213
214 const stack_n = config.stack_trace_frames;
215 const one_trace_size = @sizeOf(usize) * stack_n;
216 const traces_per_slot = 2;
217
218 pub const Error = mem.Allocator.Error;
219
220 /// Avoids creating buckets that would only be able to store a small
221 /// number of slots. Value of 1 means 2 is the minimum slot count.
222 const minimum_slots_per_bucket_log2 = 1;
223 const small_bucket_count = math.log2(page_size) - minimum_slots_per_bucket_log2;
224 const largest_bucket_object_size = 1 << (small_bucket_count - 1);
225 const LargestSizeClassInt = std.math.IntFittingRange(0, largest_bucket_object_size);
226
227 const bucketCompare = struct {
228 fn compare(a: *BucketHeader, b: *BucketHeader) std.math.Order {
229 return std.math.order(@intFromPtr(a.page), @intFromPtr(b.page));
230 }
231 }.compare;
232
233 const LargeAlloc = struct {
234 bytes: []u8,
235 requested_size: if (config.enable_memory_limit) usize else void,
236 stack_addresses: [trace_n][stack_n]usize,
237 freed: if (config.retain_metadata) bool else void,
238 alignment: if (config.never_unmap and config.retain_metadata) mem.Alignment else void,
239
240 const trace_n = if (config.retain_metadata) traces_per_slot else 1;
241
242 fn dumpStackTrace(self: *LargeAlloc, trace_kind: TraceKind) void {
243 std.debug.dumpStackTrace(self.getStackTrace(trace_kind));
244 }
245
246 fn getStackTrace(self: *LargeAlloc, trace_kind: TraceKind) std.builtin.StackTrace {
247 assert(@intFromEnum(trace_kind) < trace_n);
248 const stack_addresses = &self.stack_addresses[@intFromEnum(trace_kind)];
249 var len: usize = 0;
250 while (len < stack_n and stack_addresses[len] != 0) {
251 len += 1;
252 }
253 return .{
254 .instruction_addresses = stack_addresses,
255 .index = len,
256 };
257 }
258
259 fn captureStackTrace(self: *LargeAlloc, ret_addr: usize, trace_kind: TraceKind) void {
260 assert(@intFromEnum(trace_kind) < trace_n);
261 const stack_addresses = &self.stack_addresses[@intFromEnum(trace_kind)];
262 collectStackTrace(ret_addr, stack_addresses);
263 }
264 };
265 const LargeAllocTable = std.AutoHashMapUnmanaged(usize, LargeAlloc);
266
267 /// Bucket: In memory, in order:
268 /// * BucketHeader
269 /// * bucket_used_bits: [N]usize, // 1 bit for every slot
270 /// -- below only exists when config.safety is true --
271 /// * requested_sizes: [N]LargestSizeClassInt // 1 int for every slot
272 /// * log2_ptr_aligns: [N]u8 // 1 byte for every slot
273 /// -- above only exists when config.safety is true --
274 /// * stack_trace_addresses: [N]usize, // traces_per_slot for every allocation
275 const BucketHeader = struct {
276 allocated_count: SlotIndex,
277 freed_count: SlotIndex,
278 prev: ?*BucketHeader,
279 canary: usize = config.canary,
280
281 fn fromPage(page_addr: usize, slot_count: usize) *BucketHeader {
282 const unaligned = page_addr + page_size - bucketSize(slot_count);
283 return @ptrFromInt(unaligned & ~(@as(usize, @alignOf(BucketHeader)) - 1));
284 }
285
286 fn usedBits(bucket: *BucketHeader, index: usize) *usize {
287 const ptr: [*]u8 = @ptrCast(bucket);
288 const bits: [*]usize = @alignCast(@ptrCast(ptr + @sizeOf(BucketHeader)));
289 return &bits[index];
290 }
291
292 fn requestedSizes(bucket: *BucketHeader, slot_count: usize) []LargestSizeClassInt {
293 if (!config.safety) @compileError("requested size is only stored when safety is enabled");
294 const start_ptr = @as([*]u8, @ptrCast(bucket)) + bucketRequestedSizesStart(slot_count);
295 const sizes = @as([*]LargestSizeClassInt, @ptrCast(@alignCast(start_ptr)));
296 return sizes[0..slot_count];
297 }
298
299 fn log2PtrAligns(bucket: *BucketHeader, slot_count: usize) []mem.Alignment {
300 if (!config.safety) @compileError("requested size is only stored when safety is enabled");
301 const aligns_ptr = @as([*]u8, @ptrCast(bucket)) + bucketAlignsStart(slot_count);
302 return @ptrCast(aligns_ptr[0..slot_count]);
303 }
304
305 fn stackTracePtr(
306 bucket: *BucketHeader,
307 slot_count: usize,
308 slot_index: SlotIndex,
309 trace_kind: TraceKind,
310 ) *[stack_n]usize {
311 const start_ptr = @as([*]u8, @ptrCast(bucket)) + bucketStackFramesStart(slot_count);
312 const addr = start_ptr + one_trace_size * traces_per_slot * slot_index +
313 @intFromEnum(trace_kind) * @as(usize, one_trace_size);
314 return @ptrCast(@alignCast(addr));
315 }
316
317 fn captureStackTrace(
318 bucket: *BucketHeader,
319 ret_addr: usize,
320 slot_count: usize,
321 slot_index: SlotIndex,
322 trace_kind: TraceKind,
323 ) void {
324 // Initialize them to 0. When determining the count we must look
325 // for non zero addresses.
326 const stack_addresses = bucket.stackTracePtr(slot_count, slot_index, trace_kind);
327 collectStackTrace(ret_addr, stack_addresses);
328 }
329 };
330
331 pub fn allocator(self: *Self) Allocator {
332 return .{
333 .ptr = self,
334 .vtable = &.{
335 .alloc = alloc,
336 .resize = resize,
337 .remap = remap,
338 .free = free,
339 },
340 };
341 }
342
343 fn bucketStackTrace(
344 bucket: *BucketHeader,
345 slot_count: usize,
346 slot_index: SlotIndex,
347 trace_kind: TraceKind,
348 ) StackTrace {
349 const stack_addresses = bucket.stackTracePtr(slot_count, slot_index, trace_kind);
350 var len: usize = 0;
351 while (len < stack_n and stack_addresses[len] != 0) {
352 len += 1;
353 }
354 return .{
355 .instruction_addresses = stack_addresses,
356 .index = len,
357 };
358 }
359
360 fn bucketRequestedSizesStart(slot_count: usize) usize {
361 if (!config.safety) @compileError("requested sizes are not stored unless safety is enabled");
362 return mem.alignForward(
363 usize,
364 @sizeOf(BucketHeader) + usedBitsSize(slot_count),
365 @alignOf(LargestSizeClassInt),
366 );
367 }
368
369 fn bucketAlignsStart(slot_count: usize) usize {
370 if (!config.safety) @compileError("requested sizes are not stored unless safety is enabled");
371 return bucketRequestedSizesStart(slot_count) + (@sizeOf(LargestSizeClassInt) * slot_count);
372 }
373
374 fn bucketStackFramesStart(slot_count: usize) usize {
375 const unaligned_start = if (config.safety)
376 bucketAlignsStart(slot_count) + slot_count
377 else
378 @sizeOf(BucketHeader) + usedBitsSize(slot_count);
379 return mem.alignForward(usize, unaligned_start, @alignOf(usize));
380 }
381
382 fn bucketSize(slot_count: usize) usize {
383 return bucketStackFramesStart(slot_count) + one_trace_size * traces_per_slot * slot_count;
384 }
385
386 /// This is executed only at compile-time to prepopulate a lookup table.
387 fn calculateSlotCount(size_class_index: usize) SlotIndex {
388 const size_class = @as(usize, 1) << @as(Log2USize, @intCast(size_class_index));
389 var lower: usize = 1 << minimum_slots_per_bucket_log2;
390 var upper: usize = (page_size - bucketSize(lower)) / size_class;
391 while (upper > lower) {
392 const proposed: usize = lower + (upper - lower) / 2;
393 if (proposed == lower) return lower;
394 const slots_end = proposed * size_class;
395 const header_begin = mem.alignForward(usize, slots_end, @alignOf(BucketHeader));
396 const end = header_begin + bucketSize(proposed);
397 if (end > page_size) {
398 upper = proposed - 1;
399 } else {
400 lower = proposed;
401 }
402 }
403 const slots_end = lower * size_class;
404 const header_begin = mem.alignForward(usize, slots_end, @alignOf(BucketHeader));
405 const end = header_begin + bucketSize(lower);
406 assert(end <= page_size);
407 return lower;
408 }
409
410 fn usedBitsCount(slot_count: usize) usize {
411 return (slot_count + (@bitSizeOf(usize) - 1)) / @bitSizeOf(usize);
412 }
413
414 fn usedBitsSize(slot_count: usize) usize {
415 return usedBitsCount(slot_count) * @sizeOf(usize);
416 }
417
418 fn detectLeaksInBucket(bucket: *BucketHeader, size_class_index: usize, used_bits_count: usize) bool {
419 const size_class = @as(usize, 1) << @as(Log2USize, @intCast(size_class_index));
420 const slot_count = slot_counts[size_class_index];
421 var leaks = false;
422 for (0..used_bits_count) |used_bits_byte| {
423 const used_int = bucket.usedBits(used_bits_byte).*;
424 if (used_int != 0) {
425 for (0..@bitSizeOf(usize)) |bit_index_usize| {
426 const bit_index: Log2USize = @intCast(bit_index_usize);
427 const is_used = @as(u1, @truncate(used_int >> bit_index)) != 0;
428 if (is_used) {
429 const slot_index: SlotIndex = @intCast(used_bits_byte * @bitSizeOf(usize) + bit_index);
430 const stack_trace = bucketStackTrace(bucket, slot_count, slot_index, .alloc);
431 const page_addr = @intFromPtr(bucket) & ~(page_size - 1);
432 const addr = page_addr + slot_index * size_class;
433 log.err("memory address 0x{x} leaked: {}", .{ addr, stack_trace });
434 leaks = true;
435 }
436 }
437 }
438 }
439 return leaks;
440 }
441
442 /// Emits log messages for leaks and then returns whether there were any leaks.
443 pub fn detectLeaks(self: *Self) bool {
444 var leaks = false;
445
446 for (self.buckets, 0..) |init_optional_bucket, size_class_index| {
447 var optional_bucket = init_optional_bucket;
448 const slot_count = slot_counts[size_class_index];
449 const used_bits_count = usedBitsCount(slot_count);
450 while (optional_bucket) |bucket| {
451 leaks = detectLeaksInBucket(bucket, size_class_index, used_bits_count) or leaks;
452 optional_bucket = bucket.prev;
453 }
454 }
455
456 var it = self.large_allocations.valueIterator();
457 while (it.next()) |large_alloc| {
458 if (config.retain_metadata and large_alloc.freed) continue;
459 const stack_trace = large_alloc.getStackTrace(.alloc);
460 log.err("memory address 0x{x} leaked: {}", .{
461 @intFromPtr(large_alloc.bytes.ptr), stack_trace,
462 });
463 leaks = true;
464 }
465 return leaks;
466 }
467
468 fn freeRetainedMetadata(self: *Self) void {
469 comptime assert(config.retain_metadata);
470 if (config.never_unmap) {
471 // free large allocations that were intentionally leaked by never_unmap
472 var it = self.large_allocations.iterator();
473 while (it.next()) |large| {
474 if (large.value_ptr.freed) {
475 self.backing_allocator.rawFree(large.value_ptr.bytes, large.value_ptr.alignment, @returnAddress());
476 }
477 }
478 }
479 }
480
481 pub fn flushRetainedMetadata(self: *Self) void {
482 comptime assert(config.retain_metadata);
483 self.freeRetainedMetadata();
484 // also remove entries from large_allocations
485 var it = self.large_allocations.iterator();
486 while (it.next()) |large| {
487 if (large.value_ptr.freed) {
488 _ = self.large_allocations.remove(@intFromPtr(large.value_ptr.bytes.ptr));
489 }
490 }
491 }
492
493 /// Returns `std.heap.Check.leak` if there were leaks; `std.heap.Check.ok` otherwise.
494 pub fn deinit(self: *Self) std.heap.Check {
495 const leaks = if (config.safety) self.detectLeaks() else false;
496 if (config.retain_metadata) self.freeRetainedMetadata();
497 self.large_allocations.deinit(self.backing_allocator);
498 self.* = undefined;
499 return if (leaks) .leak else .ok;
500 }
501
502 fn collectStackTrace(first_trace_addr: usize, addresses: *[stack_n]usize) void {
503 if (stack_n == 0) return;
504 @memset(addresses, 0);
505 var stack_trace: StackTrace = .{
506 .instruction_addresses = addresses,
507 .index = 0,
508 };
509 std.debug.captureStackTrace(first_trace_addr, &stack_trace);
510 }
511
512 fn reportDoubleFree(ret_addr: usize, alloc_stack_trace: StackTrace, free_stack_trace: StackTrace) void {
513 var addresses: [stack_n]usize = @splat(0);
514 var second_free_stack_trace: StackTrace = .{
515 .instruction_addresses = &addresses,
516 .index = 0,
517 };
518 std.debug.captureStackTrace(ret_addr, &second_free_stack_trace);
519 log.err("Double free detected. Allocation: {} First free: {} Second free: {}", .{
520 alloc_stack_trace, free_stack_trace, second_free_stack_trace,
521 });
522 }
523
524 /// This function assumes the object is in the large object storage regardless
525 /// of the parameters.
526 fn resizeLarge(
527 self: *Self,
528 old_mem: []u8,
529 alignment: mem.Alignment,
530 new_size: usize,
531 ret_addr: usize,
532 may_move: bool,
533 ) ?[*]u8 {
534 if (config.retain_metadata and may_move) {
535 // Before looking up the entry (since this could invalidate
536 // it), we must reserve space for the new entry in case the
537 // allocation is relocated.
538 self.large_allocations.ensureUnusedCapacity(self.backing_allocator, 1) catch return null;
539 }
540
541 const entry = self.large_allocations.getEntry(@intFromPtr(old_mem.ptr)) orelse {
542 if (config.safety) {
543 @panic("Invalid free");
544 } else {
545 unreachable;
546 }
547 };
548
549 if (config.retain_metadata and entry.value_ptr.freed) {
550 if (config.safety) {
551 reportDoubleFree(ret_addr, entry.value_ptr.getStackTrace(.alloc), entry.value_ptr.getStackTrace(.free));
552 @panic("Unrecoverable double free");
553 } else {
554 unreachable;
555 }
556 }
557
558 if (config.safety and old_mem.len != entry.value_ptr.bytes.len) {
559 var addresses: [stack_n]usize = [1]usize{0} ** stack_n;
560 var free_stack_trace: StackTrace = .{
561 .instruction_addresses = &addresses,
562 .index = 0,
563 };
564 std.debug.captureStackTrace(ret_addr, &free_stack_trace);
565 log.err("Allocation size {d} bytes does not match free size {d}. Allocation: {} Free: {}", .{
566 entry.value_ptr.bytes.len,
567 old_mem.len,
568 entry.value_ptr.getStackTrace(.alloc),
569 free_stack_trace,
570 });
571 }
572
573 // If this would move the allocation into a small size class,
574 // refuse the request, because it would require creating small
575 // allocation metadata.
576 const new_size_class_index: usize = @max(@bitSizeOf(usize) - @clz(new_size - 1), @intFromEnum(alignment));
577 if (new_size_class_index < self.buckets.len) return null;
578
579 // Do memory limit accounting with requested sizes rather than what
580 // backing_allocator returns because if we want to return
581 // error.OutOfMemory, we have to leave allocation untouched, and
582 // that is impossible to guarantee after calling
583 // backing_allocator.rawResize.
584 const prev_req_bytes = self.total_requested_bytes;
585 if (config.enable_memory_limit) {
586 const new_req_bytes = prev_req_bytes + new_size - entry.value_ptr.requested_size;
587 if (new_req_bytes > prev_req_bytes and new_req_bytes > self.requested_memory_limit) {
588 return null;
589 }
590 self.total_requested_bytes = new_req_bytes;
591 }
592
593 const opt_resized_ptr = if (may_move)
594 self.backing_allocator.rawRemap(old_mem, alignment, new_size, ret_addr)
595 else if (self.backing_allocator.rawResize(old_mem, alignment, new_size, ret_addr))
596 old_mem.ptr
597 else
598 null;
599
600 const resized_ptr = opt_resized_ptr orelse {
601 if (config.enable_memory_limit) {
602 self.total_requested_bytes = prev_req_bytes;
603 }
604 return null;
605 };
606
607 if (config.enable_memory_limit) {
608 entry.value_ptr.requested_size = new_size;
609 }
610
611 if (config.verbose_log) {
612 log.info("large resize {d} bytes at {*} to {d} at {*}", .{
613 old_mem.len, old_mem.ptr, new_size, resized_ptr,
614 });
615 }
616 entry.value_ptr.bytes = resized_ptr[0..new_size];
617 if (config.resize_stack_traces)
618 entry.value_ptr.captureStackTrace(ret_addr, .alloc);
619
620 // Update the key of the hash map if the memory was relocated.
621 if (resized_ptr != old_mem.ptr) {
622 const large_alloc = entry.value_ptr.*;
623 if (config.retain_metadata) {
624 entry.value_ptr.freed = true;
625 entry.value_ptr.captureStackTrace(ret_addr, .free);
626 } else {
627 self.large_allocations.removeByPtr(entry.key_ptr);
628 }
629
630 const gop = self.large_allocations.getOrPutAssumeCapacity(@intFromPtr(resized_ptr));
631 if (config.retain_metadata and !config.never_unmap) {
632 // Backing allocator may be reusing memory that we're retaining metadata for
633 assert(!gop.found_existing or gop.value_ptr.freed);
634 } else {
635 assert(!gop.found_existing); // This would mean the kernel double-mapped pages.
636 }
637 gop.value_ptr.* = large_alloc;
638 }
639
640 return resized_ptr;
641 }
642
643 /// This function assumes the object is in the large object storage regardless
644 /// of the parameters.
645 fn freeLarge(
646 self: *Self,
647 old_mem: []u8,
648 alignment: mem.Alignment,
649 ret_addr: usize,
650 ) void {
651 const entry = self.large_allocations.getEntry(@intFromPtr(old_mem.ptr)) orelse {
652 if (config.safety) {
653 @panic("Invalid free");
654 } else {
655 unreachable;
656 }
657 };
658
659 if (config.retain_metadata and entry.value_ptr.freed) {
660 if (config.safety) {
661 reportDoubleFree(ret_addr, entry.value_ptr.getStackTrace(.alloc), entry.value_ptr.getStackTrace(.free));
662 return;
663 } else {
664 unreachable;
665 }
666 }
667
668 if (config.safety and old_mem.len != entry.value_ptr.bytes.len) {
669 var addresses: [stack_n]usize = [1]usize{0} ** stack_n;
670 var free_stack_trace = StackTrace{
671 .instruction_addresses = &addresses,
672 .index = 0,
673 };
674 std.debug.captureStackTrace(ret_addr, &free_stack_trace);
675 log.err("Allocation size {d} bytes does not match free size {d}. Allocation: {} Free: {}", .{
676 entry.value_ptr.bytes.len,
677 old_mem.len,
678 entry.value_ptr.getStackTrace(.alloc),
679 free_stack_trace,
680 });
681 }
682
683 if (!config.never_unmap) {
684 self.backing_allocator.rawFree(old_mem, alignment, ret_addr);
685 }
686
687 if (config.enable_memory_limit) {
688 self.total_requested_bytes -= entry.value_ptr.requested_size;
689 }
690
691 if (config.verbose_log) {
692 log.info("large free {d} bytes at {*}", .{ old_mem.len, old_mem.ptr });
693 }
694
695 if (!config.retain_metadata) {
696 assert(self.large_allocations.remove(@intFromPtr(old_mem.ptr)));
697 } else {
698 entry.value_ptr.freed = true;
699 entry.value_ptr.captureStackTrace(ret_addr, .free);
700 }
701 }
702
703 fn alloc(context: *anyopaque, len: usize, alignment: mem.Alignment, ret_addr: usize) ?[*]u8 {
704 const self: *Self = @ptrCast(@alignCast(context));
705 self.mutex.lock();
706 defer self.mutex.unlock();
707
708 if (config.enable_memory_limit) {
709 const new_req_bytes = self.total_requested_bytes + len;
710 if (new_req_bytes > self.requested_memory_limit) return null;
711 self.total_requested_bytes = new_req_bytes;
712 }
713
714 const size_class_index: usize = @max(@bitSizeOf(usize) - @clz(len - 1), @intFromEnum(alignment));
715 if (size_class_index >= self.buckets.len) {
716 @branchHint(.unlikely);
717 self.large_allocations.ensureUnusedCapacity(self.backing_allocator, 1) catch return null;
718 const ptr = self.backing_allocator.rawAlloc(len, alignment, ret_addr) orelse return null;
719 const slice = ptr[0..len];
720
721 const gop = self.large_allocations.getOrPutAssumeCapacity(@intFromPtr(slice.ptr));
722 if (config.retain_metadata and !config.never_unmap) {
723 // Backing allocator may be reusing memory that we're retaining metadata for
724 assert(!gop.found_existing or gop.value_ptr.freed);
725 } else {
726 assert(!gop.found_existing); // This would mean the kernel double-mapped pages.
727 }
728 gop.value_ptr.bytes = slice;
729 if (config.enable_memory_limit)
730 gop.value_ptr.requested_size = len;
731 gop.value_ptr.captureStackTrace(ret_addr, .alloc);
732 if (config.retain_metadata) {
733 gop.value_ptr.freed = false;
734 if (config.never_unmap) {
735 gop.value_ptr.alignment = alignment;
736 }
737 }
738
739 if (config.verbose_log) {
740 log.info("large alloc {d} bytes at {*}", .{ slice.len, slice.ptr });
741 }
742 return slice.ptr;
743 }
744
745 const slot_count = slot_counts[size_class_index];
746
747 if (self.buckets[size_class_index]) |bucket| {
748 @branchHint(.likely);
749 const slot_index = bucket.allocated_count;
750 if (slot_index < slot_count) {
751 @branchHint(.likely);
752 bucket.allocated_count = slot_index + 1;
753 const used_bits_byte = bucket.usedBits(slot_index / @bitSizeOf(usize));
754 const used_bit_index: Log2USize = @intCast(slot_index % @bitSizeOf(usize));
755 used_bits_byte.* |= (@as(usize, 1) << used_bit_index);
756 const size_class = @as(usize, 1) << @as(Log2USize, @intCast(size_class_index));
757 if (config.stack_trace_frames > 0) {
758 bucket.captureStackTrace(ret_addr, slot_count, slot_index, .alloc);
759 }
760 if (config.safety) {
761 bucket.requestedSizes(slot_count)[slot_index] = @intCast(len);
762 bucket.log2PtrAligns(slot_count)[slot_index] = alignment;
763 }
764 const page_addr = @intFromPtr(bucket) & ~(page_size - 1);
765 const addr = page_addr + slot_index * size_class;
766 if (config.verbose_log) {
767 log.info("small alloc {d} bytes at 0x{x}", .{ len, addr });
768 }
769 return @ptrFromInt(addr);
770 }
771 }
772
773 const page = self.backing_allocator.rawAlloc(page_size, page_align, @returnAddress()) orelse
774 return null;
775 const bucket: *BucketHeader = .fromPage(@intFromPtr(page), slot_count);
776 bucket.* = .{
777 .allocated_count = 1,
778 .freed_count = 0,
779 .prev = self.buckets[size_class_index],
780 };
781 self.buckets[size_class_index] = bucket;
782
783 if (!config.backing_allocator_zeroes) {
784 @memset(@as([*]usize, @as(*[1]usize, bucket.usedBits(0)))[0..usedBitsCount(slot_count)], 0);
785 if (config.safety) @memset(bucket.requestedSizes(slot_count), 0);
786 }
787
788 bucket.usedBits(0).* = 0b1;
789
790 if (config.stack_trace_frames > 0) {
791 bucket.captureStackTrace(ret_addr, slot_count, 0, .alloc);
792 }
793
794 if (config.safety) {
795 bucket.requestedSizes(slot_count)[0] = @intCast(len);
796 bucket.log2PtrAligns(slot_count)[0] = alignment;
797 }
798
799 if (config.verbose_log) {
800 log.info("small alloc {d} bytes at 0x{x}", .{ len, @intFromPtr(page) });
801 }
802
803 return page;
804 }
805
806 fn resize(
807 context: *anyopaque,
808 memory: []u8,
809 alignment: mem.Alignment,
810 new_len: usize,
811 return_address: usize,
812 ) bool {
813 const self: *Self = @ptrCast(@alignCast(context));
814 self.mutex.lock();
815 defer self.mutex.unlock();
816
817 const size_class_index: usize = @max(@bitSizeOf(usize) - @clz(memory.len - 1), @intFromEnum(alignment));
818 if (size_class_index >= self.buckets.len) {
819 return self.resizeLarge(memory, alignment, new_len, return_address, false) != null;
820 } else {
821 return resizeSmall(self, memory, alignment, new_len, return_address, size_class_index);
822 }
823 }
824
825 fn remap(
826 context: *anyopaque,
827 memory: []u8,
828 alignment: mem.Alignment,
829 new_len: usize,
830 return_address: usize,
831 ) ?[*]u8 {
832 const self: *Self = @ptrCast(@alignCast(context));
833 self.mutex.lock();
834 defer self.mutex.unlock();
835
836 const size_class_index: usize = @max(@bitSizeOf(usize) - @clz(memory.len - 1), @intFromEnum(alignment));
837 if (size_class_index >= self.buckets.len) {
838 return self.resizeLarge(memory, alignment, new_len, return_address, true);
839 } else {
840 return if (resizeSmall(self, memory, alignment, new_len, return_address, size_class_index)) memory.ptr else null;
841 }
842 }
843
844 fn free(
845 context: *anyopaque,
846 old_memory: []u8,
847 alignment: mem.Alignment,
848 return_address: usize,
849 ) void {
850 const self: *Self = @ptrCast(@alignCast(context));
851 self.mutex.lock();
852 defer self.mutex.unlock();
853
854 assert(old_memory.len != 0);
855
856 const size_class_index: usize = @max(@bitSizeOf(usize) - @clz(old_memory.len - 1), @intFromEnum(alignment));
857 if (size_class_index >= self.buckets.len) {
858 @branchHint(.unlikely);
859 self.freeLarge(old_memory, alignment, return_address);
860 return;
861 }
862
863 const slot_count = slot_counts[size_class_index];
864 const freed_addr = @intFromPtr(old_memory.ptr);
865 const page_addr = freed_addr & ~(page_size - 1);
866 const bucket: *BucketHeader = .fromPage(page_addr, slot_count);
867 if (bucket.canary != config.canary) @panic("Invalid free");
868 const page_offset = freed_addr - page_addr;
869 const size_class = @as(usize, 1) << @as(Log2USize, @intCast(size_class_index));
870 const slot_index: SlotIndex = @intCast(page_offset / size_class);
871 const used_byte_index = slot_index / @bitSizeOf(usize);
872 const used_bit_index: Log2USize = @intCast(slot_index % @bitSizeOf(usize));
873 const used_byte = bucket.usedBits(used_byte_index);
874 const is_used = @as(u1, @truncate(used_byte.* >> used_bit_index)) != 0;
875 if (!is_used) {
876 if (config.safety) {
877 reportDoubleFree(
878 return_address,
879 bucketStackTrace(bucket, slot_count, slot_index, .alloc),
880 bucketStackTrace(bucket, slot_count, slot_index, .free),
881 );
882 // Recoverable since this is a free.
883 return;
884 } else {
885 unreachable;
886 }
887 }
888
889 // Definitely an in-use small alloc now.
890 if (config.safety) {
891 const requested_size = bucket.requestedSizes(slot_count)[slot_index];
892 if (requested_size == 0) @panic("Invalid free");
893 const slot_alignment = bucket.log2PtrAligns(slot_count)[slot_index];
894 if (old_memory.len != requested_size or alignment != slot_alignment) {
895 var addresses: [stack_n]usize = [1]usize{0} ** stack_n;
896 var free_stack_trace: StackTrace = .{
897 .instruction_addresses = &addresses,
898 .index = 0,
899 };
900 std.debug.captureStackTrace(return_address, &free_stack_trace);
901 if (old_memory.len != requested_size) {
902 log.err("Allocation size {d} bytes does not match free size {d}. Allocation: {} Free: {}", .{
903 requested_size,
904 old_memory.len,
905 bucketStackTrace(bucket, slot_count, slot_index, .alloc),
906 free_stack_trace,
907 });
908 }
909 if (alignment != slot_alignment) {
910 log.err("Allocation alignment {d} does not match free alignment {d}. Allocation: {} Free: {}", .{
911 slot_alignment.toByteUnits(),
912 alignment.toByteUnits(),
913 bucketStackTrace(bucket, slot_count, slot_index, .alloc),
914 free_stack_trace,
915 });
916 }
917 }
918 }
919
920 if (config.enable_memory_limit) {
921 self.total_requested_bytes -= old_memory.len;
922 }
923
924 if (config.stack_trace_frames > 0) {
925 // Capture stack trace to be the "first free", in case a double free happens.
926 bucket.captureStackTrace(return_address, slot_count, slot_index, .free);
927 }
928
929 used_byte.* &= ~(@as(usize, 1) << used_bit_index);
930 if (config.safety) {
931 bucket.requestedSizes(slot_count)[slot_index] = 0;
932 }
933 bucket.freed_count += 1;
934 if (bucket.freed_count == bucket.allocated_count) {
935 if (self.buckets[size_class_index] == bucket) {
936 self.buckets[size_class_index] = null;
937 }
938 if (!config.never_unmap) {
939 const page: [*]align(page_size) u8 = @ptrFromInt(page_addr);
940 self.backing_allocator.rawFree(page[0..page_size], page_align, @returnAddress());
941 }
942 }
943 if (config.verbose_log) {
944 log.info("small free {d} bytes at {*}", .{ old_memory.len, old_memory.ptr });
945 }
946 }
947
948 fn resizeSmall(
949 self: *Self,
950 memory: []u8,
951 alignment: mem.Alignment,
952 new_len: usize,
953 return_address: usize,
954 size_class_index: usize,
955 ) bool {
956 const new_size_class_index: usize = @max(@bitSizeOf(usize) - @clz(new_len - 1), @intFromEnum(alignment));
957 if (!config.safety) return new_size_class_index == size_class_index;
958 const slot_count = slot_counts[size_class_index];
959 const memory_addr = @intFromPtr(memory.ptr);
960 const page_addr = memory_addr & ~(page_size - 1);
961 const bucket: *BucketHeader = .fromPage(page_addr, slot_count);
962 if (bucket.canary != config.canary) @panic("Invalid free");
963 const page_offset = memory_addr - page_addr;
964 const size_class = @as(usize, 1) << @as(Log2USize, @intCast(size_class_index));
965 const slot_index: SlotIndex = @intCast(page_offset / size_class);
966 const used_byte_index = slot_index / @bitSizeOf(usize);
967 const used_bit_index: Log2USize = @intCast(slot_index % @bitSizeOf(usize));
968 const used_byte = bucket.usedBits(used_byte_index);
969 const is_used = @as(u1, @truncate(used_byte.* >> used_bit_index)) != 0;
970 if (!is_used) {
971 reportDoubleFree(
972 return_address,
973 bucketStackTrace(bucket, slot_count, slot_index, .alloc),
974 bucketStackTrace(bucket, slot_count, slot_index, .free),
975 );
976 // Recoverable since this is a free.
977 return false;
978 }
979
980 // Definitely an in-use small alloc now.
981 const requested_size = bucket.requestedSizes(slot_count)[slot_index];
982 if (requested_size == 0) @panic("Invalid free");
983 const slot_alignment = bucket.log2PtrAligns(slot_count)[slot_index];
984 if (memory.len != requested_size or alignment != slot_alignment) {
985 var addresses: [stack_n]usize = [1]usize{0} ** stack_n;
986 var free_stack_trace: StackTrace = .{
987 .instruction_addresses = &addresses,
988 .index = 0,
989 };
990 std.debug.captureStackTrace(return_address, &free_stack_trace);
991 if (memory.len != requested_size) {
992 log.err("Allocation size {d} bytes does not match free size {d}. Allocation: {} Free: {}", .{
993 requested_size,
994 memory.len,
995 bucketStackTrace(bucket, slot_count, slot_index, .alloc),
996 free_stack_trace,
997 });
998 }
999 if (alignment != slot_alignment) {
1000 log.err("Allocation alignment {d} does not match free alignment {d}. Allocation: {} Free: {}", .{
1001 slot_alignment.toByteUnits(),
1002 alignment.toByteUnits(),
1003 bucketStackTrace(bucket, slot_count, slot_index, .alloc),
1004 free_stack_trace,
1005 });
1006 }
1007 }
1008
1009 if (new_size_class_index != size_class_index) return false;
1010
1011 const prev_req_bytes = self.total_requested_bytes;
1012 if (config.enable_memory_limit) {
1013 const new_req_bytes = prev_req_bytes - memory.len + new_len;
1014 if (new_req_bytes > prev_req_bytes and new_req_bytes > self.requested_memory_limit) {
1015 return false;
1016 }
1017 self.total_requested_bytes = new_req_bytes;
1018 }
1019
1020 if (memory.len > new_len) @memset(memory[new_len..], undefined);
1021 if (config.verbose_log)
1022 log.info("small resize {d} bytes at {*} to {d}", .{ memory.len, memory.ptr, new_len });
1023
1024 if (config.safety)
1025 bucket.requestedSizes(slot_count)[slot_index] = @intCast(new_len);
1026
1027 if (config.resize_stack_traces)
1028 bucket.captureStackTrace(return_address, slot_count, slot_index, .alloc);
1029
1030 return true;
1031 }
1032 };
1033}
1034
1035const TraceKind = enum {
1036 alloc,
1037 free,
1038};
1039
1040const test_config = Config{};
1041
1042test "small allocations - free in same order" {
1043 var gpa = DebugAllocator(test_config){};
1044 defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak");
1045 const allocator = gpa.allocator();
1046
1047 var list = std.ArrayList(*u64).init(std.testing.allocator);
1048 defer list.deinit();
1049
1050 var i: usize = 0;
1051 while (i < 513) : (i += 1) {
1052 const ptr = try allocator.create(u64);
1053 try list.append(ptr);
1054 }
1055
1056 for (list.items) |ptr| {
1057 allocator.destroy(ptr);
1058 }
1059}
1060
1061test "small allocations - free in reverse order" {
1062 var gpa = DebugAllocator(test_config){};
1063 defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak");
1064 const allocator = gpa.allocator();
1065
1066 var list = std.ArrayList(*u64).init(std.testing.allocator);
1067 defer list.deinit();
1068
1069 var i: usize = 0;
1070 while (i < 513) : (i += 1) {
1071 const ptr = try allocator.create(u64);
1072 try list.append(ptr);
1073 }
1074
1075 while (list.popOrNull()) |ptr| {
1076 allocator.destroy(ptr);
1077 }
1078}
1079
1080test "large allocations" {
1081 var gpa = DebugAllocator(test_config){};
1082 defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak");
1083 const allocator = gpa.allocator();
1084
1085 const ptr1 = try allocator.alloc(u64, 42768);
1086 const ptr2 = try allocator.alloc(u64, 52768);
1087 allocator.free(ptr1);
1088 const ptr3 = try allocator.alloc(u64, 62768);
1089 allocator.free(ptr3);
1090 allocator.free(ptr2);
1091}
1092
1093test "very large allocation" {
1094 var gpa = DebugAllocator(test_config){};
1095 defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak");
1096 const allocator = gpa.allocator();
1097
1098 try std.testing.expectError(error.OutOfMemory, allocator.alloc(u8, math.maxInt(usize)));
1099}
1100
1101test "realloc" {
1102 var gpa = DebugAllocator(test_config){};
1103 defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak");
1104 const allocator = gpa.allocator();
1105
1106 var slice = try allocator.alignedAlloc(u8, @alignOf(u32), 1);
1107 defer allocator.free(slice);
1108 slice[0] = 0x12;
1109
1110 // This reallocation should keep its pointer address.
1111 const old_slice = slice;
1112 slice = try allocator.realloc(slice, 2);
1113 try std.testing.expect(old_slice.ptr == slice.ptr);
1114 try std.testing.expect(slice[0] == 0x12);
1115 slice[1] = 0x34;
1116
1117 // This requires upgrading to a larger size class
1118 slice = try allocator.realloc(slice, 17);
1119 try std.testing.expect(slice[0] == 0x12);
1120 try std.testing.expect(slice[1] == 0x34);
1121}
1122
1123test "shrink" {
1124 var gpa: DebugAllocator(test_config) = .{};
1125 defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak");
1126 const allocator = gpa.allocator();
1127
1128 var slice = try allocator.alloc(u8, 20);
1129 defer allocator.free(slice);
1130
1131 @memset(slice, 0x11);
1132
1133 try std.testing.expect(allocator.resize(slice, 17));
1134 slice = slice[0..17];
1135
1136 for (slice) |b| {
1137 try std.testing.expect(b == 0x11);
1138 }
1139
1140 // Does not cross size class boundaries when shrinking.
1141 try std.testing.expect(!allocator.resize(slice, 16));
1142}
1143
1144test "large object - grow" {
1145 if (builtin.target.isWasm()) {
1146 // Not expected to pass on targets that do not have memory mapping.
1147 return error.SkipZigTest;
1148 }
1149 var gpa: DebugAllocator(test_config) = .{};
1150 defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak");
1151 const allocator = gpa.allocator();
1152
1153 var slice1 = try allocator.alloc(u8, default_page_size * 2 - 20);
1154 defer allocator.free(slice1);
1155
1156 const old = slice1;
1157 slice1 = try allocator.realloc(slice1, default_page_size * 2 - 10);
1158 try std.testing.expect(slice1.ptr == old.ptr);
1159
1160 slice1 = try allocator.realloc(slice1, default_page_size * 2);
1161 try std.testing.expect(slice1.ptr == old.ptr);
1162
1163 slice1 = try allocator.realloc(slice1, default_page_size * 2 + 1);
1164}
1165
1166test "realloc small object to large object" {
1167 var gpa = DebugAllocator(test_config){};
1168 defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak");
1169 const allocator = gpa.allocator();
1170
1171 var slice = try allocator.alloc(u8, 70);
1172 defer allocator.free(slice);
1173 slice[0] = 0x12;
1174 slice[60] = 0x34;
1175
1176 // This requires upgrading to a large object
1177 const large_object_size = default_page_size * 2 + 50;
1178 slice = try allocator.realloc(slice, large_object_size);
1179 try std.testing.expect(slice[0] == 0x12);
1180 try std.testing.expect(slice[60] == 0x34);
1181}
1182
1183test "shrink large object to large object" {
1184 var gpa: DebugAllocator(test_config) = .{};
1185 defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak");
1186 const allocator = gpa.allocator();
1187
1188 var slice = try allocator.alloc(u8, default_page_size * 2 + 50);
1189 defer allocator.free(slice);
1190 slice[0] = 0x12;
1191 slice[60] = 0x34;
1192
1193 if (!allocator.resize(slice, default_page_size * 2 + 1)) return;
1194 slice = slice.ptr[0 .. default_page_size * 2 + 1];
1195 try std.testing.expect(slice[0] == 0x12);
1196 try std.testing.expect(slice[60] == 0x34);
1197
1198 try std.testing.expect(allocator.resize(slice, default_page_size * 2 + 1));
1199 slice = slice[0 .. default_page_size * 2 + 1];
1200 try std.testing.expect(slice[0] == 0x12);
1201 try std.testing.expect(slice[60] == 0x34);
1202
1203 slice = try allocator.realloc(slice, default_page_size * 2);
1204 try std.testing.expect(slice[0] == 0x12);
1205 try std.testing.expect(slice[60] == 0x34);
1206}
1207
1208test "shrink large object to large object with larger alignment" {
1209 if (!builtin.link_libc and builtin.os.tag == .wasi) return error.SkipZigTest; // https://github.com/ziglang/zig/issues/22731
1210
1211 var gpa = DebugAllocator(test_config){};
1212 defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak");
1213 const allocator = gpa.allocator();
1214
1215 var debug_buffer: [1000]u8 = undefined;
1216 var fba = std.heap.FixedBufferAllocator.init(&debug_buffer);
1217 const debug_allocator = fba.allocator();
1218
1219 const alloc_size = default_page_size * 2 + 50;
1220 var slice = try allocator.alignedAlloc(u8, 16, alloc_size);
1221 defer allocator.free(slice);
1222
1223 const big_alignment: usize = default_page_size * 2;
1224 // This loop allocates until we find a page that is not aligned to the big
1225 // alignment. Then we shrink the allocation after the loop, but increase the
1226 // alignment to the higher one, that we know will force it to realloc.
1227 var stuff_to_free = std.ArrayList([]align(16) u8).init(debug_allocator);
1228 while (mem.isAligned(@intFromPtr(slice.ptr), big_alignment)) {
1229 try stuff_to_free.append(slice);
1230 slice = try allocator.alignedAlloc(u8, 16, alloc_size);
1231 }
1232 while (stuff_to_free.popOrNull()) |item| {
1233 allocator.free(item);
1234 }
1235 slice[0] = 0x12;
1236 slice[60] = 0x34;
1237
1238 slice = try allocator.reallocAdvanced(slice, big_alignment, alloc_size / 2);
1239 try std.testing.expect(slice[0] == 0x12);
1240 try std.testing.expect(slice[60] == 0x34);
1241}
1242
1243test "realloc large object to small object" {
1244 var gpa = DebugAllocator(test_config){};
1245 defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak");
1246 const allocator = gpa.allocator();
1247
1248 var slice = try allocator.alloc(u8, default_page_size * 2 + 50);
1249 defer allocator.free(slice);
1250 slice[0] = 0x12;
1251 slice[16] = 0x34;
1252
1253 slice = try allocator.realloc(slice, 19);
1254 try std.testing.expect(slice[0] == 0x12);
1255 try std.testing.expect(slice[16] == 0x34);
1256}
1257
1258test "overridable mutexes" {
1259 var gpa = DebugAllocator(.{ .MutexType = std.Thread.Mutex }){
1260 .backing_allocator = std.testing.allocator,
1261 .mutex = std.Thread.Mutex{},
1262 };
1263 defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak");
1264 const allocator = gpa.allocator();
1265
1266 const ptr = try allocator.create(i32);
1267 defer allocator.destroy(ptr);
1268}
1269
1270test "non-page-allocator backing allocator" {
1271 var gpa: DebugAllocator(.{
1272 .backing_allocator_zeroes = false,
1273 }) = .{
1274 .backing_allocator = std.testing.allocator,
1275 };
1276 defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak");
1277 const allocator = gpa.allocator();
1278
1279 const ptr = try allocator.create(i32);
1280 defer allocator.destroy(ptr);
1281}
1282
1283test "realloc large object to larger alignment" {
1284 if (!builtin.link_libc and builtin.os.tag == .wasi) return error.SkipZigTest; // https://github.com/ziglang/zig/issues/22731
1285
1286 var gpa = DebugAllocator(test_config){};
1287 defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak");
1288 const allocator = gpa.allocator();
1289
1290 var debug_buffer: [1000]u8 = undefined;
1291 var fba = std.heap.FixedBufferAllocator.init(&debug_buffer);
1292 const debug_allocator = fba.allocator();
1293
1294 var slice = try allocator.alignedAlloc(u8, 16, default_page_size * 2 + 50);
1295 defer allocator.free(slice);
1296
1297 const big_alignment: usize = default_page_size * 2;
1298 // This loop allocates until we find a page that is not aligned to the big alignment.
1299 var stuff_to_free = std.ArrayList([]align(16) u8).init(debug_allocator);
1300 while (mem.isAligned(@intFromPtr(slice.ptr), big_alignment)) {
1301 try stuff_to_free.append(slice);
1302 slice = try allocator.alignedAlloc(u8, 16, default_page_size * 2 + 50);
1303 }
1304 while (stuff_to_free.popOrNull()) |item| {
1305 allocator.free(item);
1306 }
1307 slice[0] = 0x12;
1308 slice[16] = 0x34;
1309
1310 slice = try allocator.reallocAdvanced(slice, 32, default_page_size * 2 + 100);
1311 try std.testing.expect(slice[0] == 0x12);
1312 try std.testing.expect(slice[16] == 0x34);
1313
1314 slice = try allocator.reallocAdvanced(slice, 32, default_page_size * 2 + 25);
1315 try std.testing.expect(slice[0] == 0x12);
1316 try std.testing.expect(slice[16] == 0x34);
1317
1318 slice = try allocator.reallocAdvanced(slice, big_alignment, default_page_size * 2 + 100);
1319 try std.testing.expect(slice[0] == 0x12);
1320 try std.testing.expect(slice[16] == 0x34);
1321}
1322
1323test "large object rejects shrinking to small" {
1324 if (builtin.target.isWasm()) {
1325 // Not expected to pass on targets that do not have memory mapping.
1326 return error.SkipZigTest;
1327 }
1328
1329 var failing_allocator = std.testing.FailingAllocator.init(std.heap.page_allocator, .{ .fail_index = 3 });
1330 var gpa: DebugAllocator(.{}) = .{
1331 .backing_allocator = failing_allocator.allocator(),
1332 };
1333 defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak");
1334 const allocator = gpa.allocator();
1335
1336 var slice = try allocator.alloc(u8, default_page_size * 2 + 50);
1337 defer allocator.free(slice);
1338 slice[0] = 0x12;
1339 slice[3] = 0x34;
1340
1341 try std.testing.expect(!allocator.resize(slice, 4));
1342 try std.testing.expect(slice[0] == 0x12);
1343 try std.testing.expect(slice[3] == 0x34);
1344}
1345
1346test "objects of size 1024 and 2048" {
1347 var gpa = DebugAllocator(test_config){};
1348 defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak");
1349 const allocator = gpa.allocator();
1350
1351 const slice = try allocator.alloc(u8, 1025);
1352 const slice2 = try allocator.alloc(u8, 3000);
1353
1354 allocator.free(slice);
1355 allocator.free(slice2);
1356}
1357
1358test "setting a memory cap" {
1359 var gpa = DebugAllocator(.{ .enable_memory_limit = true }){};
1360 defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak");
1361 const allocator = gpa.allocator();
1362
1363 gpa.requested_memory_limit = 1010;
1364
1365 const small = try allocator.create(i32);
1366 try std.testing.expect(gpa.total_requested_bytes == 4);
1367
1368 const big = try allocator.alloc(u8, 1000);
1369 try std.testing.expect(gpa.total_requested_bytes == 1004);
1370
1371 try std.testing.expectError(error.OutOfMemory, allocator.create(u64));
1372
1373 allocator.destroy(small);
1374 try std.testing.expect(gpa.total_requested_bytes == 1000);
1375
1376 allocator.free(big);
1377 try std.testing.expect(gpa.total_requested_bytes == 0);
1378
1379 const exact = try allocator.alloc(u8, 1010);
1380 try std.testing.expect(gpa.total_requested_bytes == 1010);
1381 allocator.free(exact);
1382}
1383
1384test "large allocations count requested size not backing size" {
1385 var gpa: DebugAllocator(.{ .enable_memory_limit = true }) = .{};
1386 const allocator = gpa.allocator();
1387
1388 var buf = try allocator.alignedAlloc(u8, 1, default_page_size + 1);
1389 try std.testing.expectEqual(default_page_size + 1, gpa.total_requested_bytes);
1390 buf = try allocator.realloc(buf, 1);
1391 try std.testing.expectEqual(1, gpa.total_requested_bytes);
1392 buf = try allocator.realloc(buf, 2);
1393 try std.testing.expectEqual(2, gpa.total_requested_bytes);
1394}
1395
1396test "retain metadata and never unmap" {
1397 var gpa = std.heap.DebugAllocator(.{
1398 .safety = true,
1399 .never_unmap = true,
1400 .retain_metadata = true,
1401 }){};
1402 defer std.debug.assert(gpa.deinit() == .ok);
1403 const allocator = gpa.allocator();
1404
1405 const alloc = try allocator.alloc(u8, 8);
1406 allocator.free(alloc);
1407
1408 const alloc2 = try allocator.alloc(u8, 8);
1409 allocator.free(alloc2);
1410}
lib/std/heap/general_purpose_allocator.zig deleted-1500
...@@ -1,1500 +0,0 @@
1//! # General Purpose Allocator
2//!
3//! ## Design Priorities
4//!
5//! ### `OptimizationMode.debug` and `OptimizationMode.release_safe`:
6//!
7//! * Detect double free, and emit stack trace of:
8//! - Where it was first allocated
9//! - Where it was freed the first time
10//! - Where it was freed the second time
11//!
12//! * Detect leaks and emit stack trace of:
13//! - Where it was allocated
14//!
15//! * When a page of memory is no longer needed, give it back to resident memory
16//! as soon as possible, so that it causes page faults when used.
17//!
18//! * Do not re-use memory slots, so that memory safety is upheld. For small
19//! allocations, this is handled here; for larger ones it is handled in the
20//! backing allocator (by default `std.heap.page_allocator`).
21//!
22//! * Make pointer math errors unlikely to harm memory from
23//! unrelated allocations.
24//!
25//! * It's OK for these mechanisms to cost some extra overhead bytes.
26//!
27//! * It's OK for performance cost for these mechanisms.
28//!
29//! * Rogue memory writes should not harm the allocator's state.
30//!
31//! * Cross platform. Operates based on a backing allocator which makes it work
32//! everywhere, even freestanding.
33//!
34//! * Compile-time configuration.
35//!
36//! ### `OptimizationMode.release_fast` (note: not much work has gone into this use case yet):
37//!
38//! * Low fragmentation is primary concern
39//! * Performance of worst-case latency is secondary concern
40//! * Performance of average-case latency is next
41//! * Finally, having freed memory unmapped, and pointer math errors unlikely to
42//! harm memory from unrelated allocations are nice-to-haves.
43//!
44//! ### `OptimizationMode.release_small` (note: not much work has gone into this use case yet):
45//!
46//! * Small binary code size of the executable is the primary concern.
47//! * Next, defer to the `.release_fast` priority list.
48//!
49//! ## Basic Design:
50//!
51//! Small allocations are divided into buckets:
52//!
53//! ```
54//! index obj_size
55//! 0 1
56//! 1 2
57//! 2 4
58//! 3 8
59//! 4 16
60//! 5 32
61//! 6 64
62//! 7 128
63//! 8 256
64//! 9 512
65//! 10 1024
66//! 11 2048
67//! ```
68//!
69//! The main allocator state has an array of all the "current" buckets for each
70//! size class. Each slot in the array can be null, meaning the bucket for that
71//! size class is not allocated. When the first object is allocated for a given
72//! size class, it allocates 1 page of memory from the OS. This page is
73//! divided into "slots" - one per allocated object. Along with the page of memory
74//! for object slots, as many pages as necessary are allocated to store the
75//! BucketHeader, followed by "used bits", and two stack traces for each slot
76//! (allocation trace and free trace).
77//!
78//! The "used bits" are 1 bit per slot representing whether the slot is used.
79//! Allocations use the data to iterate to find a free slot. Frees assert that the
80//! corresponding bit is 1 and set it to 0.
81//!
82//! Buckets have prev and next pointers. When there is only one bucket for a given
83//! size class, both prev and next point to itself. When all slots of a bucket are
84//! used, a new bucket is allocated, and enters the doubly linked list. The main
85//! allocator state tracks the "current" bucket for each size class. Leak detection
86//! currently only checks the current bucket.
87//!
88//! Resizing detects if the size class is unchanged or smaller, in which case the same
89//! pointer is returned unmodified. If a larger size class is required,
90//! `error.OutOfMemory` is returned.
91//!
92//! Large objects are allocated directly using the backing allocator and their metadata is stored
93//! in a `std.HashMap` using the backing allocator.
94
95const std = @import("std");
96const builtin = @import("builtin");
97const log = std.log.scoped(.gpa);
98const math = std.math;
99const assert = std.debug.assert;
100const mem = std.mem;
101const Allocator = std.mem.Allocator;
102const page_size = std.mem.page_size;
103const StackTrace = std.builtin.StackTrace;
104
105/// Integer type for pointing to slots in a small allocation
106const SlotIndex = std.meta.Int(.unsigned, math.log2(page_size) + 1);
107
108const default_test_stack_trace_frames: usize = if (builtin.is_test) 10 else 6;
109const default_sys_stack_trace_frames: usize = if (std.debug.sys_can_stack_trace) default_test_stack_trace_frames else 0;
110const default_stack_trace_frames: usize = switch (builtin.mode) {
111 .Debug => default_sys_stack_trace_frames,
112 else => 0,
113};
114
115pub const Config = struct {
116 /// Number of stack frames to capture.
117 stack_trace_frames: usize = default_stack_trace_frames,
118
119 /// If true, the allocator will have two fields:
120 /// * `total_requested_bytes` which tracks the total allocated bytes of memory requested.
121 /// * `requested_memory_limit` which causes allocations to return `error.OutOfMemory`
122 /// when the `total_requested_bytes` exceeds this limit.
123 /// If false, these fields will be `void`.
124 enable_memory_limit: bool = false,
125
126 /// Whether to enable safety checks.
127 safety: bool = std.debug.runtime_safety,
128
129 /// Whether the allocator may be used simultaneously from multiple threads.
130 thread_safe: bool = !builtin.single_threaded,
131
132 /// What type of mutex you'd like to use, for thread safety.
133 /// when specified, the mutex type must have the same shape as `std.Thread.Mutex` and
134 /// `DummyMutex`, and have no required fields. Specifying this field causes
135 /// the `thread_safe` field to be ignored.
136 ///
137 /// when null (default):
138 /// * the mutex type defaults to `std.Thread.Mutex` when thread_safe is enabled.
139 /// * the mutex type defaults to `DummyMutex` otherwise.
140 MutexType: ?type = null,
141
142 /// This is a temporary debugging trick you can use to turn segfaults into more helpful
143 /// logged error messages with stack trace details. The downside is that every allocation
144 /// will be leaked, unless used with retain_metadata!
145 never_unmap: bool = false,
146
147 /// This is a temporary debugging aid that retains metadata about allocations indefinitely.
148 /// This allows a greater range of double frees to be reported. All metadata is freed when
149 /// deinit is called. When used with never_unmap, deliberately leaked memory is also freed
150 /// during deinit. Currently should be used with never_unmap to avoid segfaults.
151 /// TODO https://github.com/ziglang/zig/issues/4298 will allow use without never_unmap
152 retain_metadata: bool = false,
153
154 /// Enables emitting info messages with the size and address of every allocation.
155 verbose_log: bool = false,
156};
157
158pub const Check = enum { ok, leak };
159
160/// Default initialization of this struct is deprecated; use `.init` instead.
161pub fn GeneralPurposeAllocator(comptime config: Config) type {
162 return struct {
163 backing_allocator: Allocator = std.heap.page_allocator,
164 buckets: [small_bucket_count]Buckets = [1]Buckets{Buckets{}} ** small_bucket_count,
165 cur_buckets: [small_bucket_count]?*BucketHeader = [1]?*BucketHeader{null} ** small_bucket_count,
166 large_allocations: LargeAllocTable = .{},
167 empty_buckets: if (config.retain_metadata) Buckets else void =
168 if (config.retain_metadata) Buckets{} else {},
169 bucket_node_pool: std.heap.MemoryPool(Buckets.Node) = std.heap.MemoryPool(Buckets.Node).init(std.heap.page_allocator),
170
171 total_requested_bytes: @TypeOf(total_requested_bytes_init) = total_requested_bytes_init,
172 requested_memory_limit: @TypeOf(requested_memory_limit_init) = requested_memory_limit_init,
173
174 mutex: @TypeOf(mutex_init) = mutex_init,
175
176 const Self = @This();
177
178 /// The initial state of a `GeneralPurposeAllocator`, containing no allocations and backed by the system page allocator.
179 pub const init: Self = .{
180 .backing_allocator = std.heap.page_allocator,
181 .buckets = [1]Buckets{.{}} ** small_bucket_count,
182 .cur_buckets = [1]?*BucketHeader{null} ** small_bucket_count,
183 .large_allocations = .{},
184 .empty_buckets = if (config.retain_metadata) .{} else {},
185 .bucket_node_pool = .init(std.heap.page_allocator),
186 };
187
188 const total_requested_bytes_init = if (config.enable_memory_limit) @as(usize, 0) else {};
189 const requested_memory_limit_init = if (config.enable_memory_limit) @as(usize, math.maxInt(usize)) else {};
190
191 const mutex_init = if (config.MutexType) |T|
192 T{}
193 else if (config.thread_safe)
194 std.Thread.Mutex{}
195 else
196 DummyMutex{};
197
198 const DummyMutex = struct {
199 fn lock(_: *DummyMutex) void {}
200 fn unlock(_: *DummyMutex) void {}
201 };
202
203 const stack_n = config.stack_trace_frames;
204 const one_trace_size = @sizeOf(usize) * stack_n;
205 const traces_per_slot = 2;
206
207 pub const Error = mem.Allocator.Error;
208
209 const small_bucket_count = math.log2(page_size);
210 const largest_bucket_object_size = 1 << (small_bucket_count - 1);
211 const LargestSizeClassInt = std.math.IntFittingRange(0, largest_bucket_object_size);
212
213 const bucketCompare = struct {
214 fn compare(a: *BucketHeader, b: *BucketHeader) std.math.Order {
215 return std.math.order(@intFromPtr(a.page), @intFromPtr(b.page));
216 }
217 }.compare;
218 const Buckets = std.Treap(*BucketHeader, bucketCompare);
219
220 const LargeAlloc = struct {
221 bytes: []u8,
222 requested_size: if (config.enable_memory_limit) usize else void,
223 stack_addresses: [trace_n][stack_n]usize,
224 freed: if (config.retain_metadata) bool else void,
225 log2_ptr_align: if (config.never_unmap and config.retain_metadata) u8 else void,
226
227 const trace_n = if (config.retain_metadata) traces_per_slot else 1;
228
229 fn dumpStackTrace(self: *LargeAlloc, trace_kind: TraceKind) void {
230 std.debug.dumpStackTrace(self.getStackTrace(trace_kind));
231 }
232
233 fn getStackTrace(self: *LargeAlloc, trace_kind: TraceKind) std.builtin.StackTrace {
234 assert(@intFromEnum(trace_kind) < trace_n);
235 const stack_addresses = &self.stack_addresses[@intFromEnum(trace_kind)];
236 var len: usize = 0;
237 while (len < stack_n and stack_addresses[len] != 0) {
238 len += 1;
239 }
240 return .{
241 .instruction_addresses = stack_addresses,
242 .index = len,
243 };
244 }
245
246 fn captureStackTrace(self: *LargeAlloc, ret_addr: usize, trace_kind: TraceKind) void {
247 assert(@intFromEnum(trace_kind) < trace_n);
248 const stack_addresses = &self.stack_addresses[@intFromEnum(trace_kind)];
249 collectStackTrace(ret_addr, stack_addresses);
250 }
251 };
252 const LargeAllocTable = std.AutoHashMapUnmanaged(usize, LargeAlloc);
253
254 // Bucket: In memory, in order:
255 // * BucketHeader
256 // * bucket_used_bits: [N]u8, // 1 bit for every slot; 1 byte for every 8 slots
257 // -- below only exists when config.safety is true --
258 // * requested_sizes: [N]LargestSizeClassInt // 1 int for every slot
259 // * log2_ptr_aligns: [N]u8 // 1 byte for every slot
260 // -- above only exists when config.safety is true --
261 // * stack_trace_addresses: [N]usize, // traces_per_slot for every allocation
262
263 const BucketHeader = struct {
264 page: [*]align(page_size) u8,
265 alloc_cursor: SlotIndex,
266 used_count: SlotIndex,
267
268 fn usedBits(bucket: *BucketHeader, index: usize) *u8 {
269 return @as(*u8, @ptrFromInt(@intFromPtr(bucket) + @sizeOf(BucketHeader) + index));
270 }
271
272 fn requestedSizes(bucket: *BucketHeader, size_class: usize) []LargestSizeClassInt {
273 if (!config.safety) @compileError("requested size is only stored when safety is enabled");
274 const start_ptr = @as([*]u8, @ptrCast(bucket)) + bucketRequestedSizesStart(size_class);
275 const sizes = @as([*]LargestSizeClassInt, @ptrCast(@alignCast(start_ptr)));
276 const slot_count = @divExact(page_size, size_class);
277 return sizes[0..slot_count];
278 }
279
280 fn log2PtrAligns(bucket: *BucketHeader, size_class: usize) []u8 {
281 if (!config.safety) @compileError("requested size is only stored when safety is enabled");
282 const aligns_ptr = @as([*]u8, @ptrCast(bucket)) + bucketAlignsStart(size_class);
283 const slot_count = @divExact(page_size, size_class);
284 return aligns_ptr[0..slot_count];
285 }
286
287 fn stackTracePtr(
288 bucket: *BucketHeader,
289 size_class: usize,
290 slot_index: SlotIndex,
291 trace_kind: TraceKind,
292 ) *[stack_n]usize {
293 const start_ptr = @as([*]u8, @ptrCast(bucket)) + bucketStackFramesStart(size_class);
294 const addr = start_ptr + one_trace_size * traces_per_slot * slot_index +
295 @intFromEnum(trace_kind) * @as(usize, one_trace_size);
296 return @ptrCast(@alignCast(addr));
297 }
298
299 fn captureStackTrace(
300 bucket: *BucketHeader,
301 ret_addr: usize,
302 size_class: usize,
303 slot_index: SlotIndex,
304 trace_kind: TraceKind,
305 ) void {
306 // Initialize them to 0. When determining the count we must look
307 // for non zero addresses.
308 const stack_addresses = bucket.stackTracePtr(size_class, slot_index, trace_kind);
309 collectStackTrace(ret_addr, stack_addresses);
310 }
311
312 /// Only valid for buckets within `empty_buckets`, and relies on the `alloc_cursor`
313 /// of empty buckets being set to `slot_count` when they are added to `empty_buckets`
314 fn emptyBucketSizeClass(bucket: *BucketHeader) usize {
315 return @divExact(page_size, bucket.alloc_cursor);
316 }
317 };
318
319 pub fn allocator(self: *Self) Allocator {
320 return .{
321 .ptr = self,
322 .vtable = &.{
323 .alloc = alloc,
324 .resize = resize,
325 .free = free,
326 },
327 };
328 }
329
330 fn bucketStackTrace(
331 bucket: *BucketHeader,
332 size_class: usize,
333 slot_index: SlotIndex,
334 trace_kind: TraceKind,
335 ) StackTrace {
336 const stack_addresses = bucket.stackTracePtr(size_class, slot_index, trace_kind);
337 var len: usize = 0;
338 while (len < stack_n and stack_addresses[len] != 0) {
339 len += 1;
340 }
341 return StackTrace{
342 .instruction_addresses = stack_addresses,
343 .index = len,
344 };
345 }
346
347 fn bucketRequestedSizesStart(size_class: usize) usize {
348 if (!config.safety) @compileError("requested sizes are not stored unless safety is enabled");
349 return mem.alignForward(
350 usize,
351 @sizeOf(BucketHeader) + usedBitsCount(size_class),
352 @alignOf(LargestSizeClassInt),
353 );
354 }
355
356 fn bucketAlignsStart(size_class: usize) usize {
357 if (!config.safety) @compileError("requested sizes are not stored unless safety is enabled");
358 const slot_count = @divExact(page_size, size_class);
359 return bucketRequestedSizesStart(size_class) + (@sizeOf(LargestSizeClassInt) * slot_count);
360 }
361
362 fn bucketStackFramesStart(size_class: usize) usize {
363 const unaligned_start = if (config.safety) blk: {
364 const slot_count = @divExact(page_size, size_class);
365 break :blk bucketAlignsStart(size_class) + slot_count;
366 } else @sizeOf(BucketHeader) + usedBitsCount(size_class);
367 return mem.alignForward(
368 usize,
369 unaligned_start,
370 @alignOf(usize),
371 );
372 }
373
374 fn bucketSize(size_class: usize) usize {
375 const slot_count = @divExact(page_size, size_class);
376 return bucketStackFramesStart(size_class) + one_trace_size * traces_per_slot * slot_count;
377 }
378
379 fn usedBitsCount(size_class: usize) usize {
380 const slot_count = @divExact(page_size, size_class);
381 if (slot_count < 8) return 1;
382 return @divExact(slot_count, 8);
383 }
384
385 fn detectLeaksInBucket(
386 bucket: *BucketHeader,
387 size_class: usize,
388 used_bits_count: usize,
389 ) bool {
390 var leaks = false;
391 var used_bits_byte: usize = 0;
392 while (used_bits_byte < used_bits_count) : (used_bits_byte += 1) {
393 const used_byte = bucket.usedBits(used_bits_byte).*;
394 if (used_byte != 0) {
395 var bit_index: u3 = 0;
396 while (true) : (bit_index += 1) {
397 const is_used = @as(u1, @truncate(used_byte >> bit_index)) != 0;
398 if (is_used) {
399 const slot_index = @as(SlotIndex, @intCast(used_bits_byte * 8 + bit_index));
400 const stack_trace = bucketStackTrace(bucket, size_class, slot_index, .alloc);
401 const addr = bucket.page + slot_index * size_class;
402 log.err("memory address 0x{x} leaked: {}", .{
403 @intFromPtr(addr), stack_trace,
404 });
405 leaks = true;
406 }
407 if (bit_index == math.maxInt(u3))
408 break;
409 }
410 }
411 }
412 return leaks;
413 }
414
415 /// Emits log messages for leaks and then returns whether there were any leaks.
416 pub fn detectLeaks(self: *Self) bool {
417 var leaks = false;
418
419 for (&self.buckets, 0..) |*buckets, bucket_i| {
420 if (buckets.root == null) continue;
421 const size_class = @as(usize, 1) << @as(math.Log2Int(usize), @intCast(bucket_i));
422 const used_bits_count = usedBitsCount(size_class);
423 var it = buckets.inorderIterator();
424 while (it.next()) |node| {
425 const bucket = node.key;
426 leaks = detectLeaksInBucket(bucket, size_class, used_bits_count) or leaks;
427 }
428 }
429 var it = self.large_allocations.valueIterator();
430 while (it.next()) |large_alloc| {
431 if (config.retain_metadata and large_alloc.freed) continue;
432 const stack_trace = large_alloc.getStackTrace(.alloc);
433 log.err("memory address 0x{x} leaked: {}", .{
434 @intFromPtr(large_alloc.bytes.ptr), stack_trace,
435 });
436 leaks = true;
437 }
438 return leaks;
439 }
440
441 fn freeBucket(self: *Self, bucket: *BucketHeader, size_class: usize) void {
442 const bucket_size = bucketSize(size_class);
443 const bucket_slice = @as([*]align(@alignOf(BucketHeader)) u8, @ptrCast(bucket))[0..bucket_size];
444 self.backing_allocator.free(bucket_slice);
445 }
446
447 fn freeRetainedMetadata(self: *Self) void {
448 if (config.retain_metadata) {
449 if (config.never_unmap) {
450 // free large allocations that were intentionally leaked by never_unmap
451 var it = self.large_allocations.iterator();
452 while (it.next()) |large| {
453 if (large.value_ptr.freed) {
454 self.backing_allocator.rawFree(large.value_ptr.bytes, large.value_ptr.log2_ptr_align, @returnAddress());
455 }
456 }
457 }
458 // free retained metadata for small allocations
459 while (self.empty_buckets.getMin()) |node| {
460 // remove the node from the tree before destroying it
461 var entry = self.empty_buckets.getEntryForExisting(node);
462 entry.set(null);
463
464 var bucket = node.key;
465 if (config.never_unmap) {
466 // free page that was intentionally leaked by never_unmap
467 self.backing_allocator.free(bucket.page[0..page_size]);
468 }
469 // alloc_cursor was set to slot count when bucket added to empty_buckets
470 self.freeBucket(bucket, bucket.emptyBucketSizeClass());
471 self.bucket_node_pool.destroy(node);
472 }
473 self.empty_buckets.root = null;
474 }
475 }
476
477 pub fn flushRetainedMetadata(self: *Self) void {
478 if (!config.retain_metadata) {
479 @compileError("'flushRetainedMetadata' requires 'config.retain_metadata = true'");
480 }
481 self.freeRetainedMetadata();
482 // also remove entries from large_allocations
483 var it = self.large_allocations.iterator();
484 while (it.next()) |large| {
485 if (large.value_ptr.freed) {
486 _ = self.large_allocations.remove(@intFromPtr(large.value_ptr.bytes.ptr));
487 }
488 }
489 }
490
491 /// Returns `Check.leak` if there were leaks; `Check.ok` otherwise.
492 pub fn deinit(self: *Self) Check {
493 const leaks = if (config.safety) self.detectLeaks() else false;
494 if (config.retain_metadata) {
495 self.freeRetainedMetadata();
496 }
497 self.large_allocations.deinit(self.backing_allocator);
498 self.bucket_node_pool.deinit();
499 self.* = undefined;
500 return @as(Check, @enumFromInt(@intFromBool(leaks)));
501 }
502
503 fn collectStackTrace(first_trace_addr: usize, addresses: *[stack_n]usize) void {
504 if (stack_n == 0) return;
505 @memset(addresses, 0);
506 var stack_trace = StackTrace{
507 .instruction_addresses = addresses,
508 .index = 0,
509 };
510 std.debug.captureStackTrace(first_trace_addr, &stack_trace);
511 }
512
513 fn reportDoubleFree(ret_addr: usize, alloc_stack_trace: StackTrace, free_stack_trace: StackTrace) void {
514 var addresses: [stack_n]usize = [1]usize{0} ** stack_n;
515 var second_free_stack_trace = StackTrace{
516 .instruction_addresses = &addresses,
517 .index = 0,
518 };
519 std.debug.captureStackTrace(ret_addr, &second_free_stack_trace);
520 log.err("Double free detected. Allocation: {} First free: {} Second free: {}", .{
521 alloc_stack_trace, free_stack_trace, second_free_stack_trace,
522 });
523 }
524
525 const Slot = struct {
526 bucket: *BucketHeader,
527 slot_index: usize,
528 ptr: [*]u8,
529 };
530
531 fn allocSlot(self: *Self, size_class: usize, trace_addr: usize) Error!Slot {
532 const bucket_index = math.log2(size_class);
533 var buckets = &self.buckets[bucket_index];
534 const slot_count = @divExact(page_size, size_class);
535 if (self.cur_buckets[bucket_index] == null or self.cur_buckets[bucket_index].?.alloc_cursor == slot_count) {
536 const new_bucket = try self.createBucket(size_class);
537 errdefer self.freeBucket(new_bucket, size_class);
538 const node = try self.bucket_node_pool.create();
539 node.key = new_bucket;
540 var entry = buckets.getEntryFor(new_bucket);
541 std.debug.assert(entry.node == null);
542 entry.set(node);
543 self.cur_buckets[bucket_index] = node.key;
544 }
545 const bucket = self.cur_buckets[bucket_index].?;
546
547 const slot_index = bucket.alloc_cursor;
548 bucket.alloc_cursor += 1;
549
550 const used_bits_byte = bucket.usedBits(slot_index / 8);
551 const used_bit_index: u3 = @as(u3, @intCast(slot_index % 8)); // TODO cast should be unnecessary
552 used_bits_byte.* |= (@as(u8, 1) << used_bit_index);
553 bucket.used_count += 1;
554 bucket.captureStackTrace(trace_addr, size_class, slot_index, .alloc);
555 return .{
556 .bucket = bucket,
557 .slot_index = slot_index,
558 .ptr = bucket.page + slot_index * size_class,
559 };
560 }
561
562 fn searchBucket(
563 buckets: *Buckets,
564 addr: usize,
565 current_bucket: ?*BucketHeader,
566 ) ?*BucketHeader {
567 const search_page: [*]align(page_size) u8 = @ptrFromInt(mem.alignBackward(usize, addr, page_size));
568 if (current_bucket != null and current_bucket.?.page == search_page) {
569 return current_bucket;
570 }
571 var search_header: BucketHeader = undefined;
572 search_header.page = search_page;
573 const entry = buckets.getEntryFor(&search_header);
574 return if (entry.node) |node| node.key else null;
575 }
576
577 /// This function assumes the object is in the large object storage regardless
578 /// of the parameters.
579 fn resizeLarge(
580 self: *Self,
581 old_mem: []u8,
582 log2_old_align: u8,
583 new_size: usize,
584 ret_addr: usize,
585 ) bool {
586 const entry = self.large_allocations.getEntry(@intFromPtr(old_mem.ptr)) orelse {
587 if (config.safety) {
588 @panic("Invalid free");
589 } else {
590 unreachable;
591 }
592 };
593
594 if (config.retain_metadata and entry.value_ptr.freed) {
595 if (config.safety) {
596 reportDoubleFree(ret_addr, entry.value_ptr.getStackTrace(.alloc), entry.value_ptr.getStackTrace(.free));
597 @panic("Unrecoverable double free");
598 } else {
599 unreachable;
600 }
601 }
602
603 if (config.safety and old_mem.len != entry.value_ptr.bytes.len) {
604 var addresses: [stack_n]usize = [1]usize{0} ** stack_n;
605 var free_stack_trace = StackTrace{
606 .instruction_addresses = &addresses,
607 .index = 0,
608 };
609 std.debug.captureStackTrace(ret_addr, &free_stack_trace);
610 log.err("Allocation size {d} bytes does not match free size {d}. Allocation: {} Free: {}", .{
611 entry.value_ptr.bytes.len,
612 old_mem.len,
613 entry.value_ptr.getStackTrace(.alloc),
614 free_stack_trace,
615 });
616 }
617
618 // Do memory limit accounting with requested sizes rather than what
619 // backing_allocator returns because if we want to return
620 // error.OutOfMemory, we have to leave allocation untouched, and
621 // that is impossible to guarantee after calling
622 // backing_allocator.rawResize.
623 const prev_req_bytes = self.total_requested_bytes;
624 if (config.enable_memory_limit) {
625 const new_req_bytes = prev_req_bytes + new_size - entry.value_ptr.requested_size;
626 if (new_req_bytes > prev_req_bytes and new_req_bytes > self.requested_memory_limit) {
627 return false;
628 }
629 self.total_requested_bytes = new_req_bytes;
630 }
631
632 if (!self.backing_allocator.rawResize(old_mem, log2_old_align, new_size, ret_addr)) {
633 if (config.enable_memory_limit) {
634 self.total_requested_bytes = prev_req_bytes;
635 }
636 return false;
637 }
638
639 if (config.enable_memory_limit) {
640 entry.value_ptr.requested_size = new_size;
641 }
642
643 if (config.verbose_log) {
644 log.info("large resize {d} bytes at {*} to {d}", .{
645 old_mem.len, old_mem.ptr, new_size,
646 });
647 }
648 entry.value_ptr.bytes = old_mem.ptr[0..new_size];
649 entry.value_ptr.captureStackTrace(ret_addr, .alloc);
650 return true;
651 }
652
653 /// This function assumes the object is in the large object storage regardless
654 /// of the parameters.
655 fn freeLarge(
656 self: *Self,
657 old_mem: []u8,
658 log2_old_align: u8,
659 ret_addr: usize,
660 ) void {
661 const entry = self.large_allocations.getEntry(@intFromPtr(old_mem.ptr)) orelse {
662 if (config.safety) {
663 @panic("Invalid free");
664 } else {
665 unreachable;
666 }
667 };
668
669 if (config.retain_metadata and entry.value_ptr.freed) {
670 if (config.safety) {
671 reportDoubleFree(ret_addr, entry.value_ptr.getStackTrace(.alloc), entry.value_ptr.getStackTrace(.free));
672 return;
673 } else {
674 unreachable;
675 }
676 }
677
678 if (config.safety and old_mem.len != entry.value_ptr.bytes.len) {
679 var addresses: [stack_n]usize = [1]usize{0} ** stack_n;
680 var free_stack_trace = StackTrace{
681 .instruction_addresses = &addresses,
682 .index = 0,
683 };
684 std.debug.captureStackTrace(ret_addr, &free_stack_trace);
685 log.err("Allocation size {d} bytes does not match free size {d}. Allocation: {} Free: {}", .{
686 entry.value_ptr.bytes.len,
687 old_mem.len,
688 entry.value_ptr.getStackTrace(.alloc),
689 free_stack_trace,
690 });
691 }
692
693 if (!config.never_unmap) {
694 self.backing_allocator.rawFree(old_mem, log2_old_align, ret_addr);
695 }
696
697 if (config.enable_memory_limit) {
698 self.total_requested_bytes -= entry.value_ptr.requested_size;
699 }
700
701 if (config.verbose_log) {
702 log.info("large free {d} bytes at {*}", .{ old_mem.len, old_mem.ptr });
703 }
704
705 if (!config.retain_metadata) {
706 assert(self.large_allocations.remove(@intFromPtr(old_mem.ptr)));
707 } else {
708 entry.value_ptr.freed = true;
709 entry.value_ptr.captureStackTrace(ret_addr, .free);
710 }
711 }
712
713 pub fn setRequestedMemoryLimit(self: *Self, limit: usize) void {
714 self.requested_memory_limit = limit;
715 }
716
717 fn resize(
718 ctx: *anyopaque,
719 old_mem: []u8,
720 log2_old_align_u8: u8,
721 new_size: usize,
722 ret_addr: usize,
723 ) bool {
724 const self: *Self = @ptrCast(@alignCast(ctx));
725 const log2_old_align = @as(Allocator.Log2Align, @intCast(log2_old_align_u8));
726 self.mutex.lock();
727 defer self.mutex.unlock();
728
729 assert(old_mem.len != 0);
730
731 const aligned_size = @max(old_mem.len, @as(usize, 1) << log2_old_align);
732 if (aligned_size > largest_bucket_object_size) {
733 return self.resizeLarge(old_mem, log2_old_align, new_size, ret_addr);
734 }
735 const size_class_hint = math.ceilPowerOfTwoAssert(usize, aligned_size);
736
737 var bucket_index = math.log2(size_class_hint);
738 var size_class: usize = size_class_hint;
739 const bucket = while (bucket_index < small_bucket_count) : (bucket_index += 1) {
740 if (searchBucket(&self.buckets[bucket_index], @intFromPtr(old_mem.ptr), self.cur_buckets[bucket_index])) |bucket| {
741 break bucket;
742 }
743 size_class *= 2;
744 } else blk: {
745 if (config.retain_metadata) {
746 if (!self.large_allocations.contains(@intFromPtr(old_mem.ptr))) {
747 // object not in active buckets or a large allocation, so search empty buckets
748 if (searchBucket(&self.empty_buckets, @intFromPtr(old_mem.ptr), null)) |bucket| {
749 size_class = bucket.emptyBucketSizeClass();
750 // bucket is empty so is_used below will always be false and we exit there
751 break :blk bucket;
752 } else {
753 @panic("Invalid free");
754 }
755 }
756 }
757 return self.resizeLarge(old_mem, log2_old_align, new_size, ret_addr);
758 };
759 const byte_offset = @intFromPtr(old_mem.ptr) - @intFromPtr(bucket.page);
760 const slot_index = @as(SlotIndex, @intCast(byte_offset / size_class));
761 const used_byte_index = slot_index / 8;
762 const used_bit_index = @as(u3, @intCast(slot_index % 8));
763 const used_byte = bucket.usedBits(used_byte_index);
764 const is_used = @as(u1, @truncate(used_byte.* >> used_bit_index)) != 0;
765 if (!is_used) {
766 if (config.safety) {
767 reportDoubleFree(ret_addr, bucketStackTrace(bucket, size_class, slot_index, .alloc), bucketStackTrace(bucket, size_class, slot_index, .free));
768 @panic("Unrecoverable double free");
769 } else {
770 unreachable;
771 }
772 }
773
774 // Definitely an in-use small alloc now.
775 if (config.safety) {
776 const requested_size = bucket.requestedSizes(size_class)[slot_index];
777 if (requested_size == 0) @panic("Invalid free");
778 const log2_ptr_align = bucket.log2PtrAligns(size_class)[slot_index];
779 if (old_mem.len != requested_size or log2_old_align != log2_ptr_align) {
780 var addresses: [stack_n]usize = [1]usize{0} ** stack_n;
781 var free_stack_trace = StackTrace{
782 .instruction_addresses = &addresses,
783 .index = 0,
784 };
785 std.debug.captureStackTrace(ret_addr, &free_stack_trace);
786 if (old_mem.len != requested_size) {
787 log.err("Allocation size {d} bytes does not match resize size {d}. Allocation: {} Resize: {}", .{
788 requested_size,
789 old_mem.len,
790 bucketStackTrace(bucket, size_class, slot_index, .alloc),
791 free_stack_trace,
792 });
793 }
794 if (log2_old_align != log2_ptr_align) {
795 log.err("Allocation alignment {d} does not match resize alignment {d}. Allocation: {} Resize: {}", .{
796 @as(usize, 1) << @as(math.Log2Int(usize), @intCast(log2_ptr_align)),
797 @as(usize, 1) << @as(math.Log2Int(usize), @intCast(log2_old_align)),
798 bucketStackTrace(bucket, size_class, slot_index, .alloc),
799 free_stack_trace,
800 });
801 }
802 }
803 }
804 const prev_req_bytes = self.total_requested_bytes;
805 if (config.enable_memory_limit) {
806 const new_req_bytes = prev_req_bytes + new_size - old_mem.len;
807 if (new_req_bytes > prev_req_bytes and new_req_bytes > self.requested_memory_limit) {
808 return false;
809 }
810 self.total_requested_bytes = new_req_bytes;
811 }
812
813 const new_aligned_size = @max(new_size, @as(usize, 1) << log2_old_align);
814 const new_size_class = math.ceilPowerOfTwoAssert(usize, new_aligned_size);
815 if (new_size_class <= size_class) {
816 if (old_mem.len > new_size) {
817 @memset(old_mem[new_size..], undefined);
818 }
819 if (config.verbose_log) {
820 log.info("small resize {d} bytes at {*} to {d}", .{
821 old_mem.len, old_mem.ptr, new_size,
822 });
823 }
824 if (config.safety) {
825 bucket.requestedSizes(size_class)[slot_index] = @intCast(new_size);
826 }
827 return true;
828 }
829
830 if (config.enable_memory_limit) {
831 self.total_requested_bytes = prev_req_bytes;
832 }
833 return false;
834 }
835
836 fn free(
837 ctx: *anyopaque,
838 old_mem: []u8,
839 log2_old_align_u8: u8,
840 ret_addr: usize,
841 ) void {
842 const self: *Self = @ptrCast(@alignCast(ctx));
843 const log2_old_align = @as(Allocator.Log2Align, @intCast(log2_old_align_u8));
844 self.mutex.lock();
845 defer self.mutex.unlock();
846
847 assert(old_mem.len != 0);
848
849 const aligned_size = @max(old_mem.len, @as(usize, 1) << log2_old_align);
850 if (aligned_size > largest_bucket_object_size) {
851 self.freeLarge(old_mem, log2_old_align, ret_addr);
852 return;
853 }
854 const size_class_hint = math.ceilPowerOfTwoAssert(usize, aligned_size);
855
856 var bucket_index = math.log2(size_class_hint);
857 var size_class: usize = size_class_hint;
858 const bucket = while (bucket_index < small_bucket_count) : (bucket_index += 1) {
859 if (searchBucket(&self.buckets[bucket_index], @intFromPtr(old_mem.ptr), self.cur_buckets[bucket_index])) |bucket| {
860 break bucket;
861 }
862 size_class *= 2;
863 } else blk: {
864 if (config.retain_metadata) {
865 if (!self.large_allocations.contains(@intFromPtr(old_mem.ptr))) {
866 // object not in active buckets or a large allocation, so search empty buckets
867 if (searchBucket(&self.empty_buckets, @intFromPtr(old_mem.ptr), null)) |bucket| {
868 size_class = bucket.emptyBucketSizeClass();
869 // bucket is empty so is_used below will always be false and we exit there
870 break :blk bucket;
871 } else {
872 @panic("Invalid free");
873 }
874 }
875 }
876 self.freeLarge(old_mem, log2_old_align, ret_addr);
877 return;
878 };
879 const byte_offset = @intFromPtr(old_mem.ptr) - @intFromPtr(bucket.page);
880 const slot_index = @as(SlotIndex, @intCast(byte_offset / size_class));
881 const used_byte_index = slot_index / 8;
882 const used_bit_index = @as(u3, @intCast(slot_index % 8));
883 const used_byte = bucket.usedBits(used_byte_index);
884 const is_used = @as(u1, @truncate(used_byte.* >> used_bit_index)) != 0;
885 if (!is_used) {
886 if (config.safety) {
887 reportDoubleFree(ret_addr, bucketStackTrace(bucket, size_class, slot_index, .alloc), bucketStackTrace(bucket, size_class, slot_index, .free));
888 // Recoverable if this is a free.
889 return;
890 } else {
891 unreachable;
892 }
893 }
894
895 // Definitely an in-use small alloc now.
896 if (config.safety) {
897 const requested_size = bucket.requestedSizes(size_class)[slot_index];
898 if (requested_size == 0) @panic("Invalid free");
899 const log2_ptr_align = bucket.log2PtrAligns(size_class)[slot_index];
900 if (old_mem.len != requested_size or log2_old_align != log2_ptr_align) {
901 var addresses: [stack_n]usize = [1]usize{0} ** stack_n;
902 var free_stack_trace = StackTrace{
903 .instruction_addresses = &addresses,
904 .index = 0,
905 };
906 std.debug.captureStackTrace(ret_addr, &free_stack_trace);
907 if (old_mem.len != requested_size) {
908 log.err("Allocation size {d} bytes does not match free size {d}. Allocation: {} Free: {}", .{
909 requested_size,
910 old_mem.len,
911 bucketStackTrace(bucket, size_class, slot_index, .alloc),
912 free_stack_trace,
913 });
914 }
915 if (log2_old_align != log2_ptr_align) {
916 log.err("Allocation alignment {d} does not match free alignment {d}. Allocation: {} Free: {}", .{
917 @as(usize, 1) << @as(math.Log2Int(usize), @intCast(log2_ptr_align)),
918 @as(usize, 1) << @as(math.Log2Int(usize), @intCast(log2_old_align)),
919 bucketStackTrace(bucket, size_class, slot_index, .alloc),
920 free_stack_trace,
921 });
922 }
923 }
924 }
925
926 if (config.enable_memory_limit) {
927 self.total_requested_bytes -= old_mem.len;
928 }
929
930 // Capture stack trace to be the "first free", in case a double free happens.
931 bucket.captureStackTrace(ret_addr, size_class, slot_index, .free);
932
933 used_byte.* &= ~(@as(u8, 1) << used_bit_index);
934 bucket.used_count -= 1;
935 if (config.safety) {
936 bucket.requestedSizes(size_class)[slot_index] = 0;
937 }
938 if (bucket.used_count == 0) {
939 var entry = self.buckets[bucket_index].getEntryFor(bucket);
940 // save the node for destruction/insertion into in empty_buckets
941 const node = entry.node.?;
942 entry.set(null);
943 if (self.cur_buckets[bucket_index] == bucket) {
944 self.cur_buckets[bucket_index] = null;
945 }
946 if (!config.never_unmap) {
947 self.backing_allocator.free(bucket.page[0..page_size]);
948 }
949 if (!config.retain_metadata) {
950 self.freeBucket(bucket, size_class);
951 self.bucket_node_pool.destroy(node);
952 } else {
953 // move alloc_cursor to end so we can tell size_class later
954 const slot_count = @divExact(page_size, size_class);
955 bucket.alloc_cursor = @as(SlotIndex, @truncate(slot_count));
956 var empty_entry = self.empty_buckets.getEntryFor(node.key);
957 empty_entry.set(node);
958 }
959 } else {
960 @memset(old_mem, undefined);
961 }
962 if (config.verbose_log) {
963 log.info("small free {d} bytes at {*}", .{ old_mem.len, old_mem.ptr });
964 }
965 }
966
967 // Returns true if an allocation of `size` bytes is within the specified
968 // limits if enable_memory_limit is true
969 fn isAllocationAllowed(self: *Self, size: usize) bool {
970 if (config.enable_memory_limit) {
971 const new_req_bytes = self.total_requested_bytes + size;
972 if (new_req_bytes > self.requested_memory_limit)
973 return false;
974 self.total_requested_bytes = new_req_bytes;
975 }
976
977 return true;
978 }
979
980 fn alloc(ctx: *anyopaque, len: usize, log2_ptr_align: u8, ret_addr: usize) ?[*]u8 {
981 const self: *Self = @ptrCast(@alignCast(ctx));
982 self.mutex.lock();
983 defer self.mutex.unlock();
984 if (!self.isAllocationAllowed(len)) return null;
985 return allocInner(self, len, @as(Allocator.Log2Align, @intCast(log2_ptr_align)), ret_addr) catch return null;
986 }
987
988 fn allocInner(
989 self: *Self,
990 len: usize,
991 log2_ptr_align: Allocator.Log2Align,
992 ret_addr: usize,
993 ) Allocator.Error![*]u8 {
994 const new_aligned_size = @max(len, @as(usize, 1) << @as(Allocator.Log2Align, @intCast(log2_ptr_align)));
995 if (new_aligned_size > largest_bucket_object_size) {
996 try self.large_allocations.ensureUnusedCapacity(self.backing_allocator, 1);
997 const ptr = self.backing_allocator.rawAlloc(len, log2_ptr_align, ret_addr) orelse
998 return error.OutOfMemory;
999 const slice = ptr[0..len];
1000
1001 const gop = self.large_allocations.getOrPutAssumeCapacity(@intFromPtr(slice.ptr));
1002 if (config.retain_metadata and !config.never_unmap) {
1003 // Backing allocator may be reusing memory that we're retaining metadata for
1004 assert(!gop.found_existing or gop.value_ptr.freed);
1005 } else {
1006 assert(!gop.found_existing); // This would mean the kernel double-mapped pages.
1007 }
1008 gop.value_ptr.bytes = slice;
1009 if (config.enable_memory_limit)
1010 gop.value_ptr.requested_size = len;
1011 gop.value_ptr.captureStackTrace(ret_addr, .alloc);
1012 if (config.retain_metadata) {
1013 gop.value_ptr.freed = false;
1014 if (config.never_unmap) {
1015 gop.value_ptr.log2_ptr_align = log2_ptr_align;
1016 }
1017 }
1018
1019 if (config.verbose_log) {
1020 log.info("large alloc {d} bytes at {*}", .{ slice.len, slice.ptr });
1021 }
1022 return slice.ptr;
1023 }
1024
1025 const new_size_class = math.ceilPowerOfTwoAssert(usize, new_aligned_size);
1026 const slot = try self.allocSlot(new_size_class, ret_addr);
1027 if (config.safety) {
1028 slot.bucket.requestedSizes(new_size_class)[slot.slot_index] = @intCast(len);
1029 slot.bucket.log2PtrAligns(new_size_class)[slot.slot_index] = log2_ptr_align;
1030 }
1031 if (config.verbose_log) {
1032 log.info("small alloc {d} bytes at {*}", .{ len, slot.ptr });
1033 }
1034 return slot.ptr;
1035 }
1036
1037 fn createBucket(self: *Self, size_class: usize) Error!*BucketHeader {
1038 const page = try self.backing_allocator.alignedAlloc(u8, page_size, page_size);
1039 errdefer self.backing_allocator.free(page);
1040
1041 const bucket_size = bucketSize(size_class);
1042 const bucket_bytes = try self.backing_allocator.alignedAlloc(u8, @alignOf(BucketHeader), bucket_size);
1043 const ptr = @as(*BucketHeader, @ptrCast(bucket_bytes.ptr));
1044 ptr.* = BucketHeader{
1045 .page = page.ptr,
1046 .alloc_cursor = 0,
1047 .used_count = 0,
1048 };
1049 // Set the used bits to all zeroes
1050 @memset(@as([*]u8, @as(*[1]u8, ptr.usedBits(0)))[0..usedBitsCount(size_class)], 0);
1051 if (config.safety) {
1052 // Set the requested sizes to zeroes
1053 @memset(mem.sliceAsBytes(ptr.requestedSizes(size_class)), 0);
1054 }
1055 return ptr;
1056 }
1057 };
1058}
1059
1060const TraceKind = enum {
1061 alloc,
1062 free,
1063};
1064
1065const test_config = Config{};
1066
1067test "small allocations - free in same order" {
1068 var gpa = GeneralPurposeAllocator(test_config){};
1069 defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak");
1070 const allocator = gpa.allocator();
1071
1072 var list = std.ArrayList(*u64).init(std.testing.allocator);
1073 defer list.deinit();
1074
1075 var i: usize = 0;
1076 while (i < 513) : (i += 1) {
1077 const ptr = try allocator.create(u64);
1078 try list.append(ptr);
1079 }
1080
1081 for (list.items) |ptr| {
1082 allocator.destroy(ptr);
1083 }
1084}
1085
1086test "small allocations - free in reverse order" {
1087 var gpa = GeneralPurposeAllocator(test_config){};
1088 defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak");
1089 const allocator = gpa.allocator();
1090
1091 var list = std.ArrayList(*u64).init(std.testing.allocator);
1092 defer list.deinit();
1093
1094 var i: usize = 0;
1095 while (i < 513) : (i += 1) {
1096 const ptr = try allocator.create(u64);
1097 try list.append(ptr);
1098 }
1099
1100 while (list.popOrNull()) |ptr| {
1101 allocator.destroy(ptr);
1102 }
1103}
1104
1105test "large allocations" {
1106 var gpa = GeneralPurposeAllocator(test_config){};
1107 defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak");
1108 const allocator = gpa.allocator();
1109
1110 const ptr1 = try allocator.alloc(u64, 42768);
1111 const ptr2 = try allocator.alloc(u64, 52768);
1112 allocator.free(ptr1);
1113 const ptr3 = try allocator.alloc(u64, 62768);
1114 allocator.free(ptr3);
1115 allocator.free(ptr2);
1116}
1117
1118test "very large allocation" {
1119 var gpa = GeneralPurposeAllocator(test_config){};
1120 defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak");
1121 const allocator = gpa.allocator();
1122
1123 try std.testing.expectError(error.OutOfMemory, allocator.alloc(u8, math.maxInt(usize)));
1124}
1125
1126test "realloc" {
1127 var gpa = GeneralPurposeAllocator(test_config){};
1128 defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak");
1129 const allocator = gpa.allocator();
1130
1131 var slice = try allocator.alignedAlloc(u8, @alignOf(u32), 1);
1132 defer allocator.free(slice);
1133 slice[0] = 0x12;
1134
1135 // This reallocation should keep its pointer address.
1136 const old_slice = slice;
1137 slice = try allocator.realloc(slice, 2);
1138 try std.testing.expect(old_slice.ptr == slice.ptr);
1139 try std.testing.expect(slice[0] == 0x12);
1140 slice[1] = 0x34;
1141
1142 // This requires upgrading to a larger size class
1143 slice = try allocator.realloc(slice, 17);
1144 try std.testing.expect(slice[0] == 0x12);
1145 try std.testing.expect(slice[1] == 0x34);
1146}
1147
1148test "shrink" {
1149 var gpa = GeneralPurposeAllocator(test_config){};
1150 defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak");
1151 const allocator = gpa.allocator();
1152
1153 var slice = try allocator.alloc(u8, 20);
1154 defer allocator.free(slice);
1155
1156 @memset(slice, 0x11);
1157
1158 try std.testing.expect(allocator.resize(slice, 17));
1159 slice = slice[0..17];
1160
1161 for (slice) |b| {
1162 try std.testing.expect(b == 0x11);
1163 }
1164
1165 try std.testing.expect(allocator.resize(slice, 16));
1166 slice = slice[0..16];
1167
1168 for (slice) |b| {
1169 try std.testing.expect(b == 0x11);
1170 }
1171}
1172
1173test "large object - grow" {
1174 if (builtin.target.isWasm()) {
1175 // Not expected to pass on targets that do not have memory mapping.
1176 return error.SkipZigTest;
1177 }
1178 var gpa: GeneralPurposeAllocator(test_config) = .{};
1179 defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak");
1180 const allocator = gpa.allocator();
1181
1182 var slice1 = try allocator.alloc(u8, page_size * 2 - 20);
1183 defer allocator.free(slice1);
1184
1185 const old = slice1;
1186 slice1 = try allocator.realloc(slice1, page_size * 2 - 10);
1187 try std.testing.expect(slice1.ptr == old.ptr);
1188
1189 slice1 = try allocator.realloc(slice1, page_size * 2);
1190 try std.testing.expect(slice1.ptr == old.ptr);
1191
1192 slice1 = try allocator.realloc(slice1, page_size * 2 + 1);
1193}
1194
1195test "realloc small object to large object" {
1196 var gpa = GeneralPurposeAllocator(test_config){};
1197 defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak");
1198 const allocator = gpa.allocator();
1199
1200 var slice = try allocator.alloc(u8, 70);
1201 defer allocator.free(slice);
1202 slice[0] = 0x12;
1203 slice[60] = 0x34;
1204
1205 // This requires upgrading to a large object
1206 const large_object_size = page_size * 2 + 50;
1207 slice = try allocator.realloc(slice, large_object_size);
1208 try std.testing.expect(slice[0] == 0x12);
1209 try std.testing.expect(slice[60] == 0x34);
1210}
1211
1212test "shrink large object to large object" {
1213 var gpa = GeneralPurposeAllocator(test_config){};
1214 defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak");
1215 const allocator = gpa.allocator();
1216
1217 var slice = try allocator.alloc(u8, page_size * 2 + 50);
1218 defer allocator.free(slice);
1219 slice[0] = 0x12;
1220 slice[60] = 0x34;
1221
1222 if (!allocator.resize(slice, page_size * 2 + 1)) return;
1223 slice = slice.ptr[0 .. page_size * 2 + 1];
1224 try std.testing.expect(slice[0] == 0x12);
1225 try std.testing.expect(slice[60] == 0x34);
1226
1227 try std.testing.expect(allocator.resize(slice, page_size * 2 + 1));
1228 slice = slice[0 .. page_size * 2 + 1];
1229 try std.testing.expect(slice[0] == 0x12);
1230 try std.testing.expect(slice[60] == 0x34);
1231
1232 slice = try allocator.realloc(slice, page_size * 2);
1233 try std.testing.expect(slice[0] == 0x12);
1234 try std.testing.expect(slice[60] == 0x34);
1235}
1236
1237test "shrink large object to large object with larger alignment" {
1238 if (!builtin.link_libc and builtin.os.tag == .wasi) return error.SkipZigTest; // https://github.com/ziglang/zig/issues/22731
1239
1240 var gpa = GeneralPurposeAllocator(test_config){};
1241 defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak");
1242 const allocator = gpa.allocator();
1243
1244 var debug_buffer: [1000]u8 = undefined;
1245 var fba = std.heap.FixedBufferAllocator.init(&debug_buffer);
1246 const debug_allocator = fba.allocator();
1247
1248 const alloc_size = page_size * 2 + 50;
1249 var slice = try allocator.alignedAlloc(u8, 16, alloc_size);
1250 defer allocator.free(slice);
1251
1252 const big_alignment: usize = switch (builtin.os.tag) {
1253 .windows => page_size * 32, // Windows aligns to 64K.
1254 else => page_size * 2,
1255 };
1256 // This loop allocates until we find a page that is not aligned to the big
1257 // alignment. Then we shrink the allocation after the loop, but increase the
1258 // alignment to the higher one, that we know will force it to realloc.
1259 var stuff_to_free = std.ArrayList([]align(16) u8).init(debug_allocator);
1260 while (mem.isAligned(@intFromPtr(slice.ptr), big_alignment)) {
1261 try stuff_to_free.append(slice);
1262 slice = try allocator.alignedAlloc(u8, 16, alloc_size);
1263 }
1264 while (stuff_to_free.popOrNull()) |item| {
1265 allocator.free(item);
1266 }
1267 slice[0] = 0x12;
1268 slice[60] = 0x34;
1269
1270 slice = try allocator.reallocAdvanced(slice, big_alignment, alloc_size / 2);
1271 try std.testing.expect(slice[0] == 0x12);
1272 try std.testing.expect(slice[60] == 0x34);
1273}
1274
1275test "realloc large object to small object" {
1276 var gpa = GeneralPurposeAllocator(test_config){};
1277 defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak");
1278 const allocator = gpa.allocator();
1279
1280 var slice = try allocator.alloc(u8, page_size * 2 + 50);
1281 defer allocator.free(slice);
1282 slice[0] = 0x12;
1283 slice[16] = 0x34;
1284
1285 slice = try allocator.realloc(slice, 19);
1286 try std.testing.expect(slice[0] == 0x12);
1287 try std.testing.expect(slice[16] == 0x34);
1288}
1289
1290test "overridable mutexes" {
1291 var gpa = GeneralPurposeAllocator(.{ .MutexType = std.Thread.Mutex }){
1292 .backing_allocator = std.testing.allocator,
1293 .mutex = std.Thread.Mutex{},
1294 };
1295 defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak");
1296 const allocator = gpa.allocator();
1297
1298 const ptr = try allocator.create(i32);
1299 defer allocator.destroy(ptr);
1300}
1301
1302test "non-page-allocator backing allocator" {
1303 var gpa = GeneralPurposeAllocator(.{}){ .backing_allocator = std.testing.allocator };
1304 defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak");
1305 const allocator = gpa.allocator();
1306
1307 const ptr = try allocator.create(i32);
1308 defer allocator.destroy(ptr);
1309}
1310
1311test "realloc large object to larger alignment" {
1312 if (!builtin.link_libc and builtin.os.tag == .wasi) return error.SkipZigTest; // https://github.com/ziglang/zig/issues/22731
1313
1314 var gpa = GeneralPurposeAllocator(test_config){};
1315 defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak");
1316 const allocator = gpa.allocator();
1317
1318 var debug_buffer: [1000]u8 = undefined;
1319 var fba = std.heap.FixedBufferAllocator.init(&debug_buffer);
1320 const debug_allocator = fba.allocator();
1321
1322 var slice = try allocator.alignedAlloc(u8, 16, page_size * 2 + 50);
1323 defer allocator.free(slice);
1324
1325 const big_alignment: usize = switch (builtin.os.tag) {
1326 .windows => page_size * 32, // Windows aligns to 64K.
1327 else => page_size * 2,
1328 };
1329 // This loop allocates until we find a page that is not aligned to the big alignment.
1330 var stuff_to_free = std.ArrayList([]align(16) u8).init(debug_allocator);
1331 while (mem.isAligned(@intFromPtr(slice.ptr), big_alignment)) {
1332 try stuff_to_free.append(slice);
1333 slice = try allocator.alignedAlloc(u8, 16, page_size * 2 + 50);
1334 }
1335 while (stuff_to_free.popOrNull()) |item| {
1336 allocator.free(item);
1337 }
1338 slice[0] = 0x12;
1339 slice[16] = 0x34;
1340
1341 slice = try allocator.reallocAdvanced(slice, 32, page_size * 2 + 100);
1342 try std.testing.expect(slice[0] == 0x12);
1343 try std.testing.expect(slice[16] == 0x34);
1344
1345 slice = try allocator.reallocAdvanced(slice, 32, page_size * 2 + 25);
1346 try std.testing.expect(slice[0] == 0x12);
1347 try std.testing.expect(slice[16] == 0x34);
1348
1349 slice = try allocator.reallocAdvanced(slice, big_alignment, page_size * 2 + 100);
1350 try std.testing.expect(slice[0] == 0x12);
1351 try std.testing.expect(slice[16] == 0x34);
1352}
1353
1354test "large object shrinks to small but allocation fails during shrink" {
1355 if (builtin.target.isWasm()) {
1356 // Not expected to pass on targets that do not have memory mapping.
1357 return error.SkipZigTest;
1358 }
1359
1360 var failing_allocator = std.testing.FailingAllocator.init(std.heap.page_allocator, .{ .fail_index = 3 });
1361 var gpa = GeneralPurposeAllocator(.{}){ .backing_allocator = failing_allocator.allocator() };
1362 defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak");
1363 const allocator = gpa.allocator();
1364
1365 var slice = try allocator.alloc(u8, page_size * 2 + 50);
1366 defer allocator.free(slice);
1367 slice[0] = 0x12;
1368 slice[3] = 0x34;
1369
1370 // Next allocation will fail in the backing allocator of the GeneralPurposeAllocator
1371
1372 try std.testing.expect(allocator.resize(slice, 4));
1373 slice = slice[0..4];
1374 try std.testing.expect(slice[0] == 0x12);
1375 try std.testing.expect(slice[3] == 0x34);
1376}
1377
1378test "objects of size 1024 and 2048" {
1379 var gpa = GeneralPurposeAllocator(test_config){};
1380 defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak");
1381 const allocator = gpa.allocator();
1382
1383 const slice = try allocator.alloc(u8, 1025);
1384 const slice2 = try allocator.alloc(u8, 3000);
1385
1386 allocator.free(slice);
1387 allocator.free(slice2);
1388}
1389
1390test "setting a memory cap" {
1391 var gpa = GeneralPurposeAllocator(.{ .enable_memory_limit = true }){};
1392 defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak");
1393 const allocator = gpa.allocator();
1394
1395 gpa.setRequestedMemoryLimit(1010);
1396
1397 const small = try allocator.create(i32);
1398 try std.testing.expect(gpa.total_requested_bytes == 4);
1399
1400 const big = try allocator.alloc(u8, 1000);
1401 try std.testing.expect(gpa.total_requested_bytes == 1004);
1402
1403 try std.testing.expectError(error.OutOfMemory, allocator.create(u64));
1404
1405 allocator.destroy(small);
1406 try std.testing.expect(gpa.total_requested_bytes == 1000);
1407
1408 allocator.free(big);
1409 try std.testing.expect(gpa.total_requested_bytes == 0);
1410
1411 const exact = try allocator.alloc(u8, 1010);
1412 try std.testing.expect(gpa.total_requested_bytes == 1010);
1413 allocator.free(exact);
1414}
1415
1416test "double frees" {
1417 // use a GPA to back a GPA to check for leaks of the latter's metadata
1418 var backing_gpa = GeneralPurposeAllocator(.{ .safety = true }){};
1419 defer std.testing.expect(backing_gpa.deinit() == .ok) catch @panic("leak");
1420
1421 const GPA = GeneralPurposeAllocator(.{ .safety = true, .never_unmap = true, .retain_metadata = true });
1422 var gpa = GPA{ .backing_allocator = backing_gpa.allocator() };
1423 defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak");
1424 const allocator = gpa.allocator();
1425
1426 // detect a small allocation double free, even though bucket is emptied
1427 const index: usize = 6;
1428 const size_class: usize = @as(usize, 1) << 6;
1429 const small = try allocator.alloc(u8, size_class);
1430 try std.testing.expect(GPA.searchBucket(&gpa.buckets[index], @intFromPtr(small.ptr), gpa.cur_buckets[index]) != null);
1431 allocator.free(small);
1432 try std.testing.expect(GPA.searchBucket(&gpa.buckets[index], @intFromPtr(small.ptr), gpa.cur_buckets[index]) == null);
1433 try std.testing.expect(GPA.searchBucket(&gpa.empty_buckets, @intFromPtr(small.ptr), null) != null);
1434
1435 // detect a large allocation double free
1436 const large = try allocator.alloc(u8, 2 * page_size);
1437 try std.testing.expect(gpa.large_allocations.contains(@intFromPtr(large.ptr)));
1438 try std.testing.expectEqual(gpa.large_allocations.getEntry(@intFromPtr(large.ptr)).?.value_ptr.bytes, large);
1439 allocator.free(large);
1440 try std.testing.expect(gpa.large_allocations.contains(@intFromPtr(large.ptr)));
1441 try std.testing.expect(gpa.large_allocations.getEntry(@intFromPtr(large.ptr)).?.value_ptr.freed);
1442
1443 const normal_small = try allocator.alloc(u8, size_class);
1444 defer allocator.free(normal_small);
1445 const normal_large = try allocator.alloc(u8, 2 * page_size);
1446 defer allocator.free(normal_large);
1447
1448 // check that flushing retained metadata doesn't disturb live allocations
1449 gpa.flushRetainedMetadata();
1450 try std.testing.expect(gpa.empty_buckets.root == null);
1451 try std.testing.expect(GPA.searchBucket(&gpa.buckets[index], @intFromPtr(normal_small.ptr), gpa.cur_buckets[index]) != null);
1452 try std.testing.expect(gpa.large_allocations.contains(@intFromPtr(normal_large.ptr)));
1453 try std.testing.expect(!gpa.large_allocations.contains(@intFromPtr(large.ptr)));
1454}
1455
1456test "empty bucket size class" {
1457 const GPA = GeneralPurposeAllocator(.{ .safety = true, .never_unmap = true, .retain_metadata = true });
1458 var gpa = GPA{};
1459 defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak");
1460 const allocator = gpa.allocator();
1461
1462 // allocate and free to create an empty bucket
1463 const size_class: usize = @as(usize, 1) << 6;
1464 const small = try allocator.alloc(u8, size_class);
1465 allocator.free(small);
1466
1467 // the metadata tracking system relies on alloc_cursor of empty buckets
1468 // being set to the slot count so that we can get back the size class.
1469 const empty_bucket = GPA.searchBucket(&gpa.empty_buckets, @intFromPtr(small.ptr), null).?;
1470 try std.testing.expect(empty_bucket.emptyBucketSizeClass() == size_class);
1471}
1472
1473test "bug 9995 fix, large allocs count requested size not backing size" {
1474 // with AtLeast, buffer likely to be larger than requested, especially when shrinking
1475 var gpa = GeneralPurposeAllocator(.{ .enable_memory_limit = true }){};
1476 const allocator = gpa.allocator();
1477
1478 var buf = try allocator.alignedAlloc(u8, 1, page_size + 1);
1479 try std.testing.expect(gpa.total_requested_bytes == page_size + 1);
1480 buf = try allocator.realloc(buf, 1);
1481 try std.testing.expect(gpa.total_requested_bytes == 1);
1482 buf = try allocator.realloc(buf, 2);
1483 try std.testing.expect(gpa.total_requested_bytes == 2);
1484}
1485
1486test "retain metadata and never unmap" {
1487 var gpa = std.heap.GeneralPurposeAllocator(.{
1488 .safety = true,
1489 .never_unmap = true,
1490 .retain_metadata = true,
1491 }){};
1492 defer std.debug.assert(gpa.deinit() == .ok);
1493 const allocator = gpa.allocator();
1494
1495 const alloc = try allocator.alloc(u8, 8);
1496 allocator.free(alloc);
1497
1498 const alloc2 = try allocator.alloc(u8, 8);
1499 allocator.free(alloc2);
1500}
lib/std/heap/log_to_writer_allocator.zig deleted-118
...@@ -1,118 +0,0 @@
1const std = @import("../std.zig");
2const Allocator = std.mem.Allocator;
3
4/// This allocator is used in front of another allocator and logs to the provided writer
5/// on every call to the allocator. Writer errors are ignored.
6pub fn LogToWriterAllocator(comptime Writer: type) type {
7 return struct {
8 parent_allocator: Allocator,
9 writer: Writer,
10
11 const Self = @This();
12
13 pub fn init(parent_allocator: Allocator, writer: Writer) Self {
14 return Self{
15 .parent_allocator = parent_allocator,
16 .writer = writer,
17 };
18 }
19
20 pub fn allocator(self: *Self) Allocator {
21 return .{
22 .ptr = self,
23 .vtable = &.{
24 .alloc = alloc,
25 .resize = resize,
26 .free = free,
27 },
28 };
29 }
30
31 fn alloc(
32 ctx: *anyopaque,
33 len: usize,
34 log2_ptr_align: u8,
35 ra: usize,
36 ) ?[*]u8 {
37 const self: *Self = @ptrCast(@alignCast(ctx));
38 self.writer.print("alloc : {}", .{len}) catch {};
39 const result = self.parent_allocator.rawAlloc(len, log2_ptr_align, ra);
40 if (result != null) {
41 self.writer.print(" success!\n", .{}) catch {};
42 } else {
43 self.writer.print(" failure!\n", .{}) catch {};
44 }
45 return result;
46 }
47
48 fn resize(
49 ctx: *anyopaque,
50 buf: []u8,
51 log2_buf_align: u8,
52 new_len: usize,
53 ra: usize,
54 ) bool {
55 const self: *Self = @ptrCast(@alignCast(ctx));
56 if (new_len <= buf.len) {
57 self.writer.print("shrink: {} to {}\n", .{ buf.len, new_len }) catch {};
58 } else {
59 self.writer.print("expand: {} to {}", .{ buf.len, new_len }) catch {};
60 }
61
62 if (self.parent_allocator.rawResize(buf, log2_buf_align, new_len, ra)) {
63 if (new_len > buf.len) {
64 self.writer.print(" success!\n", .{}) catch {};
65 }
66 return true;
67 }
68
69 std.debug.assert(new_len > buf.len);
70 self.writer.print(" failure!\n", .{}) catch {};
71 return false;
72 }
73
74 fn free(
75 ctx: *anyopaque,
76 buf: []u8,
77 log2_buf_align: u8,
78 ra: usize,
79 ) void {
80 const self: *Self = @ptrCast(@alignCast(ctx));
81 self.writer.print("free : {}\n", .{buf.len}) catch {};
82 self.parent_allocator.rawFree(buf, log2_buf_align, ra);
83 }
84 };
85}
86
87/// This allocator is used in front of another allocator and logs to the provided writer
88/// on every call to the allocator. Writer errors are ignored.
89pub fn logToWriterAllocator(
90 parent_allocator: Allocator,
91 writer: anytype,
92) LogToWriterAllocator(@TypeOf(writer)) {
93 return LogToWriterAllocator(@TypeOf(writer)).init(parent_allocator, writer);
94}
95
96test "LogToWriterAllocator" {
97 var log_buf: [255]u8 = undefined;
98 var fbs = std.io.fixedBufferStream(&log_buf);
99
100 var allocator_buf: [10]u8 = undefined;
101 var fixedBufferAllocator = std.mem.validationWrap(std.heap.FixedBufferAllocator.init(&allocator_buf));
102 var allocator_state = logToWriterAllocator(fixedBufferAllocator.allocator(), fbs.writer());
103 const allocator = allocator_state.allocator();
104
105 var a = try allocator.alloc(u8, 10);
106 try std.testing.expect(allocator.resize(a, 5));
107 a = a[0..5];
108 try std.testing.expect(!allocator.resize(a, 20));
109 allocator.free(a);
110
111 try std.testing.expectEqualSlices(u8,
112 \\alloc : 10 success!
113 \\shrink: 10 to 5
114 \\expand: 5 to 20 failure!
115 \\free : 5
116 \\
117 , fbs.getWritten());
118}
lib/std/heap/logging_allocator.zig deleted-133
...@@ -1,133 +0,0 @@
1const std = @import("../std.zig");
2const Allocator = std.mem.Allocator;
3
4/// This allocator is used in front of another allocator and logs to `std.log`
5/// on every call to the allocator.
6/// For logging to a `std.io.Writer` see `std.heap.LogToWriterAllocator`
7pub fn LoggingAllocator(
8 comptime success_log_level: std.log.Level,
9 comptime failure_log_level: std.log.Level,
10) type {
11 return ScopedLoggingAllocator(.default, success_log_level, failure_log_level);
12}
13
14/// This allocator is used in front of another allocator and logs to `std.log`
15/// with the given scope on every call to the allocator.
16/// For logging to a `std.io.Writer` see `std.heap.LogToWriterAllocator`
17pub fn ScopedLoggingAllocator(
18 comptime scope: @Type(.enum_literal),
19 comptime success_log_level: std.log.Level,
20 comptime failure_log_level: std.log.Level,
21) type {
22 const log = std.log.scoped(scope);
23
24 return struct {
25 parent_allocator: Allocator,
26
27 const Self = @This();
28
29 pub fn init(parent_allocator: Allocator) Self {
30 return .{
31 .parent_allocator = parent_allocator,
32 };
33 }
34
35 pub fn allocator(self: *Self) Allocator {
36 return .{
37 .ptr = self,
38 .vtable = &.{
39 .alloc = alloc,
40 .resize = resize,
41 .free = free,
42 },
43 };
44 }
45
46 // This function is required as the `std.log.log` function is not public
47 inline fn logHelper(comptime log_level: std.log.Level, comptime format: []const u8, args: anytype) void {
48 switch (log_level) {
49 .err => log.err(format, args),
50 .warn => log.warn(format, args),
51 .info => log.info(format, args),
52 .debug => log.debug(format, args),
53 }
54 }
55
56 fn alloc(
57 ctx: *anyopaque,
58 len: usize,
59 log2_ptr_align: u8,
60 ra: usize,
61 ) ?[*]u8 {
62 const self: *Self = @ptrCast(@alignCast(ctx));
63 const result = self.parent_allocator.rawAlloc(len, log2_ptr_align, ra);
64 if (result != null) {
65 logHelper(
66 success_log_level,
67 "alloc - success - len: {}, ptr_align: {}",
68 .{ len, log2_ptr_align },
69 );
70 } else {
71 logHelper(
72 failure_log_level,
73 "alloc - failure: OutOfMemory - len: {}, ptr_align: {}",
74 .{ len, log2_ptr_align },
75 );
76 }
77 return result;
78 }
79
80 fn resize(
81 ctx: *anyopaque,
82 buf: []u8,
83 log2_buf_align: u8,
84 new_len: usize,
85 ra: usize,
86 ) bool {
87 const self: *Self = @ptrCast(@alignCast(ctx));
88 if (self.parent_allocator.rawResize(buf, log2_buf_align, new_len, ra)) {
89 if (new_len <= buf.len) {
90 logHelper(
91 success_log_level,
92 "shrink - success - {} to {}, buf_align: {}",
93 .{ buf.len, new_len, log2_buf_align },
94 );
95 } else {
96 logHelper(
97 success_log_level,
98 "expand - success - {} to {}, buf_align: {}",
99 .{ buf.len, new_len, log2_buf_align },
100 );
101 }
102
103 return true;
104 }
105
106 std.debug.assert(new_len > buf.len);
107 logHelper(
108 failure_log_level,
109 "expand - failure - {} to {}, buf_align: {}",
110 .{ buf.len, new_len, log2_buf_align },
111 );
112 return false;
113 }
114
115 fn free(
116 ctx: *anyopaque,
117 buf: []u8,
118 log2_buf_align: u8,
119 ra: usize,
120 ) void {
121 const self: *Self = @ptrCast(@alignCast(ctx));
122 self.parent_allocator.rawFree(buf, log2_buf_align, ra);
123 logHelper(success_log_level, "free - len: {}", .{buf.len});
124 }
125 };
126}
127
128/// This allocator is used in front of another allocator and logs to `std.log`
129/// on every call to the allocator.
130/// For logging to a `std.io.Writer` see `std.heap.LogToWriterAllocator`
131pub fn loggingAllocator(parent_allocator: Allocator) LoggingAllocator(.debug, .err) {
132 return LoggingAllocator(.debug, .err).init(parent_allocator);
133}
lib/std/heap/sbrk_allocator.zig+4-3
...@@ -3,6 +3,7 @@ const builtin = @import("builtin");...@@ -3,6 +3,7 @@ const builtin = @import("builtin");
3const math = std.math;3const math = std.math;
4const Allocator = std.mem.Allocator;4const Allocator = std.mem.Allocator;
5const mem = std.mem;5const mem = std.mem;
6const heap = std.heap;
6const assert = std.debug.assert;7const assert = std.debug.assert;
78
8pub fn SbrkAllocator(comptime sbrk: *const fn (n: usize) usize) type {9pub fn SbrkAllocator(comptime sbrk: *const fn (n: usize) usize) type {
...@@ -18,7 +19,7 @@ pub fn SbrkAllocator(comptime sbrk: *const fn (n: usize) usize) type {...@@ -18,7 +19,7 @@ pub fn SbrkAllocator(comptime sbrk: *const fn (n: usize) usize) type {
18 const max_usize = math.maxInt(usize);19 const max_usize = math.maxInt(usize);
19 const ushift = math.Log2Int(usize);20 const ushift = math.Log2Int(usize);
20 const bigpage_size = 64 * 1024;21 const bigpage_size = 64 * 1024;
21 const pages_per_bigpage = bigpage_size / mem.page_size;22 const pages_per_bigpage = bigpage_size / heap.pageSize();
22 const bigpage_count = max_usize / bigpage_size;23 const bigpage_count = max_usize / bigpage_size;
2324
24 /// Because of storing free list pointers, the minimum size class is 3.25 /// Because of storing free list pointers, the minimum size class is 3.
...@@ -58,7 +59,7 @@ pub fn SbrkAllocator(comptime sbrk: *const fn (n: usize) usize) type {...@@ -58,7 +59,7 @@ pub fn SbrkAllocator(comptime sbrk: *const fn (n: usize) usize) type {
58 }59 }
5960
60 const next_addr = next_addrs[class];61 const next_addr = next_addrs[class];
61 if (next_addr % mem.page_size == 0) {62 if (next_addr % heap.pageSize() == 0) {
62 const addr = allocBigPages(1);63 const addr = allocBigPages(1);
63 if (addr == 0) return null;64 if (addr == 0) return null;
64 //std.debug.print("allocated fresh slot_size={d} class={d} addr=0x{x}\n", .{65 //std.debug.print("allocated fresh slot_size={d} class={d} addr=0x{x}\n", .{
...@@ -153,7 +154,7 @@ pub fn SbrkAllocator(comptime sbrk: *const fn (n: usize) usize) type {...@@ -153,7 +154,7 @@ pub fn SbrkAllocator(comptime sbrk: *const fn (n: usize) usize) type {
153 big_frees[class] = node.*;154 big_frees[class] = node.*;
154 return top_free_ptr;155 return top_free_ptr;
155 }156 }
156 return sbrk(pow2_pages * pages_per_bigpage * mem.page_size);157 return sbrk(pow2_pages * pages_per_bigpage * heap.pageSize());
157 }158 }
158 };159 };
159}160}
lib/std/mem.zig+89-56
...@@ -8,26 +8,6 @@ const testing = std.testing;...@@ -8,26 +8,6 @@ const testing = std.testing;
8const Endian = std.builtin.Endian;8const Endian = std.builtin.Endian;
9const native_endian = builtin.cpu.arch.endian();9const native_endian = builtin.cpu.arch.endian();
1010
11/// Compile time known minimum page size.
12/// https://github.com/ziglang/zig/issues/4082
13pub const page_size = switch (builtin.cpu.arch) {
14 .wasm32, .wasm64 => 64 * 1024,
15 .aarch64 => switch (builtin.os.tag) {
16 .macos, .ios, .watchos, .tvos, .visionos => 16 * 1024,
17 else => 4 * 1024,
18 },
19 .sparc64 => 8 * 1024,
20 .loongarch32, .loongarch64 => switch (builtin.os.tag) {
21 // Linux default KConfig value is 16KiB
22 .linux => 16 * 1024,
23 // FIXME:
24 // There is no other OS supported yet. Use the same value
25 // as Linux for now.
26 else => 16 * 1024,
27 },
28 else => 4 * 1024,
29};
30
31/// The standard library currently thoroughly depends on byte size11/// The standard library currently thoroughly depends on byte size
32/// being 8 bits. (see the use of u8 throughout allocation code as12/// being 8 bits. (see the use of u8 throughout allocation code as
33/// the "byte" type.) Code which depends on this can reference this13/// the "byte" type.) Code which depends on this can reference this
...@@ -38,6 +18,60 @@ pub const byte_size_in_bits = 8;...@@ -38,6 +18,60 @@ pub const byte_size_in_bits = 8;
3818
39pub const Allocator = @import("mem/Allocator.zig");19pub const Allocator = @import("mem/Allocator.zig");
4020
21/// Stored as a power-of-two.
22pub const Alignment = enum(math.Log2Int(usize)) {
23 @"1" = 0,
24 @"2" = 1,
25 @"4" = 2,
26 @"8" = 3,
27 @"16" = 4,
28 @"32" = 5,
29 @"64" = 6,
30 _,
31
32 pub fn toByteUnits(a: Alignment) usize {
33 return @as(usize, 1) << @intFromEnum(a);
34 }
35
36 pub fn fromByteUnits(n: usize) Alignment {
37 assert(std.math.isPowerOfTwo(n));
38 return @enumFromInt(@ctz(n));
39 }
40
41 pub fn order(lhs: Alignment, rhs: Alignment) std.math.Order {
42 return std.math.order(@intFromEnum(lhs), @intFromEnum(rhs));
43 }
44
45 pub fn compare(lhs: Alignment, op: std.math.CompareOperator, rhs: Alignment) bool {
46 return std.math.compare(@intFromEnum(lhs), op, @intFromEnum(rhs));
47 }
48
49 pub fn max(lhs: Alignment, rhs: Alignment) Alignment {
50 return @enumFromInt(@max(@intFromEnum(lhs), @intFromEnum(rhs)));
51 }
52
53 pub fn min(lhs: Alignment, rhs: Alignment) Alignment {
54 return @enumFromInt(@min(@intFromEnum(lhs), @intFromEnum(rhs)));
55 }
56
57 /// Return next address with this alignment.
58 pub fn forward(a: Alignment, address: usize) usize {
59 const x = (@as(usize, 1) << @intFromEnum(a)) - 1;
60 return (address + x) & ~x;
61 }
62
63 /// Return previous address with this alignment.
64 pub fn backward(a: Alignment, address: usize) usize {
65 const x = (@as(usize, 1) << @intFromEnum(a)) - 1;
66 return address & ~x;
67 }
68
69 /// Return whether address is aligned to this amount.
70 pub fn check(a: Alignment, address: usize) bool {
71 return @ctz(address) >= @intFromEnum(a);
72 }
73};
74
41/// Detects and asserts if the std.mem.Allocator interface is violated by the caller75/// Detects and asserts if the std.mem.Allocator interface is violated by the caller
42/// or the allocator.76/// or the allocator.
43pub fn ValidationAllocator(comptime T: type) type {77pub fn ValidationAllocator(comptime T: type) type {
...@@ -58,6 +92,7 @@ pub fn ValidationAllocator(comptime T: type) type {...@@ -58,6 +92,7 @@ pub fn ValidationAllocator(comptime T: type) type {
58 .vtable = &.{92 .vtable = &.{
59 .alloc = alloc,93 .alloc = alloc,
60 .resize = resize,94 .resize = resize,
95 .remap = remap,
61 .free = free,96 .free = free,
62 },97 },
63 };98 };
...@@ -71,41 +106,54 @@ pub fn ValidationAllocator(comptime T: type) type {...@@ -71,41 +106,54 @@ pub fn ValidationAllocator(comptime T: type) type {
71 pub fn alloc(106 pub fn alloc(
72 ctx: *anyopaque,107 ctx: *anyopaque,
73 n: usize,108 n: usize,
74 log2_ptr_align: u8,109 alignment: mem.Alignment,
75 ret_addr: usize,110 ret_addr: usize,
76 ) ?[*]u8 {111 ) ?[*]u8 {
77 assert(n > 0);112 assert(n > 0);
78 const self: *Self = @ptrCast(@alignCast(ctx));113 const self: *Self = @ptrCast(@alignCast(ctx));
79 const underlying = self.getUnderlyingAllocatorPtr();114 const underlying = self.getUnderlyingAllocatorPtr();
80 const result = underlying.rawAlloc(n, log2_ptr_align, ret_addr) orelse115 const result = underlying.rawAlloc(n, alignment, ret_addr) orelse
81 return null;116 return null;
82 assert(mem.isAlignedLog2(@intFromPtr(result), log2_ptr_align));117 assert(alignment.check(@intFromPtr(result)));
83 return result;118 return result;
84 }119 }
85120
86 pub fn resize(121 pub fn resize(
87 ctx: *anyopaque,122 ctx: *anyopaque,
88 buf: []u8,123 buf: []u8,
89 log2_buf_align: u8,124 alignment: Alignment,
90 new_len: usize,125 new_len: usize,
91 ret_addr: usize,126 ret_addr: usize,
92 ) bool {127 ) bool {
93 const self: *Self = @ptrCast(@alignCast(ctx));128 const self: *Self = @ptrCast(@alignCast(ctx));
94 assert(buf.len > 0);129 assert(buf.len > 0);
95 const underlying = self.getUnderlyingAllocatorPtr();130 const underlying = self.getUnderlyingAllocatorPtr();
96 return underlying.rawResize(buf, log2_buf_align, new_len, ret_addr);131 return underlying.rawResize(buf, alignment, new_len, ret_addr);
132 }
133
134 pub fn remap(
135 ctx: *anyopaque,
136 buf: []u8,
137 alignment: Alignment,
138 new_len: usize,
139 ret_addr: usize,
140 ) ?[*]u8 {
141 const self: *Self = @ptrCast(@alignCast(ctx));
142 assert(buf.len > 0);
143 const underlying = self.getUnderlyingAllocatorPtr();
144 return underlying.rawRemap(buf, alignment, new_len, ret_addr);
97 }145 }
98146
99 pub fn free(147 pub fn free(
100 ctx: *anyopaque,148 ctx: *anyopaque,
101 buf: []u8,149 buf: []u8,
102 log2_buf_align: u8,150 alignment: Alignment,
103 ret_addr: usize,151 ret_addr: usize,
104 ) void {152 ) void {
105 const self: *Self = @ptrCast(@alignCast(ctx));153 const self: *Self = @ptrCast(@alignCast(ctx));
106 assert(buf.len > 0);154 assert(buf.len > 0);
107 const underlying = self.getUnderlyingAllocatorPtr();155 const underlying = self.getUnderlyingAllocatorPtr();
108 underlying.rawFree(buf, log2_buf_align, ret_addr);156 underlying.rawFree(buf, alignment, ret_addr);
109 }157 }
110158
111 pub fn reset(self: *Self) void {159 pub fn reset(self: *Self) void {
...@@ -133,27 +181,9 @@ pub fn alignAllocLen(full_len: usize, alloc_len: usize, len_align: u29) usize {...@@ -133,27 +181,9 @@ pub fn alignAllocLen(full_len: usize, alloc_len: usize, len_align: u29) usize {
133 return adjusted;181 return adjusted;
134}182}
135183
136const fail_allocator = Allocator{
137 .ptr = undefined,
138 .vtable = &failAllocator_vtable,
139};
140
141const failAllocator_vtable = Allocator.VTable{
142 .alloc = failAllocatorAlloc,
143 .resize = Allocator.noResize,
144 .free = Allocator.noFree,
145};
146
147fn failAllocatorAlloc(_: *anyopaque, n: usize, log2_alignment: u8, ra: usize) ?[*]u8 {
148 _ = n;
149 _ = log2_alignment;
150 _ = ra;
151 return null;
152}
153
154test "Allocator basics" {184test "Allocator basics" {
155 try testing.expectError(error.OutOfMemory, fail_allocator.alloc(u8, 1));185 try testing.expectError(error.OutOfMemory, testing.failing_allocator.alloc(u8, 1));
156 try testing.expectError(error.OutOfMemory, fail_allocator.allocSentinel(u8, 1, 0));186 try testing.expectError(error.OutOfMemory, testing.failing_allocator.allocSentinel(u8, 1, 0));
157}187}
158188
159test "Allocator.resize" {189test "Allocator.resize" {
...@@ -1068,16 +1098,18 @@ pub fn indexOfSentinel(comptime T: type, comptime sentinel: T, p: [*:sentinel]co...@@ -1068,16 +1098,18 @@ pub fn indexOfSentinel(comptime T: type, comptime sentinel: T, p: [*:sentinel]co
1068 // as we don't read into a new page. This should be the case for most architectures1098 // as we don't read into a new page. This should be the case for most architectures
1069 // which use paged memory, however should be confirmed before adding a new arch below.1099 // which use paged memory, however should be confirmed before adding a new arch below.
1070 .aarch64, .x86, .x86_64 => if (std.simd.suggestVectorLength(T)) |block_len| {1100 .aarch64, .x86, .x86_64 => if (std.simd.suggestVectorLength(T)) |block_len| {
1101 const page_size = std.heap.pageSize();
1071 const block_size = @sizeOf(T) * block_len;1102 const block_size = @sizeOf(T) * block_len;
1072 const Block = @Vector(block_len, T);1103 const Block = @Vector(block_len, T);
1073 const mask: Block = @splat(sentinel);1104 const mask: Block = @splat(sentinel);
10741105
1075 comptime std.debug.assert(std.mem.page_size % block_size == 0);1106 comptime assert(std.heap.page_size_max % @sizeOf(Block) == 0);
1107 assert(page_size % @sizeOf(Block) == 0);
10761108
1077 // First block may be unaligned1109 // First block may be unaligned
1078 const start_addr = @intFromPtr(&p[i]);1110 const start_addr = @intFromPtr(&p[i]);
1079 const offset_in_page = start_addr & (std.mem.page_size - 1);1111 const offset_in_page = start_addr & (page_size - 1);
1080 if (offset_in_page <= std.mem.page_size - block_size) {1112 if (offset_in_page <= page_size - @sizeOf(Block)) {
1081 // Will not read past the end of a page, full block.1113 // Will not read past the end of a page, full block.
1082 const block: Block = p[i..][0..block_len].*;1114 const block: Block = p[i..][0..block_len].*;
1083 const matches = block == mask;1115 const matches = block == mask;
...@@ -1097,7 +1129,7 @@ pub fn indexOfSentinel(comptime T: type, comptime sentinel: T, p: [*:sentinel]co...@@ -1097,7 +1129,7 @@ pub fn indexOfSentinel(comptime T: type, comptime sentinel: T, p: [*:sentinel]co
1097 }1129 }
1098 }1130 }
10991131
1100 std.debug.assert(std.mem.isAligned(@intFromPtr(&p[i]), block_size));1132 assert(std.mem.isAligned(@intFromPtr(&p[i]), block_size));
1101 while (true) {1133 while (true) {
1102 const block: *const Block = @ptrCast(@alignCast(p[i..][0..block_len]));1134 const block: *const Block = @ptrCast(@alignCast(p[i..][0..block_len]));
1103 const matches = block.* == mask;1135 const matches = block.* == mask;
...@@ -1120,23 +1152,24 @@ pub fn indexOfSentinel(comptime T: type, comptime sentinel: T, p: [*:sentinel]co...@@ -1120,23 +1152,24 @@ pub fn indexOfSentinel(comptime T: type, comptime sentinel: T, p: [*:sentinel]co
1120test "indexOfSentinel vector paths" {1152test "indexOfSentinel vector paths" {
1121 const Types = [_]type{ u8, u16, u32, u64 };1153 const Types = [_]type{ u8, u16, u32, u64 };
1122 const allocator = std.testing.allocator;1154 const allocator = std.testing.allocator;
1155 const page_size = std.heap.pageSize();
11231156
1124 inline for (Types) |T| {1157 inline for (Types) |T| {
1125 const block_len = std.simd.suggestVectorLength(T) orelse continue;1158 const block_len = std.simd.suggestVectorLength(T) orelse continue;
11261159
1127 // Allocate three pages so we guarantee a page-crossing address with a full page after1160 // Allocate three pages so we guarantee a page-crossing address with a full page after
1128 const memory = try allocator.alloc(T, 3 * std.mem.page_size / @sizeOf(T));1161 const memory = try allocator.alloc(T, 3 * page_size / @sizeOf(T));
1129 defer allocator.free(memory);1162 defer allocator.free(memory);
1130 @memset(memory, 0xaa);1163 @memset(memory, 0xaa);
11311164
1132 // Find starting page-alignment = 01165 // Find starting page-alignment = 0
1133 var start: usize = 0;1166 var start: usize = 0;
1134 const start_addr = @intFromPtr(&memory);1167 const start_addr = @intFromPtr(&memory);
1135 start += (std.mem.alignForward(usize, start_addr, std.mem.page_size) - start_addr) / @sizeOf(T);1168 start += (std.mem.alignForward(usize, start_addr, page_size) - start_addr) / @sizeOf(T);
1136 try testing.expect(start < std.mem.page_size / @sizeOf(T));1169 try testing.expect(start < page_size / @sizeOf(T));
11371170
1138 // Validate all sub-block alignments1171 // Validate all sub-block alignments
1139 const search_len = std.mem.page_size / @sizeOf(T);1172 const search_len = page_size / @sizeOf(T);
1140 memory[start + search_len] = 0;1173 memory[start + search_len] = 0;
1141 for (0..block_len) |offset| {1174 for (0..block_len) |offset| {
1142 try testing.expectEqual(search_len - offset, indexOfSentinel(T, 0, @ptrCast(&memory[start + offset])));1175 try testing.expectEqual(search_len - offset, indexOfSentinel(T, 0, @ptrCast(&memory[start + offset])));
...@@ -1144,7 +1177,7 @@ test "indexOfSentinel vector paths" {...@@ -1144,7 +1177,7 @@ test "indexOfSentinel vector paths" {
1144 memory[start + search_len] = 0xaa;1177 memory[start + search_len] = 0xaa;
11451178
1146 // Validate page boundary crossing1179 // Validate page boundary crossing
1147 const start_page_boundary = start + (std.mem.page_size / @sizeOf(T));1180 const start_page_boundary = start + (page_size / @sizeOf(T));
1148 memory[start_page_boundary + block_len] = 0;1181 memory[start_page_boundary + block_len] = 0;
1149 for (0..block_len) |offset| {1182 for (0..block_len) |offset| {
1150 try testing.expectEqual(2 * block_len - offset, indexOfSentinel(T, 0, @ptrCast(&memory[start_page_boundary - block_len + offset])));1183 try testing.expectEqual(2 * block_len - offset, indexOfSentinel(T, 0, @ptrCast(&memory[start_page_boundary - block_len + offset])));
lib/std/mem/Allocator.zig+153-77
...@@ -6,29 +6,34 @@ const math = std.math;...@@ -6,29 +6,34 @@ const math = std.math;
6const mem = std.mem;6const mem = std.mem;
7const Allocator = @This();7const Allocator = @This();
8const builtin = @import("builtin");8const builtin = @import("builtin");
9const Alignment = std.mem.Alignment;
910
10pub const Error = error{OutOfMemory};11pub const Error = error{OutOfMemory};
11pub const Log2Align = math.Log2Int(usize);12pub const Log2Align = math.Log2Int(usize);
1213
13/// The type erased pointer to the allocator implementation.14/// The type erased pointer to the allocator implementation.
14/// Any comparison of this field may result in illegal behavior, since it may be set to15///
15/// `undefined` in cases where the allocator implementation does not have any associated16/// Any comparison of this field may result in illegal behavior, since it may
16/// state.17/// be set to `undefined` in cases where the allocator implementation does not
18/// have any associated state.
17ptr: *anyopaque,19ptr: *anyopaque,
18vtable: *const VTable,20vtable: *const VTable,
1921
20pub const VTable = struct {22pub const VTable = struct {
21 /// Attempt to allocate exactly `len` bytes aligned to `1 << ptr_align`.23 /// Return a pointer to `len` bytes with specified `alignment`, or return
24 /// `null` indicating the allocation failed.
22 ///25 ///
23 /// `ret_addr` is optionally provided as the first return address of the26 /// `ret_addr` is optionally provided as the first return address of the
24 /// allocation call stack. If the value is `0` it means no return address27 /// allocation call stack. If the value is `0` it means no return address
25 /// has been provided.28 /// has been provided.
26 alloc: *const fn (ctx: *anyopaque, len: usize, ptr_align: u8, ret_addr: usize) ?[*]u8,29 alloc: *const fn (*anyopaque, len: usize, alignment: Alignment, ret_addr: usize) ?[*]u8,
2730
28 /// Attempt to expand or shrink memory in place. `buf.len` must equal the31 /// Attempt to expand or shrink memory in place.
29 /// length requested from the most recent successful call to `alloc` or32 ///
30 /// `resize`. `buf_align` must equal the same value that was passed as the33 /// `memory.len` must equal the length requested from the most recent
31 /// `ptr_align` parameter to the original `alloc` call.34 /// successful call to `alloc`, `resize`, or `remap`. `alignment` must
35 /// equal the same value that was passed as the `alignment` parameter to
36 /// the original `alloc` call.
32 ///37 ///
33 /// A result of `true` indicates the resize was successful and the38 /// A result of `true` indicates the resize was successful and the
34 /// allocation now has the same address but a size of `new_len`. `false`39 /// allocation now has the same address but a size of `new_len`. `false`
...@@ -40,72 +45,113 @@ pub const VTable = struct {...@@ -40,72 +45,113 @@ pub const VTable = struct {
40 /// `ret_addr` is optionally provided as the first return address of the45 /// `ret_addr` is optionally provided as the first return address of the
41 /// allocation call stack. If the value is `0` it means no return address46 /// allocation call stack. If the value is `0` it means no return address
42 /// has been provided.47 /// has been provided.
43 resize: *const fn (ctx: *anyopaque, buf: []u8, buf_align: u8, new_len: usize, ret_addr: usize) bool,48 resize: *const fn (*anyopaque, memory: []u8, alignment: Alignment, new_len: usize, ret_addr: usize) bool,
4449
45 /// Free and invalidate a buffer.50 /// Attempt to expand or shrink memory, allowing relocation.
51 ///
52 /// `memory.len` must equal the length requested from the most recent
53 /// successful call to `alloc`, `resize`, or `remap`. `alignment` must
54 /// equal the same value that was passed as the `alignment` parameter to
55 /// the original `alloc` call.
56 ///
57 /// A non-`null` return value indicates the resize was successful. The
58 /// allocation may have same address, or may have been relocated. In either
59 /// case, the allocation now has size of `new_len`. A `null` return value
60 /// indicates that the resize would be equivalent to allocating new memory,
61 /// copying the bytes from the old memory, and then freeing the old memory.
62 /// In such case, it is more efficient for the caller to perform the copy.
63 ///
64 /// `new_len` must be greater than zero.
46 ///65 ///
47 /// `buf.len` must equal the most recent length returned by `alloc` or66 /// `ret_addr` is optionally provided as the first return address of the
48 /// given to a successful `resize` call.67 /// allocation call stack. If the value is `0` it means no return address
68 /// has been provided.
69 remap: *const fn (*anyopaque, memory: []u8, alignment: Alignment, new_len: usize, ret_addr: usize) ?[*]u8,
70
71 /// Free and invalidate a region of memory.
49 ///72 ///
50 /// `buf_align` must equal the same value that was passed as the73 /// `memory.len` must equal the length requested from the most recent
51 /// `ptr_align` parameter to the original `alloc` call.74 /// successful call to `alloc`, `resize`, or `remap`. `alignment` must
75 /// equal the same value that was passed as the `alignment` parameter to
76 /// the original `alloc` call.
52 ///77 ///
53 /// `ret_addr` is optionally provided as the first return address of the78 /// `ret_addr` is optionally provided as the first return address of the
54 /// allocation call stack. If the value is `0` it means no return address79 /// allocation call stack. If the value is `0` it means no return address
55 /// has been provided.80 /// has been provided.
56 free: *const fn (ctx: *anyopaque, buf: []u8, buf_align: u8, ret_addr: usize) void,81 free: *const fn (*anyopaque, memory: []u8, alignment: Alignment, ret_addr: usize) void,
57};82};
5883
59pub fn noResize(84pub fn noResize(
60 self: *anyopaque,85 self: *anyopaque,
61 buf: []u8,86 memory: []u8,
62 log2_buf_align: u8,87 alignment: Alignment,
63 new_len: usize,88 new_len: usize,
64 ret_addr: usize,89 ret_addr: usize,
65) bool {90) bool {
66 _ = self;91 _ = self;
67 _ = buf;92 _ = memory;
68 _ = log2_buf_align;93 _ = alignment;
69 _ = new_len;94 _ = new_len;
70 _ = ret_addr;95 _ = ret_addr;
71 return false;96 return false;
72}97}
7398
99pub fn noRemap(
100 self: *anyopaque,
101 memory: []u8,
102 alignment: Alignment,
103 new_len: usize,
104 ret_addr: usize,
105) ?[*]u8 {
106 _ = self;
107 _ = memory;
108 _ = alignment;
109 _ = new_len;
110 _ = ret_addr;
111 return null;
112}
113
74pub fn noFree(114pub fn noFree(
75 self: *anyopaque,115 self: *anyopaque,
76 buf: []u8,116 memory: []u8,
77 log2_buf_align: u8,117 alignment: Alignment,
78 ret_addr: usize,118 ret_addr: usize,
79) void {119) void {
80 _ = self;120 _ = self;
81 _ = buf;121 _ = memory;
82 _ = log2_buf_align;122 _ = alignment;
83 _ = ret_addr;123 _ = ret_addr;
84}124}
85125
86/// This function is not intended to be called except from within the126/// This function is not intended to be called except from within the
87/// implementation of an Allocator127/// implementation of an `Allocator`.
88pub inline fn rawAlloc(self: Allocator, len: usize, ptr_align: u8, ret_addr: usize) ?[*]u8 {128pub inline fn rawAlloc(a: Allocator, len: usize, alignment: Alignment, ret_addr: usize) ?[*]u8 {
89 return self.vtable.alloc(self.ptr, len, ptr_align, ret_addr);129 return a.vtable.alloc(a.ptr, len, alignment, ret_addr);
90}130}
91131
92/// This function is not intended to be called except from within the132/// This function is not intended to be called except from within the
93/// implementation of an Allocator133/// implementation of an `Allocator`.
94pub inline fn rawResize(self: Allocator, buf: []u8, log2_buf_align: u8, new_len: usize, ret_addr: usize) bool {134pub inline fn rawResize(a: Allocator, memory: []u8, alignment: Alignment, new_len: usize, ret_addr: usize) bool {
95 return self.vtable.resize(self.ptr, buf, log2_buf_align, new_len, ret_addr);135 return a.vtable.resize(a.ptr, memory, alignment, new_len, ret_addr);
96}136}
97137
98/// This function is not intended to be called except from within the138/// This function is not intended to be called except from within the
99/// implementation of an Allocator139/// implementation of an `Allocator`.
100pub inline fn rawFree(self: Allocator, buf: []u8, log2_buf_align: u8, ret_addr: usize) void {140pub inline fn rawRemap(a: Allocator, memory: []u8, alignment: Alignment, new_len: usize, ret_addr: usize) ?[*]u8 {
101 return self.vtable.free(self.ptr, buf, log2_buf_align, ret_addr);141 return a.vtable.remap(a.ptr, memory, alignment, new_len, ret_addr);
142}
143
144/// This function is not intended to be called except from within the
145/// implementation of an `Allocator`.
146pub inline fn rawFree(a: Allocator, memory: []u8, alignment: Alignment, ret_addr: usize) void {
147 return a.vtable.free(a.ptr, memory, alignment, ret_addr);
102}148}
103149
104/// Returns a pointer to undefined memory.150/// Returns a pointer to undefined memory.
105/// Call `destroy` with the result to free the memory.151/// Call `destroy` with the result to free the memory.
106pub fn create(self: Allocator, comptime T: type) Error!*T {152pub fn create(a: Allocator, comptime T: type) Error!*T {
107 if (@sizeOf(T) == 0) return @as(*T, @ptrFromInt(math.maxInt(usize)));153 if (@sizeOf(T) == 0) return @as(*T, @ptrFromInt(math.maxInt(usize)));
108 const ptr: *T = @ptrCast(try self.allocBytesWithAlignment(@alignOf(T), @sizeOf(T), @returnAddress()));154 const ptr: *T = @ptrCast(try a.allocBytesWithAlignment(@alignOf(T), @sizeOf(T), @returnAddress()));
109 return ptr;155 return ptr;
110}156}
111157
...@@ -117,7 +163,7 @@ pub fn destroy(self: Allocator, ptr: anytype) void {...@@ -117,7 +163,7 @@ pub fn destroy(self: Allocator, ptr: anytype) void {
117 const T = info.child;163 const T = info.child;
118 if (@sizeOf(T) == 0) return;164 if (@sizeOf(T) == 0) return;
119 const non_const_ptr = @as([*]u8, @ptrCast(@constCast(ptr)));165 const non_const_ptr = @as([*]u8, @ptrCast(@constCast(ptr)));
120 self.rawFree(non_const_ptr[0..@sizeOf(T)], log2a(info.alignment), @returnAddress());166 self.rawFree(non_const_ptr[0..@sizeOf(T)], .fromByteUnits(info.alignment), @returnAddress());
121}167}
122168
123/// Allocates an array of `n` items of type `T` and sets all the169/// Allocates an array of `n` items of type `T` and sets all the
...@@ -215,46 +261,92 @@ fn allocWithSizeAndAlignment(self: Allocator, comptime size: usize, comptime ali...@@ -215,46 +261,92 @@ fn allocWithSizeAndAlignment(self: Allocator, comptime size: usize, comptime ali
215}261}
216262
217fn allocBytesWithAlignment(self: Allocator, comptime alignment: u29, byte_count: usize, return_address: usize) Error![*]align(alignment) u8 {263fn allocBytesWithAlignment(self: Allocator, comptime alignment: u29, byte_count: usize, return_address: usize) Error![*]align(alignment) u8 {
218 // The Zig Allocator interface is not intended to solve alignments beyond
219 // the minimum OS page size. For these use cases, the caller must use OS
220 // APIs directly.
221 comptime assert(alignment <= mem.page_size);
222
223 if (byte_count == 0) {264 if (byte_count == 0) {
224 const ptr = comptime std.mem.alignBackward(usize, math.maxInt(usize), alignment);265 const ptr = comptime std.mem.alignBackward(usize, math.maxInt(usize), alignment);
225 return @as([*]align(alignment) u8, @ptrFromInt(ptr));266 return @as([*]align(alignment) u8, @ptrFromInt(ptr));
226 }267 }
227268
228 const byte_ptr = self.rawAlloc(byte_count, log2a(alignment), return_address) orelse return Error.OutOfMemory;269 const byte_ptr = self.rawAlloc(byte_count, .fromByteUnits(alignment), return_address) orelse return Error.OutOfMemory;
229 // TODO: https://github.com/ziglang/zig/issues/4298
230 @memset(byte_ptr[0..byte_count], undefined);270 @memset(byte_ptr[0..byte_count], undefined);
231 return @as([*]align(alignment) u8, @alignCast(byte_ptr));271 return @alignCast(byte_ptr);
232}272}
233273
234/// Requests to modify the size of an allocation. It is guaranteed to not move274/// Request to modify the size of an allocation.
235/// the pointer, however the allocator implementation may refuse the resize275///
236/// request by returning `false`.276/// It is guaranteed to not move the pointer, however the allocator
237pub fn resize(self: Allocator, old_mem: anytype, new_n: usize) bool {277/// implementation may refuse the resize request by returning `false`.
238 const Slice = @typeInfo(@TypeOf(old_mem)).pointer;278///
279/// `allocation` may be an empty slice, in which case a new allocation is made.
280///
281/// `new_len` may be zero, in which case the allocation is freed.
282pub fn resize(self: Allocator, allocation: anytype, new_len: usize) bool {
283 const Slice = @typeInfo(@TypeOf(allocation)).pointer;
239 const T = Slice.child;284 const T = Slice.child;
240 if (new_n == 0) {285 const alignment = Slice.alignment;
241 self.free(old_mem);286 if (new_len == 0) {
287 self.free(allocation);
242 return true;288 return true;
243 }289 }
244 if (old_mem.len == 0) {290 if (allocation.len == 0) {
245 return false;291 return false;
246 }292 }
247 const old_byte_slice = mem.sliceAsBytes(old_mem);293 const old_memory = mem.sliceAsBytes(allocation);
294 // I would like to use saturating multiplication here, but LLVM cannot lower it
295 // on WebAssembly: https://github.com/ziglang/zig/issues/9660
296 //const new_len_bytes = new_len *| @sizeOf(T);
297 const new_len_bytes = math.mul(usize, @sizeOf(T), new_len) catch return false;
298 return self.rawResize(old_memory, .fromByteUnits(alignment), new_len_bytes, @returnAddress());
299}
300
301/// Request to modify the size of an allocation, allowing relocation.
302///
303/// A non-`null` return value indicates the resize was successful. The
304/// allocation may have same address, or may have been relocated. In either
305/// case, the allocation now has size of `new_len`. A `null` return value
306/// indicates that the resize would be equivalent to allocating new memory,
307/// copying the bytes from the old memory, and then freeing the old memory.
308/// In such case, it is more efficient for the caller to perform those
309/// operations.
310///
311/// `allocation` may be an empty slice, in which case a new allocation is made.
312///
313/// `new_len` may be zero, in which case the allocation is freed.
314pub fn remap(self: Allocator, allocation: anytype, new_len: usize) t: {
315 const Slice = @typeInfo(@TypeOf(allocation)).pointer;
316 break :t ?[]align(Slice.alignment) Slice.child;
317} {
318 const Slice = @typeInfo(@TypeOf(allocation)).pointer;
319 const T = Slice.child;
320 const alignment = Slice.alignment;
321 if (new_len == 0) {
322 self.free(allocation);
323 return allocation[0..0];
324 }
325 if (allocation.len == 0) {
326 return null;
327 }
328 const old_memory = mem.sliceAsBytes(allocation);
248 // I would like to use saturating multiplication here, but LLVM cannot lower it329 // I would like to use saturating multiplication here, but LLVM cannot lower it
249 // on WebAssembly: https://github.com/ziglang/zig/issues/9660330 // on WebAssembly: https://github.com/ziglang/zig/issues/9660
250 //const new_byte_count = new_n *| @sizeOf(T);331 //const new_len_bytes = new_len *| @sizeOf(T);
251 const new_byte_count = math.mul(usize, @sizeOf(T), new_n) catch return false;332 const new_len_bytes = math.mul(usize, @sizeOf(T), new_len) catch return null;
252 return self.rawResize(old_byte_slice, log2a(Slice.alignment), new_byte_count, @returnAddress());333 const new_ptr = self.rawRemap(old_memory, .fromByteUnits(alignment), new_len_bytes, @returnAddress()) orelse return null;
334 const new_memory: []align(alignment) u8 = @alignCast(new_ptr[0..new_len_bytes]);
335 return mem.bytesAsSlice(T, new_memory);
253}336}
254337
255/// This function requests a new byte size for an existing allocation, which338/// This function requests a new byte size for an existing allocation, which
256/// can be larger, smaller, or the same size as the old memory allocation.339/// can be larger, smaller, or the same size as the old memory allocation.
340///
257/// If `new_n` is 0, this is the same as `free` and it always succeeds.341/// If `new_n` is 0, this is the same as `free` and it always succeeds.
342///
343/// `old_mem` may have length zero, which makes a new allocation.
344///
345/// This function only fails on out-of-memory conditions, unlike:
346/// * `remap` which returns `null` when the `Allocator` implementation cannot
347/// do the realloc more efficiently than the caller
348/// * `resize` which returns `false` when the `Allocator` implementation cannot
349/// change the size without relocating the allocation.
258pub fn realloc(self: Allocator, old_mem: anytype, new_n: usize) t: {350pub fn realloc(self: Allocator, old_mem: anytype, new_n: usize) t: {
259 const Slice = @typeInfo(@TypeOf(old_mem)).pointer;351 const Slice = @typeInfo(@TypeOf(old_mem)).pointer;
260 break :t Error![]align(Slice.alignment) Slice.child;352 break :t Error![]align(Slice.alignment) Slice.child;
...@@ -285,18 +377,17 @@ pub fn reallocAdvanced(...@@ -285,18 +377,17 @@ pub fn reallocAdvanced(
285 const old_byte_slice = mem.sliceAsBytes(old_mem);377 const old_byte_slice = mem.sliceAsBytes(old_mem);
286 const byte_count = math.mul(usize, @sizeOf(T), new_n) catch return Error.OutOfMemory;378 const byte_count = math.mul(usize, @sizeOf(T), new_n) catch return Error.OutOfMemory;
287 // Note: can't set shrunk memory to undefined as memory shouldn't be modified on realloc failure379 // Note: can't set shrunk memory to undefined as memory shouldn't be modified on realloc failure
288 if (self.rawResize(old_byte_slice, log2a(Slice.alignment), byte_count, return_address)) {380 if (self.rawRemap(old_byte_slice, .fromByteUnits(Slice.alignment), byte_count, return_address)) |p| {
289 const new_bytes: []align(Slice.alignment) u8 = @alignCast(old_byte_slice.ptr[0..byte_count]);381 const new_bytes: []align(Slice.alignment) u8 = @alignCast(p[0..byte_count]);
290 return mem.bytesAsSlice(T, new_bytes);382 return mem.bytesAsSlice(T, new_bytes);
291 }383 }
292384
293 const new_mem = self.rawAlloc(byte_count, log2a(Slice.alignment), return_address) orelse385 const new_mem = self.rawAlloc(byte_count, .fromByteUnits(Slice.alignment), return_address) orelse
294 return error.OutOfMemory;386 return error.OutOfMemory;
295 const copy_len = @min(byte_count, old_byte_slice.len);387 const copy_len = @min(byte_count, old_byte_slice.len);
296 @memcpy(new_mem[0..copy_len], old_byte_slice[0..copy_len]);388 @memcpy(new_mem[0..copy_len], old_byte_slice[0..copy_len]);
297 // TODO https://github.com/ziglang/zig/issues/4298
298 @memset(old_byte_slice, undefined);389 @memset(old_byte_slice, undefined);
299 self.rawFree(old_byte_slice, log2a(Slice.alignment), return_address);390 self.rawFree(old_byte_slice, .fromByteUnits(Slice.alignment), return_address);
300391
301 const new_bytes: []align(Slice.alignment) u8 = @alignCast(new_mem[0..byte_count]);392 const new_bytes: []align(Slice.alignment) u8 = @alignCast(new_mem[0..byte_count]);
302 return mem.bytesAsSlice(T, new_bytes);393 return mem.bytesAsSlice(T, new_bytes);
...@@ -311,9 +402,8 @@ pub fn free(self: Allocator, memory: anytype) void {...@@ -311,9 +402,8 @@ pub fn free(self: Allocator, memory: anytype) void {
311 const bytes_len = bytes.len + if (Slice.sentinel() != null) @sizeOf(Slice.child) else 0;402 const bytes_len = bytes.len + if (Slice.sentinel() != null) @sizeOf(Slice.child) else 0;
312 if (bytes_len == 0) return;403 if (bytes_len == 0) return;
313 const non_const_ptr = @constCast(bytes.ptr);404 const non_const_ptr = @constCast(bytes.ptr);
314 // TODO: https://github.com/ziglang/zig/issues/4298
315 @memset(non_const_ptr[0..bytes_len], undefined);405 @memset(non_const_ptr[0..bytes_len], undefined);
316 self.rawFree(non_const_ptr[0..bytes_len], log2a(Slice.alignment), @returnAddress());406 self.rawFree(non_const_ptr[0..bytes_len], .fromByteUnits(Slice.alignment), @returnAddress());
317}407}
318408
319/// Copies `m` to newly allocated memory. Caller owns the memory.409/// Copies `m` to newly allocated memory. Caller owns the memory.
...@@ -330,17 +420,3 @@ pub fn dupeZ(allocator: Allocator, comptime T: type, m: []const T) Error![:0]T {...@@ -330,17 +420,3 @@ pub fn dupeZ(allocator: Allocator, comptime T: type, m: []const T) Error![:0]T {
330 new_buf[m.len] = 0;420 new_buf[m.len] = 0;
331 return new_buf[0..m.len :0];421 return new_buf[0..m.len :0];
332}422}
333
334/// TODO replace callsites with `@log2` after this proposal is implemented:
335/// https://github.com/ziglang/zig/issues/13642
336inline fn log2a(x: anytype) switch (@typeInfo(@TypeOf(x))) {
337 .int => math.Log2Int(@TypeOf(x)),
338 .comptime_int => comptime_int,
339 else => @compileError("int please"),
340} {
341 switch (@typeInfo(@TypeOf(x))) {
342 .int => return math.log2_int(@TypeOf(x), x),
343 .comptime_int => return math.log2(x),
344 else => @compileError("bad"),
345 }
346}
lib/std/os/linux.zig+18-4
...@@ -305,6 +305,13 @@ pub const MAP = switch (native_arch) {...@@ -305,6 +305,13 @@ pub const MAP = switch (native_arch) {
305 else => @compileError("missing std.os.linux.MAP constants for this architecture"),305 else => @compileError("missing std.os.linux.MAP constants for this architecture"),
306};306};
307307
308pub const MREMAP = packed struct(u32) {
309 MAYMOVE: bool = false,
310 FIXED: bool = false,
311 DONTUNMAP: bool = false,
312 _: u29 = 0,
313};
314
308pub const O = switch (native_arch) {315pub const O = switch (native_arch) {
309 .x86_64 => packed struct(u32) {316 .x86_64 => packed struct(u32) {
310 ACCMODE: ACCMODE = .RDONLY,317 ACCMODE: ACCMODE = .RDONLY,
...@@ -892,10 +899,6 @@ pub fn umount2(special: [*:0]const u8, flags: u32) usize {...@@ -892,10 +899,6 @@ pub fn umount2(special: [*:0]const u8, flags: u32) usize {
892899
893pub fn mmap(address: ?[*]u8, length: usize, prot: usize, flags: MAP, fd: i32, offset: i64) usize {900pub fn mmap(address: ?[*]u8, length: usize, prot: usize, flags: MAP, fd: i32, offset: i64) usize {
894 if (@hasField(SYS, "mmap2")) {901 if (@hasField(SYS, "mmap2")) {
895 // Make sure the offset is also specified in multiples of page size
896 if ((offset & (MMAP2_UNIT - 1)) != 0)
897 return @bitCast(-@as(isize, @intFromEnum(E.INVAL)));
898
899 return syscall6(902 return syscall6(
900 .mmap2,903 .mmap2,
901 @intFromPtr(address),904 @intFromPtr(address),
...@@ -934,6 +937,17 @@ pub fn mprotect(address: [*]const u8, length: usize, protection: usize) usize {...@@ -934,6 +937,17 @@ pub fn mprotect(address: [*]const u8, length: usize, protection: usize) usize {
934 return syscall3(.mprotect, @intFromPtr(address), length, protection);937 return syscall3(.mprotect, @intFromPtr(address), length, protection);
935}938}
936939
940pub fn mremap(old_addr: ?[*]const u8, old_len: usize, new_len: usize, flags: MREMAP, new_addr: ?[*]const u8) usize {
941 return syscall5(
942 .mremap,
943 @intFromPtr(old_addr),
944 old_len,
945 new_len,
946 @as(u32, @bitCast(flags)),
947 @intFromPtr(new_addr),
948 );
949}
950
937pub const MSF = struct {951pub const MSF = struct {
938 pub const ASYNC = 1;952 pub const ASYNC = 1;
939 pub const INVALIDATE = 2;953 pub const INVALIDATE = 2;
lib/std/os/linux/IoUring.zig+8-7
...@@ -8,6 +8,7 @@ const posix = std.posix;...@@ -8,6 +8,7 @@ const posix = std.posix;
8const linux = std.os.linux;8const linux = std.os.linux;
9const testing = std.testing;9const testing = std.testing;
10const is_linux = builtin.os.tag == .linux;10const is_linux = builtin.os.tag == .linux;
11const page_size_min = std.heap.page_size_min;
1112
12fd: posix.fd_t = -1,13fd: posix.fd_t = -1,
13sq: SubmissionQueue,14sq: SubmissionQueue,
...@@ -1341,8 +1342,8 @@ pub const SubmissionQueue = struct {...@@ -1341,8 +1342,8 @@ pub const SubmissionQueue = struct {
1341 dropped: *u32,1342 dropped: *u32,
1342 array: []u32,1343 array: []u32,
1343 sqes: []linux.io_uring_sqe,1344 sqes: []linux.io_uring_sqe,
1344 mmap: []align(mem.page_size) u8,1345 mmap: []align(page_size_min) u8,
1345 mmap_sqes: []align(mem.page_size) u8,1346 mmap_sqes: []align(page_size_min) u8,
13461347
1347 // We use `sqe_head` and `sqe_tail` in the same way as liburing:1348 // We use `sqe_head` and `sqe_tail` in the same way as liburing:
1348 // We increment `sqe_tail` (but not `tail`) for each call to `get_sqe()`.1349 // We increment `sqe_tail` (but not `tail`) for each call to `get_sqe()`.
...@@ -1460,7 +1461,7 @@ pub const BufferGroup = struct {...@@ -1460,7 +1461,7 @@ pub const BufferGroup = struct {
1460 /// Pointer to the memory shared by the kernel.1461 /// Pointer to the memory shared by the kernel.
1461 /// `buffers_count` of `io_uring_buf` structures are shared by the kernel.1462 /// `buffers_count` of `io_uring_buf` structures are shared by the kernel.
1462 /// First `io_uring_buf` is overlaid by `io_uring_buf_ring` struct.1463 /// First `io_uring_buf` is overlaid by `io_uring_buf_ring` struct.
1463 br: *align(mem.page_size) linux.io_uring_buf_ring,1464 br: *align(page_size_min) linux.io_uring_buf_ring,
1464 /// Contiguous block of memory of size (buffers_count * buffer_size).1465 /// Contiguous block of memory of size (buffers_count * buffer_size).
1465 buffers: []u8,1466 buffers: []u8,
1466 /// Size of each buffer in buffers.1467 /// Size of each buffer in buffers.
...@@ -1555,7 +1556,7 @@ pub const BufferGroup = struct {...@@ -1555,7 +1556,7 @@ pub const BufferGroup = struct {
1555/// `fd` is IO_Uring.fd for which the provided buffer ring is being registered.1556/// `fd` is IO_Uring.fd for which the provided buffer ring is being registered.
1556/// `entries` is the number of entries requested in the buffer ring, must be power of 2.1557/// `entries` is the number of entries requested in the buffer ring, must be power of 2.
1557/// `group_id` is the chosen buffer group ID, unique in IO_Uring.1558/// `group_id` is the chosen buffer group ID, unique in IO_Uring.
1558pub fn setup_buf_ring(fd: posix.fd_t, entries: u16, group_id: u16) !*align(mem.page_size) linux.io_uring_buf_ring {1559pub fn setup_buf_ring(fd: posix.fd_t, entries: u16, group_id: u16) !*align(page_size_min) linux.io_uring_buf_ring {
1559 if (entries == 0 or entries > 1 << 15) return error.EntriesNotInRange;1560 if (entries == 0 or entries > 1 << 15) return error.EntriesNotInRange;
1560 if (!std.math.isPowerOfTwo(entries)) return error.EntriesNotPowerOfTwo;1561 if (!std.math.isPowerOfTwo(entries)) return error.EntriesNotPowerOfTwo;
15611562
...@@ -1571,7 +1572,7 @@ pub fn setup_buf_ring(fd: posix.fd_t, entries: u16, group_id: u16) !*align(mem.p...@@ -1571,7 +1572,7 @@ pub fn setup_buf_ring(fd: posix.fd_t, entries: u16, group_id: u16) !*align(mem.p
1571 errdefer posix.munmap(mmap);1572 errdefer posix.munmap(mmap);
1572 assert(mmap.len == mmap_size);1573 assert(mmap.len == mmap_size);
15731574
1574 const br: *align(mem.page_size) linux.io_uring_buf_ring = @ptrCast(mmap.ptr);1575 const br: *align(page_size_min) linux.io_uring_buf_ring = @ptrCast(mmap.ptr);
1575 try register_buf_ring(fd, @intFromPtr(br), entries, group_id);1576 try register_buf_ring(fd, @intFromPtr(br), entries, group_id);
1576 return br;1577 return br;
1577}1578}
...@@ -1613,9 +1614,9 @@ fn handle_register_buf_ring_result(res: usize) !void {...@@ -1613,9 +1614,9 @@ fn handle_register_buf_ring_result(res: usize) !void {
1613}1614}
16141615
1615// Unregisters a previously registered shared buffer ring, returned from io_uring_setup_buf_ring.1616// Unregisters a previously registered shared buffer ring, returned from io_uring_setup_buf_ring.
1616pub fn free_buf_ring(fd: posix.fd_t, br: *align(mem.page_size) linux.io_uring_buf_ring, entries: u32, group_id: u16) void {1617pub fn free_buf_ring(fd: posix.fd_t, br: *align(page_size_min) linux.io_uring_buf_ring, entries: u32, group_id: u16) void {
1617 unregister_buf_ring(fd, group_id) catch {};1618 unregister_buf_ring(fd, group_id) catch {};
1618 var mmap: []align(mem.page_size) u8 = undefined;1619 var mmap: []align(page_size_min) u8 = undefined;
1619 mmap.ptr = @ptrCast(br);1620 mmap.ptr = @ptrCast(br);
1620 mmap.len = entries * @sizeOf(linux.io_uring_buf);1621 mmap.len = entries * @sizeOf(linux.io_uring_buf);
1621 posix.munmap(mmap);1622 posix.munmap(mmap);
lib/std/os/linux/tls.zig+10-9
...@@ -17,6 +17,7 @@ const assert = std.debug.assert;...@@ -17,6 +17,7 @@ const assert = std.debug.assert;
17const native_arch = @import("builtin").cpu.arch;17const native_arch = @import("builtin").cpu.arch;
18const linux = std.os.linux;18const linux = std.os.linux;
19const posix = std.posix;19const posix = std.posix;
20const page_size_min = std.heap.page_size_min;
2021
21/// Represents an ELF TLS variant.22/// Represents an ELF TLS variant.
22///23///
...@@ -484,13 +485,13 @@ pub fn prepareArea(area: []u8) usize {...@@ -484,13 +485,13 @@ pub fn prepareArea(area: []u8) usize {
484 };485 };
485}486}
486487
487// The main motivation for the size chosen here is that this is how much ends up being requested for488/// The main motivation for the size chosen here is that this is how much ends up being requested for
488// the thread-local variables of the `std.crypto.random` implementation. I'm not sure why it ends up489/// the thread-local variables of the `std.crypto.random` implementation. I'm not sure why it ends up
489// being so much; the struct itself is only 64 bytes. I think it has to do with being page-aligned490/// being so much; the struct itself is only 64 bytes. I think it has to do with being page-aligned
490// and LLVM or LLD is not smart enough to lay out the TLS data in a space-conserving way. Anyway, I491/// and LLVM or LLD is not smart enough to lay out the TLS data in a space-conserving way. Anyway, I
491// think it's fine because it's less than 3 pages of memory, and putting it in the ELF like this is492/// think it's fine because it's less than 3 pages of memory, and putting it in the ELF like this is
492// equivalent to moving the `mmap` call below into the kernel, avoiding syscall overhead.493/// equivalent to moving the `mmap` call below into the kernel, avoiding syscall overhead.
493var main_thread_area_buffer: [0x2100]u8 align(mem.page_size) = undefined;494var main_thread_area_buffer: [0x2100]u8 align(page_size_min) = undefined;
494495
495/// Computes the layout of the static TLS area, allocates the area, initializes all of its fields,496/// Computes the layout of the static TLS area, allocates the area, initializes all of its fields,
496/// and assigns the architecture-specific value to the TP register.497/// and assigns the architecture-specific value to the TP register.
...@@ -503,7 +504,7 @@ pub fn initStatic(phdrs: []elf.Phdr) void {...@@ -503,7 +504,7 @@ pub fn initStatic(phdrs: []elf.Phdr) void {
503 const area = blk: {504 const area = blk: {
504 // Fast path for the common case where the TLS data is really small, avoid an allocation and505 // Fast path for the common case where the TLS data is really small, avoid an allocation and
505 // use our local buffer.506 // use our local buffer.
506 if (area_desc.alignment <= mem.page_size and area_desc.size <= main_thread_area_buffer.len) {507 if (area_desc.alignment <= page_size_min and area_desc.size <= main_thread_area_buffer.len) {
507 break :blk main_thread_area_buffer[0..area_desc.size];508 break :blk main_thread_area_buffer[0..area_desc.size];
508 }509 }
509510
...@@ -517,7 +518,7 @@ pub fn initStatic(phdrs: []elf.Phdr) void {...@@ -517,7 +518,7 @@ pub fn initStatic(phdrs: []elf.Phdr) void {
517 );518 );
518 if (@as(isize, @bitCast(begin_addr)) < 0) @trap();519 if (@as(isize, @bitCast(begin_addr)) < 0) @trap();
519520
520 const area_ptr: [*]align(mem.page_size) u8 = @ptrFromInt(begin_addr);521 const area_ptr: [*]align(page_size_min) u8 = @ptrFromInt(begin_addr);
521522
522 // Make sure the slice is correctly aligned.523 // Make sure the slice is correctly aligned.
523 const begin_aligned_addr = alignForward(begin_addr, area_desc.alignment);524 const begin_aligned_addr = alignForward(begin_addr, area_desc.alignment);
lib/std/os/plan9.zig+2-2
...@@ -367,8 +367,8 @@ pub fn sbrk(n: usize) usize {...@@ -367,8 +367,8 @@ pub fn sbrk(n: usize) usize {
367 bloc = @intFromPtr(&ExecData.end);367 bloc = @intFromPtr(&ExecData.end);
368 bloc_max = @intFromPtr(&ExecData.end);368 bloc_max = @intFromPtr(&ExecData.end);
369 }369 }
370 const bl = std.mem.alignForward(usize, bloc, std.mem.page_size);370 const bl = std.mem.alignForward(usize, bloc, std.heap.pageSize());
371 const n_aligned = std.mem.alignForward(usize, n, std.mem.page_size);371 const n_aligned = std.mem.alignForward(usize, n, std.heap.pageSize());
372 if (bl + n_aligned > bloc_max) {372 if (bl + n_aligned > bloc_max) {
373 // we need to allocate373 // we need to allocate
374 if (brk_(bl + n_aligned) < 0) return 0;374 if (brk_(bl + n_aligned) < 0) return 0;
lib/std/os/windows.zig-12
...@@ -2016,18 +2016,6 @@ pub fn InitOnceExecuteOnce(InitOnce: *INIT_ONCE, InitFn: INIT_ONCE_FN, Parameter...@@ -2016,18 +2016,6 @@ pub fn InitOnceExecuteOnce(InitOnce: *INIT_ONCE, InitFn: INIT_ONCE_FN, Parameter
2016 assert(kernel32.InitOnceExecuteOnce(InitOnce, InitFn, Parameter, Context) != 0);2016 assert(kernel32.InitOnceExecuteOnce(InitOnce, InitFn, Parameter, Context) != 0);
2017}2017}
20182018
2019pub fn HeapFree(hHeap: HANDLE, dwFlags: DWORD, lpMem: *anyopaque) void {
2020 assert(kernel32.HeapFree(hHeap, dwFlags, lpMem) != 0);
2021}
2022
2023pub fn HeapDestroy(hHeap: HANDLE) void {
2024 assert(kernel32.HeapDestroy(hHeap) != 0);
2025}
2026
2027pub fn LocalFree(hMem: HLOCAL) void {
2028 assert(kernel32.LocalFree(hMem) == null);
2029}
2030
2031pub const SetFileTimeError = error{Unexpected};2019pub const SetFileTimeError = error{Unexpected};
20322020
2033pub fn SetFileTime(2021pub fn SetFileTime(
lib/std/os/windows/kernel32.zig+4-12
...@@ -42,6 +42,7 @@ const WCHAR = windows.WCHAR;...@@ -42,6 +42,7 @@ const WCHAR = windows.WCHAR;
42const WIN32_FIND_DATAW = windows.WIN32_FIND_DATAW;42const WIN32_FIND_DATAW = windows.WIN32_FIND_DATAW;
43const Win32Error = windows.Win32Error;43const Win32Error = windows.Win32Error;
44const WORD = windows.WORD;44const WORD = windows.WORD;
45const SYSTEM_INFO = windows.SYSTEM_INFO;
4546
46// I/O - Filesystem47// I/O - Filesystem
4748
...@@ -527,11 +528,6 @@ pub extern "kernel32" fn HeapCreate(...@@ -527,11 +528,6 @@ pub extern "kernel32" fn HeapCreate(
527 dwMaximumSize: SIZE_T,528 dwMaximumSize: SIZE_T,
528) callconv(.winapi) ?HANDLE;529) callconv(.winapi) ?HANDLE;
529530
530// TODO: Wrapper around RtlDestroyHeap (BOOLEAN -> BOOL).
531pub extern "kernel32" fn HeapDestroy(
532 hHeap: HANDLE,
533) callconv(.winapi) BOOL;
534
535// TODO: Forwarder to RtlReAllocateHeap.531// TODO: Forwarder to RtlReAllocateHeap.
536pub extern "kernel32" fn HeapReAlloc(532pub extern "kernel32" fn HeapReAlloc(
537 hHeap: HANDLE,533 hHeap: HANDLE,
...@@ -584,10 +580,6 @@ pub extern "kernel32" fn VirtualQuery(...@@ -584,10 +580,6 @@ pub extern "kernel32" fn VirtualQuery(
584 dwLength: SIZE_T,580 dwLength: SIZE_T,
585) callconv(.winapi) SIZE_T;581) callconv(.winapi) SIZE_T;
586582
587pub extern "kernel32" fn LocalFree(
588 hMem: HLOCAL,
589) callconv(.winapi) ?HLOCAL;
590
591// TODO: Getter for peb.ProcessHeap583// TODO: Getter for peb.ProcessHeap
592pub extern "kernel32" fn GetProcessHeap() callconv(.winapi) ?HANDLE;584pub extern "kernel32" fn GetProcessHeap() callconv(.winapi) ?HANDLE;
593585
...@@ -667,6 +659,6 @@ pub extern "kernel32" fn SetLastError(...@@ -667,6 +659,6 @@ pub extern "kernel32" fn SetLastError(
667// TODO:659// TODO:
668// Wrapper around KUSER_SHARED_DATA.SystemTime.660// Wrapper around KUSER_SHARED_DATA.SystemTime.
669// Much better to use NtQuerySystemTime or NtQuerySystemTimePrecise for guaranteed 0.1ns precision.661// Much better to use NtQuerySystemTime or NtQuerySystemTimePrecise for guaranteed 0.1ns precision.
670pub extern "kernel32" fn GetSystemTimeAsFileTime(662pub extern "kernel32" fn GetSystemTimeAsFileTime(lpSystemTimeAsFileTime: *FILETIME) callconv(.winapi) void;
671 lpSystemTimeAsFileTime: *FILETIME,663
672) callconv(.winapi) void;664pub extern "kernel32" fn GetSystemInfo(lpSystemInfo: *SYSTEM_INFO) callconv(.winapi) void;
lib/std/posix.zig+45-9
...@@ -24,6 +24,7 @@ const maxInt = std.math.maxInt;...@@ -24,6 +24,7 @@ const maxInt = std.math.maxInt;
24const cast = std.math.cast;24const cast = std.math.cast;
25const assert = std.debug.assert;25const assert = std.debug.assert;
26const native_os = builtin.os.tag;26const native_os = builtin.os.tag;
27const page_size_min = std.heap.page_size_min;
2728
28test {29test {
29 _ = @import("posix/test.zig");30 _ = @import("posix/test.zig");
...@@ -82,6 +83,7 @@ pub const MAP = system.MAP;...@@ -82,6 +83,7 @@ pub const MAP = system.MAP;
82pub const MAX_ADDR_LEN = system.MAX_ADDR_LEN;83pub const MAX_ADDR_LEN = system.MAX_ADDR_LEN;
83pub const MFD = system.MFD;84pub const MFD = system.MFD;
84pub const MMAP2_UNIT = system.MMAP2_UNIT;85pub const MMAP2_UNIT = system.MMAP2_UNIT;
86pub const MREMAP = system.MREMAP;
85pub const MSF = system.MSF;87pub const MSF = system.MSF;
86pub const MSG = system.MSG;88pub const MSG = system.MSG;
87pub const NAME_MAX = system.NAME_MAX;89pub const NAME_MAX = system.NAME_MAX;
...@@ -4694,7 +4696,7 @@ pub const MProtectError = error{...@@ -4694,7 +4696,7 @@ pub const MProtectError = error{
4694 OutOfMemory,4696 OutOfMemory,
4695} || UnexpectedError;4697} || UnexpectedError;
46964698
4697pub fn mprotect(memory: []align(mem.page_size) u8, protection: u32) MProtectError!void {4699pub fn mprotect(memory: []align(page_size_min) u8, protection: u32) MProtectError!void {
4698 if (native_os == .windows) {4700 if (native_os == .windows) {
4699 const win_prot: windows.DWORD = switch (@as(u3, @truncate(protection))) {4701 const win_prot: windows.DWORD = switch (@as(u3, @truncate(protection))) {
4700 0b000 => windows.PAGE_NOACCESS,4702 0b000 => windows.PAGE_NOACCESS,
...@@ -4759,21 +4761,21 @@ pub const MMapError = error{...@@ -4759,21 +4761,21 @@ pub const MMapError = error{
4759/// * SIGSEGV - Attempted write into a region mapped as read-only.4761/// * SIGSEGV - Attempted write into a region mapped as read-only.
4760/// * SIGBUS - Attempted access to a portion of the buffer that does not correspond to the file4762/// * SIGBUS - Attempted access to a portion of the buffer that does not correspond to the file
4761pub fn mmap(4763pub fn mmap(
4762 ptr: ?[*]align(mem.page_size) u8,4764 ptr: ?[*]align(page_size_min) u8,
4763 length: usize,4765 length: usize,
4764 prot: u32,4766 prot: u32,
4765 flags: system.MAP,4767 flags: system.MAP,
4766 fd: fd_t,4768 fd: fd_t,
4767 offset: u64,4769 offset: u64,
4768) MMapError![]align(mem.page_size) u8 {4770) MMapError![]align(page_size_min) u8 {
4769 const mmap_sym = if (lfs64_abi) system.mmap64 else system.mmap;4771 const mmap_sym = if (lfs64_abi) system.mmap64 else system.mmap;
4770 const rc = mmap_sym(ptr, length, prot, @bitCast(flags), fd, @bitCast(offset));4772 const rc = mmap_sym(ptr, length, prot, @bitCast(flags), fd, @bitCast(offset));
4771 const err: E = if (builtin.link_libc) blk: {4773 const err: E = if (builtin.link_libc) blk: {
4772 if (rc != std.c.MAP_FAILED) return @as([*]align(mem.page_size) u8, @ptrCast(@alignCast(rc)))[0..length];4774 if (rc != std.c.MAP_FAILED) return @as([*]align(page_size_min) u8, @ptrCast(@alignCast(rc)))[0..length];
4773 break :blk @enumFromInt(system._errno().*);4775 break :blk @enumFromInt(system._errno().*);
4774 } else blk: {4776 } else blk: {
4775 const err = errno(rc);4777 const err = errno(rc);
4776 if (err == .SUCCESS) return @as([*]align(mem.page_size) u8, @ptrFromInt(rc))[0..length];4778 if (err == .SUCCESS) return @as([*]align(page_size_min) u8, @ptrFromInt(rc))[0..length];
4777 break :blk err;4779 break :blk err;
4778 };4780 };
4779 switch (err) {4781 switch (err) {
...@@ -4799,7 +4801,7 @@ pub fn mmap(...@@ -4799,7 +4801,7 @@ pub fn mmap(
4799/// Zig's munmap function does not, for two reasons:4801/// Zig's munmap function does not, for two reasons:
4800/// * It violates the Zig principle that resource deallocation must succeed.4802/// * It violates the Zig principle that resource deallocation must succeed.
4801/// * The Windows function, VirtualFree, has this restriction.4803/// * The Windows function, VirtualFree, has this restriction.
4802pub fn munmap(memory: []align(mem.page_size) const u8) void {4804pub fn munmap(memory: []align(page_size_min) const u8) void {
4803 switch (errno(system.munmap(memory.ptr, memory.len))) {4805 switch (errno(system.munmap(memory.ptr, memory.len))) {
4804 .SUCCESS => return,4806 .SUCCESS => return,
4805 .INVAL => unreachable, // Invalid parameters.4807 .INVAL => unreachable, // Invalid parameters.
...@@ -4808,12 +4810,46 @@ pub fn munmap(memory: []align(mem.page_size) const u8) void {...@@ -4808,12 +4810,46 @@ pub fn munmap(memory: []align(mem.page_size) const u8) void {
4808 }4810 }
4809}4811}
48104812
4813pub const MRemapError = error{
4814 LockedMemoryLimitExceeded,
4815 /// Either a bug in the calling code, or the operating system abused the
4816 /// EINVAL error code.
4817 InvalidSyscallParameters,
4818 OutOfMemory,
4819} || UnexpectedError;
4820
4821pub fn mremap(
4822 old_address: ?[*]align(page_size_min) u8,
4823 old_len: usize,
4824 new_len: usize,
4825 flags: system.MREMAP,
4826 new_address: ?[*]align(page_size_min) u8,
4827) MRemapError![]align(page_size_min) u8 {
4828 const rc = system.mremap(old_address, old_len, new_len, flags, new_address);
4829 const err: E = if (builtin.link_libc) blk: {
4830 if (rc != std.c.MAP_FAILED) return @as([*]align(page_size_min) u8, @ptrCast(@alignCast(rc)))[0..new_len];
4831 break :blk @enumFromInt(system._errno().*);
4832 } else blk: {
4833 const err = errno(rc);
4834 if (err == .SUCCESS) return @as([*]align(page_size_min) u8, @ptrFromInt(rc))[0..new_len];
4835 break :blk err;
4836 };
4837 switch (err) {
4838 .SUCCESS => unreachable,
4839 .AGAIN => return error.LockedMemoryLimitExceeded,
4840 .INVAL => return error.InvalidSyscallParameters,
4841 .NOMEM => return error.OutOfMemory,
4842 .FAULT => unreachable,
4843 else => return unexpectedErrno(err),
4844 }
4845}
4846
4811pub const MSyncError = error{4847pub const MSyncError = error{
4812 UnmappedMemory,4848 UnmappedMemory,
4813 PermissionDenied,4849 PermissionDenied,
4814} || UnexpectedError;4850} || UnexpectedError;
48154851
4816pub fn msync(memory: []align(mem.page_size) u8, flags: i32) MSyncError!void {4852pub fn msync(memory: []align(page_size_min) u8, flags: i32) MSyncError!void {
4817 switch (errno(system.msync(memory.ptr, memory.len, flags))) {4853 switch (errno(system.msync(memory.ptr, memory.len, flags))) {
4818 .SUCCESS => return,4854 .SUCCESS => return,
4819 .PERM => return error.PermissionDenied,4855 .PERM => return error.PermissionDenied,
...@@ -7135,7 +7171,7 @@ pub const MincoreError = error{...@@ -7135,7 +7171,7 @@ pub const MincoreError = error{
7135} || UnexpectedError;7171} || UnexpectedError;
71367172
7137/// Determine whether pages are resident in memory.7173/// Determine whether pages are resident in memory.
7138pub fn mincore(ptr: [*]align(mem.page_size) u8, length: usize, vec: [*]u8) MincoreError!void {7174pub fn mincore(ptr: [*]align(page_size_min) u8, length: usize, vec: [*]u8) MincoreError!void {
7139 return switch (errno(system.mincore(ptr, length, vec))) {7175 return switch (errno(system.mincore(ptr, length, vec))) {
7140 .SUCCESS => {},7176 .SUCCESS => {},
7141 .AGAIN => error.SystemResources,7177 .AGAIN => error.SystemResources,
...@@ -7181,7 +7217,7 @@ pub const MadviseError = error{...@@ -7181,7 +7217,7 @@ pub const MadviseError = error{
71817217
7182/// Give advice about use of memory.7218/// Give advice about use of memory.
7183/// This syscall is optional and is sometimes configured to be disabled.7219/// This syscall is optional and is sometimes configured to be disabled.
7184pub fn madvise(ptr: [*]align(mem.page_size) u8, length: usize, advice: u32) MadviseError!void {7220pub fn madvise(ptr: [*]align(page_size_min) u8, length: usize, advice: u32) MadviseError!void {
7185 switch (errno(system.madvise(ptr, length, advice))) {7221 switch (errno(system.madvise(ptr, length, advice))) {
7186 .SUCCESS => return,7222 .SUCCESS => return,
7187 .PERM => return error.PermissionDenied,7223 .PERM => return error.PermissionDenied,
lib/std/process.zig+1-1
...@@ -1560,7 +1560,7 @@ pub fn posixGetUserInfo(name: []const u8) !UserInfo {...@@ -1560,7 +1560,7 @@ pub fn posixGetUserInfo(name: []const u8) !UserInfo {
1560 ReadGroupId,1560 ReadGroupId,
1561 };1561 };
15621562
1563 var buf: [std.mem.page_size]u8 = undefined;1563 var buf: [std.heap.page_size_min]u8 = undefined;
1564 var name_index: usize = 0;1564 var name_index: usize = 0;
1565 var state = State.Start;1565 var state = State.Start;
1566 var uid: posix.uid_t = 0;1566 var uid: posix.uid_t = 0;
lib/std/start.zig+1-1
...@@ -576,7 +576,7 @@ fn expandStackSize(phdrs: []elf.Phdr) void {...@@ -576,7 +576,7 @@ fn expandStackSize(phdrs: []elf.Phdr) void {
576 switch (phdr.p_type) {576 switch (phdr.p_type) {
577 elf.PT_GNU_STACK => {577 elf.PT_GNU_STACK => {
578 if (phdr.p_memsz == 0) break;578 if (phdr.p_memsz == 0) break;
579 assert(phdr.p_memsz % std.mem.page_size == 0);579 assert(phdr.p_memsz % std.heap.page_size_min == 0);
580580
581 // Silently fail if we are unable to get limits.581 // Silently fail if we are unable to get limits.
582 const limits = std.posix.getrlimit(.STACK) catch break;582 const limits = std.posix.getrlimit(.STACK) catch break;
lib/std/std.zig+7
...@@ -119,6 +119,13 @@ pub const Options = struct {...@@ -119,6 +119,13 @@ pub const Options = struct {
119 args: anytype,119 args: anytype,
120 ) void = log.defaultLog,120 ) void = log.defaultLog,
121121
122 /// Overrides `std.heap.page_size_min`.
123 page_size_min: ?usize = null,
124 /// Overrides `std.heap.page_size_max`.
125 page_size_max: ?usize = null,
126 /// Overrides default implementation for determining OS page size at runtime.
127 queryPageSize: fn () usize = heap.defaultQueryPageSize,
128
122 fmt_max_depth: usize = fmt.default_max_depth,129 fmt_max_depth: usize = fmt.default_max_depth,
123130
124 cryptoRandomSeed: fn (buffer: []u8) void = @import("crypto/tlcsprng.zig").defaultRandomSeed,131 cryptoRandomSeed: fn (buffer: []u8) void = @import("crypto/tlcsprng.zig").defaultRandomSeed,
lib/std/testing.zig+15-9
...@@ -7,21 +7,27 @@ const math = std.math;...@@ -7,21 +7,27 @@ const math = std.math;
7/// Initialized on startup. Read-only after that.7/// Initialized on startup. Read-only after that.
8pub var random_seed: u32 = 0;8pub var random_seed: u32 = 0;
99
10pub const FailingAllocator = @import("testing/failing_allocator.zig").FailingAllocator;10pub const FailingAllocator = @import("testing/FailingAllocator.zig");
11pub const failing_allocator = failing_allocator_instance.allocator();
12var failing_allocator_instance = FailingAllocator.init(base_allocator_instance.allocator(), .{
13 .fail_index = 0,
14});
15var base_allocator_instance = std.heap.FixedBufferAllocator.init("");
1116
12/// This should only be used in temporary test programs.17/// This should only be used in temporary test programs.
13pub const allocator = allocator_instance.allocator();18pub const allocator = allocator_instance.allocator();
14pub var allocator_instance: std.heap.GeneralPurposeAllocator(.{}) = b: {19pub var allocator_instance: std.heap.GeneralPurposeAllocator(.{
15 if (!builtin.is_test)20 .stack_trace_frames = if (std.debug.sys_can_stack_trace) 10 else 0,
16 @compileError("Cannot use testing allocator outside of test block");21 .resize_stack_traces = true,
22 // A unique value so that when a default-constructed
23 // GeneralPurposeAllocator is incorrectly passed to testing allocator, or
24 // vice versa, panic occurs.
25 .canary = @truncate(0x2731e675c3a701ba),
26}) = b: {
27 if (!builtin.is_test) @compileError("testing allocator used when not testing");
17 break :b .init;28 break :b .init;
18};29};
1930
20pub const failing_allocator = failing_allocator_instance.allocator();
21pub var failing_allocator_instance = FailingAllocator.init(base_allocator_instance.allocator(), .{ .fail_index = 0 });
22
23pub var base_allocator_instance = std.heap.FixedBufferAllocator.init("");
24
25/// TODO https://github.com/ziglang/zig/issues/573831/// TODO https://github.com/ziglang/zig/issues/5738
26pub var log_level = std.log.Level.warn;32pub var log_level = std.log.Level.warn;
2733
lib/std/testing/FailingAllocator.zig created+161
...@@ -0,0 +1,161 @@
1//! Allocator that fails after N allocations, useful for making sure out of
2//! memory conditions are handled correctly.
3//!
4//! To use this, first initialize it and get an allocator with
5//!
6//! `const failing_allocator = &FailingAllocator.init(<allocator>,
7//! <config>).allocator;`
8//!
9//! Then use `failing_allocator` anywhere you would have used a
10//! different allocator.
11const std = @import("../std.zig");
12const mem = std.mem;
13const FailingAllocator = @This();
14
15alloc_index: usize,
16resize_index: usize,
17internal_allocator: mem.Allocator,
18allocated_bytes: usize,
19freed_bytes: usize,
20allocations: usize,
21deallocations: usize,
22stack_addresses: [num_stack_frames]usize,
23has_induced_failure: bool,
24fail_index: usize,
25resize_fail_index: usize,
26
27const num_stack_frames = if (std.debug.sys_can_stack_trace) 16 else 0;
28
29pub const Config = struct {
30 /// The number of successful allocations you can expect from this allocator.
31 /// The next allocation will fail. For example, with `fail_index` equal to
32 /// 2, the following test will pass:
33 ///
34 /// var a = try failing_alloc.create(i32);
35 /// var b = try failing_alloc.create(i32);
36 /// testing.expectError(error.OutOfMemory, failing_alloc.create(i32));
37 fail_index: usize = std.math.maxInt(usize),
38
39 /// Number of successful resizes to expect from this allocator. The next resize will fail.
40 resize_fail_index: usize = std.math.maxInt(usize),
41};
42
43pub fn init(internal_allocator: mem.Allocator, config: Config) FailingAllocator {
44 return FailingAllocator{
45 .internal_allocator = internal_allocator,
46 .alloc_index = 0,
47 .resize_index = 0,
48 .allocated_bytes = 0,
49 .freed_bytes = 0,
50 .allocations = 0,
51 .deallocations = 0,
52 .stack_addresses = undefined,
53 .has_induced_failure = false,
54 .fail_index = config.fail_index,
55 .resize_fail_index = config.resize_fail_index,
56 };
57}
58
59pub fn allocator(self: *FailingAllocator) mem.Allocator {
60 return .{
61 .ptr = self,
62 .vtable = &.{
63 .alloc = alloc,
64 .resize = resize,
65 .remap = remap,
66 .free = free,
67 },
68 };
69}
70
71fn alloc(
72 ctx: *anyopaque,
73 len: usize,
74 alignment: mem.Alignment,
75 return_address: usize,
76) ?[*]u8 {
77 const self: *FailingAllocator = @ptrCast(@alignCast(ctx));
78 if (self.alloc_index == self.fail_index) {
79 if (!self.has_induced_failure) {
80 @memset(&self.stack_addresses, 0);
81 var stack_trace = std.builtin.StackTrace{
82 .instruction_addresses = &self.stack_addresses,
83 .index = 0,
84 };
85 std.debug.captureStackTrace(return_address, &stack_trace);
86 self.has_induced_failure = true;
87 }
88 return null;
89 }
90 const result = self.internal_allocator.rawAlloc(len, alignment, return_address) orelse
91 return null;
92 self.allocated_bytes += len;
93 self.allocations += 1;
94 self.alloc_index += 1;
95 return result;
96}
97
98fn resize(
99 ctx: *anyopaque,
100 memory: []u8,
101 alignment: mem.Alignment,
102 new_len: usize,
103 ra: usize,
104) bool {
105 const self: *FailingAllocator = @ptrCast(@alignCast(ctx));
106 if (self.resize_index == self.resize_fail_index)
107 return false;
108 if (!self.internal_allocator.rawResize(memory, alignment, new_len, ra))
109 return false;
110 if (new_len < memory.len) {
111 self.freed_bytes += memory.len - new_len;
112 } else {
113 self.allocated_bytes += new_len - memory.len;
114 }
115 self.resize_index += 1;
116 return true;
117}
118
119fn remap(
120 ctx: *anyopaque,
121 memory: []u8,
122 alignment: mem.Alignment,
123 new_len: usize,
124 ra: usize,
125) ?[*]u8 {
126 const self: *FailingAllocator = @ptrCast(@alignCast(ctx));
127 if (self.resize_index == self.resize_fail_index) return null;
128 const new_ptr = self.internal_allocator.rawRemap(memory, alignment, new_len, ra) orelse return null;
129 if (new_len < memory.len) {
130 self.freed_bytes += memory.len - new_len;
131 } else {
132 self.allocated_bytes += new_len - memory.len;
133 }
134 self.resize_index += 1;
135 return new_ptr;
136}
137
138fn free(
139 ctx: *anyopaque,
140 old_mem: []u8,
141 alignment: mem.Alignment,
142 ra: usize,
143) void {
144 const self: *FailingAllocator = @ptrCast(@alignCast(ctx));
145 self.internal_allocator.rawFree(old_mem, alignment, ra);
146 self.deallocations += 1;
147 self.freed_bytes += old_mem.len;
148}
149
150/// Only valid once `has_induced_failure == true`
151pub fn getStackTrace(self: *FailingAllocator) std.builtin.StackTrace {
152 std.debug.assert(self.has_induced_failure);
153 var len: usize = 0;
154 while (len < self.stack_addresses.len and self.stack_addresses[len] != 0) {
155 len += 1;
156 }
157 return .{
158 .instruction_addresses = &self.stack_addresses,
159 .index = len,
160 };
161}
lib/std/testing/failing_allocator.zig deleted-142
...@@ -1,142 +0,0 @@
1const std = @import("../std.zig");
2const mem = std.mem;
3
4pub const Config = struct {
5 /// The number of successful allocations you can expect from this allocator.
6 /// The next allocation will fail. For example, with `fail_index` equal to
7 /// 2, the following test will pass:
8 ///
9 /// var a = try failing_alloc.create(i32);
10 /// var b = try failing_alloc.create(i32);
11 /// testing.expectError(error.OutOfMemory, failing_alloc.create(i32));
12 fail_index: usize = std.math.maxInt(usize),
13
14 /// Number of successful resizes to expect from this allocator. The next resize will fail.
15 resize_fail_index: usize = std.math.maxInt(usize),
16};
17
18/// Allocator that fails after N allocations, useful for making sure out of
19/// memory conditions are handled correctly.
20///
21/// To use this, first initialize it and get an allocator with
22///
23/// `const failing_allocator = &FailingAllocator.init(<allocator>,
24/// <config>).allocator;`
25///
26/// Then use `failing_allocator` anywhere you would have used a
27/// different allocator.
28pub const FailingAllocator = struct {
29 alloc_index: usize,
30 resize_index: usize,
31 internal_allocator: mem.Allocator,
32 allocated_bytes: usize,
33 freed_bytes: usize,
34 allocations: usize,
35 deallocations: usize,
36 stack_addresses: [num_stack_frames]usize,
37 has_induced_failure: bool,
38 fail_index: usize,
39 resize_fail_index: usize,
40
41 const num_stack_frames = if (std.debug.sys_can_stack_trace) 16 else 0;
42
43 pub fn init(internal_allocator: mem.Allocator, config: Config) FailingAllocator {
44 return FailingAllocator{
45 .internal_allocator = internal_allocator,
46 .alloc_index = 0,
47 .resize_index = 0,
48 .allocated_bytes = 0,
49 .freed_bytes = 0,
50 .allocations = 0,
51 .deallocations = 0,
52 .stack_addresses = undefined,
53 .has_induced_failure = false,
54 .fail_index = config.fail_index,
55 .resize_fail_index = config.resize_fail_index,
56 };
57 }
58
59 pub fn allocator(self: *FailingAllocator) mem.Allocator {
60 return .{
61 .ptr = self,
62 .vtable = &.{
63 .alloc = alloc,
64 .resize = resize,
65 .free = free,
66 },
67 };
68 }
69
70 fn alloc(
71 ctx: *anyopaque,
72 len: usize,
73 log2_ptr_align: u8,
74 return_address: usize,
75 ) ?[*]u8 {
76 const self: *FailingAllocator = @ptrCast(@alignCast(ctx));
77 if (self.alloc_index == self.fail_index) {
78 if (!self.has_induced_failure) {
79 @memset(&self.stack_addresses, 0);
80 var stack_trace = std.builtin.StackTrace{
81 .instruction_addresses = &self.stack_addresses,
82 .index = 0,
83 };
84 std.debug.captureStackTrace(return_address, &stack_trace);
85 self.has_induced_failure = true;
86 }
87 return null;
88 }
89 const result = self.internal_allocator.rawAlloc(len, log2_ptr_align, return_address) orelse
90 return null;
91 self.allocated_bytes += len;
92 self.allocations += 1;
93 self.alloc_index += 1;
94 return result;
95 }
96
97 fn resize(
98 ctx: *anyopaque,
99 old_mem: []u8,
100 log2_old_align: u8,
101 new_len: usize,
102 ra: usize,
103 ) bool {
104 const self: *FailingAllocator = @ptrCast(@alignCast(ctx));
105 if (self.resize_index == self.resize_fail_index)
106 return false;
107 if (!self.internal_allocator.rawResize(old_mem, log2_old_align, new_len, ra))
108 return false;
109 if (new_len < old_mem.len) {
110 self.freed_bytes += old_mem.len - new_len;
111 } else {
112 self.allocated_bytes += new_len - old_mem.len;
113 }
114 self.resize_index += 1;
115 return true;
116 }
117
118 fn free(
119 ctx: *anyopaque,
120 old_mem: []u8,
121 log2_old_align: u8,
122 ra: usize,
123 ) void {
124 const self: *FailingAllocator = @ptrCast(@alignCast(ctx));
125 self.internal_allocator.rawFree(old_mem, log2_old_align, ra);
126 self.deallocations += 1;
127 self.freed_bytes += old_mem.len;
128 }
129
130 /// Only valid once `has_induced_failure == true`
131 pub fn getStackTrace(self: *FailingAllocator) std.builtin.StackTrace {
132 std.debug.assert(self.has_induced_failure);
133 var len: usize = 0;
134 while (len < self.stack_addresses.len and self.stack_addresses[len] != 0) {
135 len += 1;
136 }
137 return .{
138 .instruction_addresses = &self.stack_addresses,
139 .index = len,
140 };
141 }
142};
lib/std/zip.zig+1-1
...@@ -162,7 +162,7 @@ pub fn decompress(...@@ -162,7 +162,7 @@ pub fn decompress(
162 var total_uncompressed: u64 = 0;162 var total_uncompressed: u64 = 0;
163 switch (method) {163 switch (method) {
164 .store => {164 .store => {
165 var buf: [std.mem.page_size]u8 = undefined;165 var buf: [4096]u8 = undefined;
166 while (true) {166 while (true) {
167 const len = try reader.read(&buf);167 const len = try reader.read(&buf);
168 if (len == 0) break;168 if (len == 0) break;
src/Package/Fetch.zig+1-1
...@@ -1249,7 +1249,7 @@ fn unzip(f: *Fetch, out_dir: fs.Dir, reader: anytype) RunError!UnpackResult {...@@ -1249,7 +1249,7 @@ fn unzip(f: *Fetch, out_dir: fs.Dir, reader: anytype) RunError!UnpackResult {
1249 .{@errorName(err)},1249 .{@errorName(err)},
1250 ));1250 ));
1251 defer zip_file.close();1251 defer zip_file.close();
1252 var buf: [std.mem.page_size]u8 = undefined;1252 var buf: [4096]u8 = undefined;
1253 while (true) {1253 while (true) {
1254 const len = reader.readAll(&buf) catch |err| return f.fail(f.location_tok, try eb.printString(1254 const len = reader.readAll(&buf) catch |err| return f.fail(f.location_tok, try eb.printString(
1255 "read zip stream failed: {s}",1255 "read zip stream failed: {s}",
test/compare_output.zig-43
...@@ -493,49 +493,6 @@ pub fn addCases(cases: *tests.CompareOutputContext) void {...@@ -493,49 +493,6 @@ pub fn addCases(cases: *tests.CompareOutputContext) void {
493 \\493 \\
494 );494 );
495495
496 // It is required to override the log function in order to print to stdout instead of stderr
497 cases.add("std.heap.LoggingAllocator logs to std.log",
498 \\const std = @import("std");
499 \\
500 \\pub const std_options: std.Options = .{
501 \\ .log_level = .debug,
502 \\ .logFn = log,
503 \\};
504 \\
505 \\pub fn main() !void {
506 \\ var allocator_buf: [10]u8 = undefined;
507 \\ const fba = std.heap.FixedBufferAllocator.init(&allocator_buf);
508 \\ var fba_wrapped = std.mem.validationWrap(fba);
509 \\ var logging_allocator = std.heap.loggingAllocator(fba_wrapped.allocator());
510 \\ const allocator = logging_allocator.allocator();
511 \\
512 \\ var a = try allocator.alloc(u8, 10);
513 \\ try std.testing.expect(allocator.resize(a, 5));
514 \\ a = a[0..5];
515 \\ try std.testing.expect(a.len == 5);
516 \\ try std.testing.expect(!allocator.resize(a, 20));
517 \\ allocator.free(a);
518 \\}
519 \\
520 \\pub fn log(
521 \\ comptime level: std.log.Level,
522 \\ comptime scope: @TypeOf(.EnumLiteral),
523 \\ comptime format: []const u8,
524 \\ args: anytype,
525 \\) void {
526 \\ const level_txt = comptime level.asText();
527 \\ const prefix2 = if (scope == .default) ": " else "(" ++ @tagName(scope) ++ "): ";
528 \\ const stdout = std.io.getStdOut().writer();
529 \\ nosuspend stdout.print(level_txt ++ prefix2 ++ format ++ "\n", args) catch return;
530 \\}
531 ,
532 \\debug: alloc - success - len: 10, ptr_align: 0
533 \\debug: shrink - success - 10 to 5, buf_align: 0
534 \\error: expand - failure - 5 to 20, buf_align: 0
535 \\debug: free - len: 5
536 \\
537 );
538
539 cases.add("valid carriage return example", "const io = @import(\"std\").io;\r\n" ++ // Testing CRLF line endings are valid496 cases.add("valid carriage return example", "const io = @import(\"std\").io;\r\n" ++ // Testing CRLF line endings are valid
540 "\r\n" ++497 "\r\n" ++
541 "pub \r fn main() void {\r\n" ++ // Testing isolated carriage return as whitespace is valid498 "pub \r fn main() void {\r\n" ++ // Testing isolated carriage return as whitespace is valid