Commit 6a6e72fff820fb641aa1b00700f6835430dae72e | zig

author	Andrew Kelley <andrew@ziglang.org> 2025-02-07 06:21:51-08:00
committer	GitHub <noreply@github.com> 2025-02-07 06:21:51-08:00
log	6a6e72fff820fb641aa1b00700f6835430dae72e
tree	ea70863e08ba9167cfe954287691cce98716d918
parent	8ad0732954df80f0f9a0248525c2bded7e82ba27
parent	b8f5cfed457726a77082b7ffe6672b6066c0a66e
signature	Signed by PGP key B5690EEEBB952194

Merge pull request #20511 from archbirdplus

runtime page size detection
rework GeneralPurposeAllocator to reduce active mapping count
Allocator VTable API update

44 files changed, 2981 insertions(+), 2710 deletions(-)

lib/fuzzer.zig+1-1

...	...	@@ -480,7 +480,7 @@ pub const MemoryMappedList = struct {
480	480	/// of this ArrayList in accordance with the respective documentation. In
481	481	/// all cases, "invalidated" means that the memory has been passed to this
482	482	/// allocator's resize or free function.
483		items: []align(std.mem.page_size) volatile u8,
	483	items: []align(std.heap.page_size_min) volatile u8,
484	484	/// How many bytes this list can hold without allocating additional memory.
485	485	capacity: usize,
486	486

lib/std/Build/Fuzz/WebServer.zig+1-1

...	...	@@ -41,7 +41,7 @@ const fuzzer_arch_os_abi = "wasm32-freestanding";
41	41	const fuzzer_cpu_features = "baseline+atomics+bulk_memory+multivalue+mutable_globals+nontrapping_fptoint+reference_types+sign_ext";
42	42
43	43	const CoverageMap = struct {
44		mapped_memory: []align(std.mem.page_size) const u8,
	44	mapped_memory: []align(std.heap.page_size_min) const u8,
45	45	coverage: Coverage,
46	46	source_locations: []Coverage.SourceLocation,
47	47	/// Elements are indexes into `source_locations` pointing to the unit tests that are being fuzz tested.

lib/std/Thread.zig+3-3

...	...	@@ -769,7 +769,7 @@ const PosixThreadImpl = struct {
769	769	// Use the same set of parameters used by the libc-less impl.
770	770	const stack_size = @max(config.stack_size, 16 * 1024);
771	771	assert(c.pthread_attr_setstacksize(&attr, stack_size) == .SUCCESS);
772		assert(c.pthread_attr_setguardsize(&attr, std.mem.page_size) == .SUCCESS);
	772	assert(c.pthread_attr_setguardsize(&attr, std.heap.pageSize()) == .SUCCESS);
773	773
774	774	var handle: c.pthread_t = undefined;
775	775	switch (c.pthread_create(
...	...	@@ -1155,7 +1155,7 @@ const LinuxThreadImpl = struct {
1155	1155	completion: Completion = Completion.init(.running),
1156	1156	child_tid: std.atomic.Value(i32) = std.atomic.Value(i32).init(1),
1157	1157	parent_tid: i32 = undefined,
1158		mapped: []align(std.mem.page_size) u8,
	1158	mapped: []align(std.heap.page_size_min) u8,
1159	1159
1160	1160	/// Calls `munmap(mapped.ptr, mapped.len)` then `exit(1)` without touching the stack (which lives in `mapped.ptr`).
1161	1161	/// Ported over from musl libc's pthread detached implementation:
...	...	@@ -1362,7 +1362,7 @@ const LinuxThreadImpl = struct {
1362	1362	};
1363	1363
1364	1364	fn spawn(config: SpawnConfig, comptime f: anytype, args: anytype) !Impl {
1365		const page_size = std.mem.page_size;
	1365	const page_size = std.heap.pageSize();
1366	1366	const Args = @TypeOf(args);
1367	1367	const Instance = struct {
1368	1368	fn_args: Args,

lib/std/array_list.zig+18-19

...	...	@@ -105,21 +105,19 @@ pub fn ArrayListAligned(comptime T: type, comptime alignment: ?u29) type {
105	105	return result;
106	106	}
107	107
108		/// The caller owns the returned memory. Empties this ArrayList,
109		/// Its capacity is cleared, making deinit() safe but unnecessary to call.
	108	/// The caller owns the returned memory. Empties this ArrayList.
	109	/// Its capacity is cleared, making `deinit` safe but unnecessary to call.
110	110	pub fn toOwnedSlice(self: *Self) Allocator.Error!Slice {
111	111	const allocator = self.allocator;
112	112
113	113	const old_memory = self.allocatedSlice();
114		if (allocator.resize(old_memory, self.items.len)) {
115		const result = self.items;
	114	if (allocator.remap(old_memory, self.items.len)) \|new_items\| {
116	115	self.* = init(allocator);
117		return result;
	116	return new_items;
118	117	}
119	118
120	119	const new_memory = try allocator.alignedAlloc(T, alignment, self.items.len);
121	120	@memcpy(new_memory, self.items);
122		@memset(self.items, undefined);
123	121	self.clearAndFree();
124	122	return new_memory;
125	123	}
...	...	@@ -185,8 +183,9 @@ pub fn ArrayListAligned(comptime T: type, comptime alignment: ?u29) type {
185	183	// extra capacity.
186	184	const new_capacity = growCapacity(self.capacity, new_len);
187	185	const old_memory = self.allocatedSlice();
188		if (self.allocator.resize(old_memory, new_capacity)) {
189		self.capacity = new_capacity;
	186	if (self.allocator.remap(old_memory, new_capacity)) \|new_memory\| {
	187	self.items.ptr = new_memory.ptr;
	188	self.capacity = new_memory.len;
190	189	return addManyAtAssumeCapacity(self, index, count);
191	190	}
192	191
...	...	@@ -468,8 +467,9 @@ pub fn ArrayListAligned(comptime T: type, comptime alignment: ?u29) type {
468	467	// the allocator implementation would pointlessly copy our
469	468	// extra capacity.
470	469	const old_memory = self.allocatedSlice();
471		if (self.allocator.resize(old_memory, new_capacity)) {
472		self.capacity = new_capacity;
	470	if (self.allocator.remap(old_memory, new_capacity)) \|new_memory\| {
	471	self.items.ptr = new_memory.ptr;
	472	self.capacity = new_memory.len;
473	473	} else {
474	474	const new_memory = try self.allocator.alignedAlloc(T, alignment, new_capacity);
475	475	@memcpy(new_memory[0..self.items.len], self.items);
...	...	@@ -707,15 +707,13 @@ pub fn ArrayListAlignedUnmanaged(comptime T: type, comptime alignment: ?u29) typ
707	707	/// Its capacity is cleared, making deinit() safe but unnecessary to call.
708	708	pub fn toOwnedSlice(self: *Self, allocator: Allocator) Allocator.Error!Slice {
709	709	const old_memory = self.allocatedSlice();
710		if (allocator.resize(old_memory, self.items.len)) {
711		const result = self.items;
	710	if (allocator.remap(old_memory, self.items.len)) \|new_items\| {
712	711	self.* = .empty;
713		return result;
	712	return new_items;
714	713	}
715	714
716	715	const new_memory = try allocator.alignedAlloc(T, alignment, self.items.len);
717	716	@memcpy(new_memory, self.items);
718		@memset(self.items, undefined);
719	717	self.clearAndFree(allocator);
720	718	return new_memory;
721	719	}
...	...	@@ -1031,9 +1029,9 @@ pub fn ArrayListAlignedUnmanaged(comptime T: type, comptime alignment: ?u29) typ
1031	1029	}
1032	1030
1033	1031	const old_memory = self.allocatedSlice();
1034		if (allocator.resize(old_memory, new_len)) {
1035		self.capacity = new_len;
1036		self.items.len = new_len;
	1032	if (allocator.remap(old_memory, new_len)) \|new_items\| {
	1033	self.capacity = new_items.len;
	1034	self.items = new_items;
1037	1035	return;
1038	1036	}
1039	1037
...	...	@@ -1099,8 +1097,9 @@ pub fn ArrayListAlignedUnmanaged(comptime T: type, comptime alignment: ?u29) typ
1099	1097	// the allocator implementation would pointlessly copy our
1100	1098	// extra capacity.
1101	1099	const old_memory = self.allocatedSlice();
1102		if (allocator.resize(old_memory, new_capacity)) {
1103		self.capacity = new_capacity;
	1100	if (allocator.remap(old_memory, new_capacity)) \|new_memory\| {
	1101	self.items.ptr = new_memory.ptr;
	1102	self.capacity = new_memory.len;
1104	1103	} else {
1105	1104	const new_memory = try allocator.alignedAlloc(T, alignment, new_capacity);
1106	1105	@memcpy(new_memory[0..self.items.len], self.items);

lib/std/c.zig+50-7

...	...	@@ -3,7 +3,7 @@ const builtin = @import("builtin");
3	3	const c = @This();
4	4	const maxInt = std.math.maxInt;
5	5	const assert = std.debug.assert;
6		const page_size = std.mem.page_size;
	6	const page_size = std.heap.page_size_min;
7	7	const native_abi = builtin.abi;
8	8	const native_arch = builtin.cpu.arch;
9	9	const native_os = builtin.os.tag;
...	...	@@ -2227,6 +2227,39 @@ pub const SC = switch (native_os) {
2227	2227	.linux => linux.SC,
2228	2228	else => void,
2229	2229	};
	2230
	2231	pub const _SC = switch (native_os) {
	2232	.driverkit, .ios, .macos, .tvos, .visionos, .watchos => enum(c_int) {
	2233	PAGESIZE = 29,
	2234	},
	2235	.dragonfly => enum(c_int) {
	2236	PAGESIZE = 47,
	2237	},
	2238	.freebsd => enum(c_int) {
	2239	PAGESIZE = 47,
	2240	},
	2241	.fuchsia => enum(c_int) {
	2242	PAGESIZE = 30,
	2243	},
	2244	.haiku => enum(c_int) {
	2245	PAGESIZE = 27,
	2246	},
	2247	.linux => enum(c_int) {
	2248	PAGESIZE = 30,
	2249	},
	2250	.netbsd => enum(c_int) {
	2251	PAGESIZE = 28,
	2252	},
	2253	.openbsd => enum(c_int) {
	2254	PAGESIZE = 28,
	2255	},
	2256	.solaris, .illumos => enum(c_int) {
	2257	PAGESIZE = 11,
	2258	NPROCESSORS_ONLN = 15,
	2259	},
	2260	else => void,
	2261	};
	2262
2230	2263	pub const SEEK = switch (native_os) {
2231	2264	.linux => linux.SEEK,
2232	2265	.emscripten => emscripten.SEEK,
...	...	@@ -7834,6 +7867,11 @@ pub const MAP = switch (native_os) {
7834	7867	else => void,
7835	7868	};
7836	7869
	7870	pub const MREMAP = switch (native_os) {
	7871	.linux => linux.MREMAP,
	7872	else => void,
	7873	};
	7874
7837	7875	/// Used by libc to communicate failure. Not actually part of the underlying syscall.
7838	7876	pub const MAP_FAILED: *anyopaque = @ptrFromInt(maxInt(usize));
7839	7877
...	...	@@ -9232,7 +9270,7 @@ pub extern "c" fn getpwnam(name: [:0]const u8) ?passwd;
9232	9270	pub extern "c" fn getpwuid(uid: uid_t) ?*passwd;
9233	9271	pub extern "c" fn getrlimit64(resource: rlimit_resource, rlim: *rlimit) c_int;
9234	9272	pub extern "c" fn lseek64(fd: fd_t, offset: i64, whence: c_int) i64;
9235		pub extern "c" fn mmap64(addr: ?align(std.mem.page_size) anyopaque, len: usize, prot: c_uint, flags: c_uint, fd: fd_t, offset: i64) anyopaque;
	9273	pub extern "c" fn mmap64(addr: ?align(page_size) anyopaque, len: usize, prot: c_uint, flags: c_uint, fd: fd_t, offset: i64) anyopaque;
9236	9274	pub extern "c" fn open64(path: [*:0]const u8, oflag: O, ...) c_int;
9237	9275	pub extern "c" fn openat64(fd: c_int, path: [*:0]const u8, oflag: O, ...) c_int;
9238	9276	pub extern "c" fn pread64(fd: fd_t, buf: [*]u8, nbyte: usize, offset: i64) isize;
...	...	@@ -9324,13 +9362,13 @@ pub extern "c" fn signalfd(fd: fd_t, mask: *const sigset_t, flags: u32) c_int;
9324	9362
9325	9363	pub extern "c" fn prlimit(pid: pid_t, resource: rlimit_resource, new_limit: const rlimit, old_limit: rlimit) c_int;
9326	9364	pub extern "c" fn mincore(
9327		addr: *align(std.mem.page_size) anyopaque,
	9365	addr: *align(page_size) anyopaque,
9328	9366	length: usize,
9329	9367	vec: [*]u8,
9330	9368	) c_int;
9331	9369
9332	9370	pub extern "c" fn madvise(
9333		addr: *align(std.mem.page_size) anyopaque,
	9371	addr: *align(page_size) anyopaque,
9334	9372	length: usize,
9335	9373	advice: u32,
9336	9374	) c_int;
...	...	@@ -9428,6 +9466,10 @@ pub const posix_memalign = switch (native_os) {
9428	9466	.dragonfly, .netbsd, .freebsd, .solaris, .openbsd, .linux, .macos, .ios, .tvos, .watchos, .visionos => private.posix_memalign,
9429	9467	else => {},
9430	9468	};
	9469	pub const sysconf = switch (native_os) {
	9470	.solaris => solaris.sysconf,
	9471	else => private.sysconf,
	9472	};
9431	9473
9432	9474	pub const sf_hdtr = switch (native_os) {
9433	9475	.freebsd, .macos, .ios, .tvos, .watchos, .visionos => extern struct {
...	...	@@ -9471,6 +9513,7 @@ pub extern "c" fn write(fd: fd_t, buf: [*]const u8, nbyte: usize) isize;
9471	9513	pub extern "c" fn pwrite(fd: fd_t, buf: [*]const u8, nbyte: usize, offset: off_t) isize;
9472	9514	pub extern "c" fn mmap(addr: ?align(page_size) anyopaque, len: usize, prot: c_uint, flags: MAP, fd: fd_t, offset: off_t) anyopaque;
9473	9515	pub extern "c" fn munmap(addr: *align(page_size) const anyopaque, len: usize) c_int;
	9516	pub extern "c" fn mremap(addr: ?align(page_size) const anyopaque, old_len: usize, new_len: usize, flags: MREMAP, ...) anyopaque;
9474	9517	pub extern "c" fn mprotect(addr: *align(page_size) anyopaque, len: usize, prot: c_uint) c_int;
9475	9518	pub extern "c" fn link(oldpath: [:0]const u8, newpath: [:0]const u8) c_int;
9476	9519	pub extern "c" fn linkat(oldfd: fd_t, oldpath: [:0]const u8, newfd: fd_t, newpath: [:0]const u8, flags: c_int) c_int;
...	...	@@ -9823,7 +9866,6 @@ pub const SCM = solaris.SCM;
9823	9866	pub const SETCONTEXT = solaris.SETCONTEXT;
9824	9867	pub const SETUSTACK = solaris.GETUSTACK;
9825	9868	pub const SFD = solaris.SFD;
9826		pub const _SC = solaris._SC;
9827	9869	pub const cmsghdr = solaris.cmsghdr;
9828	9870	pub const ctid_t = solaris.ctid_t;
9829	9871	pub const file_obj = solaris.file_obj;
...	...	@@ -9840,7 +9882,6 @@ pub const priority = solaris.priority;
9840	9882	pub const procfs = solaris.procfs;
9841	9883	pub const projid_t = solaris.projid_t;
9842	9884	pub const signalfd_siginfo = solaris.signalfd_siginfo;
9843		pub const sysconf = solaris.sysconf;
9844	9885	pub const taskid_t = solaris.taskid_t;
9845	9886	pub const zoneid_t = solaris.zoneid_t;
9846	9887
...	...	@@ -9997,6 +10038,7 @@ pub const host_t = darwin.host_t;
9997	10038	pub const ipc_space_t = darwin.ipc_space_t;
9998	10039	pub const ipc_space_port_t = darwin.ipc_space_port_t;
9999	10040	pub const kern_return_t = darwin.kern_return_t;
	10041	pub const vm_size_t = darwin.vm_size_t;
10000	10042	pub const kevent64 = darwin.kevent64;
10001	10043	pub const kevent64_s = darwin.kevent64_s;
10002	10044	pub const mach_absolute_time = darwin.mach_absolute_time;
...	...	@@ -10168,6 +10210,7 @@ const private = struct {
10168	10210	extern "c" fn socket(domain: c_uint, sock_type: c_uint, protocol: c_uint) c_int;
10169	10211	extern "c" fn stat(noalias path: [:0]const u8, noalias buf: Stat) c_int;
10170	10212	extern "c" fn sigaltstack(ss: ?stack_t, old_ss: ?stack_t) c_int;
	10213	extern "c" fn sysconf(sc: c_int) c_long;
10171	10214
10172	10215	extern "c" fn pthread_setname_np(thread: pthread_t, name: [*:0]const u8) c_int;
10173	10216	extern "c" fn getcontext(ucp: *ucontext_t) c_int;
...	...	@@ -10202,7 +10245,7 @@ const private = struct {
10202	10245	extern "c" fn __getrusage50(who: c_int, usage: *rusage) c_int;
10203	10246	extern "c" fn __gettimeofday50(noalias tv: ?timeval, noalias tz: ?timezone) c_int;
10204	10247	extern "c" fn __libc_thr_yield() c_int;
10205		extern "c" fn __msync13(addr: *align(std.mem.page_size) const anyopaque, len: usize, flags: c_int) c_int;
	10248	extern "c" fn __msync13(addr: *align(page_size) const anyopaque, len: usize, flags: c_int) c_int;
10206	10249	extern "c" fn __nanosleep50(rqtp: const timespec, rmtp: ?timespec) c_int;
10207	10250	extern "c" fn __sigaction14(sig: c_int, noalias act: ?const Sigaction, noalias oact: ?Sigaction) c_int;
10208	10251	extern "c" fn __sigfillset14(set: ?*sigset_t) void;

lib/std/c/solaris.zig-4

...	...	@@ -154,10 +154,6 @@ pub const AF_SUN = struct {
154	154	pub const NOPLM = 0x00000004;
155	155	};
156	156
157		pub const _SC = struct {
158		pub const NPROCESSORS_ONLN = 15;
159		};
160
161	157	pub const procfs = struct {
162	158	pub const misc_header = extern struct {
163	159	size: u32,

lib/std/crypto/tlcsprng.zig+5-5

...	...	@@ -42,7 +42,7 @@ var install_atfork_handler = std.once(struct {
42	42	}
43	43	}.do);
44	44
45		threadlocal var wipe_mem: []align(mem.page_size) u8 = &[_]u8{};
	45	threadlocal var wipe_mem: []align(std.heap.page_size_min) u8 = &[_]u8{};
46	46
47	47	fn tlsCsprngFill(_: *anyopaque, buffer: []u8) void {
48	48	if (os_has_arc4random) {
...	...	@@ -77,7 +77,7 @@ fn tlsCsprngFill(_: *anyopaque, buffer: []u8) void {
77	77	} else {
78	78	// Use a static thread-local buffer.
79	79	const S = struct {
80		threadlocal var buf: Context align(mem.page_size) = .{
	80	threadlocal var buf: Context align(std.heap.page_size_min) = .{
81	81	.init_state = .uninitialized,
82	82	.rng = undefined,
83	83	};
...	...	@@ -85,7 +85,7 @@ fn tlsCsprngFill(_: *anyopaque, buffer: []u8) void {
85	85	wipe_mem = mem.asBytes(&S.buf);
86	86	}
87	87	}
88		const ctx = @as(*Context, @ptrCast(wipe_mem.ptr));
	88	const ctx: *Context = @ptrCast(wipe_mem.ptr);
89	89
90	90	switch (ctx.init_state) {
91	91	.uninitialized => {
...	...	@@ -141,7 +141,7 @@ fn childAtForkHandler() callconv(.c) void {
141	141	}
142	142
143	143	fn fillWithCsprng(buffer: []u8) void {
144		const ctx = @as(*Context, @ptrCast(wipe_mem.ptr));
	144	const ctx: *Context = @ptrCast(wipe_mem.ptr);
145	145	return ctx.rng.fill(buffer);
146	146	}
147	147
...	...	@@ -157,7 +157,7 @@ fn initAndFill(buffer: []u8) void {
157	157	// the `std.options.cryptoRandomSeed` function is provided.
158	158	std.options.cryptoRandomSeed(&seed);
159	159
160		const ctx = @as(*Context, @ptrCast(wipe_mem.ptr));
	160	const ctx: *Context = @ptrCast(wipe_mem.ptr);
161	161	ctx.rng = Rng.init(seed);
162	162	std.crypto.secureZero(u8, &seed);
163	163

lib/std/debug.zig+8-8

...	...	@@ -1134,7 +1134,7 @@ fn printLineFromFileAnyOs(out_stream: anytype, source_location: SourceLocation)
1134	1134	defer f.close();
1135	1135	// TODO fstat and make sure that the file has the correct size
1136	1136
1137		var buf: [mem.page_size]u8 = undefined;
	1137	var buf: [4096]u8 = undefined;
1138	1138	var amt_read = try f.read(buf[0..]);
1139	1139	const line_start = seek: {
1140	1140	var current_line_start: usize = 0;
...	...	@@ -1237,7 +1237,7 @@ test printLineFromFileAnyOs {
1237	1237
1238	1238	const overlap = 10;
1239	1239	var writer = file.writer();
1240		try writer.writeByteNTimes('a', mem.page_size - overlap);
	1240	try writer.writeByteNTimes('a', std.heap.page_size_min - overlap);
1241	1241	try writer.writeByte('\n');
1242	1242	try writer.writeByteNTimes('a', overlap);
1243	1243
...	...	@@ -1252,10 +1252,10 @@ test printLineFromFileAnyOs {
1252	1252	defer allocator.free(path);
1253	1253
1254	1254	var writer = file.writer();
1255		try writer.writeByteNTimes('a', mem.page_size);
	1255	try writer.writeByteNTimes('a', std.heap.page_size_max);
1256	1256
1257	1257	try printLineFromFileAnyOs(output_stream, .{ .file_name = path, .line = 1, .column = 0 });
1258		try expectEqualStrings(("a" ** mem.page_size) ++ "\n", output.items);
	1258	try expectEqualStrings(("a" ** std.heap.page_size_max) ++ "\n", output.items);
1259	1259	output.clearRetainingCapacity();
1260	1260	}
1261	1261	{
...	...	@@ -1265,18 +1265,18 @@ test printLineFromFileAnyOs {
1265	1265	defer allocator.free(path);
1266	1266
1267	1267	var writer = file.writer();
1268		try writer.writeByteNTimes('a', 3 * mem.page_size);
	1268	try writer.writeByteNTimes('a', 3 * std.heap.page_size_max);
1269	1269
1270	1270	try expectError(error.EndOfFile, printLineFromFileAnyOs(output_stream, .{ .file_name = path, .line = 2, .column = 0 }));
1271	1271
1272	1272	try printLineFromFileAnyOs(output_stream, .{ .file_name = path, .line = 1, .column = 0 });
1273		try expectEqualStrings(("a" ** (3 * mem.page_size)) ++ "\n", output.items);
	1273	try expectEqualStrings(("a" ** (3 * std.heap.page_size_max)) ++ "\n", output.items);
1274	1274	output.clearRetainingCapacity();
1275	1275
1276	1276	try writer.writeAll("a\na");
1277	1277
1278	1278	try printLineFromFileAnyOs(output_stream, .{ .file_name = path, .line = 1, .column = 0 });
1279		try expectEqualStrings(("a" ** (3 * mem.page_size)) ++ "a\n", output.items);
	1279	try expectEqualStrings(("a" ** (3 * std.heap.page_size_max)) ++ "a\n", output.items);
1280	1280	output.clearRetainingCapacity();
1281	1281
1282	1282	try printLineFromFileAnyOs(output_stream, .{ .file_name = path, .line = 2, .column = 0 });
...	...	@@ -1290,7 +1290,7 @@ test printLineFromFileAnyOs {
1290	1290	defer allocator.free(path);
1291	1291
1292	1292	var writer = file.writer();
1293		const real_file_start = 3 * mem.page_size;
	1293	const real_file_start = 3 * std.heap.page_size_min;
1294	1294	try writer.writeByteNTimes('\n', real_file_start);
1295	1295	try writer.writeAll("abc\ndef");
1296	1296

lib/std/debug/Dwarf.zig+5-5

...	...	@@ -2120,8 +2120,8 @@ fn pcRelBase(field_ptr: usize, pc_rel_offset: i64) !usize {
2120	2120	pub const ElfModule = struct {
2121	2121	base_address: usize,
2122	2122	dwarf: Dwarf,
2123		mapped_memory: []align(std.mem.page_size) const u8,
2124		external_mapped_memory: ?[]align(std.mem.page_size) const u8,
	2123	mapped_memory: []align(std.heap.page_size_min) const u8,
	2124	external_mapped_memory: ?[]align(std.heap.page_size_min) const u8,
2125	2125
2126	2126	pub fn deinit(self: *@This(), allocator: Allocator) void {
2127	2127	self.dwarf.deinit(allocator);
...	...	@@ -2167,11 +2167,11 @@ pub const ElfModule = struct {
2167	2167	/// sections from an external file.
2168	2168	pub fn load(
2169	2169	gpa: Allocator,
2170		mapped_mem: []align(std.mem.page_size) const u8,
	2170	mapped_mem: []align(std.heap.page_size_min) const u8,
2171	2171	build_id: ?[]const u8,
2172	2172	expected_crc: ?u32,
2173	2173	parent_sections: *Dwarf.SectionArray,
2174		parent_mapped_mem: ?[]align(std.mem.page_size) const u8,
	2174	parent_mapped_mem: ?[]align(std.heap.page_size_min) const u8,
2175	2175	elf_filename: ?[]const u8,
2176	2176	) LoadError!Dwarf.ElfModule {
2177	2177	if (expected_crc) \|crc\| if (crc != std.hash.crc.Crc32.hash(mapped_mem)) return error.InvalidDebugInfo;
...	...	@@ -2423,7 +2423,7 @@ pub const ElfModule = struct {
2423	2423	build_id: ?[]const u8,
2424	2424	expected_crc: ?u32,
2425	2425	parent_sections: *Dwarf.SectionArray,
2426		parent_mapped_mem: ?[]align(std.mem.page_size) const u8,
	2426	parent_mapped_mem: ?[]align(std.heap.page_size_min) const u8,
2427	2427	) LoadError!Dwarf.ElfModule {
2428	2428	const elf_file = elf_file_path.root_dir.handle.openFile(elf_file_path.sub_path, .{}) catch \|err\| switch (err) {
2429	2429	error.FileNotFound => return missing(),

lib/std/debug/Info.zig-1

...	...	@@ -10,7 +10,6 @@ const std = @import("../std.zig");
10	10	const Allocator = std.mem.Allocator;
11	11	const Path = std.Build.Cache.Path;
12	12	const Dwarf = std.debug.Dwarf;
13		const page_size = std.mem.page_size;
14	13	const assert = std.debug.assert;
15	14	const Coverage = std.debug.Coverage;
16	15	const SourceLocation = std.debug.Coverage.SourceLocation;

lib/std/debug/MemoryAccessor.zig+5-4

...	...	@@ -7,7 +7,7 @@ const native_os = builtin.os.tag;
7	7	const std = @import("../std.zig");
8	8	const posix = std.posix;
9	9	const File = std.fs.File;
10		const page_size = std.mem.page_size;
	10	const page_size_min = std.heap.page_size_min;
11	11
12	12	const MemoryAccessor = @This();
13	13
...	...	@@ -93,9 +93,10 @@ pub fn isValidMemory(address: usize) bool {
93	93	// We are unable to determine validity of memory for freestanding targets
94	94	if (native_os == .freestanding or native_os == .other or native_os == .uefi) return true;
95	95
96		const aligned_address = address & ~@as(usize, @intCast((page_size - 1)));
	96	const page_size = std.heap.pageSize();
	97	const aligned_address = address & ~(page_size - 1);
97	98	if (aligned_address == 0) return false;
98		const aligned_memory = @as([*]align(page_size) u8, @ptrFromInt(aligned_address))[0..page_size];
	99	const aligned_memory = @as([*]align(page_size_min) u8, @ptrFromInt(aligned_address))[0..page_size];
99	100
100	101	if (native_os == .windows) {
101	102	const windows = std.os.windows;
...	...	@@ -104,7 +105,7 @@ pub fn isValidMemory(address: usize) bool {
104	105
105	106	// The only error this function can throw is ERROR_INVALID_PARAMETER.
106	107	// supply an address that invalid i'll be thrown.
107		const rc = windows.VirtualQuery(aligned_memory, &memory_info, aligned_memory.len) catch {
	108	const rc = windows.VirtualQuery(@ptrCast(aligned_memory), &memory_info, aligned_memory.len) catch {
108	109	return false;
109	110	};
110	111

lib/std/debug/SelfInfo.zig+3-3

...	...	@@ -504,7 +504,7 @@ pub const Module = switch (native_os) {
504	504	.macos, .ios, .watchos, .tvos, .visionos => struct {
505	505	base_address: usize,
506	506	vmaddr_slide: usize,
507		mapped_memory: []align(mem.page_size) const u8,
	507	mapped_memory: []align(std.heap.page_size_min) const u8,
508	508	symbols: []const MachoSymbol,
509	509	strings: [:0]const u8,
510	510	ofiles: OFileTable,
...	...	@@ -1046,7 +1046,7 @@ pub fn readElfDebugInfo(
1046	1046	build_id: ?[]const u8,
1047	1047	expected_crc: ?u32,
1048	1048	parent_sections: *Dwarf.SectionArray,
1049		parent_mapped_mem: ?[]align(mem.page_size) const u8,
	1049	parent_mapped_mem: ?[]align(std.heap.page_size_min) const u8,
1050	1050	) !Dwarf.ElfModule {
1051	1051	nosuspend {
1052	1052	const elf_file = (if (elf_filename) \|filename\| blk: {
...	...	@@ -1088,7 +1088,7 @@ const MachoSymbol = struct {
1088	1088
1089	1089	/// Takes ownership of file, even on error.
1090	1090	/// TODO it's weird to take ownership even on error, rework this code.
1091		fn mapWholeFile(file: File) ![]align(mem.page_size) const u8 {
	1091	fn mapWholeFile(file: File) ![]align(std.heap.page_size_min) const u8 {
1092	1092	nosuspend {
1093	1093	defer file.close();
1094	1094

lib/std/dynamic_library.zig+7-5

...	...	@@ -143,7 +143,7 @@ pub const ElfDynLib = struct {
143	143	hashtab: [*]posix.Elf_Symndx,
144	144	versym: ?[*]elf.Versym,
145	145	verdef: ?*elf.Verdef,
146		memory: []align(mem.page_size) u8,
	146	memory: []align(std.heap.page_size_min) u8,
147	147
148	148	pub const Error = ElfDynLibError;
149	149
...	...	@@ -219,11 +219,13 @@ pub const ElfDynLib = struct {
219	219	const stat = try file.stat();
220	220	const size = std.math.cast(usize, stat.size) orelse return error.FileTooBig;
221	221
	222	const page_size = std.heap.pageSize();
	223
222	224	// This one is to read the ELF info. We do more mmapping later
223	225	// corresponding to the actual LOAD sections.
224	226	const file_bytes = try posix.mmap(
225	227	null,
226		mem.alignForward(usize, size, mem.page_size),
	228	mem.alignForward(usize, size, page_size),
227	229	posix.PROT.READ,
228	230	.{ .TYPE = .PRIVATE },
229	231	fd,
...	...	@@ -284,10 +286,10 @@ pub const ElfDynLib = struct {
284	286	elf.PT_LOAD => {
285	287	// The VirtAddr may not be page-aligned; in such case there will be
286	288	// extra nonsense mapped before/after the VirtAddr,MemSiz
287		const aligned_addr = (base + ph.p_vaddr) & ~(@as(usize, mem.page_size) - 1);
	289	const aligned_addr = (base + ph.p_vaddr) & ~(@as(usize, page_size) - 1);
288	290	const extra_bytes = (base + ph.p_vaddr) - aligned_addr;
289		const extended_memsz = mem.alignForward(usize, ph.p_memsz + extra_bytes, mem.page_size);
290		const ptr = @as([*]align(mem.page_size) u8, @ptrFromInt(aligned_addr));
	291	const extended_memsz = mem.alignForward(usize, ph.p_memsz + extra_bytes, page_size);
	292	const ptr = @as([*]align(std.heap.page_size_min) u8, @ptrFromInt(aligned_addr));
291	293	const prot = elfToMmapProt(ph.p_flags);
292	294	if ((ph.p_flags & elf.PF_W) == 0) {
293	295	// If it does not need write access, it can be mapped from the fd.

lib/std/fifo.zig+1-1

...	...	@@ -91,7 +91,7 @@ pub fn LinearFifo(
91	91	mem.copyForwards(T, self.buf[0..self.count], self.buf[self.head..][0..self.count]);
92	92	self.head = 0;
93	93	} else {
94		var tmp: [mem.page_size / 2 / @sizeOf(T)]T = undefined;
	94	var tmp: [4096 / 2 / @sizeOf(T)]T = undefined;
95	95
96	96	while (self.head != 0) {
97	97	const n = @min(self.head, tmp.len);

lib/std/hash_map.zig+20

...	...	@@ -413,10 +413,15 @@ pub fn HashMap(
413	413	/// If there is an `Entry` with a matching key, it is deleted from
414	414	/// the hash map, and this function returns true. Otherwise this
415	415	/// function returns false.
	416	///
	417	/// TODO: answer the question in these doc comments, does this
	418	/// increase the unused capacity by one?
416	419	pub fn remove(self: *Self, key: K) bool {
417	420	return self.unmanaged.removeContext(key, self.ctx);
418	421	}
419	422
	423	/// TODO: answer the question in these doc comments, does this
	424	/// increase the unused capacity by one?
420	425	pub fn removeAdapted(self: *Self, key: anytype, ctx: anytype) bool {
421	426	return self.unmanaged.removeAdapted(key, ctx);
422	427	}
...	...	@@ -424,6 +429,9 @@ pub fn HashMap(
424	429	/// Delete the entry with key pointed to by key_ptr from the hash map.
425	430	/// key_ptr is assumed to be a valid pointer to a key that is present
426	431	/// in the hash map.
	432	///
	433	/// TODO: answer the question in these doc comments, does this
	434	/// increase the unused capacity by one?
427	435	pub fn removeByPtr(self: Self, key_ptr: K) void {
428	436	self.unmanaged.removeByPtr(key_ptr);
429	437	}
...	...	@@ -1225,14 +1233,23 @@ pub fn HashMapUnmanaged(
1225	1233	/// If there is an `Entry` with a matching key, it is deleted from
1226	1234	/// the hash map, and this function returns true. Otherwise this
1227	1235	/// function returns false.
	1236	///
	1237	/// TODO: answer the question in these doc comments, does this
	1238	/// increase the unused capacity by one?
1228	1239	pub fn remove(self: *Self, key: K) bool {
1229	1240	if (@sizeOf(Context) != 0)
1230	1241	@compileError("Cannot infer context " ++ @typeName(Context) ++ ", call removeContext instead.");
1231	1242	return self.removeContext(key, undefined);
1232	1243	}
	1244
	1245	/// TODO: answer the question in these doc comments, does this
	1246	/// increase the unused capacity by one?
1233	1247	pub fn removeContext(self: *Self, key: K, ctx: Context) bool {
1234	1248	return self.removeAdapted(key, ctx);
1235	1249	}
	1250
	1251	/// TODO: answer the question in these doc comments, does this
	1252	/// increase the unused capacity by one?
1236	1253	pub fn removeAdapted(self: *Self, key: anytype, ctx: anytype) bool {
1237	1254	if (self.getIndex(key, ctx)) \|idx\| {
1238	1255	self.removeByIndex(idx);
...	...	@@ -1245,6 +1262,9 @@ pub fn HashMapUnmanaged(
1245	1262	/// Delete the entry with key pointed to by key_ptr from the hash map.
1246	1263	/// key_ptr is assumed to be a valid pointer to a key that is present
1247	1264	/// in the hash map.
	1265	///
	1266	/// TODO: answer the question in these doc comments, does this
	1267	/// increase the unused capacity by one?
1248	1268	pub fn removeByPtr(self: Self, key_ptr: K) void {
1249	1269	// TODO: replace with pointer subtraction once supported by zig
1250	1270	// if @sizeOf(K) == 0 then there is at most one item in the hash

lib/std/heap.zig+504-417

...	...	@@ -8,19 +8,20 @@ const c = std.c;
8	8	const Allocator = std.mem.Allocator;
9	9	const windows = std.os.windows;
10	10
11		pub const LoggingAllocator = @import("heap/logging_allocator.zig").LoggingAllocator;
12		pub const loggingAllocator = @import("heap/logging_allocator.zig").loggingAllocator;
13		pub const ScopedLoggingAllocator = @import("heap/logging_allocator.zig").ScopedLoggingAllocator;
14		pub const LogToWriterAllocator = @import("heap/log_to_writer_allocator.zig").LogToWriterAllocator;
15		pub const logToWriterAllocator = @import("heap/log_to_writer_allocator.zig").logToWriterAllocator;
16	11	pub const ArenaAllocator = @import("heap/arena_allocator.zig").ArenaAllocator;
17		pub const GeneralPurposeAllocatorConfig = @import("heap/general_purpose_allocator.zig").Config;
18		pub const GeneralPurposeAllocator = @import("heap/general_purpose_allocator.zig").GeneralPurposeAllocator;
19		pub const Check = @import("heap/general_purpose_allocator.zig").Check;
20	12	pub const WasmAllocator = @import("heap/WasmAllocator.zig");
21	13	pub const PageAllocator = @import("heap/PageAllocator.zig");
22	14	pub const ThreadSafeAllocator = @import("heap/ThreadSafeAllocator.zig");
23	15	pub const SbrkAllocator = @import("heap/sbrk_allocator.zig").SbrkAllocator;
	16	pub const FixedBufferAllocator = @import("heap/FixedBufferAllocator.zig");
	17
	18	pub const DebugAllocatorConfig = @import("heap/debug_allocator.zig").Config;
	19	pub const DebugAllocator = @import("heap/debug_allocator.zig").DebugAllocator;
	20	pub const Check = enum { ok, leak };
	21	/// Deprecated; to be removed after 0.14.0 is tagged.
	22	pub const GeneralPurposeAllocatorConfig = DebugAllocatorConfig;
	23	/// Deprecated; to be removed after 0.14.0 is tagged.
	24	pub const GeneralPurposeAllocator = DebugAllocator;
24	25
25	26	const memory_pool = @import("heap/memory_pool.zig");
26	27	pub const MemoryPool = memory_pool.MemoryPool;
...	...	@@ -29,7 +30,97 @@ pub const MemoryPoolExtra = memory_pool.MemoryPoolExtra;
29	30	pub const MemoryPoolOptions = memory_pool.Options;
30	31
31	32	/// TODO Utilize this on Windows.
32		pub var next_mmap_addr_hint: ?[*]align(mem.page_size) u8 = null;
	33	pub var next_mmap_addr_hint: ?[*]align(page_size_min) u8 = null;
	34
	35	/// comptime-known minimum page size of the target.
	36	///
	37	/// All pointers from `mmap` or `VirtualAlloc` are aligned to at least
	38	/// `page_size_min`, but their actual alignment may be bigger.
	39	///
	40	/// This value can be overridden via `std.options.page_size_min`.
	41	///
	42	/// On many systems, the actual page size can only be determined at runtime
	43	/// with `pageSize`.
	44	pub const page_size_min: usize = std.options.page_size_min orelse (page_size_min_default orelse if (builtin.os.tag == .freestanding or builtin.os.tag == .other)
	45	@compileError("freestanding/other page_size_min must provided with std.options.page_size_min")
	46	else
	47	@compileError(@tagName(builtin.cpu.arch) ++ "-" ++ @tagName(builtin.os.tag) ++ " has unknown page_size_min; populate std.options.page_size_min"));
	48
	49	/// comptime-known maximum page size of the target.
	50	///
	51	/// Targeting a system with a larger page size may require overriding
	52	/// `std.options.page_size_max`, as well as providing a corresponding linker
	53	/// option.
	54	///
	55	/// The actual page size can only be determined at runtime with `pageSize`.
	56	pub const page_size_max: usize = std.options.page_size_max orelse (page_size_max_default orelse if (builtin.os.tag == .freestanding or builtin.os.tag == .other)
	57	@compileError("freestanding/other page_size_max must provided with std.options.page_size_max")
	58	else
	59	@compileError(@tagName(builtin.cpu.arch) ++ "-" ++ @tagName(builtin.os.tag) ++ " has unknown page_size_max; populate std.options.page_size_max"));
	60
	61	/// If the page size is comptime-known, return value is comptime.
	62	/// Otherwise, calls `std.options.queryPageSize` which by default queries the
	63	/// host operating system at runtime.
	64	pub inline fn pageSize() usize {
	65	if (page_size_min == page_size_max) return page_size_min;
	66	return std.options.queryPageSize();
	67	}
	68
	69	test pageSize {
	70	assert(std.math.isPowerOfTwo(pageSize()));
	71	}
	72
	73	/// The default implementation of `std.options.queryPageSize`.
	74	/// Asserts that the page size is within `page_size_min` and `page_size_max`
	75	pub fn defaultQueryPageSize() usize {
	76	const global = struct {
	77	var cached_result: std.atomic.Value(usize) = .init(0);
	78	};
	79	var size = global.cached_result.load(.unordered);
	80	if (size > 0) return size;
	81	size = switch (builtin.os.tag) {
	82	.linux => if (builtin.link_libc) @intCast(std.c.sysconf(@intFromEnum(std.c._SC.PAGESIZE))) else std.os.linux.getauxval(std.elf.AT_PAGESZ),
	83	.driverkit, .ios, .macos, .tvos, .visionos, .watchos => blk: {
	84	const task_port = std.c.mach_task_self();
	85	// mach_task_self may fail "if there are any resource failures or other errors".
	86	if (task_port == std.c.TASK_NULL)
	87	break :blk 0;
	88	var info_count = std.c.TASK_VM_INFO_COUNT;
	89	var vm_info: std.c.task_vm_info_data_t = undefined;
	90	vm_info.page_size = 0;
	91	_ = std.c.task_info(
	92	task_port,
	93	std.c.TASK_VM_INFO,
	94	@as(std.c.task_info_t, @ptrCast(&vm_info)),
	95	&info_count,
	96	);
	97	assert(vm_info.page_size != 0);
	98	break :blk @intCast(vm_info.page_size);
	99	},
	100	.windows => blk: {
	101	var info: std.os.windows.SYSTEM_INFO = undefined;
	102	std.os.windows.kernel32.GetSystemInfo(&info);
	103	break :blk info.dwPageSize;
	104	},
	105	else => if (builtin.link_libc)
	106	@intCast(std.c.sysconf(@intFromEnum(std.c._SC.PAGESIZE)))
	107	else if (builtin.os.tag == .freestanding or builtin.os.tag == .other)
	108	@compileError("unsupported target: freestanding/other")
	109	else
	110	@compileError("pageSize on " ++ @tagName(builtin.cpu.arch) ++ "-" ++ @tagName(builtin.os.tag) ++ " is not supported without linking libc, using the default implementation"),
	111	};
	112
	113	assert(size >= page_size_min);
	114	assert(size <= page_size_max);
	115	global.cached_result.store(size, .unordered);
	116
	117	return size;
	118	}
	119
	120	test defaultQueryPageSize {
	121	if (builtin.cpu.arch.isWasm()) return error.SkipZigTest;
	122	assert(std.math.isPowerOfTwo(defaultQueryPageSize()));
	123	}
33	124
34	125	const CAllocator = struct {
35	126	comptime {
...	...	@@ -38,6 +129,13 @@ const CAllocator = struct {
38	129	}
39	130	}
40	131
	132	const vtable: Allocator.VTable = .{
	133	.alloc = alloc,
	134	.resize = resize,
	135	.remap = remap,
	136	.free = free,
	137	};
	138
41	139	pub const supports_malloc_size = @TypeOf(malloc_size) != void;
42	140	pub const malloc_size = if (@TypeOf(c.malloc_size) != void)
43	141	c.malloc_size
...	...	@@ -53,29 +151,29 @@ const CAllocator = struct {
53	151	};
54	152
55	153	fn getHeader(ptr: []u8) [*]u8 {
56		return @as([]u8, @ptrFromInt(@intFromPtr(ptr) - @sizeOf(usize)));
	154	return @alignCast(@ptrCast(ptr - @sizeOf(usize)));
57	155	}
58	156
59		fn alignedAlloc(len: usize, log2_align: u8) ?[*]u8 {
60		const alignment = @as(usize, 1) << @as(Allocator.Log2Align, @intCast(log2_align));
	157	fn alignedAlloc(len: usize, alignment: mem.Alignment) ?[*]u8 {
	158	const alignment_bytes = alignment.toByteUnits();
61	159	if (supports_posix_memalign) {
62	160	// The posix_memalign only accepts alignment values that are a
63	161	// multiple of the pointer size
64		const eff_alignment = @max(alignment, @sizeOf(usize));
	162	const effective_alignment = @max(alignment_bytes, @sizeOf(usize));
65	163
66	164	var aligned_ptr: ?*anyopaque = undefined;
67		if (c.posix_memalign(&aligned_ptr, eff_alignment, len) != 0)
	165	if (c.posix_memalign(&aligned_ptr, effective_alignment, len) != 0)
68	166	return null;
69	167
70		return @as([*]u8, @ptrCast(aligned_ptr));
	168	return @ptrCast(aligned_ptr);
71	169	}
72	170
73	171	// Thin wrapper around regular malloc, overallocate to account for
74	172	// alignment padding and store the original malloc()'ed pointer before
75	173	// the aligned address.
76		const unaligned_ptr = @as([*]u8, @ptrCast(c.malloc(len + alignment - 1 + @sizeOf(usize)) orelse return null));
	174	const unaligned_ptr = @as([*]u8, @ptrCast(c.malloc(len + alignment_bytes - 1 + @sizeOf(usize)) orelse return null));
77	175	const unaligned_addr = @intFromPtr(unaligned_ptr);
78		const aligned_addr = mem.alignForward(usize, unaligned_addr + @sizeOf(usize), alignment);
	176	const aligned_addr = mem.alignForward(usize, unaligned_addr + @sizeOf(usize), alignment_bytes);
79	177	const aligned_ptr = unaligned_ptr + (aligned_addr - unaligned_addr);
80	178	getHeader(aligned_ptr).* = unaligned_ptr;
81	179
...	...	@@ -104,22 +202,22 @@ const CAllocator = struct {
104	202	fn alloc(
105	203	_: *anyopaque,
106	204	len: usize,
107		log2_align: u8,
	205	alignment: mem.Alignment,
108	206	return_address: usize,
109	207	) ?[*]u8 {
110	208	_ = return_address;
111	209	assert(len > 0);
112		return alignedAlloc(len, log2_align);
	210	return alignedAlloc(len, alignment);
113	211	}
114	212
115	213	fn resize(
116	214	_: *anyopaque,
117	215	buf: []u8,
118		log2_buf_align: u8,
	216	alignment: mem.Alignment,
119	217	new_len: usize,
120	218	return_address: usize,
121	219	) bool {
122		_ = log2_buf_align;
	220	_ = alignment;
123	221	_ = return_address;
124	222	if (new_len <= buf.len) {
125	223	return true;
...	...	@@ -133,13 +231,25 @@ const CAllocator = struct {
133	231	return false;
134	232	}
135	233
	234	fn remap(
	235	context: *anyopaque,
	236	memory: []u8,
	237	alignment: mem.Alignment,
	238	new_len: usize,
	239	return_address: usize,
	240	) ?[*]u8 {
	241	// realloc would potentially return a new allocation that does not
	242	// respect the original alignment.
	243	return if (resize(context, memory, alignment, new_len, return_address)) memory.ptr else null;
	244	}
	245
136	246	fn free(
137	247	_: *anyopaque,
138	248	buf: []u8,
139		log2_buf_align: u8,
	249	alignment: mem.Alignment,
140	250	return_address: usize,
141	251	) void {
142		_ = log2_buf_align;
	252	_ = alignment;
143	253	_ = return_address;
144	254	alignedFree(buf.ptr);
145	255	}
...	...	@@ -148,78 +258,83 @@ const CAllocator = struct {
148	258	/// Supports the full Allocator interface, including alignment, and exploiting
149	259	/// `malloc_usable_size` if available. For an allocator that directly calls
150	260	/// `malloc`/`free`, see `raw_c_allocator`.
151		pub const c_allocator = Allocator{
	261	pub const c_allocator: Allocator = .{
152	262	.ptr = undefined,
153		.vtable = &c_allocator_vtable,
154		};
155		const c_allocator_vtable = Allocator.VTable{
156		.alloc = CAllocator.alloc,
157		.resize = CAllocator.resize,
158		.free = CAllocator.free,
	263	.vtable = &CAllocator.vtable,
159	264	};
160	265
161		/// Asserts allocations are within `@alignOf(std.c.max_align_t)` and directly calls
162		/// `malloc`/`free`. Does not attempt to utilize `malloc_usable_size`.
	266	/// Asserts allocations are within `@alignOf(std.c.max_align_t)` and directly
	267	/// calls `malloc`/`free`. Does not attempt to utilize `malloc_usable_size`.
163	268	/// This allocator is safe to use as the backing allocator with
164		/// `ArenaAllocator` for example and is more optimal in such a case
165		/// than `c_allocator`.
166		pub const raw_c_allocator = Allocator{
	269	/// `ArenaAllocator` for example and is more optimal in such a case than
	270	/// `c_allocator`.
	271	pub const raw_c_allocator: Allocator = .{
167	272	.ptr = undefined,
168	273	.vtable = &raw_c_allocator_vtable,
169	274	};
170		const raw_c_allocator_vtable = Allocator.VTable{
	275	const raw_c_allocator_vtable: Allocator.VTable = .{
171	276	.alloc = rawCAlloc,
172	277	.resize = rawCResize,
	278	.remap = rawCRemap,
173	279	.free = rawCFree,
174	280	};
175	281
176	282	fn rawCAlloc(
177		_: *anyopaque,
	283	context: *anyopaque,
178	284	len: usize,
179		log2_ptr_align: u8,
180		ret_addr: usize,
	285	alignment: mem.Alignment,
	286	return_address: usize,
181	287	) ?[*]u8 {
182		_ = ret_addr;
183		assert(log2_ptr_align <= comptime std.math.log2_int(usize, @alignOf(std.c.max_align_t)));
	288	_ = context;
	289	_ = return_address;
	290	assert(alignment.compare(.lte, comptime .fromByteUnits(@alignOf(std.c.max_align_t))));
184	291	// Note that this pointer cannot be aligncasted to max_align_t because if
185	292	// len is < max_align_t then the alignment can be smaller. For example, if
186	293	// max_align_t is 16, but the user requests 8 bytes, there is no built-in
187	294	// type in C that is size 8 and has 16 byte alignment, so the alignment may
188	295	// be 8 bytes rather than 16. Similarly if only 1 byte is requested, malloc
189	296	// is allowed to return a 1-byte aligned pointer.
190		return @as(?[*]u8, @ptrCast(c.malloc(len)));
	297	return @ptrCast(c.malloc(len));
191	298	}
192	299
193	300	fn rawCResize(
194		_: *anyopaque,
195		buf: []u8,
196		log2_old_align: u8,
	301	context: *anyopaque,
	302	memory: []u8,
	303	alignment: mem.Alignment,
197	304	new_len: usize,
198		ret_addr: usize,
	305	return_address: usize,
199	306	) bool {
200		_ = log2_old_align;
201		_ = ret_addr;
202
203		if (new_len <= buf.len)
204		return true;
205
206		if (CAllocator.supports_malloc_size) {
207		const full_len = CAllocator.malloc_size(buf.ptr);
208		if (new_len <= full_len) return true;
209		}
210
	307	_ = context;
	308	_ = memory;
	309	_ = alignment;
	310	_ = new_len;
	311	_ = return_address;
211	312	return false;
212	313	}
213	314
	315	fn rawCRemap(
	316	context: *anyopaque,
	317	memory: []u8,
	318	alignment: mem.Alignment,
	319	new_len: usize,
	320	return_address: usize,
	321	) ?[*]u8 {
	322	_ = context;
	323	_ = alignment;
	324	_ = return_address;
	325	return @ptrCast(c.realloc(memory.ptr, new_len));
	326	}
	327
214	328	fn rawCFree(
215		_: *anyopaque,
216		buf: []u8,
217		log2_old_align: u8,
218		ret_addr: usize,
	329	context: *anyopaque,
	330	memory: []u8,
	331	alignment: mem.Alignment,
	332	return_address: usize,
219	333	) void {
220		_ = log2_old_align;
221		_ = ret_addr;
222		c.free(buf.ptr);
	334	_ = context;
	335	_ = alignment;
	336	_ = return_address;
	337	c.free(memory.ptr);
223	338	}
224	339
225	340	/// On operating systems that support memory mapping, this allocator makes a
...	...	@@ -253,252 +368,6 @@ pub const wasm_allocator: Allocator = .{
253	368	.vtable = &WasmAllocator.vtable,
254	369	};
255	370
256		/// Verifies that the adjusted length will still map to the full length
257		pub fn alignPageAllocLen(full_len: usize, len: usize) usize {
258		const aligned_len = mem.alignAllocLen(full_len, len);
259		assert(mem.alignForward(usize, aligned_len, mem.page_size) == full_len);
260		return aligned_len;
261		}
262
263		pub const HeapAllocator = switch (builtin.os.tag) {
264		.windows => struct {
265		heap_handle: ?HeapHandle,
266
267		const HeapHandle = windows.HANDLE;
268
269		pub fn init() HeapAllocator {
270		return HeapAllocator{
271		.heap_handle = null,
272		};
273		}
274
275		pub fn allocator(self: *HeapAllocator) Allocator {
276		return .{
277		.ptr = self,
278		.vtable = &.{
279		.alloc = alloc,
280		.resize = resize,
281		.free = free,
282		},
283		};
284		}
285
286		pub fn deinit(self: *HeapAllocator) void {
287		if (self.heap_handle) \|heap_handle\| {
288		windows.HeapDestroy(heap_handle);
289		}
290		}
291
292		fn getRecordPtr(buf: []u8) *align(1) usize {
293		return @as(*align(1) usize, @ptrFromInt(@intFromPtr(buf.ptr) + buf.len));
294		}
295
296		fn alloc(
297		ctx: *anyopaque,
298		n: usize,
299		log2_ptr_align: u8,
300		return_address: usize,
301		) ?[*]u8 {
302		_ = return_address;
303		const self: *HeapAllocator = @ptrCast(@alignCast(ctx));
304
305		const ptr_align = @as(usize, 1) << @as(Allocator.Log2Align, @intCast(log2_ptr_align));
306		const amt = n + ptr_align - 1 + @sizeOf(usize);
307		const optional_heap_handle = @atomicLoad(?HeapHandle, &self.heap_handle, .seq_cst);
308		const heap_handle = optional_heap_handle orelse blk: {
309		const options = if (builtin.single_threaded) windows.HEAP_NO_SERIALIZE else 0;
310		const hh = windows.kernel32.HeapCreate(options, amt, 0) orelse return null;
311		const other_hh = @cmpxchgStrong(?HeapHandle, &self.heap_handle, null, hh, .seq_cst, .seq_cst) orelse break :blk hh;
312		windows.HeapDestroy(hh);
313		break :blk other_hh.?; // can't be null because of the cmpxchg
314		};
315		const ptr = windows.kernel32.HeapAlloc(heap_handle, 0, amt) orelse return null;
316		const root_addr = @intFromPtr(ptr);
317		const aligned_addr = mem.alignForward(usize, root_addr, ptr_align);
318		const buf = @as([*]u8, @ptrFromInt(aligned_addr))[0..n];
319		getRecordPtr(buf).* = root_addr;
320		return buf.ptr;
321		}
322
323		fn resize(
324		ctx: *anyopaque,
325		buf: []u8,
326		log2_buf_align: u8,
327		new_size: usize,
328		return_address: usize,
329		) bool {
330		_ = log2_buf_align;
331		_ = return_address;
332		const self: *HeapAllocator = @ptrCast(@alignCast(ctx));
333
334		const root_addr = getRecordPtr(buf).*;
335		const align_offset = @intFromPtr(buf.ptr) - root_addr;
336		const amt = align_offset + new_size + @sizeOf(usize);
337		const new_ptr = windows.kernel32.HeapReAlloc(
338		self.heap_handle.?,
339		windows.HEAP_REALLOC_IN_PLACE_ONLY,
340		@as(*anyopaque, @ptrFromInt(root_addr)),
341		amt,
342		) orelse return false;
343		assert(new_ptr == @as(*anyopaque, @ptrFromInt(root_addr)));
344		getRecordPtr(buf.ptr[0..new_size]).* = root_addr;
345		return true;
346		}
347
348		fn free(
349		ctx: *anyopaque,
350		buf: []u8,
351		log2_buf_align: u8,
352		return_address: usize,
353		) void {
354		_ = log2_buf_align;
355		_ = return_address;
356		const self: *HeapAllocator = @ptrCast(@alignCast(ctx));
357		windows.HeapFree(self.heap_handle.?, 0, @as(anyopaque, @ptrFromInt(getRecordPtr(buf).)));
358		}
359		},
360		else => @compileError("Unsupported OS"),
361		};
362
363		fn sliceContainsPtr(container: []u8, ptr: [*]u8) bool {
364		return @intFromPtr(ptr) >= @intFromPtr(container.ptr) and
365		@intFromPtr(ptr) < (@intFromPtr(container.ptr) + container.len);
366		}
367
368		fn sliceContainsSlice(container: []u8, slice: []u8) bool {
369		return @intFromPtr(slice.ptr) >= @intFromPtr(container.ptr) and
370		(@intFromPtr(slice.ptr) + slice.len) <= (@intFromPtr(container.ptr) + container.len);
371		}
372
373		pub const FixedBufferAllocator = struct {
374		end_index: usize,
375		buffer: []u8,
376
377		pub fn init(buffer: []u8) FixedBufferAllocator {
378		return FixedBufferAllocator{
379		.buffer = buffer,
380		.end_index = 0,
381		};
382		}
383
384		/// WARNING using this at the same time as the interface returned by `threadSafeAllocator` is not thread safe
385		pub fn allocator(self: *FixedBufferAllocator) Allocator {
386		return .{
387		.ptr = self,
388		.vtable = &.{
389		.alloc = alloc,
390		.resize = resize,
391		.free = free,
392		},
393		};
394		}
395
396		/// Provides a lock free thread safe `Allocator` interface to the underlying `FixedBufferAllocator`
397		/// WARNING using this at the same time as the interface returned by `allocator` is not thread safe
398		pub fn threadSafeAllocator(self: *FixedBufferAllocator) Allocator {
399		return .{
400		.ptr = self,
401		.vtable = &.{
402		.alloc = threadSafeAlloc,
403		.resize = Allocator.noResize,
404		.free = Allocator.noFree,
405		},
406		};
407		}
408
409		pub fn ownsPtr(self: FixedBufferAllocator, ptr: []u8) bool {
410		return sliceContainsPtr(self.buffer, ptr);
411		}
412
413		pub fn ownsSlice(self: *FixedBufferAllocator, slice: []u8) bool {
414		return sliceContainsSlice(self.buffer, slice);
415		}
416
417		/// NOTE: this will not work in all cases, if the last allocation had an adjusted_index
418		/// then we won't be able to determine what the last allocation was. This is because
419		/// the alignForward operation done in alloc is not reversible.
420		pub fn isLastAllocation(self: *FixedBufferAllocator, buf: []u8) bool {
421		return buf.ptr + buf.len == self.buffer.ptr + self.end_index;
422		}
423
424		fn alloc(ctx: anyopaque, n: usize, log2_ptr_align: u8, ra: usize) ?[]u8 {
425		const self: *FixedBufferAllocator = @ptrCast(@alignCast(ctx));
426		_ = ra;
427		const ptr_align = @as(usize, 1) << @as(Allocator.Log2Align, @intCast(log2_ptr_align));
428		const adjust_off = mem.alignPointerOffset(self.buffer.ptr + self.end_index, ptr_align) orelse return null;
429		const adjusted_index = self.end_index + adjust_off;
430		const new_end_index = adjusted_index + n;
431		if (new_end_index > self.buffer.len) return null;
432		self.end_index = new_end_index;
433		return self.buffer.ptr + adjusted_index;
434		}
435
436		fn resize(
437		ctx: *anyopaque,
438		buf: []u8,
439		log2_buf_align: u8,
440		new_size: usize,
441		return_address: usize,
442		) bool {
443		const self: *FixedBufferAllocator = @ptrCast(@alignCast(ctx));
444		_ = log2_buf_align;
445		_ = return_address;
446		assert(@inComptime() or self.ownsSlice(buf));
447
448		if (!self.isLastAllocation(buf)) {
449		if (new_size > buf.len) return false;
450		return true;
451		}
452
453		if (new_size <= buf.len) {
454		const sub = buf.len - new_size;
455		self.end_index -= sub;
456		return true;
457		}
458
459		const add = new_size - buf.len;
460		if (add + self.end_index > self.buffer.len) return false;
461
462		self.end_index += add;
463		return true;
464		}
465
466		fn free(
467		ctx: *anyopaque,
468		buf: []u8,
469		log2_buf_align: u8,
470		return_address: usize,
471		) void {
472		const self: *FixedBufferAllocator = @ptrCast(@alignCast(ctx));
473		_ = log2_buf_align;
474		_ = return_address;
475		assert(@inComptime() or self.ownsSlice(buf));
476
477		if (self.isLastAllocation(buf)) {
478		self.end_index -= buf.len;
479		}
480		}
481
482		fn threadSafeAlloc(ctx: anyopaque, n: usize, log2_ptr_align: u8, ra: usize) ?[]u8 {
483		const self: *FixedBufferAllocator = @ptrCast(@alignCast(ctx));
484		_ = ra;
485		const ptr_align = @as(usize, 1) << @as(Allocator.Log2Align, @intCast(log2_ptr_align));
486		var end_index = @atomicLoad(usize, &self.end_index, .seq_cst);
487		while (true) {
488		const adjust_off = mem.alignPointerOffset(self.buffer.ptr + end_index, ptr_align) orelse return null;
489		const adjusted_index = end_index + adjust_off;
490		const new_end_index = adjusted_index + n;
491		if (new_end_index > self.buffer.len) return null;
492		end_index = @cmpxchgWeak(usize, &self.end_index, end_index, new_end_index, .seq_cst, .seq_cst) orelse
493		return self.buffer[adjusted_index..new_end_index].ptr;
494		}
495		}
496
497		pub fn reset(self: *FixedBufferAllocator) void {
498		self.end_index = 0;
499		}
500		};
501
502	371	/// Returns a `StackFallbackAllocator` allocating using either a
503	372	/// `FixedBufferAllocator` on an array of size `size` and falling back to
504	373	/// `fallback_allocator` if that fails.
...	...	@@ -537,6 +406,7 @@ pub fn StackFallbackAllocator(comptime size: usize) type {
537	406	.vtable = &.{
538	407	.alloc = alloc,
539	408	.resize = resize,
	409	.remap = remap,
540	410	.free = free,
541	411	},
542	412	};
...	...	@@ -551,40 +421,55 @@ pub fn StackFallbackAllocator(comptime size: usize) type {
551	421	fn alloc(
552	422	ctx: *anyopaque,
553	423	len: usize,
554		log2_ptr_align: u8,
	424	alignment: mem.Alignment,
555	425	ra: usize,
556	426	) ?[*]u8 {
557	427	const self: *Self = @ptrCast(@alignCast(ctx));
558		return FixedBufferAllocator.alloc(&self.fixed_buffer_allocator, len, log2_ptr_align, ra) orelse
559		return self.fallback_allocator.rawAlloc(len, log2_ptr_align, ra);
	428	return FixedBufferAllocator.alloc(&self.fixed_buffer_allocator, len, alignment, ra) orelse
	429	return self.fallback_allocator.rawAlloc(len, alignment, ra);
560	430	}
561	431
562	432	fn resize(
563	433	ctx: *anyopaque,
564	434	buf: []u8,
565		log2_buf_align: u8,
	435	alignment: mem.Alignment,
566	436	new_len: usize,
567	437	ra: usize,
568	438	) bool {
569	439	const self: *Self = @ptrCast(@alignCast(ctx));
570	440	if (self.fixed_buffer_allocator.ownsPtr(buf.ptr)) {
571		return FixedBufferAllocator.resize(&self.fixed_buffer_allocator, buf, log2_buf_align, new_len, ra);
	441	return FixedBufferAllocator.resize(&self.fixed_buffer_allocator, buf, alignment, new_len, ra);
572	442	} else {
573		return self.fallback_allocator.rawResize(buf, log2_buf_align, new_len, ra);
	443	return self.fallback_allocator.rawResize(buf, alignment, new_len, ra);
	444	}
	445	}
	446
	447	fn remap(
	448	context: *anyopaque,
	449	memory: []u8,
	450	alignment: mem.Alignment,
	451	new_len: usize,
	452	return_address: usize,
	453	) ?[*]u8 {
	454	const self: *Self = @ptrCast(@alignCast(context));
	455	if (self.fixed_buffer_allocator.ownsPtr(memory.ptr)) {
	456	return FixedBufferAllocator.remap(&self.fixed_buffer_allocator, memory, alignment, new_len, return_address);
	457	} else {
	458	return self.fallback_allocator.rawRemap(memory, alignment, new_len, return_address);
574	459	}
575	460	}
576	461
577	462	fn free(
578	463	ctx: *anyopaque,
579	464	buf: []u8,
580		log2_buf_align: u8,
	465	alignment: mem.Alignment,
581	466	ra: usize,
582	467	) void {
583	468	const self: *Self = @ptrCast(@alignCast(ctx));
584	469	if (self.fixed_buffer_allocator.ownsPtr(buf.ptr)) {
585		return FixedBufferAllocator.free(&self.fixed_buffer_allocator, buf, log2_buf_align, ra);
	470	return FixedBufferAllocator.free(&self.fixed_buffer_allocator, buf, alignment, ra);
586	471	} else {
587		return self.fallback_allocator.rawFree(buf, log2_buf_align, ra);
	472	return self.fallback_allocator.rawFree(buf, alignment, ra);
588	473	}
589	474	}
590	475	};
...	...	@@ -605,7 +490,7 @@ test "raw_c_allocator" {
605	490	}
606	491	}
607	492
608		test "PageAllocator" {
	493	test PageAllocator {
609	494	const allocator = page_allocator;
610	495	try testAllocator(allocator);
611	496	try testAllocatorAligned(allocator);
...	...	@@ -615,35 +500,19 @@ test "PageAllocator" {
615	500	}
616	501
617	502	if (builtin.os.tag == .windows) {
618		const slice = try allocator.alignedAlloc(u8, mem.page_size, 128);
	503	const slice = try allocator.alignedAlloc(u8, page_size_min, 128);
619	504	slice[0] = 0x12;
620	505	slice[127] = 0x34;
621	506	allocator.free(slice);
622	507	}
623	508	{
624		var buf = try allocator.alloc(u8, mem.page_size + 1);
	509	var buf = try allocator.alloc(u8, pageSize() + 1);
625	510	defer allocator.free(buf);
626	511	buf = try allocator.realloc(buf, 1); // shrink past the page boundary
627	512	}
628	513	}
629	514
630		test "HeapAllocator" {
631		if (builtin.os.tag == .windows) {
632		// https://github.com/ziglang/zig/issues/13702
633		if (builtin.cpu.arch == .aarch64) return error.SkipZigTest;
634
635		var heap_allocator = HeapAllocator.init();
636		defer heap_allocator.deinit();
637		const allocator = heap_allocator.allocator();
638
639		try testAllocator(allocator);
640		try testAllocatorAligned(allocator);
641		try testAllocatorLargeAlignment(allocator);
642		try testAllocatorAlignedShrink(allocator);
643		}
644		}
645
646		test "ArenaAllocator" {
	515	test ArenaAllocator {
647	516	var arena_allocator = ArenaAllocator.init(page_allocator);
648	517	defer arena_allocator.deinit();
649	518	const allocator = arena_allocator.allocator();
...	...	@@ -654,38 +523,6 @@ test "ArenaAllocator" {
654	523	try testAllocatorAlignedShrink(allocator);
655	524	}
656	525
657		var test_fixed_buffer_allocator_memory: [800000 * @sizeOf(u64)]u8 = undefined;
658		test "FixedBufferAllocator" {
659		var fixed_buffer_allocator = mem.validationWrap(FixedBufferAllocator.init(test_fixed_buffer_allocator_memory[0..]));
660		const allocator = fixed_buffer_allocator.allocator();
661
662		try testAllocator(allocator);
663		try testAllocatorAligned(allocator);
664		try testAllocatorLargeAlignment(allocator);
665		try testAllocatorAlignedShrink(allocator);
666		}
667
668		test "FixedBufferAllocator.reset" {
669		var buf: [8]u8 align(@alignOf(u64)) = undefined;
670		var fba = FixedBufferAllocator.init(buf[0..]);
671		const allocator = fba.allocator();
672
673		const X = 0xeeeeeeeeeeeeeeee;
674		const Y = 0xffffffffffffffff;
675
676		const x = try allocator.create(u64);
677		x.* = X;
678		try testing.expectError(error.OutOfMemory, allocator.create(u64));
679
680		fba.reset();
681		const y = try allocator.create(u64);
682		y.* = Y;
683
684		// we expect Y to have overwritten X.
685		try testing.expect(x.* == y.*);
686		try testing.expect(y.* == Y);
687		}
688
689	526	test "StackFallbackAllocator" {
690	527	{
691	528	var stack_allocator = stackFallback(4096, std.testing.allocator);
...	...	@@ -705,46 +542,6 @@ test "StackFallbackAllocator" {
705	542	}
706	543	}
707	544
708		test "FixedBufferAllocator Reuse memory on realloc" {
709		var small_fixed_buffer: [10]u8 = undefined;
710		// check if we re-use the memory
711		{
712		var fixed_buffer_allocator = FixedBufferAllocator.init(small_fixed_buffer[0..]);
713		const allocator = fixed_buffer_allocator.allocator();
714
715		const slice0 = try allocator.alloc(u8, 5);
716		try testing.expect(slice0.len == 5);
717		const slice1 = try allocator.realloc(slice0, 10);
718		try testing.expect(slice1.ptr == slice0.ptr);
719		try testing.expect(slice1.len == 10);
720		try testing.expectError(error.OutOfMemory, allocator.realloc(slice1, 11));
721		}
722		// check that we don't re-use the memory if it's not the most recent block
723		{
724		var fixed_buffer_allocator = FixedBufferAllocator.init(small_fixed_buffer[0..]);
725		const allocator = fixed_buffer_allocator.allocator();
726
727		var slice0 = try allocator.alloc(u8, 2);
728		slice0[0] = 1;
729		slice0[1] = 2;
730		const slice1 = try allocator.alloc(u8, 2);
731		const slice2 = try allocator.realloc(slice0, 4);
732		try testing.expect(slice0.ptr != slice2.ptr);
733		try testing.expect(slice1.ptr != slice2.ptr);
734		try testing.expect(slice2[0] == 1);
735		try testing.expect(slice2[1] == 2);
736		}
737		}
738
739		test "Thread safe FixedBufferAllocator" {
740		var fixed_buffer_allocator = FixedBufferAllocator.init(test_fixed_buffer_allocator_memory[0..]);
741
742		try testAllocator(fixed_buffer_allocator.threadSafeAllocator());
743		try testAllocatorAligned(fixed_buffer_allocator.threadSafeAllocator());
744		try testAllocatorLargeAlignment(fixed_buffer_allocator.threadSafeAllocator());
745		try testAllocatorAlignedShrink(fixed_buffer_allocator.threadSafeAllocator());
746		}
747
748	545	/// This one should not try alignments that exceed what C malloc can handle.
749	546	pub fn testAllocator(base_allocator: mem.Allocator) !void {
750	547	var validationAllocator = mem.validationWrap(base_allocator);
...	...	@@ -824,7 +621,7 @@ pub fn testAllocatorLargeAlignment(base_allocator: mem.Allocator) !void {
824	621	var validationAllocator = mem.validationWrap(base_allocator);
825	622	const allocator = validationAllocator.allocator();
826	623
827		const large_align: usize = mem.page_size / 2;
	624	const large_align: usize = page_size_min / 2;
828	625
829	626	var align_mask: usize = undefined;
830	627	align_mask = @shlWithOverflow(~@as(usize, 0), @as(Allocator.Log2Align, @ctz(large_align)))[0];
...	...	@@ -857,7 +654,7 @@ pub fn testAllocatorAlignedShrink(base_allocator: mem.Allocator) !void {
857	654	var fib = FixedBufferAllocator.init(&debug_buffer);
858	655	const debug_allocator = fib.allocator();
859	656
860		const alloc_size = mem.page_size * 2 + 50;
	657	const alloc_size = pageSize() * 2 + 50;
861	658	var slice = try allocator.alignedAlloc(u8, 16, alloc_size);
862	659	defer allocator.free(slice);
863	660
...	...	@@ -866,7 +663,7 @@ pub fn testAllocatorAlignedShrink(base_allocator: mem.Allocator) !void {
866	663	// which is 16 pages, hence the 32. This test may require to increase
867	664	// the size of the allocations feeding the `allocator` parameter if they
868	665	// fail, because of this high over-alignment we want to have.
869		while (@intFromPtr(slice.ptr) == mem.alignForward(usize, @intFromPtr(slice.ptr), mem.page_size * 32)) {
	666	while (@intFromPtr(slice.ptr) == mem.alignForward(usize, @intFromPtr(slice.ptr), pageSize() * 32)) {
870	667	try stuff_to_free.append(slice);
871	668	slice = try allocator.alignedAlloc(u8, 16, alloc_size);
872	669	}
...	...	@@ -881,13 +678,303 @@ pub fn testAllocatorAlignedShrink(base_allocator: mem.Allocator) !void {
881	678	try testing.expect(slice[60] == 0x34);
882	679	}
883	680
	681	const page_size_min_default: ?usize = switch (builtin.os.tag) {
	682	.driverkit, .ios, .macos, .tvos, .visionos, .watchos => switch (builtin.cpu.arch) {
	683	.x86_64 => 4 << 10,
	684	.aarch64 => 16 << 10,
	685	else => null,
	686	},
	687	.windows => switch (builtin.cpu.arch) {
	688	// -- <https://devblogs.microsoft.com/oldnewthing/20210510-00/?p=105200>
	689	.x86, .x86_64 => 4 << 10,
	690	// SuperH => 4 << 10,
	691	.mips, .mipsel, .mips64, .mips64el => 4 << 10,
	692	.powerpc, .powerpcle, .powerpc64, .powerpc64le => 4 << 10,
	693	// DEC Alpha => 8 << 10,
	694	// Itanium => 8 << 10,
	695	.thumb, .thumbeb, .arm, .armeb, .aarch64, .aarch64_be => 4 << 10,
	696	else => null,
	697	},
	698	.wasi => switch (builtin.cpu.arch) {
	699	.wasm32, .wasm64 => 64 << 10,
	700	else => null,
	701	},
	702	// https://github.com/tianocore/edk2/blob/b158dad150bf02879668f72ce306445250838201/MdePkg/Include/Uefi/UefiBaseType.h#L180-L187
	703	.uefi => 4 << 10,
	704	.freebsd => switch (builtin.cpu.arch) {
	705	// FreeBSD/sys/*
	706	.x86, .x86_64 => 4 << 10,
	707	.thumb, .thumbeb, .arm, .armeb => 4 << 10,
	708	.aarch64, .aarch64_be => 4 << 10,
	709	.powerpc, .powerpc64, .powerpc64le, .powerpcle => 4 << 10,
	710	.riscv32, .riscv64 => 4 << 10,
	711	else => null,
	712	},
	713	.netbsd => switch (builtin.cpu.arch) {
	714	// NetBSD/sys/arch/*
	715	.x86, .x86_64 => 4 << 10,
	716	.thumb, .thumbeb, .arm, .armeb => 4 << 10,
	717	.aarch64, .aarch64_be => 4 << 10,
	718	.mips, .mipsel, .mips64, .mips64el => 4 << 10,
	719	.powerpc, .powerpc64, .powerpc64le, .powerpcle => 4 << 10,
	720	.sparc => 4 << 10,
	721	.sparc64 => 8 << 10,
	722	.riscv32, .riscv64 => 4 << 10,
	723	// Sun-2
	724	.m68k => 2 << 10,
	725	else => null,
	726	},
	727	.dragonfly => switch (builtin.cpu.arch) {
	728	.x86, .x86_64 => 4 << 10,
	729	else => null,
	730	},
	731	.openbsd => switch (builtin.cpu.arch) {
	732	// OpenBSD/sys/arch/*
	733	.x86, .x86_64 => 4 << 10,
	734	.thumb, .thumbeb, .arm, .armeb, .aarch64, .aarch64_be => 4 << 10,
	735	.mips64, .mips64el => 4 << 10,
	736	.powerpc, .powerpc64, .powerpc64le, .powerpcle => 4 << 10,
	737	.riscv64 => 4 << 10,
	738	.sparc64 => 8 << 10,
	739	else => null,
	740	},
	741	.solaris, .illumos => switch (builtin.cpu.arch) {
	742	// src/uts/*/sys/machparam.h
	743	.x86, .x86_64 => 4 << 10,
	744	.sparc, .sparc64 => 8 << 10,
	745	else => null,
	746	},
	747	.fuchsia => switch (builtin.cpu.arch) {
	748	// fuchsia/kernel/arch/*/include/arch/defines.h
	749	.x86_64 => 4 << 10,
	750	.aarch64, .aarch64_be => 4 << 10,
	751	.riscv64 => 4 << 10,
	752	else => null,
	753	},
	754	// https://github.com/SerenityOS/serenity/blob/62b938b798dc009605b5df8a71145942fc53808b/Kernel/API/POSIX/sys/limits.h#L11-L13
	755	.serenity => 4 << 10,
	756	.haiku => switch (builtin.cpu.arch) {
	757	// haiku/headers/posix/arch/*/limits.h
	758	.thumb, .thumbeb, .arm, .armeb => 4 << 10,
	759	.aarch64, .aarch64_be => 4 << 10,
	760	.m68k => 4 << 10,
	761	.mips, .mipsel, .mips64, .mips64el => 4 << 10,
	762	.powerpc, .powerpc64, .powerpc64le, .powerpcle => 4 << 10,
	763	.riscv64 => 4 << 10,
	764	.sparc64 => 8 << 10,
	765	.x86, .x86_64 => 4 << 10,
	766	else => null,
	767	},
	768	.hurd => switch (builtin.cpu.arch) {
	769	// gnumach//include/mach//vm_param.h
	770	.x86, .x86_64 => 4 << 10,
	771	.aarch64 => null,
	772	else => null,
	773	},
	774	.plan9 => switch (builtin.cpu.arch) {
	775	// 9front/sys/src/9/*/mem.h
	776	.x86, .x86_64 => 4 << 10,
	777	.thumb, .thumbeb, .arm, .armeb => 4 << 10,
	778	.aarch64, .aarch64_be => 4 << 10,
	779	.mips, .mipsel, .mips64, .mips64el => 4 << 10,
	780	.powerpc, .powerpcle, .powerpc64, .powerpc64le => 4 << 10,
	781	.sparc => 4 << 10,
	782	else => null,
	783	},
	784	.ps3 => switch (builtin.cpu.arch) {
	785	// cell/SDK_doc/en/html/C_and_C++_standard_libraries/stdlib.html
	786	.powerpc64 => 1 << 20, // 1 MiB
	787	else => null,
	788	},
	789	.ps4 => switch (builtin.cpu.arch) {
	790	// https://github.com/ps4dev/ps4sdk/blob/4df9d001b66ae4ec07d9a51b62d1e4c5e270eecc/include/machine/param.h#L95
	791	.x86, .x86_64 => 4 << 10,
	792	else => null,
	793	},
	794	.ps5 => switch (builtin.cpu.arch) {
	795	// https://github.com/PS5Dev/PS5SDK/blob/a2e03a2a0231a3a3397fa6cd087a01ca6d04f273/include/machine/param.h#L95
	796	.x86, .x86_64 => 16 << 10,
	797	else => null,
	798	},
	799	// system/lib/libc/musl/arch/emscripten/bits/limits.h
	800	.emscripten => 64 << 10,
	801	.linux => switch (builtin.cpu.arch) {
	802	// Linux/arch/*/Kconfig
	803	.arc => 4 << 10,
	804	.thumb, .thumbeb, .arm, .armeb => 4 << 10,
	805	.aarch64, .aarch64_be => 4 << 10,
	806	.csky => 4 << 10,
	807	.hexagon => 4 << 10,
	808	.loongarch32, .loongarch64 => 4 << 10,
	809	.m68k => 4 << 10,
	810	.mips, .mipsel, .mips64, .mips64el => 4 << 10,
	811	.powerpc, .powerpc64, .powerpc64le, .powerpcle => 4 << 10,
	812	.riscv32, .riscv64 => 4 << 10,
	813	.s390x => 4 << 10,
	814	.sparc => 4 << 10,
	815	.sparc64 => 8 << 10,
	816	.x86, .x86_64 => 4 << 10,
	817	.xtensa => 4 << 10,
	818	else => null,
	819	},
	820	.freestanding => switch (builtin.cpu.arch) {
	821	.wasm32, .wasm64 => 64 << 10,
	822	else => null,
	823	},
	824	else => null,
	825	};
	826
	827	const page_size_max_default: ?usize = switch (builtin.os.tag) {
	828	.driverkit, .ios, .macos, .tvos, .visionos, .watchos => switch (builtin.cpu.arch) {
	829	.x86_64 => 4 << 10,
	830	.aarch64 => 16 << 10,
	831	else => null,
	832	},
	833	.windows => switch (builtin.cpu.arch) {
	834	// -- <https://devblogs.microsoft.com/oldnewthing/20210510-00/?p=105200>
	835	.x86, .x86_64 => 4 << 10,
	836	// SuperH => 4 << 10,
	837	.mips, .mipsel, .mips64, .mips64el => 4 << 10,
	838	.powerpc, .powerpcle, .powerpc64, .powerpc64le => 4 << 10,
	839	// DEC Alpha => 8 << 10,
	840	// Itanium => 8 << 10,
	841	.thumb, .thumbeb, .arm, .armeb, .aarch64, .aarch64_be => 4 << 10,
	842	else => null,
	843	},
	844	.wasi => switch (builtin.cpu.arch) {
	845	.wasm32, .wasm64 => 64 << 10,
	846	else => null,
	847	},
	848	// https://github.com/tianocore/edk2/blob/b158dad150bf02879668f72ce306445250838201/MdePkg/Include/Uefi/UefiBaseType.h#L180-L187
	849	.uefi => 4 << 10,
	850	.freebsd => switch (builtin.cpu.arch) {
	851	// FreeBSD/sys/*
	852	.x86, .x86_64 => 4 << 10,
	853	.thumb, .thumbeb, .arm, .armeb => 4 << 10,
	854	.aarch64, .aarch64_be => 4 << 10,
	855	.powerpc, .powerpc64, .powerpc64le, .powerpcle => 4 << 10,
	856	.riscv32, .riscv64 => 4 << 10,
	857	else => null,
	858	},
	859	.netbsd => switch (builtin.cpu.arch) {
	860	// NetBSD/sys/arch/*
	861	.x86, .x86_64 => 4 << 10,
	862	.thumb, .thumbeb, .arm, .armeb => 4 << 10,
	863	.aarch64, .aarch64_be => 64 << 10,
	864	.mips, .mipsel, .mips64, .mips64el => 16 << 10,
	865	.powerpc, .powerpc64, .powerpc64le, .powerpcle => 16 << 10,
	866	.sparc => 8 << 10,
	867	.sparc64 => 8 << 10,
	868	.riscv32, .riscv64 => 4 << 10,
	869	.m68k => 8 << 10,
	870	else => null,
	871	},
	872	.dragonfly => switch (builtin.cpu.arch) {
	873	.x86, .x86_64 => 4 << 10,
	874	else => null,
	875	},
	876	.openbsd => switch (builtin.cpu.arch) {
	877	// OpenBSD/sys/arch/*
	878	.x86, .x86_64 => 4 << 10,
	879	.thumb, .thumbeb, .arm, .armeb, .aarch64, .aarch64_be => 4 << 10,
	880	.mips64, .mips64el => 16 << 10,
	881	.powerpc, .powerpc64, .powerpc64le, .powerpcle => 4 << 10,
	882	.riscv64 => 4 << 10,
	883	.sparc64 => 8 << 10,
	884	else => null,
	885	},
	886	.solaris, .illumos => switch (builtin.cpu.arch) {
	887	// src/uts/*/sys/machparam.h
	888	.x86, .x86_64 => 4 << 10,
	889	.sparc, .sparc64 => 8 << 10,
	890	else => null,
	891	},
	892	.fuchsia => switch (builtin.cpu.arch) {
	893	// fuchsia/kernel/arch/*/include/arch/defines.h
	894	.x86_64 => 4 << 10,
	895	.aarch64, .aarch64_be => 4 << 10,
	896	.riscv64 => 4 << 10,
	897	else => null,
	898	},
	899	// https://github.com/SerenityOS/serenity/blob/62b938b798dc009605b5df8a71145942fc53808b/Kernel/API/POSIX/sys/limits.h#L11-L13
	900	.serenity => 4 << 10,
	901	.haiku => switch (builtin.cpu.arch) {
	902	// haiku/headers/posix/arch/*/limits.h
	903	.thumb, .thumbeb, .arm, .armeb => 4 << 10,
	904	.aarch64, .aarch64_be => 4 << 10,
	905	.m68k => 4 << 10,
	906	.mips, .mipsel, .mips64, .mips64el => 4 << 10,
	907	.powerpc, .powerpc64, .powerpc64le, .powerpcle => 4 << 10,
	908	.riscv64 => 4 << 10,
	909	.sparc64 => 8 << 10,
	910	.x86, .x86_64 => 4 << 10,
	911	else => null,
	912	},
	913	.hurd => switch (builtin.cpu.arch) {
	914	// gnumach//include/mach//vm_param.h
	915	.x86, .x86_64 => 4 << 10,
	916	.aarch64 => null,
	917	else => null,
	918	},
	919	.plan9 => switch (builtin.cpu.arch) {
	920	// 9front/sys/src/9/*/mem.h
	921	.x86, .x86_64 => 4 << 10,
	922	.thumb, .thumbeb, .arm, .armeb => 4 << 10,
	923	.aarch64, .aarch64_be => 64 << 10,
	924	.mips, .mipsel, .mips64, .mips64el => 16 << 10,
	925	.powerpc, .powerpcle, .powerpc64, .powerpc64le => 4 << 10,
	926	.sparc => 4 << 10,
	927	else => null,
	928	},
	929	.ps3 => switch (builtin.cpu.arch) {
	930	// cell/SDK_doc/en/html/C_and_C++_standard_libraries/stdlib.html
	931	.powerpc64 => 1 << 20, // 1 MiB
	932	else => null,
	933	},
	934	.ps4 => switch (builtin.cpu.arch) {
	935	// https://github.com/ps4dev/ps4sdk/blob/4df9d001b66ae4ec07d9a51b62d1e4c5e270eecc/include/machine/param.h#L95
	936	.x86, .x86_64 => 4 << 10,
	937	else => null,
	938	},
	939	.ps5 => switch (builtin.cpu.arch) {
	940	// https://github.com/PS5Dev/PS5SDK/blob/a2e03a2a0231a3a3397fa6cd087a01ca6d04f273/include/machine/param.h#L95
	941	.x86, .x86_64 => 16 << 10,
	942	else => null,
	943	},
	944	// system/lib/libc/musl/arch/emscripten/bits/limits.h
	945	.emscripten => 64 << 10,
	946	.linux => switch (builtin.cpu.arch) {
	947	// Linux/arch/*/Kconfig
	948	.arc => 16 << 10,
	949	.thumb, .thumbeb, .arm, .armeb => 4 << 10,
	950	.aarch64, .aarch64_be => 64 << 10,
	951	.csky => 4 << 10,
	952	.hexagon => 256 << 10,
	953	.loongarch32, .loongarch64 => 64 << 10,
	954	.m68k => 8 << 10,
	955	.mips, .mipsel, .mips64, .mips64el => 64 << 10,
	956	.powerpc, .powerpc64, .powerpc64le, .powerpcle => 256 << 10,
	957	.riscv32, .riscv64 => 4 << 10,
	958	.s390x => 4 << 10,
	959	.sparc => 4 << 10,
	960	.sparc64 => 8 << 10,
	961	.x86, .x86_64 => 4 << 10,
	962	.xtensa => 4 << 10,
	963	else => null,
	964	},
	965	.freestanding => switch (builtin.cpu.arch) {
	966	.wasm32, .wasm64 => 64 << 10,
	967	else => null,
	968	},
	969	else => null,
	970	};
	971
884	972	test {
885		_ = LoggingAllocator;
886		_ = LogToWriterAllocator;
887		_ = ScopedLoggingAllocator;
888	973	_ = @import("heap/memory_pool.zig");
889	974	_ = ArenaAllocator;
890	975	_ = GeneralPurposeAllocator;
	976	_ = FixedBufferAllocator;
	977	_ = ThreadSafeAllocator;
891	978	if (builtin.target.isWasm()) {
892	979	_ = WasmAllocator;
893	980	}

lib/std/heap/FixedBufferAllocator.zig created+230

...	...	@@ -0,0 +1,230 @@
	1	const std = @import("../std.zig");
	2	const Allocator = std.mem.Allocator;
	3	const assert = std.debug.assert;
	4	const mem = std.mem;
	5
	6	const FixedBufferAllocator = @This();
	7
	8	end_index: usize,
	9	buffer: []u8,
	10
	11	pub fn init(buffer: []u8) FixedBufferAllocator {
	12	return .{
	13	.buffer = buffer,
	14	.end_index = 0,
	15	};
	16	}
	17
	18	/// Using this at the same time as the interface returned by `threadSafeAllocator` is not thread safe.
	19	pub fn allocator(self: *FixedBufferAllocator) Allocator {
	20	return .{
	21	.ptr = self,
	22	.vtable = &.{
	23	.alloc = alloc,
	24	.resize = resize,
	25	.remap = remap,
	26	.free = free,
	27	},
	28	};
	29	}
	30
	31	/// Provides a lock free thread safe `Allocator` interface to the underlying `FixedBufferAllocator`
	32	///
	33	/// Using this at the same time as the interface returned by `allocator` is not thread safe.
	34	pub fn threadSafeAllocator(self: *FixedBufferAllocator) Allocator {
	35	return .{
	36	.ptr = self,
	37	.vtable = &.{
	38	.alloc = threadSafeAlloc,
	39	.resize = Allocator.noResize,
	40	.remap = Allocator.noRemap,
	41	.free = Allocator.noFree,
	42	},
	43	};
	44	}
	45
	46	pub fn ownsPtr(self: FixedBufferAllocator, ptr: []u8) bool {
	47	return sliceContainsPtr(self.buffer, ptr);
	48	}
	49
	50	pub fn ownsSlice(self: *FixedBufferAllocator, slice: []u8) bool {
	51	return sliceContainsSlice(self.buffer, slice);
	52	}
	53
	54	/// This has false negatives when the last allocation had an
	55	/// adjusted_index. In such case we won't be able to determine what the
	56	/// last allocation was because the alignForward operation done in alloc is
	57	/// not reversible.
	58	pub fn isLastAllocation(self: *FixedBufferAllocator, buf: []u8) bool {
	59	return buf.ptr + buf.len == self.buffer.ptr + self.end_index;
	60	}
	61
	62	pub fn alloc(ctx: anyopaque, n: usize, alignment: mem.Alignment, ra: usize) ?[]u8 {
	63	const self: *FixedBufferAllocator = @ptrCast(@alignCast(ctx));
	64	_ = ra;
	65	const ptr_align = alignment.toByteUnits();
	66	const adjust_off = mem.alignPointerOffset(self.buffer.ptr + self.end_index, ptr_align) orelse return null;
	67	const adjusted_index = self.end_index + adjust_off;
	68	const new_end_index = adjusted_index + n;
	69	if (new_end_index > self.buffer.len) return null;
	70	self.end_index = new_end_index;
	71	return self.buffer.ptr + adjusted_index;
	72	}
	73
	74	pub fn resize(
	75	ctx: *anyopaque,
	76	buf: []u8,
	77	alignment: mem.Alignment,
	78	new_size: usize,
	79	return_address: usize,
	80	) bool {
	81	const self: *FixedBufferAllocator = @ptrCast(@alignCast(ctx));
	82	_ = alignment;
	83	_ = return_address;
	84	assert(@inComptime() or self.ownsSlice(buf));
	85
	86	if (!self.isLastAllocation(buf)) {
	87	if (new_size > buf.len) return false;
	88	return true;
	89	}
	90
	91	if (new_size <= buf.len) {
	92	const sub = buf.len - new_size;
	93	self.end_index -= sub;
	94	return true;
	95	}
	96
	97	const add = new_size - buf.len;
	98	if (add + self.end_index > self.buffer.len) return false;
	99
	100	self.end_index += add;
	101	return true;
	102	}
	103
	104	pub fn remap(
	105	context: *anyopaque,
	106	memory: []u8,
	107	alignment: mem.Alignment,
	108	new_len: usize,
	109	return_address: usize,
	110	) ?[*]u8 {
	111	return if (resize(context, memory, alignment, new_len, return_address)) memory.ptr else null;
	112	}
	113
	114	pub fn free(
	115	ctx: *anyopaque,
	116	buf: []u8,
	117	alignment: mem.Alignment,
	118	return_address: usize,
	119	) void {
	120	const self: *FixedBufferAllocator = @ptrCast(@alignCast(ctx));
	121	_ = alignment;
	122	_ = return_address;
	123	assert(@inComptime() or self.ownsSlice(buf));
	124
	125	if (self.isLastAllocation(buf)) {
	126	self.end_index -= buf.len;
	127	}
	128	}
	129
	130	fn threadSafeAlloc(ctx: anyopaque, n: usize, alignment: mem.Alignment, ra: usize) ?[]u8 {
	131	const self: *FixedBufferAllocator = @ptrCast(@alignCast(ctx));
	132	_ = ra;
	133	const ptr_align = alignment.toByteUnits();
	134	var end_index = @atomicLoad(usize, &self.end_index, .seq_cst);
	135	while (true) {
	136	const adjust_off = mem.alignPointerOffset(self.buffer.ptr + end_index, ptr_align) orelse return null;
	137	const adjusted_index = end_index + adjust_off;
	138	const new_end_index = adjusted_index + n;
	139	if (new_end_index > self.buffer.len) return null;
	140	end_index = @cmpxchgWeak(usize, &self.end_index, end_index, new_end_index, .seq_cst, .seq_cst) orelse
	141	return self.buffer[adjusted_index..new_end_index].ptr;
	142	}
	143	}
	144
	145	pub fn reset(self: *FixedBufferAllocator) void {
	146	self.end_index = 0;
	147	}
	148
	149	fn sliceContainsPtr(container: []u8, ptr: [*]u8) bool {
	150	return @intFromPtr(ptr) >= @intFromPtr(container.ptr) and
	151	@intFromPtr(ptr) < (@intFromPtr(container.ptr) + container.len);
	152	}
	153
	154	fn sliceContainsSlice(container: []u8, slice: []u8) bool {
	155	return @intFromPtr(slice.ptr) >= @intFromPtr(container.ptr) and
	156	(@intFromPtr(slice.ptr) + slice.len) <= (@intFromPtr(container.ptr) + container.len);
	157	}
	158
	159	var test_fixed_buffer_allocator_memory: [800000 * @sizeOf(u64)]u8 = undefined;
	160
	161	test FixedBufferAllocator {
	162	var fixed_buffer_allocator = mem.validationWrap(FixedBufferAllocator.init(test_fixed_buffer_allocator_memory[0..]));
	163	const a = fixed_buffer_allocator.allocator();
	164
	165	try std.heap.testAllocator(a);
	166	try std.heap.testAllocatorAligned(a);
	167	try std.heap.testAllocatorLargeAlignment(a);
	168	try std.heap.testAllocatorAlignedShrink(a);
	169	}
	170
	171	test reset {
	172	var buf: [8]u8 align(@alignOf(u64)) = undefined;
	173	var fba = FixedBufferAllocator.init(buf[0..]);
	174	const a = fba.allocator();
	175
	176	const X = 0xeeeeeeeeeeeeeeee;
	177	const Y = 0xffffffffffffffff;
	178
	179	const x = try a.create(u64);
	180	x.* = X;
	181	try std.testing.expectError(error.OutOfMemory, a.create(u64));
	182
	183	fba.reset();
	184	const y = try a.create(u64);
	185	y.* = Y;
	186
	187	// we expect Y to have overwritten X.
	188	try std.testing.expect(x.* == y.*);
	189	try std.testing.expect(y.* == Y);
	190	}
	191
	192	test "reuse memory on realloc" {
	193	var small_fixed_buffer: [10]u8 = undefined;
	194	// check if we re-use the memory
	195	{
	196	var fixed_buffer_allocator = FixedBufferAllocator.init(small_fixed_buffer[0..]);
	197	const a = fixed_buffer_allocator.allocator();
	198
	199	const slice0 = try a.alloc(u8, 5);
	200	try std.testing.expect(slice0.len == 5);
	201	const slice1 = try a.realloc(slice0, 10);
	202	try std.testing.expect(slice1.ptr == slice0.ptr);
	203	try std.testing.expect(slice1.len == 10);
	204	try std.testing.expectError(error.OutOfMemory, a.realloc(slice1, 11));
	205	}
	206	// check that we don't re-use the memory if it's not the most recent block
	207	{
	208	var fixed_buffer_allocator = FixedBufferAllocator.init(small_fixed_buffer[0..]);
	209	const a = fixed_buffer_allocator.allocator();
	210
	211	var slice0 = try a.alloc(u8, 2);
	212	slice0[0] = 1;
	213	slice0[1] = 2;
	214	const slice1 = try a.alloc(u8, 2);
	215	const slice2 = try a.realloc(slice0, 4);
	216	try std.testing.expect(slice0.ptr != slice2.ptr);
	217	try std.testing.expect(slice1.ptr != slice2.ptr);
	218	try std.testing.expect(slice2[0] == 1);
	219	try std.testing.expect(slice2[1] == 2);
	220	}
	221	}
	222
	223	test "thread safe version" {
	224	var fixed_buffer_allocator = FixedBufferAllocator.init(test_fixed_buffer_allocator_memory[0..]);
	225
	226	try std.heap.testAllocator(fixed_buffer_allocator.threadSafeAllocator());
	227	try std.heap.testAllocatorAligned(fixed_buffer_allocator.threadSafeAllocator());
	228	try std.heap.testAllocatorLargeAlignment(fixed_buffer_allocator.threadSafeAllocator());
	229	try std.heap.testAllocatorAlignedShrink(fixed_buffer_allocator.threadSafeAllocator());
	230	}

lib/std/heap/PageAllocator.zig+128-52

...	...	@@ -7,107 +7,183 @@ const assert = std.debug.assert;
7	7	const native_os = builtin.os.tag;
8	8	const windows = std.os.windows;
9	9	const posix = std.posix;
	10	const page_size_min = std.heap.page_size_min;
10	11
11		pub const vtable = Allocator.VTable{
	12	pub const vtable: Allocator.VTable = .{
12	13	.alloc = alloc,
13	14	.resize = resize,
	15	.remap = remap,
14	16	.free = free,
15	17	};
16	18
17		fn alloc(_: anyopaque, n: usize, log2_align: u8, ra: usize) ?[]u8 {
	19	fn alloc(context: anyopaque, n: usize, alignment: mem.Alignment, ra: usize) ?[]u8 {
	20	_ = context;
18	21	_ = ra;
19		_ = log2_align;
20	22	assert(n > 0);
21		if (n > maxInt(usize) - (mem.page_size - 1)) return null;
	23
	24	const page_size = std.heap.pageSize();
	25	if (n >= maxInt(usize) - page_size) return null;
	26	const alignment_bytes = alignment.toByteUnits();
22	27
23	28	if (native_os == .windows) {
	29	// According to official documentation, VirtualAlloc aligns to page
	30	// boundary, however, empirically it reserves pages on a 64K boundary.
	31	// Since it is very likely the requested alignment will be honored,
	32	// this logic first tries a call with exactly the size requested,
	33	// before falling back to the loop below.
	34	// https://devblogs.microsoft.com/oldnewthing/?p=42223
24	35	const addr = windows.VirtualAlloc(
25	36	null,
26
27	37	// VirtualAlloc will round the length to a multiple of page size.
28		// VirtualAlloc docs: If the lpAddress parameter is NULL, this value is rounded up to the next page boundary
	38	// "If the lpAddress parameter is NULL, this value is rounded up to
	39	// the next page boundary".
29	40	n,
30
31	41	windows.MEM_COMMIT \| windows.MEM_RESERVE,
32	42	windows.PAGE_READWRITE,
33	43	) catch return null;
34		return @ptrCast(addr);
	44
	45	if (mem.isAligned(@intFromPtr(addr), alignment_bytes))
	46	return @ptrCast(addr);
	47
	48	// Fallback: reserve a range of memory large enough to find a
	49	// sufficiently aligned address, then free the entire range and
	50	// immediately allocate the desired subset. Another thread may have won
	51	// the race to map the target range, in which case a retry is needed.
	52	windows.VirtualFree(addr, 0, windows.MEM_RELEASE);
	53
	54	const overalloc_len = n + alignment_bytes - page_size;
	55	const aligned_len = mem.alignForward(usize, n, page_size);
	56
	57	while (true) {
	58	const reserved_addr = windows.VirtualAlloc(
	59	null,
	60	overalloc_len,
	61	windows.MEM_RESERVE,
	62	windows.PAGE_NOACCESS,
	63	) catch return null;
	64	const aligned_addr = mem.alignForward(usize, @intFromPtr(reserved_addr), alignment_bytes);
	65	windows.VirtualFree(reserved_addr, 0, windows.MEM_RELEASE);
	66	const ptr = windows.VirtualAlloc(
	67	@ptrFromInt(aligned_addr),
	68	aligned_len,
	69	windows.MEM_COMMIT \| windows.MEM_RESERVE,
	70	windows.PAGE_READWRITE,
	71	) catch continue;
	72	return @ptrCast(ptr);
	73	}
35	74	}
36	75
37		const aligned_len = mem.alignForward(usize, n, mem.page_size);
	76	const aligned_len = mem.alignForward(usize, n, page_size);
	77	const max_drop_len = alignment_bytes - @min(alignment_bytes, page_size);
	78	const overalloc_len = if (max_drop_len <= aligned_len - n)
	79	aligned_len
	80	else
	81	mem.alignForward(usize, aligned_len + max_drop_len, page_size);
38	82	const hint = @atomicLoad(@TypeOf(std.heap.next_mmap_addr_hint), &std.heap.next_mmap_addr_hint, .unordered);
39	83	const slice = posix.mmap(
40	84	hint,
41		aligned_len,
	85	overalloc_len,
42	86	posix.PROT.READ \| posix.PROT.WRITE,
43	87	.{ .TYPE = .PRIVATE, .ANONYMOUS = true },
44	88	-1,
45	89	0,
46	90	) catch return null;
47		assert(mem.isAligned(@intFromPtr(slice.ptr), mem.page_size));
48		const new_hint: [*]align(mem.page_size) u8 = @alignCast(slice.ptr + aligned_len);
	91	const result_ptr = mem.alignPointer(slice.ptr, alignment_bytes) orelse return null;
	92	// Unmap the extra bytes that were only requested in order to guarantee
	93	// that the range of memory we were provided had a proper alignment in it
	94	// somewhere. The extra bytes could be at the beginning, or end, or both.
	95	const drop_len = result_ptr - slice.ptr;
	96	if (drop_len != 0) posix.munmap(slice[0..drop_len]);
	97	const remaining_len = overalloc_len - drop_len;
	98	if (remaining_len > aligned_len) posix.munmap(@alignCast(result_ptr[aligned_len..remaining_len]));
	99	const new_hint: [*]align(page_size_min) u8 = @alignCast(result_ptr + aligned_len);
49	100	_ = @cmpxchgStrong(@TypeOf(std.heap.next_mmap_addr_hint), &std.heap.next_mmap_addr_hint, hint, new_hint, .monotonic, .monotonic);
50		return slice.ptr;
	101	return result_ptr;
51	102	}
52	103
53	104	fn resize(
54		_: *anyopaque,
55		buf_unaligned: []u8,
56		log2_buf_align: u8,
57		new_size: usize,
	105	context: *anyopaque,
	106	memory: []u8,
	107	alignment: mem.Alignment,
	108	new_len: usize,
58	109	return_address: usize,
59	110	) bool {
60		_ = log2_buf_align;
	111	_ = context;
	112	_ = alignment;
61	113	_ = return_address;
62		const new_size_aligned = mem.alignForward(usize, new_size, mem.page_size);
	114	return realloc(memory, new_len, false) != null;
	115	}
	116
	117	pub fn remap(
	118	context: *anyopaque,
	119	memory: []u8,
	120	alignment: mem.Alignment,
	121	new_len: usize,
	122	return_address: usize,
	123	) ?[*]u8 {
	124	_ = context;
	125	_ = alignment;
	126	_ = return_address;
	127	return realloc(memory, new_len, true);
	128	}
	129
	130	fn free(context: *anyopaque, slice: []u8, alignment: mem.Alignment, return_address: usize) void {
	131	_ = context;
	132	_ = alignment;
	133	_ = return_address;
	134
	135	if (native_os == .windows) {
	136	windows.VirtualFree(slice.ptr, 0, windows.MEM_RELEASE);
	137	} else {
	138	const buf_aligned_len = mem.alignForward(usize, slice.len, std.heap.pageSize());
	139	posix.munmap(@alignCast(slice.ptr[0..buf_aligned_len]));
	140	}
	141	}
	142
	143	fn realloc(uncasted_memory: []u8, new_len: usize, may_move: bool) ?[*]u8 {
	144	const memory: []align(std.heap.page_size_min) u8 = @alignCast(uncasted_memory);
	145	const page_size = std.heap.pageSize();
	146	const new_size_aligned = mem.alignForward(usize, new_len, page_size);
63	147
64	148	if (native_os == .windows) {
65		if (new_size <= buf_unaligned.len) {
66		const base_addr = @intFromPtr(buf_unaligned.ptr);
67		const old_addr_end = base_addr + buf_unaligned.len;
68		const new_addr_end = mem.alignForward(usize, base_addr + new_size, mem.page_size);
	149	if (new_len <= memory.len) {
	150	const base_addr = @intFromPtr(memory.ptr);
	151	const old_addr_end = base_addr + memory.len;
	152	const new_addr_end = mem.alignForward(usize, base_addr + new_len, page_size);
69	153	if (old_addr_end > new_addr_end) {
70		// For shrinking that is not releasing, we will only
71		// decommit the pages not needed anymore.
	154	// For shrinking that is not releasing, we will only decommit
	155	// the pages not needed anymore.
72	156	windows.VirtualFree(
73		@as(*anyopaque, @ptrFromInt(new_addr_end)),
	157	@ptrFromInt(new_addr_end),
74	158	old_addr_end - new_addr_end,
75	159	windows.MEM_DECOMMIT,
76	160	);
77	161	}
78		return true;
	162	return memory.ptr;
79	163	}
80		const old_size_aligned = mem.alignForward(usize, buf_unaligned.len, mem.page_size);
	164	const old_size_aligned = mem.alignForward(usize, memory.len, page_size);
81	165	if (new_size_aligned <= old_size_aligned) {
82		return true;
	166	return memory.ptr;
83	167	}
84		return false;
	168	return null;
85	169	}
86	170
87		const buf_aligned_len = mem.alignForward(usize, buf_unaligned.len, mem.page_size);
88		if (new_size_aligned == buf_aligned_len)
89		return true;
	171	const page_aligned_len = mem.alignForward(usize, memory.len, page_size);
	172	if (new_size_aligned == page_aligned_len)
	173	return memory.ptr;
90	174
91		if (new_size_aligned < buf_aligned_len) {
92		const ptr = buf_unaligned.ptr + new_size_aligned;
93		// TODO: if the next_mmap_addr_hint is within the unmapped range, update it
94		posix.munmap(@alignCast(ptr[0 .. buf_aligned_len - new_size_aligned]));
95		return true;
	175	if (posix.MREMAP != void) {
	176	// TODO: if the next_mmap_addr_hint is within the remapped range, update it
	177	const new_memory = posix.mremap(memory.ptr, memory.len, new_len, .{ .MAYMOVE = may_move }, null) catch return null;
	178	return new_memory.ptr;
96	179	}
97	180
98		// TODO: call mremap
99		// TODO: if the next_mmap_addr_hint is within the remapped range, update it
100		return false;
101		}
102
103		fn free(_: *anyopaque, slice: []u8, log2_buf_align: u8, return_address: usize) void {
104		_ = log2_buf_align;
105		_ = return_address;
106
107		if (native_os == .windows) {
108		windows.VirtualFree(slice.ptr, 0, windows.MEM_RELEASE);
109		} else {
110		const buf_aligned_len = mem.alignForward(usize, slice.len, mem.page_size);
111		posix.munmap(@alignCast(slice.ptr[0..buf_aligned_len]));
	181	if (new_size_aligned < page_aligned_len) {
	182	const ptr = memory.ptr + new_size_aligned;
	183	// TODO: if the next_mmap_addr_hint is within the unmapped range, update it
	184	posix.munmap(@alignCast(ptr[0 .. page_aligned_len - new_size_aligned]));
	185	return memory.ptr;
112	186	}
	187
	188	return null;
113	189	}

lib/std/heap/ThreadSafeAllocator.zig+16-6

...	...	@@ -9,35 +9,45 @@ pub fn allocator(self: *ThreadSafeAllocator) Allocator {
9	9	.vtable = &.{
10	10	.alloc = alloc,
11	11	.resize = resize,
	12	.remap = remap,
12	13	.free = free,
13	14	},
14	15	};
15	16	}
16	17
17		fn alloc(ctx: anyopaque, n: usize, log2_ptr_align: u8, ra: usize) ?[]u8 {
	18	fn alloc(ctx: anyopaque, n: usize, alignment: std.mem.Alignment, ra: usize) ?[]u8 {
18	19	const self: *ThreadSafeAllocator = @ptrCast(@alignCast(ctx));
19	20	self.mutex.lock();
20	21	defer self.mutex.unlock();
21	22
22		return self.child_allocator.rawAlloc(n, log2_ptr_align, ra);
	23	return self.child_allocator.rawAlloc(n, alignment, ra);
23	24	}
24	25
25		fn resize(ctx: *anyopaque, buf: []u8, log2_buf_align: u8, new_len: usize, ret_addr: usize) bool {
	26	fn resize(ctx: *anyopaque, buf: []u8, alignment: std.mem.Alignment, new_len: usize, ret_addr: usize) bool {
26	27	const self: *ThreadSafeAllocator = @ptrCast(@alignCast(ctx));
27	28
28	29	self.mutex.lock();
29	30	defer self.mutex.unlock();
30	31
31		return self.child_allocator.rawResize(buf, log2_buf_align, new_len, ret_addr);
	32	return self.child_allocator.rawResize(buf, alignment, new_len, ret_addr);
32	33	}
33	34
34		fn free(ctx: *anyopaque, buf: []u8, log2_buf_align: u8, ret_addr: usize) void {
	35	fn remap(context: anyopaque, memory: []u8, alignment: std.mem.Alignment, new_len: usize, return_address: usize) ?[]u8 {
	36	const self: *ThreadSafeAllocator = @ptrCast(@alignCast(context));
	37
	38	self.mutex.lock();
	39	defer self.mutex.unlock();
	40
	41	return self.child_allocator.rawRemap(memory, alignment, new_len, return_address);
	42	}
	43
	44	fn free(ctx: *anyopaque, buf: []u8, alignment: std.mem.Alignment, ret_addr: usize) void {
35	45	const self: *ThreadSafeAllocator = @ptrCast(@alignCast(ctx));
36	46
37	47	self.mutex.lock();
38	48	defer self.mutex.unlock();
39	49
40		return self.child_allocator.rawFree(buf, log2_buf_align, ret_addr);
	50	return self.child_allocator.rawFree(buf, alignment, ret_addr);
41	51	}
42	52
43	53	const std = @import("../std.zig");

lib/std/heap/WasmAllocator.zig+21-11

...	...	@@ -20,6 +20,7 @@ comptime {
20	20	pub const vtable: Allocator.VTable = .{
21	21	.alloc = alloc,
22	22	.resize = resize,
	23	.remap = remap,
23	24	.free = free,
24	25	};
25	26
...	...	@@ -40,18 +41,17 @@ const size_class_count = math.log2(bigpage_size) - min_class;
40	41	/// etc.
41	42	const big_size_class_count = math.log2(bigpage_count);
42	43
43		var next_addrs = [1]usize{0} ** size_class_count;
	44	var next_addrs: [size_class_count]usize = @splat(0);
44	45	/// For each size class, points to the freed pointer.
45		var frees = [1]usize{0} ** size_class_count;
	46	var frees: [size_class_count]usize = @splat(0);
46	47	/// For each big size class, points to the freed pointer.
47		var big_frees = [1]usize{0} ** big_size_class_count;
	48	var big_frees: [big_size_class_count]usize = @splat(0);
48	49
49		fn alloc(ctx: anyopaque, len: usize, log2_align: u8, return_address: usize) ?[]u8 {
	50	fn alloc(ctx: anyopaque, len: usize, alignment: mem.Alignment, return_address: usize) ?[]u8 {
50	51	_ = ctx;
51	52	_ = return_address;
52	53	// Make room for the freelist next pointer.
53		const alignment = @as(usize, 1) << @as(Allocator.Log2Align, @intCast(log2_align));
54		const actual_len = @max(len +\| @sizeOf(usize), alignment);
	54	const actual_len = @max(len +\| @sizeOf(usize), alignment.toByteUnits());
55	55	const slot_size = math.ceilPowerOfTwo(usize, actual_len) catch return null;
56	56	const class = math.log2(slot_size) - min_class;
57	57	if (class < size_class_count) {
...	...	@@ -86,7 +86,7 @@ fn alloc(ctx: anyopaque, len: usize, log2_align: u8, return_address: usize) ?[
86	86	fn resize(
87	87	ctx: *anyopaque,
88	88	buf: []u8,
89		log2_buf_align: u8,
	89	alignment: mem.Alignment,
90	90	new_len: usize,
91	91	return_address: usize,
92	92	) bool {
...	...	@@ -94,7 +94,7 @@ fn resize(
94	94	_ = return_address;
95	95	// We don't want to move anything from one size class to another, but we
96	96	// can recover bytes in between powers of two.
97		const buf_align = @as(usize, 1) << @as(Allocator.Log2Align, @intCast(log2_buf_align));
	97	const buf_align = alignment.toByteUnits();
98	98	const old_actual_len = @max(buf.len + @sizeOf(usize), buf_align);
99	99	const new_actual_len = @max(new_len +\| @sizeOf(usize), buf_align);
100	100	const old_small_slot_size = math.ceilPowerOfTwoAssert(usize, old_actual_len);
...	...	@@ -111,15 +111,25 @@ fn resize(
111	111	}
112	112	}
113	113
	114	fn remap(
	115	context: *anyopaque,
	116	memory: []u8,
	117	alignment: mem.Alignment,
	118	new_len: usize,
	119	return_address: usize,
	120	) ?[*]u8 {
	121	return if (resize(context, memory, alignment, new_len, return_address)) memory.ptr else null;
	122	}
	123
114	124	fn free(
115	125	ctx: *anyopaque,
116	126	buf: []u8,
117		log2_buf_align: u8,
	127	alignment: mem.Alignment,
118	128	return_address: usize,
119	129	) void {
120	130	_ = ctx;
121	131	_ = return_address;
122		const buf_align = @as(usize, 1) << @as(Allocator.Log2Align, @intCast(log2_buf_align));
	132	const buf_align = alignment.toByteUnits();
123	133	const actual_len = @max(buf.len + @sizeOf(usize), buf_align);
124	134	const slot_size = math.ceilPowerOfTwoAssert(usize, actual_len);
125	135	const class = math.log2(slot_size) - min_class;
...	...	@@ -160,7 +170,7 @@ fn allocBigPages(n: usize) usize {
160	170	return @as(usize, @intCast(page_index)) * wasm.page_size;
161	171	}
162	172
163		const test_ally = Allocator{
	173	const test_ally: Allocator = .{
164	174	.ptr = undefined,
165	175	.vtable = &vtable,
166	176	};

lib/std/heap/arena_allocator.zig+25-17

...	...	@@ -29,12 +29,14 @@ pub const ArenaAllocator = struct {
29	29	.vtable = &.{
30	30	.alloc = alloc,
31	31	.resize = resize,
	32	.remap = remap,
32	33	.free = free,
33	34	},
34	35	};
35	36	}
36	37
37	38	const BufNode = std.SinglyLinkedList(usize).Node;
	39	const BufNode_alignment: mem.Alignment = .fromByteUnits(@alignOf(BufNode));
38	40
39	41	pub fn init(child_allocator: Allocator) ArenaAllocator {
40	42	return (State{}).promote(child_allocator);
...	...	@@ -47,9 +49,8 @@ pub const ArenaAllocator = struct {
47	49	while (it) \|node\| {
48	50	// this has to occur before the free because the free frees node
49	51	const next_it = node.next;
50		const align_bits = std.math.log2_int(usize, @alignOf(BufNode));
51	52	const alloc_buf = @as([*]u8, @ptrCast(node))[0..node.data];
52		self.child_allocator.rawFree(alloc_buf, align_bits, @returnAddress());
	53	self.child_allocator.rawFree(alloc_buf, BufNode_alignment, @returnAddress());
53	54	it = next_it;
54	55	}
55	56	}
...	...	@@ -120,7 +121,6 @@ pub const ArenaAllocator = struct {
120	121	return true;
121	122	}
122	123	const total_size = requested_capacity + @sizeOf(BufNode);
123		const align_bits = std.math.log2_int(usize, @alignOf(BufNode));
124	124	// Free all nodes except for the last one
125	125	var it = self.state.buffer_list.first;
126	126	const maybe_first_node = while (it) \|node\| {
...	...	@@ -129,7 +129,7 @@ pub const ArenaAllocator = struct {
129	129	if (next_it == null)
130	130	break node;
131	131	const alloc_buf = @as([*]u8, @ptrCast(node))[0..node.data];
132		self.child_allocator.rawFree(alloc_buf, align_bits, @returnAddress());
	132	self.child_allocator.rawFree(alloc_buf, BufNode_alignment, @returnAddress());
133	133	it = next_it;
134	134	} else null;
135	135	std.debug.assert(maybe_first_node == null or maybe_first_node.?.next == null);
...	...	@@ -141,16 +141,16 @@ pub const ArenaAllocator = struct {
141	141	if (first_node.data == total_size)
142	142	return true;
143	143	const first_alloc_buf = @as([*]u8, @ptrCast(first_node))[0..first_node.data];
144		if (self.child_allocator.rawResize(first_alloc_buf, align_bits, total_size, @returnAddress())) {
	144	if (self.child_allocator.rawResize(first_alloc_buf, BufNode_alignment, total_size, @returnAddress())) {
145	145	// successful resize
146	146	first_node.data = total_size;
147	147	} else {
148	148	// manual realloc
149		const new_ptr = self.child_allocator.rawAlloc(total_size, align_bits, @returnAddress()) orelse {
	149	const new_ptr = self.child_allocator.rawAlloc(total_size, BufNode_alignment, @returnAddress()) orelse {
150	150	// we failed to preheat the arena properly, signal this to the user.
151	151	return false;
152	152	};
153		self.child_allocator.rawFree(first_alloc_buf, align_bits, @returnAddress());
	153	self.child_allocator.rawFree(first_alloc_buf, BufNode_alignment, @returnAddress());
154	154	const node: *BufNode = @ptrCast(@alignCast(new_ptr));
155	155	node.* = .{ .data = total_size };
156	156	self.state.buffer_list.first = node;
...	...	@@ -163,8 +163,7 @@ pub const ArenaAllocator = struct {
163	163	const actual_min_size = minimum_size + (@sizeOf(BufNode) + 16);
164	164	const big_enough_len = prev_len + actual_min_size;
165	165	const len = big_enough_len + big_enough_len / 2;
166		const log2_align = comptime std.math.log2_int(usize, @alignOf(BufNode));
167		const ptr = self.child_allocator.rawAlloc(len, log2_align, @returnAddress()) orelse
	166	const ptr = self.child_allocator.rawAlloc(len, BufNode_alignment, @returnAddress()) orelse
168	167	return null;
169	168	const buf_node: *BufNode = @ptrCast(@alignCast(ptr));
170	169	buf_node.* = .{ .data = len };
...	...	@@ -173,11 +172,11 @@ pub const ArenaAllocator = struct {
173	172	return buf_node;
174	173	}
175	174
176		fn alloc(ctx: anyopaque, n: usize, log2_ptr_align: u8, ra: usize) ?[]u8 {
	175	fn alloc(ctx: anyopaque, n: usize, alignment: mem.Alignment, ra: usize) ?[]u8 {
177	176	const self: *ArenaAllocator = @ptrCast(@alignCast(ctx));
178	177	_ = ra;
179	178
180		const ptr_align = @as(usize, 1) << @as(Allocator.Log2Align, @intCast(log2_ptr_align));
	179	const ptr_align = alignment.toByteUnits();
181	180	var cur_node = if (self.state.buffer_list.first) \|first_node\|
182	181	first_node
183	182	else
...	...	@@ -197,8 +196,7 @@ pub const ArenaAllocator = struct {
197	196	}
198	197
199	198	const bigger_buf_size = @sizeOf(BufNode) + new_end_index;
200		const log2_align = comptime std.math.log2_int(usize, @alignOf(BufNode));
201		if (self.child_allocator.rawResize(cur_alloc_buf, log2_align, bigger_buf_size, @returnAddress())) {
	199	if (self.child_allocator.rawResize(cur_alloc_buf, BufNode_alignment, bigger_buf_size, @returnAddress())) {
202	200	cur_node.data = bigger_buf_size;
203	201	} else {
204	202	// Allocate a new node if that's not possible
...	...	@@ -207,9 +205,9 @@ pub const ArenaAllocator = struct {
207	205	}
208	206	}
209	207
210		fn resize(ctx: *anyopaque, buf: []u8, log2_buf_align: u8, new_len: usize, ret_addr: usize) bool {
	208	fn resize(ctx: *anyopaque, buf: []u8, alignment: mem.Alignment, new_len: usize, ret_addr: usize) bool {
211	209	const self: *ArenaAllocator = @ptrCast(@alignCast(ctx));
212		_ = log2_buf_align;
	210	_ = alignment;
213	211	_ = ret_addr;
214	212
215	213	const cur_node = self.state.buffer_list.first orelse return false;
...	...	@@ -231,8 +229,18 @@ pub const ArenaAllocator = struct {
231	229	}
232	230	}
233	231
234		fn free(ctx: *anyopaque, buf: []u8, log2_buf_align: u8, ret_addr: usize) void {
235		_ = log2_buf_align;
	232	fn remap(
	233	context: *anyopaque,
	234	memory: []u8,
	235	alignment: mem.Alignment,
	236	new_len: usize,
	237	return_address: usize,
	238	) ?[*]u8 {
	239	return if (resize(context, memory, alignment, new_len, return_address)) memory.ptr else null;
	240	}
	241
	242	fn free(ctx: *anyopaque, buf: []u8, alignment: mem.Alignment, ret_addr: usize) void {
	243	_ = alignment;
236	244	_ = ret_addr;
237	245
238	246	const self: *ArenaAllocator = @ptrCast(@alignCast(ctx));

lib/std/heap/debug_allocator.zig created+1410

...	...	@@ -0,0 +1,1410 @@
	1	//! An allocator that is intended to be used in Debug mode.
	2	//!
	3	//! ## Features
	4	//!
	5	//! * Captures stack traces on allocation, free, and optionally resize.
	6	//! * Double free detection, which prints all three traces (first alloc, first
	7	//! free, second free).
	8	//! * Leak detection, with stack traces.
	9	//! * Never reuses memory addresses, making it easier for Zig to detect branch
	10	//! on undefined values in case of dangling pointers. This relies on
	11	//! the backing allocator to also not reuse addresses.
	12	//! * Uses a minimum backing allocation size to avoid operating system errors
	13	//! from having too many active memory mappings.
	14	//! * When a page of memory is no longer needed, give it back to resident
	15	//! memory as soon as possible, so that it causes page faults when used.
	16	//! * Cross platform. Operates based on a backing allocator which makes it work
	17	//! everywhere, even freestanding.
	18	//! * Compile-time configuration.
	19	//!
	20	//! These features require the allocator to be quite slow and wasteful. For
	21	//! example, when allocating a single byte, the efficiency is less than 1%;
	22	//! it requires more than 100 bytes of overhead to manage the allocation for
	23	//! one byte. The efficiency gets better with larger allocations.
	24	//!
	25	//! ## Basic Design
	26	//!
	27	//! Allocations are divided into two categories, small and large.
	28	//!
	29	//! Small allocations are divided into buckets based on `page_size`:
	30	//!
	31	//! ```
	32	//! index obj_size
	33	//! 0 1
	34	//! 1 2
	35	//! 2 4
	36	//! 3 8
	37	//! 4 16
	38	//! 5 32
	39	//! 6 64
	40	//! 7 128
	41	//! 8 256
	42	//! 9 512
	43	//! 10 1024
	44	//! 11 2048
	45	//! ...
	46	//! ```
	47	//!
	48	//! This goes on for `small_bucket_count` indexes.
	49	//!
	50	//! Allocations are grouped into an object size based on max(len, alignment),
	51	//! rounded up to the next power of two.
	52	//!
	53	//! The main allocator state has an array of all the "current" buckets for each
	54	//! size class. Each slot in the array can be null, meaning the bucket for that
	55	//! size class is not allocated. When the first object is allocated for a given
	56	//! size class, it makes one `page_size` allocation from the backing allocator.
	57	//! This allocation is divided into "slots" - one per allocated object, leaving
	58	//! room for the allocation metadata (starting with `BucketHeader`), which is
	59	//! located at the very end of the "page".
	60	//!
	61	//! The allocation metadata includes "used bits" - 1 bit per slot representing
	62	//! whether the slot is used. Allocations always take the next available slot
	63	//! from the current bucket, setting the corresponding used bit, as well as
	64	//! incrementing `allocated_count`.
	65	//!
	66	//! Frees recover the allocation metadata based on the address, length, and
	67	//! alignment, relying on the backing allocation's large alignment, combined
	68	//! with the fact that allocations are never moved from small to large, or vice
	69	//! versa.
	70	//!
	71	//! When a bucket is full, a new one is allocated, containing a pointer to the
	72	//! previous one. This singly-linked list is iterated during leak detection.
	73	//!
	74	//! Resizing and remapping work the same on small allocations: if the size
	75	//! class would not change, then the operation succeeds, and the address is
	76	//! unchanged. Otherwise, the request is rejected.
	77	//!
	78	//! Large objects are allocated directly using the backing allocator. Metadata
	79	//! is stored separately in a `std.HashMap` using the backing allocator.
	80	//!
	81	//! Resizing and remapping are forwarded directly to the backing allocator,
	82	//! except where such operations would change the category from large to small.
	83
	84	const std = @import("std");
	85	const builtin = @import("builtin");
	86	const log = std.log.scoped(.gpa);
	87	const math = std.math;
	88	const assert = std.debug.assert;
	89	const mem = std.mem;
	90	const Allocator = std.mem.Allocator;
	91	const StackTrace = std.builtin.StackTrace;
	92
	93	const default_page_size: usize = @max(std.heap.page_size_max, switch (builtin.os.tag) {
	94	.windows => 64 * 1024, // Makes `std.heap.PageAllocator` take the happy path.
	95	.wasi => 64 * 1024, // Max alignment supported by `std.heap.WasmAllocator`.
	96	else => 128 * 1024, // Avoids too many active mappings when `page_size_max` is low.
	97	});
	98
	99	const Log2USize = std.math.Log2Int(usize);
	100
	101	const default_sys_stack_trace_frames: usize = if (std.debug.sys_can_stack_trace) 6 else 0;
	102	const default_stack_trace_frames: usize = switch (builtin.mode) {
	103	.Debug => default_sys_stack_trace_frames,
	104	else => 0,
	105	};
	106
	107	pub const Config = struct {
	108	/// Number of stack frames to capture.
	109	stack_trace_frames: usize = default_stack_trace_frames,
	110
	111	/// If true, the allocator will have two fields:
	112	/// * `total_requested_bytes` which tracks the total allocated bytes of memory requested.
	113	/// * `requested_memory_limit` which causes allocations to return `error.OutOfMemory`
	114	/// when the `total_requested_bytes` exceeds this limit.
	115	/// If false, these fields will be `void`.
	116	enable_memory_limit: bool = false,
	117
	118	/// Whether to enable safety checks.
	119	safety: bool = std.debug.runtime_safety,
	120
	121	/// Whether the allocator may be used simultaneously from multiple threads.
	122	thread_safe: bool = !builtin.single_threaded,
	123
	124	/// What type of mutex you'd like to use, for thread safety.
	125	/// when specified, the mutex type must have the same shape as `std.Thread.Mutex` and
	126	/// `DummyMutex`, and have no required fields. Specifying this field causes
	127	/// the `thread_safe` field to be ignored.
	128	///
	129	/// when null (default):
	130	/// * the mutex type defaults to `std.Thread.Mutex` when thread_safe is enabled.
	131	/// * the mutex type defaults to `DummyMutex` otherwise.
	132	MutexType: ?type = null,
	133
	134	/// This is a temporary debugging trick you can use to turn segfaults into more helpful
	135	/// logged error messages with stack trace details. The downside is that every allocation
	136	/// will be leaked, unless used with retain_metadata!
	137	never_unmap: bool = false,
	138
	139	/// This is a temporary debugging aid that retains metadata about allocations indefinitely.
	140	/// This allows a greater range of double frees to be reported. All metadata is freed when
	141	/// deinit is called. When used with never_unmap, deliberately leaked memory is also freed
	142	/// during deinit. Currently should be used with never_unmap to avoid segfaults.
	143	/// TODO https://github.com/ziglang/zig/issues/4298 will allow use without never_unmap
	144	retain_metadata: bool = false,
	145
	146	/// Enables emitting info messages with the size and address of every allocation.
	147	verbose_log: bool = false,
	148
	149	/// Tell whether the backing allocator returns already-zeroed memory.
	150	backing_allocator_zeroes: bool = true,
	151
	152	/// When resizing an allocation, refresh the stack trace with the resize
	153	/// callsite. Comes with a performance penalty.
	154	resize_stack_traces: bool = false,
	155
	156	/// Magic value that distinguishes allocations owned by this allocator from
	157	/// other regions of memory.
	158	canary: usize = @truncate(0x9232a6ff85dff10f),
	159
	160	/// The size of allocations requested from the backing allocator for
	161	/// subdividing into slots for small allocations.
	162	///
	163	/// Must be a power of two.
	164	page_size: usize = default_page_size,
	165	};
	166
	167	/// Default initialization of this struct is deprecated; use `.init` instead.
	168	pub fn DebugAllocator(comptime config: Config) type {
	169	return struct {
	170	backing_allocator: Allocator = std.heap.page_allocator,
	171	/// Tracks the active bucket, which is the one that has free slots in it.
	172	buckets: [small_bucket_count]?BucketHeader = [1]?BucketHeader{null} ** small_bucket_count,
	173	large_allocations: LargeAllocTable = .empty,
	174	total_requested_bytes: @TypeOf(total_requested_bytes_init) = total_requested_bytes_init,
	175	requested_memory_limit: @TypeOf(requested_memory_limit_init) = requested_memory_limit_init,
	176	mutex: @TypeOf(mutex_init) = mutex_init,
	177
	178	const Self = @This();
	179
	180	pub const init: Self = .{};
	181
	182	/// These can be derived from size_class_index but the calculation is nontrivial.
	183	const slot_counts: [small_bucket_count]SlotIndex = init: {
	184	@setEvalBranchQuota(10000);
	185	var result: [small_bucket_count]SlotIndex = undefined;
	186	for (&result, 0..) \|elem, i\| elem. = calculateSlotCount(i);
	187	break :init result;
	188	};
	189
	190	comptime {
	191	assert(math.isPowerOfTwo(page_size));
	192	}
	193
	194	const page_size = config.page_size;
	195	const page_align: mem.Alignment = .fromByteUnits(page_size);
	196	/// Integer type for pointing to slots in a small allocation
	197	const SlotIndex = std.meta.Int(.unsigned, math.log2(page_size) + 1);
	198
	199	const total_requested_bytes_init = if (config.enable_memory_limit) @as(usize, 0) else {};
	200	const requested_memory_limit_init = if (config.enable_memory_limit) @as(usize, math.maxInt(usize)) else {};
	201
	202	const mutex_init = if (config.MutexType) \|T\|
	203	T{}
	204	else if (config.thread_safe)
	205	std.Thread.Mutex{}
	206	else
	207	DummyMutex{};
	208
	209	const DummyMutex = struct {
	210	inline fn lock(_: *DummyMutex) void {}
	211	inline fn unlock(_: *DummyMutex) void {}
	212	};
	213
	214	const stack_n = config.stack_trace_frames;
	215	const one_trace_size = @sizeOf(usize) * stack_n;
	216	const traces_per_slot = 2;
	217
	218	pub const Error = mem.Allocator.Error;
	219
	220	/// Avoids creating buckets that would only be able to store a small
	221	/// number of slots. Value of 1 means 2 is the minimum slot count.
	222	const minimum_slots_per_bucket_log2 = 1;
	223	const small_bucket_count = math.log2(page_size) - minimum_slots_per_bucket_log2;
	224	const largest_bucket_object_size = 1 << (small_bucket_count - 1);
	225	const LargestSizeClassInt = std.math.IntFittingRange(0, largest_bucket_object_size);
	226
	227	const bucketCompare = struct {
	228	fn compare(a: BucketHeader, b: BucketHeader) std.math.Order {
	229	return std.math.order(@intFromPtr(a.page), @intFromPtr(b.page));
	230	}
	231	}.compare;
	232
	233	const LargeAlloc = struct {
	234	bytes: []u8,
	235	requested_size: if (config.enable_memory_limit) usize else void,
	236	stack_addresses: [trace_n][stack_n]usize,
	237	freed: if (config.retain_metadata) bool else void,
	238	alignment: if (config.never_unmap and config.retain_metadata) mem.Alignment else void,
	239
	240	const trace_n = if (config.retain_metadata) traces_per_slot else 1;
	241
	242	fn dumpStackTrace(self: *LargeAlloc, trace_kind: TraceKind) void {
	243	std.debug.dumpStackTrace(self.getStackTrace(trace_kind));
	244	}
	245
	246	fn getStackTrace(self: *LargeAlloc, trace_kind: TraceKind) std.builtin.StackTrace {
	247	assert(@intFromEnum(trace_kind) < trace_n);
	248	const stack_addresses = &self.stack_addresses[@intFromEnum(trace_kind)];
	249	var len: usize = 0;
	250	while (len < stack_n and stack_addresses[len] != 0) {
	251	len += 1;
	252	}
	253	return .{
	254	.instruction_addresses = stack_addresses,
	255	.index = len,
	256	};
	257	}
	258
	259	fn captureStackTrace(self: *LargeAlloc, ret_addr: usize, trace_kind: TraceKind) void {
	260	assert(@intFromEnum(trace_kind) < trace_n);
	261	const stack_addresses = &self.stack_addresses[@intFromEnum(trace_kind)];
	262	collectStackTrace(ret_addr, stack_addresses);
	263	}
	264	};
	265	const LargeAllocTable = std.AutoHashMapUnmanaged(usize, LargeAlloc);
	266
	267	/// Bucket: In memory, in order:
	268	/// * BucketHeader
	269	/// * bucket_used_bits: [N]usize, // 1 bit for every slot
	270	/// -- below only exists when config.safety is true --
	271	/// * requested_sizes: [N]LargestSizeClassInt // 1 int for every slot
	272	/// * log2_ptr_aligns: [N]u8 // 1 byte for every slot
	273	/// -- above only exists when config.safety is true --
	274	/// * stack_trace_addresses: [N]usize, // traces_per_slot for every allocation
	275	const BucketHeader = struct {
	276	allocated_count: SlotIndex,
	277	freed_count: SlotIndex,
	278	prev: ?*BucketHeader,
	279	canary: usize = config.canary,
	280
	281	fn fromPage(page_addr: usize, slot_count: usize) *BucketHeader {
	282	const unaligned = page_addr + page_size - bucketSize(slot_count);
	283	return @ptrFromInt(unaligned & ~(@as(usize, @alignOf(BucketHeader)) - 1));
	284	}
	285
	286	fn usedBits(bucket: BucketHeader, index: usize) usize {
	287	const ptr: [*]u8 = @ptrCast(bucket);
	288	const bits: [*]usize = @alignCast(@ptrCast(ptr + @sizeOf(BucketHeader)));
	289	return &bits[index];
	290	}
	291
	292	fn requestedSizes(bucket: *BucketHeader, slot_count: usize) []LargestSizeClassInt {
	293	if (!config.safety) @compileError("requested size is only stored when safety is enabled");
	294	const start_ptr = @as([*]u8, @ptrCast(bucket)) + bucketRequestedSizesStart(slot_count);
	295	const sizes = @as([*]LargestSizeClassInt, @ptrCast(@alignCast(start_ptr)));
	296	return sizes[0..slot_count];
	297	}
	298
	299	fn log2PtrAligns(bucket: *BucketHeader, slot_count: usize) []mem.Alignment {
	300	if (!config.safety) @compileError("requested size is only stored when safety is enabled");
	301	const aligns_ptr = @as([*]u8, @ptrCast(bucket)) + bucketAlignsStart(slot_count);
	302	return @ptrCast(aligns_ptr[0..slot_count]);
	303	}
	304
	305	fn stackTracePtr(
	306	bucket: *BucketHeader,
	307	slot_count: usize,
	308	slot_index: SlotIndex,
	309	trace_kind: TraceKind,
	310	) *[stack_n]usize {
	311	const start_ptr = @as([*]u8, @ptrCast(bucket)) + bucketStackFramesStart(slot_count);
	312	const addr = start_ptr + one_trace_size * traces_per_slot * slot_index +
	313	@intFromEnum(trace_kind) * @as(usize, one_trace_size);
	314	return @ptrCast(@alignCast(addr));
	315	}
	316
	317	fn captureStackTrace(
	318	bucket: *BucketHeader,
	319	ret_addr: usize,
	320	slot_count: usize,
	321	slot_index: SlotIndex,
	322	trace_kind: TraceKind,
	323	) void {
	324	// Initialize them to 0. When determining the count we must look
	325	// for non zero addresses.
	326	const stack_addresses = bucket.stackTracePtr(slot_count, slot_index, trace_kind);
	327	collectStackTrace(ret_addr, stack_addresses);
	328	}
	329	};
	330
	331	pub fn allocator(self: *Self) Allocator {
	332	return .{
	333	.ptr = self,
	334	.vtable = &.{
	335	.alloc = alloc,
	336	.resize = resize,
	337	.remap = remap,
	338	.free = free,
	339	},
	340	};
	341	}
	342
	343	fn bucketStackTrace(
	344	bucket: *BucketHeader,
	345	slot_count: usize,
	346	slot_index: SlotIndex,
	347	trace_kind: TraceKind,
	348	) StackTrace {
	349	const stack_addresses = bucket.stackTracePtr(slot_count, slot_index, trace_kind);
	350	var len: usize = 0;
	351	while (len < stack_n and stack_addresses[len] != 0) {
	352	len += 1;
	353	}
	354	return .{
	355	.instruction_addresses = stack_addresses,
	356	.index = len,
	357	};
	358	}
	359
	360	fn bucketRequestedSizesStart(slot_count: usize) usize {
	361	if (!config.safety) @compileError("requested sizes are not stored unless safety is enabled");
	362	return mem.alignForward(
	363	usize,
	364	@sizeOf(BucketHeader) + usedBitsSize(slot_count),
	365	@alignOf(LargestSizeClassInt),
	366	);
	367	}
	368
	369	fn bucketAlignsStart(slot_count: usize) usize {
	370	if (!config.safety) @compileError("requested sizes are not stored unless safety is enabled");
	371	return bucketRequestedSizesStart(slot_count) + (@sizeOf(LargestSizeClassInt) * slot_count);
	372	}
	373
	374	fn bucketStackFramesStart(slot_count: usize) usize {
	375	const unaligned_start = if (config.safety)
	376	bucketAlignsStart(slot_count) + slot_count
	377	else
	378	@sizeOf(BucketHeader) + usedBitsSize(slot_count);
	379	return mem.alignForward(usize, unaligned_start, @alignOf(usize));
	380	}
	381
	382	fn bucketSize(slot_count: usize) usize {
	383	return bucketStackFramesStart(slot_count) + one_trace_size * traces_per_slot * slot_count;
	384	}
	385
	386	/// This is executed only at compile-time to prepopulate a lookup table.
	387	fn calculateSlotCount(size_class_index: usize) SlotIndex {
	388	const size_class = @as(usize, 1) << @as(Log2USize, @intCast(size_class_index));
	389	var lower: usize = 1 << minimum_slots_per_bucket_log2;
	390	var upper: usize = (page_size - bucketSize(lower)) / size_class;
	391	while (upper > lower) {
	392	const proposed: usize = lower + (upper - lower) / 2;
	393	if (proposed == lower) return lower;
	394	const slots_end = proposed * size_class;
	395	const header_begin = mem.alignForward(usize, slots_end, @alignOf(BucketHeader));
	396	const end = header_begin + bucketSize(proposed);
	397	if (end > page_size) {
	398	upper = proposed - 1;
	399	} else {
	400	lower = proposed;
	401	}
	402	}
	403	const slots_end = lower * size_class;
	404	const header_begin = mem.alignForward(usize, slots_end, @alignOf(BucketHeader));
	405	const end = header_begin + bucketSize(lower);
	406	assert(end <= page_size);
	407	return lower;
	408	}
	409
	410	fn usedBitsCount(slot_count: usize) usize {
	411	return (slot_count + (@bitSizeOf(usize) - 1)) / @bitSizeOf(usize);
	412	}
	413
	414	fn usedBitsSize(slot_count: usize) usize {
	415	return usedBitsCount(slot_count) * @sizeOf(usize);
	416	}
	417
	418	fn detectLeaksInBucket(bucket: *BucketHeader, size_class_index: usize, used_bits_count: usize) bool {
	419	const size_class = @as(usize, 1) << @as(Log2USize, @intCast(size_class_index));
	420	const slot_count = slot_counts[size_class_index];
	421	var leaks = false;
	422	for (0..used_bits_count) \|used_bits_byte\| {
	423	const used_int = bucket.usedBits(used_bits_byte).*;
	424	if (used_int != 0) {
	425	for (0..@bitSizeOf(usize)) \|bit_index_usize\| {
	426	const bit_index: Log2USize = @intCast(bit_index_usize);
	427	const is_used = @as(u1, @truncate(used_int >> bit_index)) != 0;
	428	if (is_used) {
	429	const slot_index: SlotIndex = @intCast(used_bits_byte * @bitSizeOf(usize) + bit_index);
	430	const stack_trace = bucketStackTrace(bucket, slot_count, slot_index, .alloc);
	431	const page_addr = @intFromPtr(bucket) & ~(page_size - 1);
	432	const addr = page_addr + slot_index * size_class;
	433	log.err("memory address 0x{x} leaked: {}", .{ addr, stack_trace });
	434	leaks = true;
	435	}
	436	}
	437	}
	438	}
	439	return leaks;
	440	}
	441
	442	/// Emits log messages for leaks and then returns whether there were any leaks.
	443	pub fn detectLeaks(self: *Self) bool {
	444	var leaks = false;
	445
	446	for (self.buckets, 0..) \|init_optional_bucket, size_class_index\| {
	447	var optional_bucket = init_optional_bucket;
	448	const slot_count = slot_counts[size_class_index];
	449	const used_bits_count = usedBitsCount(slot_count);
	450	while (optional_bucket) \|bucket\| {
	451	leaks = detectLeaksInBucket(bucket, size_class_index, used_bits_count) or leaks;
	452	optional_bucket = bucket.prev;
	453	}
	454	}
	455
	456	var it = self.large_allocations.valueIterator();
	457	while (it.next()) \|large_alloc\| {
	458	if (config.retain_metadata and large_alloc.freed) continue;
	459	const stack_trace = large_alloc.getStackTrace(.alloc);
	460	log.err("memory address 0x{x} leaked: {}", .{
	461	@intFromPtr(large_alloc.bytes.ptr), stack_trace,
	462	});
	463	leaks = true;
	464	}
	465	return leaks;
	466	}
	467
	468	fn freeRetainedMetadata(self: *Self) void {
	469	comptime assert(config.retain_metadata);
	470	if (config.never_unmap) {
	471	// free large allocations that were intentionally leaked by never_unmap
	472	var it = self.large_allocations.iterator();
	473	while (it.next()) \|large\| {
	474	if (large.value_ptr.freed) {
	475	self.backing_allocator.rawFree(large.value_ptr.bytes, large.value_ptr.alignment, @returnAddress());
	476	}
	477	}
	478	}
	479	}
	480
	481	pub fn flushRetainedMetadata(self: *Self) void {
	482	comptime assert(config.retain_metadata);
	483	self.freeRetainedMetadata();
	484	// also remove entries from large_allocations
	485	var it = self.large_allocations.iterator();
	486	while (it.next()) \|large\| {
	487	if (large.value_ptr.freed) {
	488	_ = self.large_allocations.remove(@intFromPtr(large.value_ptr.bytes.ptr));
	489	}
	490	}
	491	}
	492
	493	/// Returns `std.heap.Check.leak` if there were leaks; `std.heap.Check.ok` otherwise.
	494	pub fn deinit(self: *Self) std.heap.Check {
	495	const leaks = if (config.safety) self.detectLeaks() else false;
	496	if (config.retain_metadata) self.freeRetainedMetadata();
	497	self.large_allocations.deinit(self.backing_allocator);
	498	self.* = undefined;
	499	return if (leaks) .leak else .ok;
	500	}
	501
	502	fn collectStackTrace(first_trace_addr: usize, addresses: *[stack_n]usize) void {
	503	if (stack_n == 0) return;
	504	@memset(addresses, 0);
	505	var stack_trace: StackTrace = .{
	506	.instruction_addresses = addresses,
	507	.index = 0,
	508	};
	509	std.debug.captureStackTrace(first_trace_addr, &stack_trace);
	510	}
	511
	512	fn reportDoubleFree(ret_addr: usize, alloc_stack_trace: StackTrace, free_stack_trace: StackTrace) void {
	513	var addresses: [stack_n]usize = @splat(0);
	514	var second_free_stack_trace: StackTrace = .{
	515	.instruction_addresses = &addresses,
	516	.index = 0,
	517	};
	518	std.debug.captureStackTrace(ret_addr, &second_free_stack_trace);
	519	log.err("Double free detected. Allocation: {} First free: {} Second free: {}", .{
	520	alloc_stack_trace, free_stack_trace, second_free_stack_trace,
	521	});
	522	}
	523
	524	/// This function assumes the object is in the large object storage regardless
	525	/// of the parameters.
	526	fn resizeLarge(
	527	self: *Self,
	528	old_mem: []u8,
	529	alignment: mem.Alignment,
	530	new_size: usize,
	531	ret_addr: usize,
	532	may_move: bool,
	533	) ?[*]u8 {
	534	if (config.retain_metadata and may_move) {
	535	// Before looking up the entry (since this could invalidate
	536	// it), we must reserve space for the new entry in case the
	537	// allocation is relocated.
	538	self.large_allocations.ensureUnusedCapacity(self.backing_allocator, 1) catch return null;
	539	}
	540
	541	const entry = self.large_allocations.getEntry(@intFromPtr(old_mem.ptr)) orelse {
	542	if (config.safety) {
	543	@panic("Invalid free");
	544	} else {
	545	unreachable;
	546	}
	547	};
	548
	549	if (config.retain_metadata and entry.value_ptr.freed) {
	550	if (config.safety) {
	551	reportDoubleFree(ret_addr, entry.value_ptr.getStackTrace(.alloc), entry.value_ptr.getStackTrace(.free));
	552	@panic("Unrecoverable double free");
	553	} else {
	554	unreachable;
	555	}
	556	}
	557
	558	if (config.safety and old_mem.len != entry.value_ptr.bytes.len) {
	559	var addresses: [stack_n]usize = [1]usize{0} ** stack_n;
	560	var free_stack_trace: StackTrace = .{
	561	.instruction_addresses = &addresses,
	562	.index = 0,
	563	};
	564	std.debug.captureStackTrace(ret_addr, &free_stack_trace);
	565	log.err("Allocation size {d} bytes does not match free size {d}. Allocation: {} Free: {}", .{
	566	entry.value_ptr.bytes.len,
	567	old_mem.len,
	568	entry.value_ptr.getStackTrace(.alloc),
	569	free_stack_trace,
	570	});
	571	}
	572
	573	// If this would move the allocation into a small size class,
	574	// refuse the request, because it would require creating small
	575	// allocation metadata.
	576	const new_size_class_index: usize = @max(@bitSizeOf(usize) - @clz(new_size - 1), @intFromEnum(alignment));
	577	if (new_size_class_index < self.buckets.len) return null;
	578
	579	// Do memory limit accounting with requested sizes rather than what
	580	// backing_allocator returns because if we want to return
	581	// error.OutOfMemory, we have to leave allocation untouched, and
	582	// that is impossible to guarantee after calling
	583	// backing_allocator.rawResize.
	584	const prev_req_bytes = self.total_requested_bytes;
	585	if (config.enable_memory_limit) {
	586	const new_req_bytes = prev_req_bytes + new_size - entry.value_ptr.requested_size;
	587	if (new_req_bytes > prev_req_bytes and new_req_bytes > self.requested_memory_limit) {
	588	return null;
	589	}
	590	self.total_requested_bytes = new_req_bytes;
	591	}
	592
	593	const opt_resized_ptr = if (may_move)
	594	self.backing_allocator.rawRemap(old_mem, alignment, new_size, ret_addr)
	595	else if (self.backing_allocator.rawResize(old_mem, alignment, new_size, ret_addr))
	596	old_mem.ptr
	597	else
	598	null;
	599
	600	const resized_ptr = opt_resized_ptr orelse {
	601	if (config.enable_memory_limit) {
	602	self.total_requested_bytes = prev_req_bytes;
	603	}
	604	return null;
	605	};
	606
	607	if (config.enable_memory_limit) {
	608	entry.value_ptr.requested_size = new_size;
	609	}
	610
	611	if (config.verbose_log) {
	612	log.info("large resize {d} bytes at {} to {d} at {}", .{
	613	old_mem.len, old_mem.ptr, new_size, resized_ptr,
	614	});
	615	}
	616	entry.value_ptr.bytes = resized_ptr[0..new_size];
	617	if (config.resize_stack_traces)
	618	entry.value_ptr.captureStackTrace(ret_addr, .alloc);
	619
	620	// Update the key of the hash map if the memory was relocated.
	621	if (resized_ptr != old_mem.ptr) {
	622	const large_alloc = entry.value_ptr.*;
	623	if (config.retain_metadata) {
	624	entry.value_ptr.freed = true;
	625	entry.value_ptr.captureStackTrace(ret_addr, .free);
	626	} else {
	627	self.large_allocations.removeByPtr(entry.key_ptr);
	628	}
	629
	630	const gop = self.large_allocations.getOrPutAssumeCapacity(@intFromPtr(resized_ptr));
	631	if (config.retain_metadata and !config.never_unmap) {
	632	// Backing allocator may be reusing memory that we're retaining metadata for
	633	assert(!gop.found_existing or gop.value_ptr.freed);
	634	} else {
	635	assert(!gop.found_existing); // This would mean the kernel double-mapped pages.
	636	}
	637	gop.value_ptr.* = large_alloc;
	638	}
	639
	640	return resized_ptr;
	641	}
	642
	643	/// This function assumes the object is in the large object storage regardless
	644	/// of the parameters.
	645	fn freeLarge(
	646	self: *Self,
	647	old_mem: []u8,
	648	alignment: mem.Alignment,
	649	ret_addr: usize,
	650	) void {
	651	const entry = self.large_allocations.getEntry(@intFromPtr(old_mem.ptr)) orelse {
	652	if (config.safety) {
	653	@panic("Invalid free");
	654	} else {
	655	unreachable;
	656	}
	657	};
	658
	659	if (config.retain_metadata and entry.value_ptr.freed) {
	660	if (config.safety) {
	661	reportDoubleFree(ret_addr, entry.value_ptr.getStackTrace(.alloc), entry.value_ptr.getStackTrace(.free));
	662	return;
	663	} else {
	664	unreachable;
	665	}
	666	}
	667
	668	if (config.safety and old_mem.len != entry.value_ptr.bytes.len) {
	669	var addresses: [stack_n]usize = [1]usize{0} ** stack_n;
	670	var free_stack_trace = StackTrace{
	671	.instruction_addresses = &addresses,
	672	.index = 0,
	673	};
	674	std.debug.captureStackTrace(ret_addr, &free_stack_trace);
	675	log.err("Allocation size {d} bytes does not match free size {d}. Allocation: {} Free: {}", .{
	676	entry.value_ptr.bytes.len,
	677	old_mem.len,
	678	entry.value_ptr.getStackTrace(.alloc),
	679	free_stack_trace,
	680	});
	681	}
	682
	683	if (!config.never_unmap) {
	684	self.backing_allocator.rawFree(old_mem, alignment, ret_addr);
	685	}
	686
	687	if (config.enable_memory_limit) {
	688	self.total_requested_bytes -= entry.value_ptr.requested_size;
	689	}
	690
	691	if (config.verbose_log) {
	692	log.info("large free {d} bytes at {*}", .{ old_mem.len, old_mem.ptr });
	693	}
	694
	695	if (!config.retain_metadata) {
	696	assert(self.large_allocations.remove(@intFromPtr(old_mem.ptr)));
	697	} else {
	698	entry.value_ptr.freed = true;
	699	entry.value_ptr.captureStackTrace(ret_addr, .free);
	700	}
	701	}
	702
	703	fn alloc(context: anyopaque, len: usize, alignment: mem.Alignment, ret_addr: usize) ?[]u8 {
	704	const self: *Self = @ptrCast(@alignCast(context));
	705	self.mutex.lock();
	706	defer self.mutex.unlock();
	707
	708	if (config.enable_memory_limit) {
	709	const new_req_bytes = self.total_requested_bytes + len;
	710	if (new_req_bytes > self.requested_memory_limit) return null;
	711	self.total_requested_bytes = new_req_bytes;
	712	}
	713
	714	const size_class_index: usize = @max(@bitSizeOf(usize) - @clz(len - 1), @intFromEnum(alignment));
	715	if (size_class_index >= self.buckets.len) {
	716	@branchHint(.unlikely);
	717	self.large_allocations.ensureUnusedCapacity(self.backing_allocator, 1) catch return null;
	718	const ptr = self.backing_allocator.rawAlloc(len, alignment, ret_addr) orelse return null;
	719	const slice = ptr[0..len];
	720
	721	const gop = self.large_allocations.getOrPutAssumeCapacity(@intFromPtr(slice.ptr));
	722	if (config.retain_metadata and !config.never_unmap) {
	723	// Backing allocator may be reusing memory that we're retaining metadata for
	724	assert(!gop.found_existing or gop.value_ptr.freed);
	725	} else {
	726	assert(!gop.found_existing); // This would mean the kernel double-mapped pages.
	727	}
	728	gop.value_ptr.bytes = slice;
	729	if (config.enable_memory_limit)
	730	gop.value_ptr.requested_size = len;
	731	gop.value_ptr.captureStackTrace(ret_addr, .alloc);
	732	if (config.retain_metadata) {
	733	gop.value_ptr.freed = false;
	734	if (config.never_unmap) {
	735	gop.value_ptr.alignment = alignment;
	736	}
	737	}
	738
	739	if (config.verbose_log) {
	740	log.info("large alloc {d} bytes at {*}", .{ slice.len, slice.ptr });
	741	}
	742	return slice.ptr;
	743	}
	744
	745	const slot_count = slot_counts[size_class_index];
	746
	747	if (self.buckets[size_class_index]) \|bucket\| {
	748	@branchHint(.likely);
	749	const slot_index = bucket.allocated_count;
	750	if (slot_index < slot_count) {
	751	@branchHint(.likely);
	752	bucket.allocated_count = slot_index + 1;
	753	const used_bits_byte = bucket.usedBits(slot_index / @bitSizeOf(usize));
	754	const used_bit_index: Log2USize = @intCast(slot_index % @bitSizeOf(usize));
	755	used_bits_byte.* \|= (@as(usize, 1) << used_bit_index);
	756	const size_class = @as(usize, 1) << @as(Log2USize, @intCast(size_class_index));
	757	if (config.stack_trace_frames > 0) {
	758	bucket.captureStackTrace(ret_addr, slot_count, slot_index, .alloc);
	759	}
	760	if (config.safety) {
	761	bucket.requestedSizes(slot_count)[slot_index] = @intCast(len);
	762	bucket.log2PtrAligns(slot_count)[slot_index] = alignment;
	763	}
	764	const page_addr = @intFromPtr(bucket) & ~(page_size - 1);
	765	const addr = page_addr + slot_index * size_class;
	766	if (config.verbose_log) {
	767	log.info("small alloc {d} bytes at 0x{x}", .{ len, addr });
	768	}
	769	return @ptrFromInt(addr);
	770	}
	771	}
	772
	773	const page = self.backing_allocator.rawAlloc(page_size, page_align, @returnAddress()) orelse
	774	return null;
	775	const bucket: *BucketHeader = .fromPage(@intFromPtr(page), slot_count);
	776	bucket.* = .{
	777	.allocated_count = 1,
	778	.freed_count = 0,
	779	.prev = self.buckets[size_class_index],
	780	};
	781	self.buckets[size_class_index] = bucket;
	782
	783	if (!config.backing_allocator_zeroes) {
	784	@memset(@as([]usize, @as([1]usize, bucket.usedBits(0)))[0..usedBitsCount(slot_count)], 0);
	785	if (config.safety) @memset(bucket.requestedSizes(slot_count), 0);
	786	}
	787
	788	bucket.usedBits(0).* = 0b1;
	789
	790	if (config.stack_trace_frames > 0) {
	791	bucket.captureStackTrace(ret_addr, slot_count, 0, .alloc);
	792	}
	793
	794	if (config.safety) {
	795	bucket.requestedSizes(slot_count)[0] = @intCast(len);
	796	bucket.log2PtrAligns(slot_count)[0] = alignment;
	797	}
	798
	799	if (config.verbose_log) {
	800	log.info("small alloc {d} bytes at 0x{x}", .{ len, @intFromPtr(page) });
	801	}
	802
	803	return page;
	804	}
	805
	806	fn resize(
	807	context: *anyopaque,
	808	memory: []u8,
	809	alignment: mem.Alignment,
	810	new_len: usize,
	811	return_address: usize,
	812	) bool {
	813	const self: *Self = @ptrCast(@alignCast(context));
	814	self.mutex.lock();
	815	defer self.mutex.unlock();
	816
	817	const size_class_index: usize = @max(@bitSizeOf(usize) - @clz(memory.len - 1), @intFromEnum(alignment));
	818	if (size_class_index >= self.buckets.len) {
	819	return self.resizeLarge(memory, alignment, new_len, return_address, false) != null;
	820	} else {
	821	return resizeSmall(self, memory, alignment, new_len, return_address, size_class_index);
	822	}
	823	}
	824
	825	fn remap(
	826	context: *anyopaque,
	827	memory: []u8,
	828	alignment: mem.Alignment,
	829	new_len: usize,
	830	return_address: usize,
	831	) ?[*]u8 {
	832	const self: *Self = @ptrCast(@alignCast(context));
	833	self.mutex.lock();
	834	defer self.mutex.unlock();
	835
	836	const size_class_index: usize = @max(@bitSizeOf(usize) - @clz(memory.len - 1), @intFromEnum(alignment));
	837	if (size_class_index >= self.buckets.len) {
	838	return self.resizeLarge(memory, alignment, new_len, return_address, true);
	839	} else {
	840	return if (resizeSmall(self, memory, alignment, new_len, return_address, size_class_index)) memory.ptr else null;
	841	}
	842	}
	843
	844	fn free(
	845	context: *anyopaque,
	846	old_memory: []u8,
	847	alignment: mem.Alignment,
	848	return_address: usize,
	849	) void {
	850	const self: *Self = @ptrCast(@alignCast(context));
	851	self.mutex.lock();
	852	defer self.mutex.unlock();
	853
	854	assert(old_memory.len != 0);
	855
	856	const size_class_index: usize = @max(@bitSizeOf(usize) - @clz(old_memory.len - 1), @intFromEnum(alignment));
	857	if (size_class_index >= self.buckets.len) {
	858	@branchHint(.unlikely);
	859	self.freeLarge(old_memory, alignment, return_address);
	860	return;
	861	}
	862
	863	const slot_count = slot_counts[size_class_index];
	864	const freed_addr = @intFromPtr(old_memory.ptr);
	865	const page_addr = freed_addr & ~(page_size - 1);
	866	const bucket: *BucketHeader = .fromPage(page_addr, slot_count);
	867	if (bucket.canary != config.canary) @panic("Invalid free");
	868	const page_offset = freed_addr - page_addr;
	869	const size_class = @as(usize, 1) << @as(Log2USize, @intCast(size_class_index));
	870	const slot_index: SlotIndex = @intCast(page_offset / size_class);
	871	const used_byte_index = slot_index / @bitSizeOf(usize);
	872	const used_bit_index: Log2USize = @intCast(slot_index % @bitSizeOf(usize));
	873	const used_byte = bucket.usedBits(used_byte_index);
	874	const is_used = @as(u1, @truncate(used_byte.* >> used_bit_index)) != 0;
	875	if (!is_used) {
	876	if (config.safety) {
	877	reportDoubleFree(
	878	return_address,
	879	bucketStackTrace(bucket, slot_count, slot_index, .alloc),
	880	bucketStackTrace(bucket, slot_count, slot_index, .free),
	881	);
	882	// Recoverable since this is a free.
	883	return;
	884	} else {
	885	unreachable;
	886	}
	887	}
	888
	889	// Definitely an in-use small alloc now.
	890	if (config.safety) {
	891	const requested_size = bucket.requestedSizes(slot_count)[slot_index];
	892	if (requested_size == 0) @panic("Invalid free");
	893	const slot_alignment = bucket.log2PtrAligns(slot_count)[slot_index];
	894	if (old_memory.len != requested_size or alignment != slot_alignment) {
	895	var addresses: [stack_n]usize = [1]usize{0} ** stack_n;
	896	var free_stack_trace: StackTrace = .{
	897	.instruction_addresses = &addresses,
	898	.index = 0,
	899	};
	900	std.debug.captureStackTrace(return_address, &free_stack_trace);
	901	if (old_memory.len != requested_size) {
	902	log.err("Allocation size {d} bytes does not match free size {d}. Allocation: {} Free: {}", .{
	903	requested_size,
	904	old_memory.len,
	905	bucketStackTrace(bucket, slot_count, slot_index, .alloc),
	906	free_stack_trace,
	907	});
	908	}
	909	if (alignment != slot_alignment) {
	910	log.err("Allocation alignment {d} does not match free alignment {d}. Allocation: {} Free: {}", .{
	911	slot_alignment.toByteUnits(),
	912	alignment.toByteUnits(),
	913	bucketStackTrace(bucket, slot_count, slot_index, .alloc),
	914	free_stack_trace,
	915	});
	916	}
	917	}
	918	}
	919
	920	if (config.enable_memory_limit) {
	921	self.total_requested_bytes -= old_memory.len;
	922	}
	923
	924	if (config.stack_trace_frames > 0) {
	925	// Capture stack trace to be the "first free", in case a double free happens.
	926	bucket.captureStackTrace(return_address, slot_count, slot_index, .free);
	927	}
	928
	929	used_byte.* &= ~(@as(usize, 1) << used_bit_index);
	930	if (config.safety) {
	931	bucket.requestedSizes(slot_count)[slot_index] = 0;
	932	}
	933	bucket.freed_count += 1;
	934	if (bucket.freed_count == bucket.allocated_count) {
	935	if (self.buckets[size_class_index] == bucket) {
	936	self.buckets[size_class_index] = null;
	937	}
	938	if (!config.never_unmap) {
	939	const page: [*]align(page_size) u8 = @ptrFromInt(page_addr);
	940	self.backing_allocator.rawFree(page[0..page_size], page_align, @returnAddress());
	941	}
	942	}
	943	if (config.verbose_log) {
	944	log.info("small free {d} bytes at {*}", .{ old_memory.len, old_memory.ptr });
	945	}
	946	}
	947
	948	fn resizeSmall(
	949	self: *Self,
	950	memory: []u8,
	951	alignment: mem.Alignment,
	952	new_len: usize,
	953	return_address: usize,
	954	size_class_index: usize,
	955	) bool {
	956	const new_size_class_index: usize = @max(@bitSizeOf(usize) - @clz(new_len - 1), @intFromEnum(alignment));
	957	if (!config.safety) return new_size_class_index == size_class_index;
	958	const slot_count = slot_counts[size_class_index];
	959	const memory_addr = @intFromPtr(memory.ptr);
	960	const page_addr = memory_addr & ~(page_size - 1);
	961	const bucket: *BucketHeader = .fromPage(page_addr, slot_count);
	962	if (bucket.canary != config.canary) @panic("Invalid free");
	963	const page_offset = memory_addr - page_addr;
	964	const size_class = @as(usize, 1) << @as(Log2USize, @intCast(size_class_index));
	965	const slot_index: SlotIndex = @intCast(page_offset / size_class);
	966	const used_byte_index = slot_index / @bitSizeOf(usize);
	967	const used_bit_index: Log2USize = @intCast(slot_index % @bitSizeOf(usize));
	968	const used_byte = bucket.usedBits(used_byte_index);
	969	const is_used = @as(u1, @truncate(used_byte.* >> used_bit_index)) != 0;
	970	if (!is_used) {
	971	reportDoubleFree(
	972	return_address,
	973	bucketStackTrace(bucket, slot_count, slot_index, .alloc),
	974	bucketStackTrace(bucket, slot_count, slot_index, .free),
	975	);
	976	// Recoverable since this is a free.
	977	return false;
	978	}
	979
	980	// Definitely an in-use small alloc now.
	981	const requested_size = bucket.requestedSizes(slot_count)[slot_index];
	982	if (requested_size == 0) @panic("Invalid free");
	983	const slot_alignment = bucket.log2PtrAligns(slot_count)[slot_index];
	984	if (memory.len != requested_size or alignment != slot_alignment) {
	985	var addresses: [stack_n]usize = [1]usize{0} ** stack_n;
	986	var free_stack_trace: StackTrace = .{
	987	.instruction_addresses = &addresses,
	988	.index = 0,
	989	};
	990	std.debug.captureStackTrace(return_address, &free_stack_trace);
	991	if (memory.len != requested_size) {
	992	log.err("Allocation size {d} bytes does not match free size {d}. Allocation: {} Free: {}", .{
	993	requested_size,
	994	memory.len,
	995	bucketStackTrace(bucket, slot_count, slot_index, .alloc),
	996	free_stack_trace,
	997	});
	998	}
	999	if (alignment != slot_alignment) {
	1000	log.err("Allocation alignment {d} does not match free alignment {d}. Allocation: {} Free: {}", .{
	1001	slot_alignment.toByteUnits(),
	1002	alignment.toByteUnits(),
	1003	bucketStackTrace(bucket, slot_count, slot_index, .alloc),
	1004	free_stack_trace,
	1005	});
	1006	}
	1007	}
	1008
	1009	if (new_size_class_index != size_class_index) return false;
	1010
	1011	const prev_req_bytes = self.total_requested_bytes;
	1012	if (config.enable_memory_limit) {
	1013	const new_req_bytes = prev_req_bytes - memory.len + new_len;
	1014	if (new_req_bytes > prev_req_bytes and new_req_bytes > self.requested_memory_limit) {
	1015	return false;
	1016	}
	1017	self.total_requested_bytes = new_req_bytes;
	1018	}
	1019
	1020	if (memory.len > new_len) @memset(memory[new_len..], undefined);
	1021	if (config.verbose_log)
	1022	log.info("small resize {d} bytes at {*} to {d}", .{ memory.len, memory.ptr, new_len });
	1023
	1024	if (config.safety)
	1025	bucket.requestedSizes(slot_count)[slot_index] = @intCast(new_len);
	1026
	1027	if (config.resize_stack_traces)
	1028	bucket.captureStackTrace(return_address, slot_count, slot_index, .alloc);
	1029
	1030	return true;
	1031	}
	1032	};
	1033	}
	1034
	1035	const TraceKind = enum {
	1036	alloc,
	1037	free,
	1038	};
	1039
	1040	const test_config = Config{};
	1041
	1042	test "small allocations - free in same order" {
	1043	var gpa = DebugAllocator(test_config){};
	1044	defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak");
	1045	const allocator = gpa.allocator();
	1046
	1047	var list = std.ArrayList(*u64).init(std.testing.allocator);
	1048	defer list.deinit();
	1049
	1050	var i: usize = 0;
	1051	while (i < 513) : (i += 1) {
	1052	const ptr = try allocator.create(u64);
	1053	try list.append(ptr);
	1054	}
	1055
	1056	for (list.items) \|ptr\| {
	1057	allocator.destroy(ptr);
	1058	}
	1059	}
	1060
	1061	test "small allocations - free in reverse order" {
	1062	var gpa = DebugAllocator(test_config){};
	1063	defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak");
	1064	const allocator = gpa.allocator();
	1065
	1066	var list = std.ArrayList(*u64).init(std.testing.allocator);
	1067	defer list.deinit();
	1068
	1069	var i: usize = 0;
	1070	while (i < 513) : (i += 1) {
	1071	const ptr = try allocator.create(u64);
	1072	try list.append(ptr);
	1073	}
	1074
	1075	while (list.popOrNull()) \|ptr\| {
	1076	allocator.destroy(ptr);
	1077	}
	1078	}
	1079
	1080	test "large allocations" {
	1081	var gpa = DebugAllocator(test_config){};
	1082	defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak");
	1083	const allocator = gpa.allocator();
	1084
	1085	const ptr1 = try allocator.alloc(u64, 42768);
	1086	const ptr2 = try allocator.alloc(u64, 52768);
	1087	allocator.free(ptr1);
	1088	const ptr3 = try allocator.alloc(u64, 62768);
	1089	allocator.free(ptr3);
	1090	allocator.free(ptr2);
	1091	}
	1092
	1093	test "very large allocation" {
	1094	var gpa = DebugAllocator(test_config){};
	1095	defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak");
	1096	const allocator = gpa.allocator();
	1097
	1098	try std.testing.expectError(error.OutOfMemory, allocator.alloc(u8, math.maxInt(usize)));
	1099	}
	1100
	1101	test "realloc" {
	1102	var gpa = DebugAllocator(test_config){};
	1103	defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak");
	1104	const allocator = gpa.allocator();
	1105
	1106	var slice = try allocator.alignedAlloc(u8, @alignOf(u32), 1);
	1107	defer allocator.free(slice);
	1108	slice[0] = 0x12;
	1109
	1110	// This reallocation should keep its pointer address.
	1111	const old_slice = slice;
	1112	slice = try allocator.realloc(slice, 2);
	1113	try std.testing.expect(old_slice.ptr == slice.ptr);
	1114	try std.testing.expect(slice[0] == 0x12);
	1115	slice[1] = 0x34;
	1116
	1117	// This requires upgrading to a larger size class
	1118	slice = try allocator.realloc(slice, 17);
	1119	try std.testing.expect(slice[0] == 0x12);
	1120	try std.testing.expect(slice[1] == 0x34);
	1121	}
	1122
	1123	test "shrink" {
	1124	var gpa: DebugAllocator(test_config) = .{};
	1125	defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak");
	1126	const allocator = gpa.allocator();
	1127
	1128	var slice = try allocator.alloc(u8, 20);
	1129	defer allocator.free(slice);
	1130
	1131	@memset(slice, 0x11);
	1132
	1133	try std.testing.expect(allocator.resize(slice, 17));
	1134	slice = slice[0..17];
	1135
	1136	for (slice) \|b\| {
	1137	try std.testing.expect(b == 0x11);
	1138	}
	1139
	1140	// Does not cross size class boundaries when shrinking.
	1141	try std.testing.expect(!allocator.resize(slice, 16));
	1142	}
	1143
	1144	test "large object - grow" {
	1145	if (builtin.target.isWasm()) {
	1146	// Not expected to pass on targets that do not have memory mapping.
	1147	return error.SkipZigTest;
	1148	}
	1149	var gpa: DebugAllocator(test_config) = .{};
	1150	defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak");
	1151	const allocator = gpa.allocator();
	1152
	1153	var slice1 = try allocator.alloc(u8, default_page_size * 2 - 20);
	1154	defer allocator.free(slice1);
	1155
	1156	const old = slice1;
	1157	slice1 = try allocator.realloc(slice1, default_page_size * 2 - 10);
	1158	try std.testing.expect(slice1.ptr == old.ptr);
	1159
	1160	slice1 = try allocator.realloc(slice1, default_page_size * 2);
	1161	try std.testing.expect(slice1.ptr == old.ptr);
	1162
	1163	slice1 = try allocator.realloc(slice1, default_page_size * 2 + 1);
	1164	}
	1165
	1166	test "realloc small object to large object" {
	1167	var gpa = DebugAllocator(test_config){};
	1168	defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak");
	1169	const allocator = gpa.allocator();
	1170
	1171	var slice = try allocator.alloc(u8, 70);
	1172	defer allocator.free(slice);
	1173	slice[0] = 0x12;
	1174	slice[60] = 0x34;
	1175
	1176	// This requires upgrading to a large object
	1177	const large_object_size = default_page_size * 2 + 50;
	1178	slice = try allocator.realloc(slice, large_object_size);
	1179	try std.testing.expect(slice[0] == 0x12);
	1180	try std.testing.expect(slice[60] == 0x34);
	1181	}
	1182
	1183	test "shrink large object to large object" {
	1184	var gpa: DebugAllocator(test_config) = .{};
	1185	defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak");
	1186	const allocator = gpa.allocator();
	1187
	1188	var slice = try allocator.alloc(u8, default_page_size * 2 + 50);
	1189	defer allocator.free(slice);
	1190	slice[0] = 0x12;
	1191	slice[60] = 0x34;
	1192
	1193	if (!allocator.resize(slice, default_page_size * 2 + 1)) return;
	1194	slice = slice.ptr[0 .. default_page_size * 2 + 1];
	1195	try std.testing.expect(slice[0] == 0x12);
	1196	try std.testing.expect(slice[60] == 0x34);
	1197
	1198	try std.testing.expect(allocator.resize(slice, default_page_size * 2 + 1));
	1199	slice = slice[0 .. default_page_size * 2 + 1];
	1200	try std.testing.expect(slice[0] == 0x12);
	1201	try std.testing.expect(slice[60] == 0x34);
	1202
	1203	slice = try allocator.realloc(slice, default_page_size * 2);
	1204	try std.testing.expect(slice[0] == 0x12);
	1205	try std.testing.expect(slice[60] == 0x34);
	1206	}
	1207
	1208	test "shrink large object to large object with larger alignment" {
	1209	if (!builtin.link_libc and builtin.os.tag == .wasi) return error.SkipZigTest; // https://github.com/ziglang/zig/issues/22731
	1210
	1211	var gpa = DebugAllocator(test_config){};
	1212	defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak");
	1213	const allocator = gpa.allocator();
	1214
	1215	var debug_buffer: [1000]u8 = undefined;
	1216	var fba = std.heap.FixedBufferAllocator.init(&debug_buffer);
	1217	const debug_allocator = fba.allocator();
	1218
	1219	const alloc_size = default_page_size * 2 + 50;
	1220	var slice = try allocator.alignedAlloc(u8, 16, alloc_size);
	1221	defer allocator.free(slice);
	1222
	1223	const big_alignment: usize = default_page_size * 2;
	1224	// This loop allocates until we find a page that is not aligned to the big
	1225	// alignment. Then we shrink the allocation after the loop, but increase the
	1226	// alignment to the higher one, that we know will force it to realloc.
	1227	var stuff_to_free = std.ArrayList([]align(16) u8).init(debug_allocator);
	1228	while (mem.isAligned(@intFromPtr(slice.ptr), big_alignment)) {
	1229	try stuff_to_free.append(slice);
	1230	slice = try allocator.alignedAlloc(u8, 16, alloc_size);
	1231	}
	1232	while (stuff_to_free.popOrNull()) \|item\| {
	1233	allocator.free(item);
	1234	}
	1235	slice[0] = 0x12;
	1236	slice[60] = 0x34;
	1237
	1238	slice = try allocator.reallocAdvanced(slice, big_alignment, alloc_size / 2);
	1239	try std.testing.expect(slice[0] == 0x12);
	1240	try std.testing.expect(slice[60] == 0x34);
	1241	}
	1242
	1243	test "realloc large object to small object" {
	1244	var gpa = DebugAllocator(test_config){};
	1245	defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak");
	1246	const allocator = gpa.allocator();
	1247
	1248	var slice = try allocator.alloc(u8, default_page_size * 2 + 50);
	1249	defer allocator.free(slice);
	1250	slice[0] = 0x12;
	1251	slice[16] = 0x34;
	1252
	1253	slice = try allocator.realloc(slice, 19);
	1254	try std.testing.expect(slice[0] == 0x12);
	1255	try std.testing.expect(slice[16] == 0x34);
	1256	}
	1257
	1258	test "overridable mutexes" {
	1259	var gpa = DebugAllocator(.{ .MutexType = std.Thread.Mutex }){
	1260	.backing_allocator = std.testing.allocator,
	1261	.mutex = std.Thread.Mutex{},
	1262	};
	1263	defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak");
	1264	const allocator = gpa.allocator();
	1265
	1266	const ptr = try allocator.create(i32);
	1267	defer allocator.destroy(ptr);
	1268	}
	1269
	1270	test "non-page-allocator backing allocator" {
	1271	var gpa: DebugAllocator(.{
	1272	.backing_allocator_zeroes = false,
	1273	}) = .{
	1274	.backing_allocator = std.testing.allocator,
	1275	};
	1276	defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak");
	1277	const allocator = gpa.allocator();
	1278
	1279	const ptr = try allocator.create(i32);
	1280	defer allocator.destroy(ptr);
	1281	}
	1282
	1283	test "realloc large object to larger alignment" {
	1284	if (!builtin.link_libc and builtin.os.tag == .wasi) return error.SkipZigTest; // https://github.com/ziglang/zig/issues/22731
	1285
	1286	var gpa = DebugAllocator(test_config){};
	1287	defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak");
	1288	const allocator = gpa.allocator();
	1289
	1290	var debug_buffer: [1000]u8 = undefined;
	1291	var fba = std.heap.FixedBufferAllocator.init(&debug_buffer);
	1292	const debug_allocator = fba.allocator();
	1293
	1294	var slice = try allocator.alignedAlloc(u8, 16, default_page_size * 2 + 50);
	1295	defer allocator.free(slice);
	1296
	1297	const big_alignment: usize = default_page_size * 2;
	1298	// This loop allocates until we find a page that is not aligned to the big alignment.
	1299	var stuff_to_free = std.ArrayList([]align(16) u8).init(debug_allocator);
	1300	while (mem.isAligned(@intFromPtr(slice.ptr), big_alignment)) {
	1301	try stuff_to_free.append(slice);
	1302	slice = try allocator.alignedAlloc(u8, 16, default_page_size * 2 + 50);
	1303	}
	1304	while (stuff_to_free.popOrNull()) \|item\| {
	1305	allocator.free(item);
	1306	}
	1307	slice[0] = 0x12;
	1308	slice[16] = 0x34;
	1309
	1310	slice = try allocator.reallocAdvanced(slice, 32, default_page_size * 2 + 100);
	1311	try std.testing.expect(slice[0] == 0x12);
	1312	try std.testing.expect(slice[16] == 0x34);
	1313
	1314	slice = try allocator.reallocAdvanced(slice, 32, default_page_size * 2 + 25);
	1315	try std.testing.expect(slice[0] == 0x12);
	1316	try std.testing.expect(slice[16] == 0x34);
	1317
	1318	slice = try allocator.reallocAdvanced(slice, big_alignment, default_page_size * 2 + 100);
	1319	try std.testing.expect(slice[0] == 0x12);
	1320	try std.testing.expect(slice[16] == 0x34);
	1321	}
	1322
	1323	test "large object rejects shrinking to small" {
	1324	if (builtin.target.isWasm()) {
	1325	// Not expected to pass on targets that do not have memory mapping.
	1326	return error.SkipZigTest;
	1327	}
	1328
	1329	var failing_allocator = std.testing.FailingAllocator.init(std.heap.page_allocator, .{ .fail_index = 3 });
	1330	var gpa: DebugAllocator(.{}) = .{
	1331	.backing_allocator = failing_allocator.allocator(),
	1332	};
	1333	defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak");
	1334	const allocator = gpa.allocator();
	1335
	1336	var slice = try allocator.alloc(u8, default_page_size * 2 + 50);
	1337	defer allocator.free(slice);
	1338	slice[0] = 0x12;
	1339	slice[3] = 0x34;
	1340
	1341	try std.testing.expect(!allocator.resize(slice, 4));
	1342	try std.testing.expect(slice[0] == 0x12);
	1343	try std.testing.expect(slice[3] == 0x34);
	1344	}
	1345
	1346	test "objects of size 1024 and 2048" {
	1347	var gpa = DebugAllocator(test_config){};
	1348	defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak");
	1349	const allocator = gpa.allocator();
	1350
	1351	const slice = try allocator.alloc(u8, 1025);
	1352	const slice2 = try allocator.alloc(u8, 3000);
	1353
	1354	allocator.free(slice);
	1355	allocator.free(slice2);
	1356	}
	1357
	1358	test "setting a memory cap" {
	1359	var gpa = DebugAllocator(.{ .enable_memory_limit = true }){};
	1360	defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak");
	1361	const allocator = gpa.allocator();
	1362
	1363	gpa.requested_memory_limit = 1010;
	1364
	1365	const small = try allocator.create(i32);
	1366	try std.testing.expect(gpa.total_requested_bytes == 4);
	1367
	1368	const big = try allocator.alloc(u8, 1000);
	1369	try std.testing.expect(gpa.total_requested_bytes == 1004);
	1370
	1371	try std.testing.expectError(error.OutOfMemory, allocator.create(u64));
	1372
	1373	allocator.destroy(small);
	1374	try std.testing.expect(gpa.total_requested_bytes == 1000);
	1375
	1376	allocator.free(big);
	1377	try std.testing.expect(gpa.total_requested_bytes == 0);
	1378
	1379	const exact = try allocator.alloc(u8, 1010);
	1380	try std.testing.expect(gpa.total_requested_bytes == 1010);
	1381	allocator.free(exact);
	1382	}
	1383
	1384	test "large allocations count requested size not backing size" {
	1385	var gpa: DebugAllocator(.{ .enable_memory_limit = true }) = .{};
	1386	const allocator = gpa.allocator();
	1387
	1388	var buf = try allocator.alignedAlloc(u8, 1, default_page_size + 1);
	1389	try std.testing.expectEqual(default_page_size + 1, gpa.total_requested_bytes);
	1390	buf = try allocator.realloc(buf, 1);
	1391	try std.testing.expectEqual(1, gpa.total_requested_bytes);
	1392	buf = try allocator.realloc(buf, 2);
	1393	try std.testing.expectEqual(2, gpa.total_requested_bytes);
	1394	}
	1395
	1396	test "retain metadata and never unmap" {
	1397	var gpa = std.heap.DebugAllocator(.{
	1398	.safety = true,
	1399	.never_unmap = true,
	1400	.retain_metadata = true,
	1401	}){};
	1402	defer std.debug.assert(gpa.deinit() == .ok);
	1403	const allocator = gpa.allocator();
	1404
	1405	const alloc = try allocator.alloc(u8, 8);
	1406	allocator.free(alloc);
	1407
	1408	const alloc2 = try allocator.alloc(u8, 8);
	1409	allocator.free(alloc2);
	1410	}

lib/std/heap/general_purpose_allocator.zig deleted-1500

...	...	@@ -1,1500 +0,0 @@
1		//! # General Purpose Allocator
2		//!
3		//! ## Design Priorities
4		//!
5		//! ### `OptimizationMode.debug` and `OptimizationMode.release_safe`:
6		//!
7		//! * Detect double free, and emit stack trace of:
8		//! - Where it was first allocated
9		//! - Where it was freed the first time
10		//! - Where it was freed the second time
11		//!
12		//! * Detect leaks and emit stack trace of:
13		//! - Where it was allocated
14		//!
15		//! * When a page of memory is no longer needed, give it back to resident memory
16		//! as soon as possible, so that it causes page faults when used.
17		//!
18		//! * Do not re-use memory slots, so that memory safety is upheld. For small
19		//! allocations, this is handled here; for larger ones it is handled in the
20		//! backing allocator (by default `std.heap.page_allocator`).
21		//!
22		//! * Make pointer math errors unlikely to harm memory from
23		//! unrelated allocations.
24		//!
25		//! * It's OK for these mechanisms to cost some extra overhead bytes.
26		//!
27		//! * It's OK for performance cost for these mechanisms.
28		//!
29		//! * Rogue memory writes should not harm the allocator's state.
30		//!
31		//! * Cross platform. Operates based on a backing allocator which makes it work
32		//! everywhere, even freestanding.
33		//!
34		//! * Compile-time configuration.
35		//!
36		//! ### `OptimizationMode.release_fast` (note: not much work has gone into this use case yet):
37		//!
38		//! * Low fragmentation is primary concern
39		//! * Performance of worst-case latency is secondary concern
40		//! * Performance of average-case latency is next
41		//! * Finally, having freed memory unmapped, and pointer math errors unlikely to
42		//! harm memory from unrelated allocations are nice-to-haves.
43		//!
44		//! ### `OptimizationMode.release_small` (note: not much work has gone into this use case yet):
45		//!
46		//! * Small binary code size of the executable is the primary concern.
47		//! * Next, defer to the `.release_fast` priority list.
48		//!
49		//! ## Basic Design:
50		//!
51		//! Small allocations are divided into buckets:
52		//!
53		//! ```
54		//! index obj_size
55		//! 0 1
56		//! 1 2
57		//! 2 4
58		//! 3 8
59		//! 4 16
60		//! 5 32
61		//! 6 64
62		//! 7 128
63		//! 8 256
64		//! 9 512
65		//! 10 1024
66		//! 11 2048
67		//! ```
68		//!
69		//! The main allocator state has an array of all the "current" buckets for each
70		//! size class. Each slot in the array can be null, meaning the bucket for that
71		//! size class is not allocated. When the first object is allocated for a given
72		//! size class, it allocates 1 page of memory from the OS. This page is
73		//! divided into "slots" - one per allocated object. Along with the page of memory
74		//! for object slots, as many pages as necessary are allocated to store the
75		//! BucketHeader, followed by "used bits", and two stack traces for each slot
76		//! (allocation trace and free trace).
77		//!
78		//! The "used bits" are 1 bit per slot representing whether the slot is used.
79		//! Allocations use the data to iterate to find a free slot. Frees assert that the
80		//! corresponding bit is 1 and set it to 0.
81		//!
82		//! Buckets have prev and next pointers. When there is only one bucket for a given
83		//! size class, both prev and next point to itself. When all slots of a bucket are
84		//! used, a new bucket is allocated, and enters the doubly linked list. The main
85		//! allocator state tracks the "current" bucket for each size class. Leak detection
86		//! currently only checks the current bucket.
87		//!
88		//! Resizing detects if the size class is unchanged or smaller, in which case the same
89		//! pointer is returned unmodified. If a larger size class is required,
90		//! `error.OutOfMemory` is returned.
91		//!
92		//! Large objects are allocated directly using the backing allocator and their metadata is stored
93		//! in a `std.HashMap` using the backing allocator.
94
95		const std = @import("std");
96		const builtin = @import("builtin");
97		const log = std.log.scoped(.gpa);
98		const math = std.math;
99		const assert = std.debug.assert;
100		const mem = std.mem;
101		const Allocator = std.mem.Allocator;
102		const page_size = std.mem.page_size;
103		const StackTrace = std.builtin.StackTrace;
104
105		/// Integer type for pointing to slots in a small allocation
106		const SlotIndex = std.meta.Int(.unsigned, math.log2(page_size) + 1);
107
108		const default_test_stack_trace_frames: usize = if (builtin.is_test) 10 else 6;
109		const default_sys_stack_trace_frames: usize = if (std.debug.sys_can_stack_trace) default_test_stack_trace_frames else 0;
110		const default_stack_trace_frames: usize = switch (builtin.mode) {
111		.Debug => default_sys_stack_trace_frames,
112		else => 0,
113		};
114
115		pub const Config = struct {
116		/// Number of stack frames to capture.
117		stack_trace_frames: usize = default_stack_trace_frames,
118
119		/// If true, the allocator will have two fields:
120		/// * `total_requested_bytes` which tracks the total allocated bytes of memory requested.
121		/// * `requested_memory_limit` which causes allocations to return `error.OutOfMemory`
122		/// when the `total_requested_bytes` exceeds this limit.
123		/// If false, these fields will be `void`.
124		enable_memory_limit: bool = false,
125
126		/// Whether to enable safety checks.
127		safety: bool = std.debug.runtime_safety,
128
129		/// Whether the allocator may be used simultaneously from multiple threads.
130		thread_safe: bool = !builtin.single_threaded,
131
132		/// What type of mutex you'd like to use, for thread safety.
133		/// when specified, the mutex type must have the same shape as `std.Thread.Mutex` and
134		/// `DummyMutex`, and have no required fields. Specifying this field causes
135		/// the `thread_safe` field to be ignored.
136		///
137		/// when null (default):
138		/// * the mutex type defaults to `std.Thread.Mutex` when thread_safe is enabled.
139		/// * the mutex type defaults to `DummyMutex` otherwise.
140		MutexType: ?type = null,
141
142		/// This is a temporary debugging trick you can use to turn segfaults into more helpful
143		/// logged error messages with stack trace details. The downside is that every allocation
144		/// will be leaked, unless used with retain_metadata!
145		never_unmap: bool = false,
146
147		/// This is a temporary debugging aid that retains metadata about allocations indefinitely.
148		/// This allows a greater range of double frees to be reported. All metadata is freed when
149		/// deinit is called. When used with never_unmap, deliberately leaked memory is also freed
150		/// during deinit. Currently should be used with never_unmap to avoid segfaults.
151		/// TODO https://github.com/ziglang/zig/issues/4298 will allow use without never_unmap
152		retain_metadata: bool = false,
153
154		/// Enables emitting info messages with the size and address of every allocation.
155		verbose_log: bool = false,
156		};
157
158		pub const Check = enum { ok, leak };
159
160		/// Default initialization of this struct is deprecated; use `.init` instead.
161		pub fn GeneralPurposeAllocator(comptime config: Config) type {
162		return struct {
163		backing_allocator: Allocator = std.heap.page_allocator,
164		buckets: [small_bucket_count]Buckets = [1]Buckets{Buckets{}} ** small_bucket_count,
165		cur_buckets: [small_bucket_count]?BucketHeader = [1]?BucketHeader{null} ** small_bucket_count,
166		large_allocations: LargeAllocTable = .{},
167		empty_buckets: if (config.retain_metadata) Buckets else void =
168		if (config.retain_metadata) Buckets{} else {},
169		bucket_node_pool: std.heap.MemoryPool(Buckets.Node) = std.heap.MemoryPool(Buckets.Node).init(std.heap.page_allocator),
170
171		total_requested_bytes: @TypeOf(total_requested_bytes_init) = total_requested_bytes_init,
172		requested_memory_limit: @TypeOf(requested_memory_limit_init) = requested_memory_limit_init,
173
174		mutex: @TypeOf(mutex_init) = mutex_init,
175
176		const Self = @This();
177
178		/// The initial state of a `GeneralPurposeAllocator`, containing no allocations and backed by the system page allocator.
179		pub const init: Self = .{
180		.backing_allocator = std.heap.page_allocator,
181		.buckets = [1]Buckets{.{}} ** small_bucket_count,
182		.cur_buckets = [1]?BucketHeader{null} * small_bucket_count,
183		.large_allocations = .{},
184		.empty_buckets = if (config.retain_metadata) .{} else {},
185		.bucket_node_pool = .init(std.heap.page_allocator),
186		};
187
188		const total_requested_bytes_init = if (config.enable_memory_limit) @as(usize, 0) else {};
189		const requested_memory_limit_init = if (config.enable_memory_limit) @as(usize, math.maxInt(usize)) else {};
190
191		const mutex_init = if (config.MutexType) \|T\|
192		T{}
193		else if (config.thread_safe)
194		std.Thread.Mutex{}
195		else
196		DummyMutex{};
197
198		const DummyMutex = struct {
199		fn lock(_: *DummyMutex) void {}
200		fn unlock(_: *DummyMutex) void {}
201		};
202
203		const stack_n = config.stack_trace_frames;
204		const one_trace_size = @sizeOf(usize) * stack_n;
205		const traces_per_slot = 2;
206
207		pub const Error = mem.Allocator.Error;
208
209		const small_bucket_count = math.log2(page_size);
210		const largest_bucket_object_size = 1 << (small_bucket_count - 1);
211		const LargestSizeClassInt = std.math.IntFittingRange(0, largest_bucket_object_size);
212
213		const bucketCompare = struct {
214		fn compare(a: BucketHeader, b: BucketHeader) std.math.Order {
215		return std.math.order(@intFromPtr(a.page), @intFromPtr(b.page));
216		}
217		}.compare;
218		const Buckets = std.Treap(*BucketHeader, bucketCompare);
219
220		const LargeAlloc = struct {
221		bytes: []u8,
222		requested_size: if (config.enable_memory_limit) usize else void,
223		stack_addresses: [trace_n][stack_n]usize,
224		freed: if (config.retain_metadata) bool else void,
225		log2_ptr_align: if (config.never_unmap and config.retain_metadata) u8 else void,
226
227		const trace_n = if (config.retain_metadata) traces_per_slot else 1;
228
229		fn dumpStackTrace(self: *LargeAlloc, trace_kind: TraceKind) void {
230		std.debug.dumpStackTrace(self.getStackTrace(trace_kind));
231		}
232
233		fn getStackTrace(self: *LargeAlloc, trace_kind: TraceKind) std.builtin.StackTrace {
234		assert(@intFromEnum(trace_kind) < trace_n);
235		const stack_addresses = &self.stack_addresses[@intFromEnum(trace_kind)];
236		var len: usize = 0;
237		while (len < stack_n and stack_addresses[len] != 0) {
238		len += 1;
239		}
240		return .{
241		.instruction_addresses = stack_addresses,
242		.index = len,
243		};
244		}
245
246		fn captureStackTrace(self: *LargeAlloc, ret_addr: usize, trace_kind: TraceKind) void {
247		assert(@intFromEnum(trace_kind) < trace_n);
248		const stack_addresses = &self.stack_addresses[@intFromEnum(trace_kind)];
249		collectStackTrace(ret_addr, stack_addresses);
250		}
251		};
252		const LargeAllocTable = std.AutoHashMapUnmanaged(usize, LargeAlloc);
253
254		// Bucket: In memory, in order:
255		// * BucketHeader
256		// * bucket_used_bits: [N]u8, // 1 bit for every slot; 1 byte for every 8 slots
257		// -- below only exists when config.safety is true --
258		// * requested_sizes: [N]LargestSizeClassInt // 1 int for every slot
259		// * log2_ptr_aligns: [N]u8 // 1 byte for every slot
260		// -- above only exists when config.safety is true --
261		// * stack_trace_addresses: [N]usize, // traces_per_slot for every allocation
262
263		const BucketHeader = struct {
264		page: [*]align(page_size) u8,
265		alloc_cursor: SlotIndex,
266		used_count: SlotIndex,
267
268		fn usedBits(bucket: BucketHeader, index: usize) u8 {
269		return @as(*u8, @ptrFromInt(@intFromPtr(bucket) + @sizeOf(BucketHeader) + index));
270		}
271
272		fn requestedSizes(bucket: *BucketHeader, size_class: usize) []LargestSizeClassInt {
273		if (!config.safety) @compileError("requested size is only stored when safety is enabled");
274		const start_ptr = @as([*]u8, @ptrCast(bucket)) + bucketRequestedSizesStart(size_class);
275		const sizes = @as([*]LargestSizeClassInt, @ptrCast(@alignCast(start_ptr)));
276		const slot_count = @divExact(page_size, size_class);
277		return sizes[0..slot_count];
278		}
279
280		fn log2PtrAligns(bucket: *BucketHeader, size_class: usize) []u8 {
281		if (!config.safety) @compileError("requested size is only stored when safety is enabled");
282		const aligns_ptr = @as([*]u8, @ptrCast(bucket)) + bucketAlignsStart(size_class);
283		const slot_count = @divExact(page_size, size_class);
284		return aligns_ptr[0..slot_count];
285		}
286
287		fn stackTracePtr(
288		bucket: *BucketHeader,
289		size_class: usize,
290		slot_index: SlotIndex,
291		trace_kind: TraceKind,
292		) *[stack_n]usize {
293		const start_ptr = @as([*]u8, @ptrCast(bucket)) + bucketStackFramesStart(size_class);
294		const addr = start_ptr + one_trace_size * traces_per_slot * slot_index +
295		@intFromEnum(trace_kind) * @as(usize, one_trace_size);
296		return @ptrCast(@alignCast(addr));
297		}
298
299		fn captureStackTrace(
300		bucket: *BucketHeader,
301		ret_addr: usize,
302		size_class: usize,
303		slot_index: SlotIndex,
304		trace_kind: TraceKind,
305		) void {
306		// Initialize them to 0. When determining the count we must look
307		// for non zero addresses.
308		const stack_addresses = bucket.stackTracePtr(size_class, slot_index, trace_kind);
309		collectStackTrace(ret_addr, stack_addresses);
310		}
311
312		/// Only valid for buckets within `empty_buckets`, and relies on the `alloc_cursor`
313		/// of empty buckets being set to `slot_count` when they are added to `empty_buckets`
314		fn emptyBucketSizeClass(bucket: *BucketHeader) usize {
315		return @divExact(page_size, bucket.alloc_cursor);
316		}
317		};
318
319		pub fn allocator(self: *Self) Allocator {
320		return .{
321		.ptr = self,
322		.vtable = &.{
323		.alloc = alloc,
324		.resize = resize,
325		.free = free,
326		},
327		};
328		}
329
330		fn bucketStackTrace(
331		bucket: *BucketHeader,
332		size_class: usize,
333		slot_index: SlotIndex,
334		trace_kind: TraceKind,
335		) StackTrace {
336		const stack_addresses = bucket.stackTracePtr(size_class, slot_index, trace_kind);
337		var len: usize = 0;
338		while (len < stack_n and stack_addresses[len] != 0) {
339		len += 1;
340		}
341		return StackTrace{
342		.instruction_addresses = stack_addresses,
343		.index = len,
344		};
345		}
346
347		fn bucketRequestedSizesStart(size_class: usize) usize {
348		if (!config.safety) @compileError("requested sizes are not stored unless safety is enabled");
349		return mem.alignForward(
350		usize,
351		@sizeOf(BucketHeader) + usedBitsCount(size_class),
352		@alignOf(LargestSizeClassInt),
353		);
354		}
355
356		fn bucketAlignsStart(size_class: usize) usize {
357		if (!config.safety) @compileError("requested sizes are not stored unless safety is enabled");
358		const slot_count = @divExact(page_size, size_class);
359		return bucketRequestedSizesStart(size_class) + (@sizeOf(LargestSizeClassInt) * slot_count);
360		}
361
362		fn bucketStackFramesStart(size_class: usize) usize {
363		const unaligned_start = if (config.safety) blk: {
364		const slot_count = @divExact(page_size, size_class);
365		break :blk bucketAlignsStart(size_class) + slot_count;
366		} else @sizeOf(BucketHeader) + usedBitsCount(size_class);
367		return mem.alignForward(
368		usize,
369		unaligned_start,
370		@alignOf(usize),
371		);
372		}
373
374		fn bucketSize(size_class: usize) usize {
375		const slot_count = @divExact(page_size, size_class);
376		return bucketStackFramesStart(size_class) + one_trace_size * traces_per_slot * slot_count;
377		}
378
379		fn usedBitsCount(size_class: usize) usize {
380		const slot_count = @divExact(page_size, size_class);
381		if (slot_count < 8) return 1;
382		return @divExact(slot_count, 8);
383		}
384
385		fn detectLeaksInBucket(
386		bucket: *BucketHeader,
387		size_class: usize,
388		used_bits_count: usize,
389		) bool {
390		var leaks = false;
391		var used_bits_byte: usize = 0;
392		while (used_bits_byte < used_bits_count) : (used_bits_byte += 1) {
393		const used_byte = bucket.usedBits(used_bits_byte).*;
394		if (used_byte != 0) {
395		var bit_index: u3 = 0;
396		while (true) : (bit_index += 1) {
397		const is_used = @as(u1, @truncate(used_byte >> bit_index)) != 0;
398		if (is_used) {
399		const slot_index = @as(SlotIndex, @intCast(used_bits_byte * 8 + bit_index));
400		const stack_trace = bucketStackTrace(bucket, size_class, slot_index, .alloc);
401		const addr = bucket.page + slot_index * size_class;
402		log.err("memory address 0x{x} leaked: {}", .{
403		@intFromPtr(addr), stack_trace,
404		});
405		leaks = true;
406		}
407		if (bit_index == math.maxInt(u3))
408		break;
409		}
410		}
411		}
412		return leaks;
413		}
414
415		/// Emits log messages for leaks and then returns whether there were any leaks.
416		pub fn detectLeaks(self: *Self) bool {
417		var leaks = false;
418
419		for (&self.buckets, 0..) \|*buckets, bucket_i\| {
420		if (buckets.root == null) continue;
421		const size_class = @as(usize, 1) << @as(math.Log2Int(usize), @intCast(bucket_i));
422		const used_bits_count = usedBitsCount(size_class);
423		var it = buckets.inorderIterator();
424		while (it.next()) \|node\| {
425		const bucket = node.key;
426		leaks = detectLeaksInBucket(bucket, size_class, used_bits_count) or leaks;
427		}
428		}
429		var it = self.large_allocations.valueIterator();
430		while (it.next()) \|large_alloc\| {
431		if (config.retain_metadata and large_alloc.freed) continue;
432		const stack_trace = large_alloc.getStackTrace(.alloc);
433		log.err("memory address 0x{x} leaked: {}", .{
434		@intFromPtr(large_alloc.bytes.ptr), stack_trace,
435		});
436		leaks = true;
437		}
438		return leaks;
439		}
440
441		fn freeBucket(self: Self, bucket: BucketHeader, size_class: usize) void {
442		const bucket_size = bucketSize(size_class);
443		const bucket_slice = @as([*]align(@alignOf(BucketHeader)) u8, @ptrCast(bucket))[0..bucket_size];
444		self.backing_allocator.free(bucket_slice);
445		}
446
447		fn freeRetainedMetadata(self: *Self) void {
448		if (config.retain_metadata) {
449		if (config.never_unmap) {
450		// free large allocations that were intentionally leaked by never_unmap
451		var it = self.large_allocations.iterator();
452		while (it.next()) \|large\| {
453		if (large.value_ptr.freed) {
454		self.backing_allocator.rawFree(large.value_ptr.bytes, large.value_ptr.log2_ptr_align, @returnAddress());
455		}
456		}
457		}
458		// free retained metadata for small allocations
459		while (self.empty_buckets.getMin()) \|node\| {
460		// remove the node from the tree before destroying it
461		var entry = self.empty_buckets.getEntryForExisting(node);
462		entry.set(null);
463
464		var bucket = node.key;
465		if (config.never_unmap) {
466		// free page that was intentionally leaked by never_unmap
467		self.backing_allocator.free(bucket.page[0..page_size]);
468		}
469		// alloc_cursor was set to slot count when bucket added to empty_buckets
470		self.freeBucket(bucket, bucket.emptyBucketSizeClass());
471		self.bucket_node_pool.destroy(node);
472		}
473		self.empty_buckets.root = null;
474		}
475		}
476
477		pub fn flushRetainedMetadata(self: *Self) void {
478		if (!config.retain_metadata) {
479		@compileError("'flushRetainedMetadata' requires 'config.retain_metadata = true'");
480		}
481		self.freeRetainedMetadata();
482		// also remove entries from large_allocations
483		var it = self.large_allocations.iterator();
484		while (it.next()) \|large\| {
485		if (large.value_ptr.freed) {
486		_ = self.large_allocations.remove(@intFromPtr(large.value_ptr.bytes.ptr));
487		}
488		}
489		}
490
491		/// Returns `Check.leak` if there were leaks; `Check.ok` otherwise.
492		pub fn deinit(self: *Self) Check {
493		const leaks = if (config.safety) self.detectLeaks() else false;
494		if (config.retain_metadata) {
495		self.freeRetainedMetadata();
496		}
497		self.large_allocations.deinit(self.backing_allocator);
498		self.bucket_node_pool.deinit();
499		self.* = undefined;
500		return @as(Check, @enumFromInt(@intFromBool(leaks)));
501		}
502
503		fn collectStackTrace(first_trace_addr: usize, addresses: *[stack_n]usize) void {
504		if (stack_n == 0) return;
505		@memset(addresses, 0);
506		var stack_trace = StackTrace{
507		.instruction_addresses = addresses,
508		.index = 0,
509		};
510		std.debug.captureStackTrace(first_trace_addr, &stack_trace);
511		}
512
513		fn reportDoubleFree(ret_addr: usize, alloc_stack_trace: StackTrace, free_stack_trace: StackTrace) void {
514		var addresses: [stack_n]usize = [1]usize{0} ** stack_n;
515		var second_free_stack_trace = StackTrace{
516		.instruction_addresses = &addresses,
517		.index = 0,
518		};
519		std.debug.captureStackTrace(ret_addr, &second_free_stack_trace);
520		log.err("Double free detected. Allocation: {} First free: {} Second free: {}", .{
521		alloc_stack_trace, free_stack_trace, second_free_stack_trace,
522		});
523		}
524
525		const Slot = struct {
526		bucket: *BucketHeader,
527		slot_index: usize,
528		ptr: [*]u8,
529		};
530
531		fn allocSlot(self: *Self, size_class: usize, trace_addr: usize) Error!Slot {
532		const bucket_index = math.log2(size_class);
533		var buckets = &self.buckets[bucket_index];
534		const slot_count = @divExact(page_size, size_class);
535		if (self.cur_buckets[bucket_index] == null or self.cur_buckets[bucket_index].?.alloc_cursor == slot_count) {
536		const new_bucket = try self.createBucket(size_class);
537		errdefer self.freeBucket(new_bucket, size_class);
538		const node = try self.bucket_node_pool.create();
539		node.key = new_bucket;
540		var entry = buckets.getEntryFor(new_bucket);
541		std.debug.assert(entry.node == null);
542		entry.set(node);
543		self.cur_buckets[bucket_index] = node.key;
544		}
545		const bucket = self.cur_buckets[bucket_index].?;
546
547		const slot_index = bucket.alloc_cursor;
548		bucket.alloc_cursor += 1;
549
550		const used_bits_byte = bucket.usedBits(slot_index / 8);
551		const used_bit_index: u3 = @as(u3, @intCast(slot_index % 8)); // TODO cast should be unnecessary
552		used_bits_byte.* \|= (@as(u8, 1) << used_bit_index);
553		bucket.used_count += 1;
554		bucket.captureStackTrace(trace_addr, size_class, slot_index, .alloc);
555		return .{
556		.bucket = bucket,
557		.slot_index = slot_index,
558		.ptr = bucket.page + slot_index * size_class,
559		};
560		}
561
562		fn searchBucket(
563		buckets: *Buckets,
564		addr: usize,
565		current_bucket: ?*BucketHeader,
566		) ?*BucketHeader {
567		const search_page: [*]align(page_size) u8 = @ptrFromInt(mem.alignBackward(usize, addr, page_size));
568		if (current_bucket != null and current_bucket.?.page == search_page) {
569		return current_bucket;
570		}
571		var search_header: BucketHeader = undefined;
572		search_header.page = search_page;
573		const entry = buckets.getEntryFor(&search_header);
574		return if (entry.node) \|node\| node.key else null;
575		}
576
577		/// This function assumes the object is in the large object storage regardless
578		/// of the parameters.
579		fn resizeLarge(
580		self: *Self,
581		old_mem: []u8,
582		log2_old_align: u8,
583		new_size: usize,
584		ret_addr: usize,
585		) bool {
586		const entry = self.large_allocations.getEntry(@intFromPtr(old_mem.ptr)) orelse {
587		if (config.safety) {
588		@panic("Invalid free");
589		} else {
590		unreachable;
591		}
592		};
593
594		if (config.retain_metadata and entry.value_ptr.freed) {
595		if (config.safety) {
596		reportDoubleFree(ret_addr, entry.value_ptr.getStackTrace(.alloc), entry.value_ptr.getStackTrace(.free));
597		@panic("Unrecoverable double free");
598		} else {
599		unreachable;
600		}
601		}
602
603		if (config.safety and old_mem.len != entry.value_ptr.bytes.len) {
604		var addresses: [stack_n]usize = [1]usize{0} ** stack_n;
605		var free_stack_trace = StackTrace{
606		.instruction_addresses = &addresses,
607		.index = 0,
608		};
609		std.debug.captureStackTrace(ret_addr, &free_stack_trace);
610		log.err("Allocation size {d} bytes does not match free size {d}. Allocation: {} Free: {}", .{
611		entry.value_ptr.bytes.len,
612		old_mem.len,
613		entry.value_ptr.getStackTrace(.alloc),
614		free_stack_trace,
615		});
616		}
617
618		// Do memory limit accounting with requested sizes rather than what
619		// backing_allocator returns because if we want to return
620		// error.OutOfMemory, we have to leave allocation untouched, and
621		// that is impossible to guarantee after calling
622		// backing_allocator.rawResize.
623		const prev_req_bytes = self.total_requested_bytes;
624		if (config.enable_memory_limit) {
625		const new_req_bytes = prev_req_bytes + new_size - entry.value_ptr.requested_size;
626		if (new_req_bytes > prev_req_bytes and new_req_bytes > self.requested_memory_limit) {
627		return false;
628		}
629		self.total_requested_bytes = new_req_bytes;
630		}
631
632		if (!self.backing_allocator.rawResize(old_mem, log2_old_align, new_size, ret_addr)) {
633		if (config.enable_memory_limit) {
634		self.total_requested_bytes = prev_req_bytes;
635		}
636		return false;
637		}
638
639		if (config.enable_memory_limit) {
640		entry.value_ptr.requested_size = new_size;
641		}
642
643		if (config.verbose_log) {
644		log.info("large resize {d} bytes at {*} to {d}", .{
645		old_mem.len, old_mem.ptr, new_size,
646		});
647		}
648		entry.value_ptr.bytes = old_mem.ptr[0..new_size];
649		entry.value_ptr.captureStackTrace(ret_addr, .alloc);
650		return true;
651		}
652
653		/// This function assumes the object is in the large object storage regardless
654		/// of the parameters.
655		fn freeLarge(
656		self: *Self,
657		old_mem: []u8,
658		log2_old_align: u8,
659		ret_addr: usize,
660		) void {
661		const entry = self.large_allocations.getEntry(@intFromPtr(old_mem.ptr)) orelse {
662		if (config.safety) {
663		@panic("Invalid free");
664		} else {
665		unreachable;
666		}
667		};
668
669		if (config.retain_metadata and entry.value_ptr.freed) {
670		if (config.safety) {
671		reportDoubleFree(ret_addr, entry.value_ptr.getStackTrace(.alloc), entry.value_ptr.getStackTrace(.free));
672		return;
673		} else {
674		unreachable;
675		}
676		}
677
678		if (config.safety and old_mem.len != entry.value_ptr.bytes.len) {
679		var addresses: [stack_n]usize = [1]usize{0} ** stack_n;
680		var free_stack_trace = StackTrace{
681		.instruction_addresses = &addresses,
682		.index = 0,
683		};
684		std.debug.captureStackTrace(ret_addr, &free_stack_trace);
685		log.err("Allocation size {d} bytes does not match free size {d}. Allocation: {} Free: {}", .{
686		entry.value_ptr.bytes.len,
687		old_mem.len,
688		entry.value_ptr.getStackTrace(.alloc),
689		free_stack_trace,
690		});
691		}
692
693		if (!config.never_unmap) {
694		self.backing_allocator.rawFree(old_mem, log2_old_align, ret_addr);
695		}
696
697		if (config.enable_memory_limit) {
698		self.total_requested_bytes -= entry.value_ptr.requested_size;
699		}
700
701		if (config.verbose_log) {
702		log.info("large free {d} bytes at {*}", .{ old_mem.len, old_mem.ptr });
703		}
704
705		if (!config.retain_metadata) {
706		assert(self.large_allocations.remove(@intFromPtr(old_mem.ptr)));
707		} else {
708		entry.value_ptr.freed = true;
709		entry.value_ptr.captureStackTrace(ret_addr, .free);
710		}
711		}
712
713		pub fn setRequestedMemoryLimit(self: *Self, limit: usize) void {
714		self.requested_memory_limit = limit;
715		}
716
717		fn resize(
718		ctx: *anyopaque,
719		old_mem: []u8,
720		log2_old_align_u8: u8,
721		new_size: usize,
722		ret_addr: usize,
723		) bool {
724		const self: *Self = @ptrCast(@alignCast(ctx));
725		const log2_old_align = @as(Allocator.Log2Align, @intCast(log2_old_align_u8));
726		self.mutex.lock();
727		defer self.mutex.unlock();
728
729		assert(old_mem.len != 0);
730
731		const aligned_size = @max(old_mem.len, @as(usize, 1) << log2_old_align);
732		if (aligned_size > largest_bucket_object_size) {
733		return self.resizeLarge(old_mem, log2_old_align, new_size, ret_addr);
734		}
735		const size_class_hint = math.ceilPowerOfTwoAssert(usize, aligned_size);
736
737		var bucket_index = math.log2(size_class_hint);
738		var size_class: usize = size_class_hint;
739		const bucket = while (bucket_index < small_bucket_count) : (bucket_index += 1) {
740		if (searchBucket(&self.buckets[bucket_index], @intFromPtr(old_mem.ptr), self.cur_buckets[bucket_index])) \|bucket\| {
741		break bucket;
742		}
743		size_class *= 2;
744		} else blk: {
745		if (config.retain_metadata) {
746		if (!self.large_allocations.contains(@intFromPtr(old_mem.ptr))) {
747		// object not in active buckets or a large allocation, so search empty buckets
748		if (searchBucket(&self.empty_buckets, @intFromPtr(old_mem.ptr), null)) \|bucket\| {
749		size_class = bucket.emptyBucketSizeClass();
750		// bucket is empty so is_used below will always be false and we exit there
751		break :blk bucket;
752		} else {
753		@panic("Invalid free");
754		}
755		}
756		}
757		return self.resizeLarge(old_mem, log2_old_align, new_size, ret_addr);
758		};
759		const byte_offset = @intFromPtr(old_mem.ptr) - @intFromPtr(bucket.page);
760		const slot_index = @as(SlotIndex, @intCast(byte_offset / size_class));
761		const used_byte_index = slot_index / 8;
762		const used_bit_index = @as(u3, @intCast(slot_index % 8));
763		const used_byte = bucket.usedBits(used_byte_index);
764		const is_used = @as(u1, @truncate(used_byte.* >> used_bit_index)) != 0;
765		if (!is_used) {
766		if (config.safety) {
767		reportDoubleFree(ret_addr, bucketStackTrace(bucket, size_class, slot_index, .alloc), bucketStackTrace(bucket, size_class, slot_index, .free));
768		@panic("Unrecoverable double free");
769		} else {
770		unreachable;
771		}
772		}
773
774		// Definitely an in-use small alloc now.
775		if (config.safety) {
776		const requested_size = bucket.requestedSizes(size_class)[slot_index];
777		if (requested_size == 0) @panic("Invalid free");
778		const log2_ptr_align = bucket.log2PtrAligns(size_class)[slot_index];
779		if (old_mem.len != requested_size or log2_old_align != log2_ptr_align) {
780		var addresses: [stack_n]usize = [1]usize{0} ** stack_n;
781		var free_stack_trace = StackTrace{
782		.instruction_addresses = &addresses,
783		.index = 0,
784		};
785		std.debug.captureStackTrace(ret_addr, &free_stack_trace);
786		if (old_mem.len != requested_size) {
787		log.err("Allocation size {d} bytes does not match resize size {d}. Allocation: {} Resize: {}", .{
788		requested_size,
789		old_mem.len,
790		bucketStackTrace(bucket, size_class, slot_index, .alloc),
791		free_stack_trace,
792		});
793		}
794		if (log2_old_align != log2_ptr_align) {
795		log.err("Allocation alignment {d} does not match resize alignment {d}. Allocation: {} Resize: {}", .{
796		@as(usize, 1) << @as(math.Log2Int(usize), @intCast(log2_ptr_align)),
797		@as(usize, 1) << @as(math.Log2Int(usize), @intCast(log2_old_align)),
798		bucketStackTrace(bucket, size_class, slot_index, .alloc),
799		free_stack_trace,
800		});
801		}
802		}
803		}
804		const prev_req_bytes = self.total_requested_bytes;
805		if (config.enable_memory_limit) {
806		const new_req_bytes = prev_req_bytes + new_size - old_mem.len;
807		if (new_req_bytes > prev_req_bytes and new_req_bytes > self.requested_memory_limit) {
808		return false;
809		}
810		self.total_requested_bytes = new_req_bytes;
811		}
812
813		const new_aligned_size = @max(new_size, @as(usize, 1) << log2_old_align);
814		const new_size_class = math.ceilPowerOfTwoAssert(usize, new_aligned_size);
815		if (new_size_class <= size_class) {
816		if (old_mem.len > new_size) {
817		@memset(old_mem[new_size..], undefined);
818		}
819		if (config.verbose_log) {
820		log.info("small resize {d} bytes at {*} to {d}", .{
821		old_mem.len, old_mem.ptr, new_size,
822		});
823		}
824		if (config.safety) {
825		bucket.requestedSizes(size_class)[slot_index] = @intCast(new_size);
826		}
827		return true;
828		}
829
830		if (config.enable_memory_limit) {
831		self.total_requested_bytes = prev_req_bytes;
832		}
833		return false;
834		}
835
836		fn free(
837		ctx: *anyopaque,
838		old_mem: []u8,
839		log2_old_align_u8: u8,
840		ret_addr: usize,
841		) void {
842		const self: *Self = @ptrCast(@alignCast(ctx));
843		const log2_old_align = @as(Allocator.Log2Align, @intCast(log2_old_align_u8));
844		self.mutex.lock();
845		defer self.mutex.unlock();
846
847		assert(old_mem.len != 0);
848
849		const aligned_size = @max(old_mem.len, @as(usize, 1) << log2_old_align);
850		if (aligned_size > largest_bucket_object_size) {
851		self.freeLarge(old_mem, log2_old_align, ret_addr);
852		return;
853		}
854		const size_class_hint = math.ceilPowerOfTwoAssert(usize, aligned_size);
855
856		var bucket_index = math.log2(size_class_hint);
857		var size_class: usize = size_class_hint;
858		const bucket = while (bucket_index < small_bucket_count) : (bucket_index += 1) {
859		if (searchBucket(&self.buckets[bucket_index], @intFromPtr(old_mem.ptr), self.cur_buckets[bucket_index])) \|bucket\| {
860		break bucket;
861		}
862		size_class *= 2;
863		} else blk: {
864		if (config.retain_metadata) {
865		if (!self.large_allocations.contains(@intFromPtr(old_mem.ptr))) {
866		// object not in active buckets or a large allocation, so search empty buckets
867		if (searchBucket(&self.empty_buckets, @intFromPtr(old_mem.ptr), null)) \|bucket\| {
868		size_class = bucket.emptyBucketSizeClass();
869		// bucket is empty so is_used below will always be false and we exit there
870		break :blk bucket;
871		} else {
872		@panic("Invalid free");
873		}
874		}
875		}
876		self.freeLarge(old_mem, log2_old_align, ret_addr);
877		return;
878		};
879		const byte_offset = @intFromPtr(old_mem.ptr) - @intFromPtr(bucket.page);
880		const slot_index = @as(SlotIndex, @intCast(byte_offset / size_class));
881		const used_byte_index = slot_index / 8;
882		const used_bit_index = @as(u3, @intCast(slot_index % 8));
883		const used_byte = bucket.usedBits(used_byte_index);
884		const is_used = @as(u1, @truncate(used_byte.* >> used_bit_index)) != 0;
885		if (!is_used) {
886		if (config.safety) {
887		reportDoubleFree(ret_addr, bucketStackTrace(bucket, size_class, slot_index, .alloc), bucketStackTrace(bucket, size_class, slot_index, .free));
888		// Recoverable if this is a free.
889		return;
890		} else {
891		unreachable;
892		}
893		}
894
895		// Definitely an in-use small alloc now.
896		if (config.safety) {
897		const requested_size = bucket.requestedSizes(size_class)[slot_index];
898		if (requested_size == 0) @panic("Invalid free");
899		const log2_ptr_align = bucket.log2PtrAligns(size_class)[slot_index];
900		if (old_mem.len != requested_size or log2_old_align != log2_ptr_align) {
901		var addresses: [stack_n]usize = [1]usize{0} ** stack_n;
902		var free_stack_trace = StackTrace{
903		.instruction_addresses = &addresses,
904		.index = 0,
905		};
906		std.debug.captureStackTrace(ret_addr, &free_stack_trace);
907		if (old_mem.len != requested_size) {
908		log.err("Allocation size {d} bytes does not match free size {d}. Allocation: {} Free: {}", .{
909		requested_size,
910		old_mem.len,
911		bucketStackTrace(bucket, size_class, slot_index, .alloc),
912		free_stack_trace,
913		});
914		}
915		if (log2_old_align != log2_ptr_align) {
916		log.err("Allocation alignment {d} does not match free alignment {d}. Allocation: {} Free: {}", .{
917		@as(usize, 1) << @as(math.Log2Int(usize), @intCast(log2_ptr_align)),
918		@as(usize, 1) << @as(math.Log2Int(usize), @intCast(log2_old_align)),
919		bucketStackTrace(bucket, size_class, slot_index, .alloc),
920		free_stack_trace,
921		});
922		}
923		}
924		}
925
926		if (config.enable_memory_limit) {
927		self.total_requested_bytes -= old_mem.len;
928		}
929
930		// Capture stack trace to be the "first free", in case a double free happens.
931		bucket.captureStackTrace(ret_addr, size_class, slot_index, .free);
932
933		used_byte.* &= ~(@as(u8, 1) << used_bit_index);
934		bucket.used_count -= 1;
935		if (config.safety) {
936		bucket.requestedSizes(size_class)[slot_index] = 0;
937		}
938		if (bucket.used_count == 0) {
939		var entry = self.buckets[bucket_index].getEntryFor(bucket);
940		// save the node for destruction/insertion into in empty_buckets
941		const node = entry.node.?;
942		entry.set(null);
943		if (self.cur_buckets[bucket_index] == bucket) {
944		self.cur_buckets[bucket_index] = null;
945		}
946		if (!config.never_unmap) {
947		self.backing_allocator.free(bucket.page[0..page_size]);
948		}
949		if (!config.retain_metadata) {
950		self.freeBucket(bucket, size_class);
951		self.bucket_node_pool.destroy(node);
952		} else {
953		// move alloc_cursor to end so we can tell size_class later
954		const slot_count = @divExact(page_size, size_class);
955		bucket.alloc_cursor = @as(SlotIndex, @truncate(slot_count));
956		var empty_entry = self.empty_buckets.getEntryFor(node.key);
957		empty_entry.set(node);
958		}
959		} else {
960		@memset(old_mem, undefined);
961		}
962		if (config.verbose_log) {
963		log.info("small free {d} bytes at {*}", .{ old_mem.len, old_mem.ptr });
964		}
965		}
966
967		// Returns true if an allocation of `size` bytes is within the specified
968		// limits if enable_memory_limit is true
969		fn isAllocationAllowed(self: *Self, size: usize) bool {
970		if (config.enable_memory_limit) {
971		const new_req_bytes = self.total_requested_bytes + size;
972		if (new_req_bytes > self.requested_memory_limit)
973		return false;
974		self.total_requested_bytes = new_req_bytes;
975		}
976
977		return true;
978		}
979
980		fn alloc(ctx: anyopaque, len: usize, log2_ptr_align: u8, ret_addr: usize) ?[]u8 {
981		const self: *Self = @ptrCast(@alignCast(ctx));
982		self.mutex.lock();
983		defer self.mutex.unlock();
984		if (!self.isAllocationAllowed(len)) return null;
985		return allocInner(self, len, @as(Allocator.Log2Align, @intCast(log2_ptr_align)), ret_addr) catch return null;
986		}
987
988		fn allocInner(
989		self: *Self,
990		len: usize,
991		log2_ptr_align: Allocator.Log2Align,
992		ret_addr: usize,
993		) Allocator.Error![*]u8 {
994		const new_aligned_size = @max(len, @as(usize, 1) << @as(Allocator.Log2Align, @intCast(log2_ptr_align)));
995		if (new_aligned_size > largest_bucket_object_size) {
996		try self.large_allocations.ensureUnusedCapacity(self.backing_allocator, 1);
997		const ptr = self.backing_allocator.rawAlloc(len, log2_ptr_align, ret_addr) orelse
998		return error.OutOfMemory;
999		const slice = ptr[0..len];
1000
1001		const gop = self.large_allocations.getOrPutAssumeCapacity(@intFromPtr(slice.ptr));
1002		if (config.retain_metadata and !config.never_unmap) {
1003		// Backing allocator may be reusing memory that we're retaining metadata for
1004		assert(!gop.found_existing or gop.value_ptr.freed);
1005		} else {
1006		assert(!gop.found_existing); // This would mean the kernel double-mapped pages.
1007		}
1008		gop.value_ptr.bytes = slice;
1009		if (config.enable_memory_limit)
1010		gop.value_ptr.requested_size = len;
1011		gop.value_ptr.captureStackTrace(ret_addr, .alloc);
1012		if (config.retain_metadata) {
1013		gop.value_ptr.freed = false;
1014		if (config.never_unmap) {
1015		gop.value_ptr.log2_ptr_align = log2_ptr_align;
1016		}
1017		}
1018
1019		if (config.verbose_log) {
1020		log.info("large alloc {d} bytes at {*}", .{ slice.len, slice.ptr });
1021		}
1022		return slice.ptr;
1023		}
1024
1025		const new_size_class = math.ceilPowerOfTwoAssert(usize, new_aligned_size);
1026		const slot = try self.allocSlot(new_size_class, ret_addr);
1027		if (config.safety) {
1028		slot.bucket.requestedSizes(new_size_class)[slot.slot_index] = @intCast(len);
1029		slot.bucket.log2PtrAligns(new_size_class)[slot.slot_index] = log2_ptr_align;
1030		}
1031		if (config.verbose_log) {
1032		log.info("small alloc {d} bytes at {*}", .{ len, slot.ptr });
1033		}
1034		return slot.ptr;
1035		}
1036
1037		fn createBucket(self: Self, size_class: usize) Error!BucketHeader {
1038		const page = try self.backing_allocator.alignedAlloc(u8, page_size, page_size);
1039		errdefer self.backing_allocator.free(page);
1040
1041		const bucket_size = bucketSize(size_class);
1042		const bucket_bytes = try self.backing_allocator.alignedAlloc(u8, @alignOf(BucketHeader), bucket_size);
1043		const ptr = @as(*BucketHeader, @ptrCast(bucket_bytes.ptr));
1044		ptr.* = BucketHeader{
1045		.page = page.ptr,
1046		.alloc_cursor = 0,
1047		.used_count = 0,
1048		};
1049		// Set the used bits to all zeroes
1050		@memset(@as([]u8, @as([1]u8, ptr.usedBits(0)))[0..usedBitsCount(size_class)], 0);
1051		if (config.safety) {
1052		// Set the requested sizes to zeroes
1053		@memset(mem.sliceAsBytes(ptr.requestedSizes(size_class)), 0);
1054		}
1055		return ptr;
1056		}
1057		};
1058		}
1059
1060		const TraceKind = enum {
1061		alloc,
1062		free,
1063		};
1064
1065		const test_config = Config{};
1066
1067		test "small allocations - free in same order" {
1068		var gpa = GeneralPurposeAllocator(test_config){};
1069		defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak");
1070		const allocator = gpa.allocator();
1071
1072		var list = std.ArrayList(*u64).init(std.testing.allocator);
1073		defer list.deinit();
1074
1075		var i: usize = 0;
1076		while (i < 513) : (i += 1) {
1077		const ptr = try allocator.create(u64);
1078		try list.append(ptr);
1079		}
1080
1081		for (list.items) \|ptr\| {
1082		allocator.destroy(ptr);
1083		}
1084		}
1085
1086		test "small allocations - free in reverse order" {
1087		var gpa = GeneralPurposeAllocator(test_config){};
1088		defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak");
1089		const allocator = gpa.allocator();
1090
1091		var list = std.ArrayList(*u64).init(std.testing.allocator);
1092		defer list.deinit();
1093
1094		var i: usize = 0;
1095		while (i < 513) : (i += 1) {
1096		const ptr = try allocator.create(u64);
1097		try list.append(ptr);
1098		}
1099
1100		while (list.popOrNull()) \|ptr\| {
1101		allocator.destroy(ptr);
1102		}
1103		}
1104
1105		test "large allocations" {
1106		var gpa = GeneralPurposeAllocator(test_config){};
1107		defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak");
1108		const allocator = gpa.allocator();
1109
1110		const ptr1 = try allocator.alloc(u64, 42768);
1111		const ptr2 = try allocator.alloc(u64, 52768);
1112		allocator.free(ptr1);
1113		const ptr3 = try allocator.alloc(u64, 62768);
1114		allocator.free(ptr3);
1115		allocator.free(ptr2);
1116		}
1117
1118		test "very large allocation" {
1119		var gpa = GeneralPurposeAllocator(test_config){};
1120		defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak");
1121		const allocator = gpa.allocator();
1122
1123		try std.testing.expectError(error.OutOfMemory, allocator.alloc(u8, math.maxInt(usize)));
1124		}
1125
1126		test "realloc" {
1127		var gpa = GeneralPurposeAllocator(test_config){};
1128		defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak");
1129		const allocator = gpa.allocator();
1130
1131		var slice = try allocator.alignedAlloc(u8, @alignOf(u32), 1);
1132		defer allocator.free(slice);
1133		slice[0] = 0x12;
1134
1135		// This reallocation should keep its pointer address.
1136		const old_slice = slice;
1137		slice = try allocator.realloc(slice, 2);
1138		try std.testing.expect(old_slice.ptr == slice.ptr);
1139		try std.testing.expect(slice[0] == 0x12);
1140		slice[1] = 0x34;
1141
1142		// This requires upgrading to a larger size class
1143		slice = try allocator.realloc(slice, 17);
1144		try std.testing.expect(slice[0] == 0x12);
1145		try std.testing.expect(slice[1] == 0x34);
1146		}
1147
1148		test "shrink" {
1149		var gpa = GeneralPurposeAllocator(test_config){};
1150		defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak");
1151		const allocator = gpa.allocator();
1152
1153		var slice = try allocator.alloc(u8, 20);
1154		defer allocator.free(slice);
1155
1156		@memset(slice, 0x11);
1157
1158		try std.testing.expect(allocator.resize(slice, 17));
1159		slice = slice[0..17];
1160
1161		for (slice) \|b\| {
1162		try std.testing.expect(b == 0x11);
1163		}
1164
1165		try std.testing.expect(allocator.resize(slice, 16));
1166		slice = slice[0..16];
1167
1168		for (slice) \|b\| {
1169		try std.testing.expect(b == 0x11);
1170		}
1171		}
1172
1173		test "large object - grow" {
1174		if (builtin.target.isWasm()) {
1175		// Not expected to pass on targets that do not have memory mapping.
1176		return error.SkipZigTest;
1177		}
1178		var gpa: GeneralPurposeAllocator(test_config) = .{};
1179		defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak");
1180		const allocator = gpa.allocator();
1181
1182		var slice1 = try allocator.alloc(u8, page_size * 2 - 20);
1183		defer allocator.free(slice1);
1184
1185		const old = slice1;
1186		slice1 = try allocator.realloc(slice1, page_size * 2 - 10);
1187		try std.testing.expect(slice1.ptr == old.ptr);
1188
1189		slice1 = try allocator.realloc(slice1, page_size * 2);
1190		try std.testing.expect(slice1.ptr == old.ptr);
1191
1192		slice1 = try allocator.realloc(slice1, page_size * 2 + 1);
1193		}
1194
1195		test "realloc small object to large object" {
1196		var gpa = GeneralPurposeAllocator(test_config){};
1197		defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak");
1198		const allocator = gpa.allocator();
1199
1200		var slice = try allocator.alloc(u8, 70);
1201		defer allocator.free(slice);
1202		slice[0] = 0x12;
1203		slice[60] = 0x34;
1204
1205		// This requires upgrading to a large object
1206		const large_object_size = page_size * 2 + 50;
1207		slice = try allocator.realloc(slice, large_object_size);
1208		try std.testing.expect(slice[0] == 0x12);
1209		try std.testing.expect(slice[60] == 0x34);
1210		}
1211
1212		test "shrink large object to large object" {
1213		var gpa = GeneralPurposeAllocator(test_config){};
1214		defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak");
1215		const allocator = gpa.allocator();
1216
1217		var slice = try allocator.alloc(u8, page_size * 2 + 50);
1218		defer allocator.free(slice);
1219		slice[0] = 0x12;
1220		slice[60] = 0x34;
1221
1222		if (!allocator.resize(slice, page_size * 2 + 1)) return;
1223		slice = slice.ptr[0 .. page_size * 2 + 1];
1224		try std.testing.expect(slice[0] == 0x12);
1225		try std.testing.expect(slice[60] == 0x34);
1226
1227		try std.testing.expect(allocator.resize(slice, page_size * 2 + 1));
1228		slice = slice[0 .. page_size * 2 + 1];
1229		try std.testing.expect(slice[0] == 0x12);
1230		try std.testing.expect(slice[60] == 0x34);
1231
1232		slice = try allocator.realloc(slice, page_size * 2);
1233		try std.testing.expect(slice[0] == 0x12);
1234		try std.testing.expect(slice[60] == 0x34);
1235		}
1236
1237		test "shrink large object to large object with larger alignment" {
1238		if (!builtin.link_libc and builtin.os.tag == .wasi) return error.SkipZigTest; // https://github.com/ziglang/zig/issues/22731
1239
1240		var gpa = GeneralPurposeAllocator(test_config){};
1241		defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak");
1242		const allocator = gpa.allocator();
1243
1244		var debug_buffer: [1000]u8 = undefined;
1245		var fba = std.heap.FixedBufferAllocator.init(&debug_buffer);
1246		const debug_allocator = fba.allocator();
1247
1248		const alloc_size = page_size * 2 + 50;
1249		var slice = try allocator.alignedAlloc(u8, 16, alloc_size);
1250		defer allocator.free(slice);
1251
1252		const big_alignment: usize = switch (builtin.os.tag) {
1253		.windows => page_size * 32, // Windows aligns to 64K.
1254		else => page_size * 2,
1255		};
1256		// This loop allocates until we find a page that is not aligned to the big
1257		// alignment. Then we shrink the allocation after the loop, but increase the
1258		// alignment to the higher one, that we know will force it to realloc.
1259		var stuff_to_free = std.ArrayList([]align(16) u8).init(debug_allocator);
1260		while (mem.isAligned(@intFromPtr(slice.ptr), big_alignment)) {
1261		try stuff_to_free.append(slice);
1262		slice = try allocator.alignedAlloc(u8, 16, alloc_size);
1263		}
1264		while (stuff_to_free.popOrNull()) \|item\| {
1265		allocator.free(item);
1266		}
1267		slice[0] = 0x12;
1268		slice[60] = 0x34;
1269
1270		slice = try allocator.reallocAdvanced(slice, big_alignment, alloc_size / 2);
1271		try std.testing.expect(slice[0] == 0x12);
1272		try std.testing.expect(slice[60] == 0x34);
1273		}
1274
1275		test "realloc large object to small object" {
1276		var gpa = GeneralPurposeAllocator(test_config){};
1277		defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak");
1278		const allocator = gpa.allocator();
1279
1280		var slice = try allocator.alloc(u8, page_size * 2 + 50);
1281		defer allocator.free(slice);
1282		slice[0] = 0x12;
1283		slice[16] = 0x34;
1284
1285		slice = try allocator.realloc(slice, 19);
1286		try std.testing.expect(slice[0] == 0x12);
1287		try std.testing.expect(slice[16] == 0x34);
1288		}
1289
1290		test "overridable mutexes" {
1291		var gpa = GeneralPurposeAllocator(.{ .MutexType = std.Thread.Mutex }){
1292		.backing_allocator = std.testing.allocator,
1293		.mutex = std.Thread.Mutex{},
1294		};
1295		defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak");
1296		const allocator = gpa.allocator();
1297
1298		const ptr = try allocator.create(i32);
1299		defer allocator.destroy(ptr);
1300		}
1301
1302		test "non-page-allocator backing allocator" {
1303		var gpa = GeneralPurposeAllocator(.{}){ .backing_allocator = std.testing.allocator };
1304		defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak");
1305		const allocator = gpa.allocator();
1306
1307		const ptr = try allocator.create(i32);
1308		defer allocator.destroy(ptr);
1309		}
1310
1311		test "realloc large object to larger alignment" {
1312		if (!builtin.link_libc and builtin.os.tag == .wasi) return error.SkipZigTest; // https://github.com/ziglang/zig/issues/22731
1313
1314		var gpa = GeneralPurposeAllocator(test_config){};
1315		defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak");
1316		const allocator = gpa.allocator();
1317
1318		var debug_buffer: [1000]u8 = undefined;
1319		var fba = std.heap.FixedBufferAllocator.init(&debug_buffer);
1320		const debug_allocator = fba.allocator();
1321
1322		var slice = try allocator.alignedAlloc(u8, 16, page_size * 2 + 50);
1323		defer allocator.free(slice);
1324
1325		const big_alignment: usize = switch (builtin.os.tag) {
1326		.windows => page_size * 32, // Windows aligns to 64K.
1327		else => page_size * 2,
1328		};
1329		// This loop allocates until we find a page that is not aligned to the big alignment.
1330		var stuff_to_free = std.ArrayList([]align(16) u8).init(debug_allocator);
1331		while (mem.isAligned(@intFromPtr(slice.ptr), big_alignment)) {
1332		try stuff_to_free.append(slice);
1333		slice = try allocator.alignedAlloc(u8, 16, page_size * 2 + 50);
1334		}
1335		while (stuff_to_free.popOrNull()) \|item\| {
1336		allocator.free(item);
1337		}
1338		slice[0] = 0x12;
1339		slice[16] = 0x34;
1340
1341		slice = try allocator.reallocAdvanced(slice, 32, page_size * 2 + 100);
1342		try std.testing.expect(slice[0] == 0x12);
1343		try std.testing.expect(slice[16] == 0x34);
1344
1345		slice = try allocator.reallocAdvanced(slice, 32, page_size * 2 + 25);
1346		try std.testing.expect(slice[0] == 0x12);
1347		try std.testing.expect(slice[16] == 0x34);
1348
1349		slice = try allocator.reallocAdvanced(slice, big_alignment, page_size * 2 + 100);
1350		try std.testing.expect(slice[0] == 0x12);
1351		try std.testing.expect(slice[16] == 0x34);
1352		}
1353
1354		test "large object shrinks to small but allocation fails during shrink" {
1355		if (builtin.target.isWasm()) {
1356		// Not expected to pass on targets that do not have memory mapping.
1357		return error.SkipZigTest;
1358		}
1359
1360		var failing_allocator = std.testing.FailingAllocator.init(std.heap.page_allocator, .{ .fail_index = 3 });
1361		var gpa = GeneralPurposeAllocator(.{}){ .backing_allocator = failing_allocator.allocator() };
1362		defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak");
1363		const allocator = gpa.allocator();
1364
1365		var slice = try allocator.alloc(u8, page_size * 2 + 50);
1366		defer allocator.free(slice);
1367		slice[0] = 0x12;
1368		slice[3] = 0x34;
1369
1370		// Next allocation will fail in the backing allocator of the GeneralPurposeAllocator
1371
1372		try std.testing.expect(allocator.resize(slice, 4));
1373		slice = slice[0..4];
1374		try std.testing.expect(slice[0] == 0x12);
1375		try std.testing.expect(slice[3] == 0x34);
1376		}
1377
1378		test "objects of size 1024 and 2048" {
1379		var gpa = GeneralPurposeAllocator(test_config){};
1380		defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak");
1381		const allocator = gpa.allocator();
1382
1383		const slice = try allocator.alloc(u8, 1025);
1384		const slice2 = try allocator.alloc(u8, 3000);
1385
1386		allocator.free(slice);
1387		allocator.free(slice2);
1388		}
1389
1390		test "setting a memory cap" {
1391		var gpa = GeneralPurposeAllocator(.{ .enable_memory_limit = true }){};
1392		defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak");
1393		const allocator = gpa.allocator();
1394
1395		gpa.setRequestedMemoryLimit(1010);
1396
1397		const small = try allocator.create(i32);
1398		try std.testing.expect(gpa.total_requested_bytes == 4);
1399
1400		const big = try allocator.alloc(u8, 1000);
1401		try std.testing.expect(gpa.total_requested_bytes == 1004);
1402
1403		try std.testing.expectError(error.OutOfMemory, allocator.create(u64));
1404
1405		allocator.destroy(small);
1406		try std.testing.expect(gpa.total_requested_bytes == 1000);
1407
1408		allocator.free(big);
1409		try std.testing.expect(gpa.total_requested_bytes == 0);
1410
1411		const exact = try allocator.alloc(u8, 1010);
1412		try std.testing.expect(gpa.total_requested_bytes == 1010);
1413		allocator.free(exact);
1414		}
1415
1416		test "double frees" {
1417		// use a GPA to back a GPA to check for leaks of the latter's metadata
1418		var backing_gpa = GeneralPurposeAllocator(.{ .safety = true }){};
1419		defer std.testing.expect(backing_gpa.deinit() == .ok) catch @panic("leak");
1420
1421		const GPA = GeneralPurposeAllocator(.{ .safety = true, .never_unmap = true, .retain_metadata = true });
1422		var gpa = GPA{ .backing_allocator = backing_gpa.allocator() };
1423		defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak");
1424		const allocator = gpa.allocator();
1425
1426		// detect a small allocation double free, even though bucket is emptied
1427		const index: usize = 6;
1428		const size_class: usize = @as(usize, 1) << 6;
1429		const small = try allocator.alloc(u8, size_class);
1430		try std.testing.expect(GPA.searchBucket(&gpa.buckets[index], @intFromPtr(small.ptr), gpa.cur_buckets[index]) != null);
1431		allocator.free(small);
1432		try std.testing.expect(GPA.searchBucket(&gpa.buckets[index], @intFromPtr(small.ptr), gpa.cur_buckets[index]) == null);
1433		try std.testing.expect(GPA.searchBucket(&gpa.empty_buckets, @intFromPtr(small.ptr), null) != null);
1434
1435		// detect a large allocation double free
1436		const large = try allocator.alloc(u8, 2 * page_size);
1437		try std.testing.expect(gpa.large_allocations.contains(@intFromPtr(large.ptr)));
1438		try std.testing.expectEqual(gpa.large_allocations.getEntry(@intFromPtr(large.ptr)).?.value_ptr.bytes, large);
1439		allocator.free(large);
1440		try std.testing.expect(gpa.large_allocations.contains(@intFromPtr(large.ptr)));
1441		try std.testing.expect(gpa.large_allocations.getEntry(@intFromPtr(large.ptr)).?.value_ptr.freed);
1442
1443		const normal_small = try allocator.alloc(u8, size_class);
1444		defer allocator.free(normal_small);
1445		const normal_large = try allocator.alloc(u8, 2 * page_size);
1446		defer allocator.free(normal_large);
1447
1448		// check that flushing retained metadata doesn't disturb live allocations
1449		gpa.flushRetainedMetadata();
1450		try std.testing.expect(gpa.empty_buckets.root == null);
1451		try std.testing.expect(GPA.searchBucket(&gpa.buckets[index], @intFromPtr(normal_small.ptr), gpa.cur_buckets[index]) != null);
1452		try std.testing.expect(gpa.large_allocations.contains(@intFromPtr(normal_large.ptr)));
1453		try std.testing.expect(!gpa.large_allocations.contains(@intFromPtr(large.ptr)));
1454		}
1455
1456		test "empty bucket size class" {
1457		const GPA = GeneralPurposeAllocator(.{ .safety = true, .never_unmap = true, .retain_metadata = true });
1458		var gpa = GPA{};
1459		defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak");
1460		const allocator = gpa.allocator();
1461
1462		// allocate and free to create an empty bucket
1463		const size_class: usize = @as(usize, 1) << 6;
1464		const small = try allocator.alloc(u8, size_class);
1465		allocator.free(small);
1466
1467		// the metadata tracking system relies on alloc_cursor of empty buckets
1468		// being set to the slot count so that we can get back the size class.
1469		const empty_bucket = GPA.searchBucket(&gpa.empty_buckets, @intFromPtr(small.ptr), null).?;
1470		try std.testing.expect(empty_bucket.emptyBucketSizeClass() == size_class);
1471		}
1472
1473		test "bug 9995 fix, large allocs count requested size not backing size" {
1474		// with AtLeast, buffer likely to be larger than requested, especially when shrinking
1475		var gpa = GeneralPurposeAllocator(.{ .enable_memory_limit = true }){};
1476		const allocator = gpa.allocator();
1477
1478		var buf = try allocator.alignedAlloc(u8, 1, page_size + 1);
1479		try std.testing.expect(gpa.total_requested_bytes == page_size + 1);
1480		buf = try allocator.realloc(buf, 1);
1481		try std.testing.expect(gpa.total_requested_bytes == 1);
1482		buf = try allocator.realloc(buf, 2);
1483		try std.testing.expect(gpa.total_requested_bytes == 2);
1484		}
1485
1486		test "retain metadata and never unmap" {
1487		var gpa = std.heap.GeneralPurposeAllocator(.{
1488		.safety = true,
1489		.never_unmap = true,
1490		.retain_metadata = true,
1491		}){};
1492		defer std.debug.assert(gpa.deinit() == .ok);
1493		const allocator = gpa.allocator();
1494
1495		const alloc = try allocator.alloc(u8, 8);
1496		allocator.free(alloc);
1497
1498		const alloc2 = try allocator.alloc(u8, 8);
1499		allocator.free(alloc2);
1500		}

lib/std/heap/log_to_writer_allocator.zig deleted-118

...	...	@@ -1,118 +0,0 @@
1		const std = @import("../std.zig");
2		const Allocator = std.mem.Allocator;
3
4		/// This allocator is used in front of another allocator and logs to the provided writer
5		/// on every call to the allocator. Writer errors are ignored.
6		pub fn LogToWriterAllocator(comptime Writer: type) type {
7		return struct {
8		parent_allocator: Allocator,
9		writer: Writer,
10
11		const Self = @This();
12
13		pub fn init(parent_allocator: Allocator, writer: Writer) Self {
14		return Self{
15		.parent_allocator = parent_allocator,
16		.writer = writer,
17		};
18		}
19
20		pub fn allocator(self: *Self) Allocator {
21		return .{
22		.ptr = self,
23		.vtable = &.{
24		.alloc = alloc,
25		.resize = resize,
26		.free = free,
27		},
28		};
29		}
30
31		fn alloc(
32		ctx: *anyopaque,
33		len: usize,
34		log2_ptr_align: u8,
35		ra: usize,
36		) ?[*]u8 {
37		const self: *Self = @ptrCast(@alignCast(ctx));
38		self.writer.print("alloc : {}", .{len}) catch {};
39		const result = self.parent_allocator.rawAlloc(len, log2_ptr_align, ra);
40		if (result != null) {
41		self.writer.print(" success!\n", .{}) catch {};
42		} else {
43		self.writer.print(" failure!\n", .{}) catch {};
44		}
45		return result;
46		}
47
48		fn resize(
49		ctx: *anyopaque,
50		buf: []u8,
51		log2_buf_align: u8,
52		new_len: usize,
53		ra: usize,
54		) bool {
55		const self: *Self = @ptrCast(@alignCast(ctx));
56		if (new_len <= buf.len) {
57		self.writer.print("shrink: {} to {}\n", .{ buf.len, new_len }) catch {};
58		} else {
59		self.writer.print("expand: {} to {}", .{ buf.len, new_len }) catch {};
60		}
61
62		if (self.parent_allocator.rawResize(buf, log2_buf_align, new_len, ra)) {
63		if (new_len > buf.len) {
64		self.writer.print(" success!\n", .{}) catch {};
65		}
66		return true;
67		}
68
69		std.debug.assert(new_len > buf.len);
70		self.writer.print(" failure!\n", .{}) catch {};
71		return false;
72		}
73
74		fn free(
75		ctx: *anyopaque,
76		buf: []u8,
77		log2_buf_align: u8,
78		ra: usize,
79		) void {
80		const self: *Self = @ptrCast(@alignCast(ctx));
81		self.writer.print("free : {}\n", .{buf.len}) catch {};
82		self.parent_allocator.rawFree(buf, log2_buf_align, ra);
83		}
84		};
85		}
86
87		/// This allocator is used in front of another allocator and logs to the provided writer
88		/// on every call to the allocator. Writer errors are ignored.
89		pub fn logToWriterAllocator(
90		parent_allocator: Allocator,
91		writer: anytype,
92		) LogToWriterAllocator(@TypeOf(writer)) {
93		return LogToWriterAllocator(@TypeOf(writer)).init(parent_allocator, writer);
94		}
95
96		test "LogToWriterAllocator" {
97		var log_buf: [255]u8 = undefined;
98		var fbs = std.io.fixedBufferStream(&log_buf);
99
100		var allocator_buf: [10]u8 = undefined;
101		var fixedBufferAllocator = std.mem.validationWrap(std.heap.FixedBufferAllocator.init(&allocator_buf));
102		var allocator_state = logToWriterAllocator(fixedBufferAllocator.allocator(), fbs.writer());
103		const allocator = allocator_state.allocator();
104
105		var a = try allocator.alloc(u8, 10);
106		try std.testing.expect(allocator.resize(a, 5));
107		a = a[0..5];
108		try std.testing.expect(!allocator.resize(a, 20));
109		allocator.free(a);
110
111		try std.testing.expectEqualSlices(u8,
112		\\alloc : 10 success!
113		\\shrink: 10 to 5
114		\\expand: 5 to 20 failure!
115		\\free : 5
116		\\
117		, fbs.getWritten());
118		}

lib/std/heap/logging_allocator.zig deleted-133

...	...	@@ -1,133 +0,0 @@
1		const std = @import("../std.zig");
2		const Allocator = std.mem.Allocator;
3
4		/// This allocator is used in front of another allocator and logs to `std.log`
5		/// on every call to the allocator.
6		/// For logging to a `std.io.Writer` see `std.heap.LogToWriterAllocator`
7		pub fn LoggingAllocator(
8		comptime success_log_level: std.log.Level,
9		comptime failure_log_level: std.log.Level,
10		) type {
11		return ScopedLoggingAllocator(.default, success_log_level, failure_log_level);
12		}
13
14		/// This allocator is used in front of another allocator and logs to `std.log`
15		/// with the given scope on every call to the allocator.
16		/// For logging to a `std.io.Writer` see `std.heap.LogToWriterAllocator`
17		pub fn ScopedLoggingAllocator(
18		comptime scope: @Type(.enum_literal),
19		comptime success_log_level: std.log.Level,
20		comptime failure_log_level: std.log.Level,
21		) type {
22		const log = std.log.scoped(scope);
23
24		return struct {
25		parent_allocator: Allocator,
26
27		const Self = @This();
28
29		pub fn init(parent_allocator: Allocator) Self {
30		return .{
31		.parent_allocator = parent_allocator,
32		};
33		}
34
35		pub fn allocator(self: *Self) Allocator {
36		return .{
37		.ptr = self,
38		.vtable = &.{
39		.alloc = alloc,
40		.resize = resize,
41		.free = free,
42		},
43		};
44		}
45
46		// This function is required as the `std.log.log` function is not public
47		inline fn logHelper(comptime log_level: std.log.Level, comptime format: []const u8, args: anytype) void {
48		switch (log_level) {
49		.err => log.err(format, args),
50		.warn => log.warn(format, args),
51		.info => log.info(format, args),
52		.debug => log.debug(format, args),
53		}
54		}
55
56		fn alloc(
57		ctx: *anyopaque,
58		len: usize,
59		log2_ptr_align: u8,
60		ra: usize,
61		) ?[*]u8 {
62		const self: *Self = @ptrCast(@alignCast(ctx));
63		const result = self.parent_allocator.rawAlloc(len, log2_ptr_align, ra);
64		if (result != null) {
65		logHelper(
66		success_log_level,
67		"alloc - success - len: {}, ptr_align: {}",
68		.{ len, log2_ptr_align },
69		);
70		} else {
71		logHelper(
72		failure_log_level,
73		"alloc - failure: OutOfMemory - len: {}, ptr_align: {}",
74		.{ len, log2_ptr_align },
75		);
76		}
77		return result;
78		}
79
80		fn resize(
81		ctx: *anyopaque,
82		buf: []u8,
83		log2_buf_align: u8,
84		new_len: usize,
85		ra: usize,
86		) bool {
87		const self: *Self = @ptrCast(@alignCast(ctx));
88		if (self.parent_allocator.rawResize(buf, log2_buf_align, new_len, ra)) {
89		if (new_len <= buf.len) {
90		logHelper(
91		success_log_level,
92		"shrink - success - {} to {}, buf_align: {}",
93		.{ buf.len, new_len, log2_buf_align },
94		);
95		} else {
96		logHelper(
97		success_log_level,
98		"expand - success - {} to {}, buf_align: {}",
99		.{ buf.len, new_len, log2_buf_align },
100		);
101		}
102
103		return true;
104		}
105
106		std.debug.assert(new_len > buf.len);
107		logHelper(
108		failure_log_level,
109		"expand - failure - {} to {}, buf_align: {}",
110		.{ buf.len, new_len, log2_buf_align },
111		);
112		return false;
113		}
114
115		fn free(
116		ctx: *anyopaque,
117		buf: []u8,
118		log2_buf_align: u8,
119		ra: usize,
120		) void {
121		const self: *Self = @ptrCast(@alignCast(ctx));
122		self.parent_allocator.rawFree(buf, log2_buf_align, ra);
123		logHelper(success_log_level, "free - len: {}", .{buf.len});
124		}
125		};
126		}
127
128		/// This allocator is used in front of another allocator and logs to `std.log`
129		/// on every call to the allocator.
130		/// For logging to a `std.io.Writer` see `std.heap.LogToWriterAllocator`
131		pub fn loggingAllocator(parent_allocator: Allocator) LoggingAllocator(.debug, .err) {
132		return LoggingAllocator(.debug, .err).init(parent_allocator);
133		}

lib/std/heap/sbrk_allocator.zig+4-3

...	...	@@ -3,6 +3,7 @@ const builtin = @import("builtin");
3	3	const math = std.math;
4	4	const Allocator = std.mem.Allocator;
5	5	const mem = std.mem;
	6	const heap = std.heap;
6	7	const assert = std.debug.assert;
7	8
8	9	pub fn SbrkAllocator(comptime sbrk: *const fn (n: usize) usize) type {
...	...	@@ -18,7 +19,7 @@ pub fn SbrkAllocator(comptime sbrk: *const fn (n: usize) usize) type {
18	19	const max_usize = math.maxInt(usize);
19	20	const ushift = math.Log2Int(usize);
20	21	const bigpage_size = 64 * 1024;
21		const pages_per_bigpage = bigpage_size / mem.page_size;
	22	const pages_per_bigpage = bigpage_size / heap.pageSize();
22	23	const bigpage_count = max_usize / bigpage_size;
23	24
24	25	/// Because of storing free list pointers, the minimum size class is 3.
...	...	@@ -58,7 +59,7 @@ pub fn SbrkAllocator(comptime sbrk: *const fn (n: usize) usize) type {
58	59	}
59	60
60	61	const next_addr = next_addrs[class];
61		if (next_addr % mem.page_size == 0) {
	62	if (next_addr % heap.pageSize() == 0) {
62	63	const addr = allocBigPages(1);
63	64	if (addr == 0) return null;
64	65	//std.debug.print("allocated fresh slot_size={d} class={d} addr=0x{x}\n", .{
...	...	@@ -153,7 +154,7 @@ pub fn SbrkAllocator(comptime sbrk: *const fn (n: usize) usize) type {
153	154	big_frees[class] = node.*;
154	155	return top_free_ptr;
155	156	}
156		return sbrk(pow2_pages * pages_per_bigpage * mem.page_size);
	157	return sbrk(pow2_pages * pages_per_bigpage * heap.pageSize());
157	158	}
158	159	};
159	160	}

lib/std/mem.zig+89-56

...	...	@@ -8,26 +8,6 @@ const testing = std.testing;
8	8	const Endian = std.builtin.Endian;
9	9	const native_endian = builtin.cpu.arch.endian();
10	10
11		/// Compile time known minimum page size.
12		/// https://github.com/ziglang/zig/issues/4082
13		pub const page_size = switch (builtin.cpu.arch) {
14		.wasm32, .wasm64 => 64 * 1024,
15		.aarch64 => switch (builtin.os.tag) {
16		.macos, .ios, .watchos, .tvos, .visionos => 16 * 1024,
17		else => 4 * 1024,
18		},
19		.sparc64 => 8 * 1024,
20		.loongarch32, .loongarch64 => switch (builtin.os.tag) {
21		// Linux default KConfig value is 16KiB
22		.linux => 16 * 1024,
23		// FIXME:
24		// There is no other OS supported yet. Use the same value
25		// as Linux for now.
26		else => 16 * 1024,
27		},
28		else => 4 * 1024,
29		};
30
31	11	/// The standard library currently thoroughly depends on byte size
32	12	/// being 8 bits. (see the use of u8 throughout allocation code as
33	13	/// the "byte" type.) Code which depends on this can reference this
...	...	@@ -38,6 +18,60 @@ pub const byte_size_in_bits = 8;
38	18
39	19	pub const Allocator = @import("mem/Allocator.zig");
40	20
	21	/// Stored as a power-of-two.
	22	pub const Alignment = enum(math.Log2Int(usize)) {
	23	@"1" = 0,
	24	@"2" = 1,
	25	@"4" = 2,
	26	@"8" = 3,
	27	@"16" = 4,
	28	@"32" = 5,
	29	@"64" = 6,
	30	_,
	31
	32	pub fn toByteUnits(a: Alignment) usize {
	33	return @as(usize, 1) << @intFromEnum(a);
	34	}
	35
	36	pub fn fromByteUnits(n: usize) Alignment {
	37	assert(std.math.isPowerOfTwo(n));
	38	return @enumFromInt(@ctz(n));
	39	}
	40
	41	pub fn order(lhs: Alignment, rhs: Alignment) std.math.Order {
	42	return std.math.order(@intFromEnum(lhs), @intFromEnum(rhs));
	43	}
	44
	45	pub fn compare(lhs: Alignment, op: std.math.CompareOperator, rhs: Alignment) bool {
	46	return std.math.compare(@intFromEnum(lhs), op, @intFromEnum(rhs));
	47	}
	48
	49	pub fn max(lhs: Alignment, rhs: Alignment) Alignment {
	50	return @enumFromInt(@max(@intFromEnum(lhs), @intFromEnum(rhs)));
	51	}
	52
	53	pub fn min(lhs: Alignment, rhs: Alignment) Alignment {
	54	return @enumFromInt(@min(@intFromEnum(lhs), @intFromEnum(rhs)));
	55	}
	56
	57	/// Return next address with this alignment.
	58	pub fn forward(a: Alignment, address: usize) usize {
	59	const x = (@as(usize, 1) << @intFromEnum(a)) - 1;
	60	return (address + x) & ~x;
	61	}
	62
	63	/// Return previous address with this alignment.
	64	pub fn backward(a: Alignment, address: usize) usize {
	65	const x = (@as(usize, 1) << @intFromEnum(a)) - 1;
	66	return address & ~x;
	67	}
	68
	69	/// Return whether address is aligned to this amount.
	70	pub fn check(a: Alignment, address: usize) bool {
	71	return @ctz(address) >= @intFromEnum(a);
	72	}
	73	};
	74
41	75	/// Detects and asserts if the std.mem.Allocator interface is violated by the caller
42	76	/// or the allocator.
43	77	pub fn ValidationAllocator(comptime T: type) type {
...	...	@@ -58,6 +92,7 @@ pub fn ValidationAllocator(comptime T: type) type {
58	92	.vtable = &.{
59	93	.alloc = alloc,
60	94	.resize = resize,
	95	.remap = remap,
61	96	.free = free,
62	97	},
63	98	};
...	...	@@ -71,41 +106,54 @@ pub fn ValidationAllocator(comptime T: type) type {
71	106	pub fn alloc(
72	107	ctx: *anyopaque,
73	108	n: usize,
74		log2_ptr_align: u8,
	109	alignment: mem.Alignment,
75	110	ret_addr: usize,
76	111	) ?[*]u8 {
77	112	assert(n > 0);
78	113	const self: *Self = @ptrCast(@alignCast(ctx));
79	114	const underlying = self.getUnderlyingAllocatorPtr();
80		const result = underlying.rawAlloc(n, log2_ptr_align, ret_addr) orelse
	115	const result = underlying.rawAlloc(n, alignment, ret_addr) orelse
81	116	return null;
82		assert(mem.isAlignedLog2(@intFromPtr(result), log2_ptr_align));
	117	assert(alignment.check(@intFromPtr(result)));
83	118	return result;
84	119	}
85	120
86	121	pub fn resize(
87	122	ctx: *anyopaque,
88	123	buf: []u8,
89		log2_buf_align: u8,
	124	alignment: Alignment,
90	125	new_len: usize,
91	126	ret_addr: usize,
92	127	) bool {
93	128	const self: *Self = @ptrCast(@alignCast(ctx));
94	129	assert(buf.len > 0);
95	130	const underlying = self.getUnderlyingAllocatorPtr();
96		return underlying.rawResize(buf, log2_buf_align, new_len, ret_addr);
	131	return underlying.rawResize(buf, alignment, new_len, ret_addr);
	132	}
	133
	134	pub fn remap(
	135	ctx: *anyopaque,
	136	buf: []u8,
	137	alignment: Alignment,
	138	new_len: usize,
	139	ret_addr: usize,
	140	) ?[*]u8 {
	141	const self: *Self = @ptrCast(@alignCast(ctx));
	142	assert(buf.len > 0);
	143	const underlying = self.getUnderlyingAllocatorPtr();
	144	return underlying.rawRemap(buf, alignment, new_len, ret_addr);
97	145	}
98	146
99	147	pub fn free(
100	148	ctx: *anyopaque,
101	149	buf: []u8,
102		log2_buf_align: u8,
	150	alignment: Alignment,
103	151	ret_addr: usize,
104	152	) void {
105	153	const self: *Self = @ptrCast(@alignCast(ctx));
106	154	assert(buf.len > 0);
107	155	const underlying = self.getUnderlyingAllocatorPtr();
108		underlying.rawFree(buf, log2_buf_align, ret_addr);
	156	underlying.rawFree(buf, alignment, ret_addr);
109	157	}
110	158
111	159	pub fn reset(self: *Self) void {
...	...	@@ -133,27 +181,9 @@ pub fn alignAllocLen(full_len: usize, alloc_len: usize, len_align: u29) usize {
133	181	return adjusted;
134	182	}
135	183
136		const fail_allocator = Allocator{
137		.ptr = undefined,
138		.vtable = &failAllocator_vtable,
139		};
140
141		const failAllocator_vtable = Allocator.VTable{
142		.alloc = failAllocatorAlloc,
143		.resize = Allocator.noResize,
144		.free = Allocator.noFree,
145		};
146
147		fn failAllocatorAlloc(_: anyopaque, n: usize, log2_alignment: u8, ra: usize) ?[]u8 {
148		_ = n;
149		_ = log2_alignment;
150		_ = ra;
151		return null;
152		}
153
154	184	test "Allocator basics" {
155		try testing.expectError(error.OutOfMemory, fail_allocator.alloc(u8, 1));
156		try testing.expectError(error.OutOfMemory, fail_allocator.allocSentinel(u8, 1, 0));
	185	try testing.expectError(error.OutOfMemory, testing.failing_allocator.alloc(u8, 1));
	186	try testing.expectError(error.OutOfMemory, testing.failing_allocator.allocSentinel(u8, 1, 0));
157	187	}
158	188
159	189	test "Allocator.resize" {
...	...	@@ -1068,16 +1098,18 @@ pub fn indexOfSentinel(comptime T: type, comptime sentinel: T, p: [*:sentinel]co
1068	1098	// as we don't read into a new page. This should be the case for most architectures
1069	1099	// which use paged memory, however should be confirmed before adding a new arch below.
1070	1100	.aarch64, .x86, .x86_64 => if (std.simd.suggestVectorLength(T)) \|block_len\| {
	1101	const page_size = std.heap.pageSize();
1071	1102	const block_size = @sizeOf(T) * block_len;
1072	1103	const Block = @Vector(block_len, T);
1073	1104	const mask: Block = @splat(sentinel);
1074	1105
1075		comptime std.debug.assert(std.mem.page_size % block_size == 0);
	1106	comptime assert(std.heap.page_size_max % @sizeOf(Block) == 0);
	1107	assert(page_size % @sizeOf(Block) == 0);
1076	1108
1077	1109	// First block may be unaligned
1078	1110	const start_addr = @intFromPtr(&p[i]);
1079		const offset_in_page = start_addr & (std.mem.page_size - 1);
1080		if (offset_in_page <= std.mem.page_size - block_size) {
	1111	const offset_in_page = start_addr & (page_size - 1);
	1112	if (offset_in_page <= page_size - @sizeOf(Block)) {
1081	1113	// Will not read past the end of a page, full block.
1082	1114	const block: Block = p[i..][0..block_len].*;
1083	1115	const matches = block == mask;
...	...	@@ -1097,7 +1129,7 @@ pub fn indexOfSentinel(comptime T: type, comptime sentinel: T, p: [*:sentinel]co
1097	1129	}
1098	1130	}
1099	1131
1100		std.debug.assert(std.mem.isAligned(@intFromPtr(&p[i]), block_size));
	1132	assert(std.mem.isAligned(@intFromPtr(&p[i]), block_size));
1101	1133	while (true) {
1102	1134	const block: *const Block = @ptrCast(@alignCast(p[i..][0..block_len]));
1103	1135	const matches = block.* == mask;
...	...	@@ -1120,23 +1152,24 @@ pub fn indexOfSentinel(comptime T: type, comptime sentinel: T, p: [*:sentinel]co
1120	1152	test "indexOfSentinel vector paths" {
1121	1153	const Types = [_]type{ u8, u16, u32, u64 };
1122	1154	const allocator = std.testing.allocator;
	1155	const page_size = std.heap.pageSize();
1123	1156
1124	1157	inline for (Types) \|T\| {
1125	1158	const block_len = std.simd.suggestVectorLength(T) orelse continue;
1126	1159
1127	1160	// Allocate three pages so we guarantee a page-crossing address with a full page after
1128		const memory = try allocator.alloc(T, 3 * std.mem.page_size / @sizeOf(T));
	1161	const memory = try allocator.alloc(T, 3 * page_size / @sizeOf(T));
1129	1162	defer allocator.free(memory);
1130	1163	@memset(memory, 0xaa);
1131	1164
1132	1165	// Find starting page-alignment = 0
1133	1166	var start: usize = 0;
1134	1167	const start_addr = @intFromPtr(&memory);
1135		start += (std.mem.alignForward(usize, start_addr, std.mem.page_size) - start_addr) / @sizeOf(T);
1136		try testing.expect(start < std.mem.page_size / @sizeOf(T));
	1168	start += (std.mem.alignForward(usize, start_addr, page_size) - start_addr) / @sizeOf(T);
	1169	try testing.expect(start < page_size / @sizeOf(T));
1137	1170
1138	1171	// Validate all sub-block alignments
1139		const search_len = std.mem.page_size / @sizeOf(T);
	1172	const search_len = page_size / @sizeOf(T);
1140	1173	memory[start + search_len] = 0;
1141	1174	for (0..block_len) \|offset\| {
1142	1175	try testing.expectEqual(search_len - offset, indexOfSentinel(T, 0, @ptrCast(&memory[start + offset])));
...	...	@@ -1144,7 +1177,7 @@ test "indexOfSentinel vector paths" {
1144	1177	memory[start + search_len] = 0xaa;
1145	1178
1146	1179	// Validate page boundary crossing
1147		const start_page_boundary = start + (std.mem.page_size / @sizeOf(T));
	1180	const start_page_boundary = start + (page_size / @sizeOf(T));
1148	1181	memory[start_page_boundary + block_len] = 0;
1149	1182	for (0..block_len) \|offset\| {
1150	1183	try testing.expectEqual(2 * block_len - offset, indexOfSentinel(T, 0, @ptrCast(&memory[start_page_boundary - block_len + offset])));

lib/std/mem/Allocator.zig+153-77

...	...	@@ -6,29 +6,34 @@ const math = std.math;
6	6	const mem = std.mem;
7	7	const Allocator = @This();
8	8	const builtin = @import("builtin");
	9	const Alignment = std.mem.Alignment;
9	10
10	11	pub const Error = error{OutOfMemory};
11	12	pub const Log2Align = math.Log2Int(usize);
12	13
13	14	/// The type erased pointer to the allocator implementation.
14		/// Any comparison of this field may result in illegal behavior, since it may be set to
15		/// `undefined` in cases where the allocator implementation does not have any associated
16		/// state.
	15	///
	16	/// Any comparison of this field may result in illegal behavior, since it may
	17	/// be set to `undefined` in cases where the allocator implementation does not
	18	/// have any associated state.
17	19	ptr: *anyopaque,
18	20	vtable: *const VTable,
19	21
20	22	pub const VTable = struct {
21		/// Attempt to allocate exactly `len` bytes aligned to `1 << ptr_align`.
	23	/// Return a pointer to `len` bytes with specified `alignment`, or return
	24	/// `null` indicating the allocation failed.
22	25	///
23	26	/// `ret_addr` is optionally provided as the first return address of the
24	27	/// allocation call stack. If the value is `0` it means no return address
25	28	/// has been provided.
26		alloc: const fn (ctx: anyopaque, len: usize, ptr_align: u8, ret_addr: usize) ?[*]u8,
	29	alloc: const fn (anyopaque, len: usize, alignment: Alignment, ret_addr: usize) ?[*]u8,
27	30
28		/// Attempt to expand or shrink memory in place. `buf.len` must equal the
29		/// length requested from the most recent successful call to `alloc` or
30		/// `resize`. `buf_align` must equal the same value that was passed as the
31		/// `ptr_align` parameter to the original `alloc` call.
	31	/// Attempt to expand or shrink memory in place.
	32	///
	33	/// `memory.len` must equal the length requested from the most recent
	34	/// successful call to `alloc`, `resize`, or `remap`. `alignment` must
	35	/// equal the same value that was passed as the `alignment` parameter to
	36	/// the original `alloc` call.
32	37	///
33	38	/// A result of `true` indicates the resize was successful and the
34	39	/// allocation now has the same address but a size of `new_len`. `false`
...	...	@@ -40,72 +45,113 @@ pub const VTable = struct {
40	45	/// `ret_addr` is optionally provided as the first return address of the
41	46	/// allocation call stack. If the value is `0` it means no return address
42	47	/// has been provided.
43		resize: const fn (ctx: anyopaque, buf: []u8, buf_align: u8, new_len: usize, ret_addr: usize) bool,
	48	resize: const fn (anyopaque, memory: []u8, alignment: Alignment, new_len: usize, ret_addr: usize) bool,
44	49
45		/// Free and invalidate a buffer.
	50	/// Attempt to expand or shrink memory, allowing relocation.
	51	///
	52	/// `memory.len` must equal the length requested from the most recent
	53	/// successful call to `alloc`, `resize`, or `remap`. `alignment` must
	54	/// equal the same value that was passed as the `alignment` parameter to
	55	/// the original `alloc` call.
	56	///
	57	/// A non-`null` return value indicates the resize was successful. The
	58	/// allocation may have same address, or may have been relocated. In either
	59	/// case, the allocation now has size of `new_len`. A `null` return value
	60	/// indicates that the resize would be equivalent to allocating new memory,
	61	/// copying the bytes from the old memory, and then freeing the old memory.
	62	/// In such case, it is more efficient for the caller to perform the copy.
	63	///
	64	/// `new_len` must be greater than zero.
46	65	///
47		/// `buf.len` must equal the most recent length returned by `alloc` or
48		/// given to a successful `resize` call.
	66	/// `ret_addr` is optionally provided as the first return address of the
	67	/// allocation call stack. If the value is `0` it means no return address
	68	/// has been provided.
	69	remap: const fn (anyopaque, memory: []u8, alignment: Alignment, new_len: usize, ret_addr: usize) ?[*]u8,
	70
	71	/// Free and invalidate a region of memory.
49	72	///
50		/// `buf_align` must equal the same value that was passed as the
51		/// `ptr_align` parameter to the original `alloc` call.
	73	/// `memory.len` must equal the length requested from the most recent
	74	/// successful call to `alloc`, `resize`, or `remap`. `alignment` must
	75	/// equal the same value that was passed as the `alignment` parameter to
	76	/// the original `alloc` call.
52	77	///
53	78	/// `ret_addr` is optionally provided as the first return address of the
54	79	/// allocation call stack. If the value is `0` it means no return address
55	80	/// has been provided.
56		free: const fn (ctx: anyopaque, buf: []u8, buf_align: u8, ret_addr: usize) void,
	81	free: const fn (anyopaque, memory: []u8, alignment: Alignment, ret_addr: usize) void,
57	82	};
58	83
59	84	pub fn noResize(
60	85	self: *anyopaque,
61		buf: []u8,
62		log2_buf_align: u8,
	86	memory: []u8,
	87	alignment: Alignment,
63	88	new_len: usize,
64	89	ret_addr: usize,
65	90	) bool {
66	91	_ = self;
67		_ = buf;
68		_ = log2_buf_align;
	92	_ = memory;
	93	_ = alignment;
69	94	_ = new_len;
70	95	_ = ret_addr;
71	96	return false;
72	97	}
73	98
	99	pub fn noRemap(
	100	self: *anyopaque,
	101	memory: []u8,
	102	alignment: Alignment,
	103	new_len: usize,
	104	ret_addr: usize,
	105	) ?[*]u8 {
	106	_ = self;
	107	_ = memory;
	108	_ = alignment;
	109	_ = new_len;
	110	_ = ret_addr;
	111	return null;
	112	}
	113
74	114	pub fn noFree(
75	115	self: *anyopaque,
76		buf: []u8,
77		log2_buf_align: u8,
	116	memory: []u8,
	117	alignment: Alignment,
78	118	ret_addr: usize,
79	119	) void {
80	120	_ = self;
81		_ = buf;
82		_ = log2_buf_align;
	121	_ = memory;
	122	_ = alignment;
83	123	_ = ret_addr;
84	124	}
85	125
86	126	/// This function is not intended to be called except from within the
87		/// implementation of an Allocator
88		pub inline fn rawAlloc(self: Allocator, len: usize, ptr_align: u8, ret_addr: usize) ?[*]u8 {
89		return self.vtable.alloc(self.ptr, len, ptr_align, ret_addr);
	127	/// implementation of an `Allocator`.
	128	pub inline fn rawAlloc(a: Allocator, len: usize, alignment: Alignment, ret_addr: usize) ?[*]u8 {
	129	return a.vtable.alloc(a.ptr, len, alignment, ret_addr);
90	130	}
91	131
92	132	/// This function is not intended to be called except from within the
93		/// implementation of an Allocator
94		pub inline fn rawResize(self: Allocator, buf: []u8, log2_buf_align: u8, new_len: usize, ret_addr: usize) bool {
95		return self.vtable.resize(self.ptr, buf, log2_buf_align, new_len, ret_addr);
	133	/// implementation of an `Allocator`.
	134	pub inline fn rawResize(a: Allocator, memory: []u8, alignment: Alignment, new_len: usize, ret_addr: usize) bool {
	135	return a.vtable.resize(a.ptr, memory, alignment, new_len, ret_addr);
96	136	}
97	137
98	138	/// This function is not intended to be called except from within the
99		/// implementation of an Allocator
100		pub inline fn rawFree(self: Allocator, buf: []u8, log2_buf_align: u8, ret_addr: usize) void {
101		return self.vtable.free(self.ptr, buf, log2_buf_align, ret_addr);
	139	/// implementation of an `Allocator`.
	140	pub inline fn rawRemap(a: Allocator, memory: []u8, alignment: Alignment, new_len: usize, ret_addr: usize) ?[*]u8 {
	141	return a.vtable.remap(a.ptr, memory, alignment, new_len, ret_addr);
	142	}
	143
	144	/// This function is not intended to be called except from within the
	145	/// implementation of an `Allocator`.
	146	pub inline fn rawFree(a: Allocator, memory: []u8, alignment: Alignment, ret_addr: usize) void {
	147	return a.vtable.free(a.ptr, memory, alignment, ret_addr);
102	148	}
103	149
104	150	/// Returns a pointer to undefined memory.
105	151	/// Call `destroy` with the result to free the memory.
106		pub fn create(self: Allocator, comptime T: type) Error!*T {
	152	pub fn create(a: Allocator, comptime T: type) Error!*T {
107	153	if (@sizeOf(T) == 0) return @as(*T, @ptrFromInt(math.maxInt(usize)));
108		const ptr: *T = @ptrCast(try self.allocBytesWithAlignment(@alignOf(T), @sizeOf(T), @returnAddress()));
	154	const ptr: *T = @ptrCast(try a.allocBytesWithAlignment(@alignOf(T), @sizeOf(T), @returnAddress()));
109	155	return ptr;
110	156	}
111	157
...	...	@@ -117,7 +163,7 @@ pub fn destroy(self: Allocator, ptr: anytype) void {
117	163	const T = info.child;
118	164	if (@sizeOf(T) == 0) return;
119	165	const non_const_ptr = @as([*]u8, @ptrCast(@constCast(ptr)));
120		self.rawFree(non_const_ptr[0..@sizeOf(T)], log2a(info.alignment), @returnAddress());
	166	self.rawFree(non_const_ptr[0..@sizeOf(T)], .fromByteUnits(info.alignment), @returnAddress());
121	167	}
122	168
123	169	/// Allocates an array of `n` items of type `T` and sets all the
...	...	@@ -215,46 +261,92 @@ fn allocWithSizeAndAlignment(self: Allocator, comptime size: usize, comptime ali
215	261	}
216	262
217	263	fn allocBytesWithAlignment(self: Allocator, comptime alignment: u29, byte_count: usize, return_address: usize) Error![*]align(alignment) u8 {
218		// The Zig Allocator interface is not intended to solve alignments beyond
219		// the minimum OS page size. For these use cases, the caller must use OS
220		// APIs directly.
221		comptime assert(alignment <= mem.page_size);
222
223	264	if (byte_count == 0) {
224	265	const ptr = comptime std.mem.alignBackward(usize, math.maxInt(usize), alignment);
225	266	return @as([*]align(alignment) u8, @ptrFromInt(ptr));
226	267	}
227	268
228		const byte_ptr = self.rawAlloc(byte_count, log2a(alignment), return_address) orelse return Error.OutOfMemory;
229		// TODO: https://github.com/ziglang/zig/issues/4298
	269	const byte_ptr = self.rawAlloc(byte_count, .fromByteUnits(alignment), return_address) orelse return Error.OutOfMemory;
230	270	@memset(byte_ptr[0..byte_count], undefined);
231		return @as([*]align(alignment) u8, @alignCast(byte_ptr));
	271	return @alignCast(byte_ptr);
232	272	}
233	273
234		/// Requests to modify the size of an allocation. It is guaranteed to not move
235		/// the pointer, however the allocator implementation may refuse the resize
236		/// request by returning `false`.
237		pub fn resize(self: Allocator, old_mem: anytype, new_n: usize) bool {
238		const Slice = @typeInfo(@TypeOf(old_mem)).pointer;
	274	/// Request to modify the size of an allocation.
	275	///
	276	/// It is guaranteed to not move the pointer, however the allocator
	277	/// implementation may refuse the resize request by returning `false`.
	278	///
	279	/// `allocation` may be an empty slice, in which case a new allocation is made.
	280	///
	281	/// `new_len` may be zero, in which case the allocation is freed.
	282	pub fn resize(self: Allocator, allocation: anytype, new_len: usize) bool {
	283	const Slice = @typeInfo(@TypeOf(allocation)).pointer;
239	284	const T = Slice.child;
240		if (new_n == 0) {
241		self.free(old_mem);
	285	const alignment = Slice.alignment;
	286	if (new_len == 0) {
	287	self.free(allocation);
242	288	return true;
243	289	}
244		if (old_mem.len == 0) {
	290	if (allocation.len == 0) {
245	291	return false;
246	292	}
247		const old_byte_slice = mem.sliceAsBytes(old_mem);
	293	const old_memory = mem.sliceAsBytes(allocation);
	294	// I would like to use saturating multiplication here, but LLVM cannot lower it
	295	// on WebAssembly: https://github.com/ziglang/zig/issues/9660
	296	//const new_len_bytes = new_len *\| @sizeOf(T);
	297	const new_len_bytes = math.mul(usize, @sizeOf(T), new_len) catch return false;
	298	return self.rawResize(old_memory, .fromByteUnits(alignment), new_len_bytes, @returnAddress());
	299	}
	300
	301	/// Request to modify the size of an allocation, allowing relocation.
	302	///
	303	/// A non-`null` return value indicates the resize was successful. The
	304	/// allocation may have same address, or may have been relocated. In either
	305	/// case, the allocation now has size of `new_len`. A `null` return value
	306	/// indicates that the resize would be equivalent to allocating new memory,
	307	/// copying the bytes from the old memory, and then freeing the old memory.
	308	/// In such case, it is more efficient for the caller to perform those
	309	/// operations.
	310	///
	311	/// `allocation` may be an empty slice, in which case a new allocation is made.
	312	///
	313	/// `new_len` may be zero, in which case the allocation is freed.
	314	pub fn remap(self: Allocator, allocation: anytype, new_len: usize) t: {
	315	const Slice = @typeInfo(@TypeOf(allocation)).pointer;
	316	break :t ?[]align(Slice.alignment) Slice.child;
	317	} {
	318	const Slice = @typeInfo(@TypeOf(allocation)).pointer;
	319	const T = Slice.child;
	320	const alignment = Slice.alignment;
	321	if (new_len == 0) {
	322	self.free(allocation);
	323	return allocation[0..0];
	324	}
	325	if (allocation.len == 0) {
	326	return null;
	327	}
	328	const old_memory = mem.sliceAsBytes(allocation);
248	329	// I would like to use saturating multiplication here, but LLVM cannot lower it
249	330	// on WebAssembly: https://github.com/ziglang/zig/issues/9660
250		//const new_byte_count = new_n *\| @sizeOf(T);
251		const new_byte_count = math.mul(usize, @sizeOf(T), new_n) catch return false;
252		return self.rawResize(old_byte_slice, log2a(Slice.alignment), new_byte_count, @returnAddress());
	331	//const new_len_bytes = new_len *\| @sizeOf(T);
	332	const new_len_bytes = math.mul(usize, @sizeOf(T), new_len) catch return null;
	333	const new_ptr = self.rawRemap(old_memory, .fromByteUnits(alignment), new_len_bytes, @returnAddress()) orelse return null;
	334	const new_memory: []align(alignment) u8 = @alignCast(new_ptr[0..new_len_bytes]);
	335	return mem.bytesAsSlice(T, new_memory);
253	336	}
254	337
255	338	/// This function requests a new byte size for an existing allocation, which
256	339	/// can be larger, smaller, or the same size as the old memory allocation.
	340	///
257	341	/// If `new_n` is 0, this is the same as `free` and it always succeeds.
	342	///
	343	/// `old_mem` may have length zero, which makes a new allocation.
	344	///
	345	/// This function only fails on out-of-memory conditions, unlike:
	346	/// * `remap` which returns `null` when the `Allocator` implementation cannot
	347	/// do the realloc more efficiently than the caller
	348	/// * `resize` which returns `false` when the `Allocator` implementation cannot
	349	/// change the size without relocating the allocation.
258	350	pub fn realloc(self: Allocator, old_mem: anytype, new_n: usize) t: {
259	351	const Slice = @typeInfo(@TypeOf(old_mem)).pointer;
260	352	break :t Error![]align(Slice.alignment) Slice.child;
...	...	@@ -285,18 +377,17 @@ pub fn reallocAdvanced(
285	377	const old_byte_slice = mem.sliceAsBytes(old_mem);
286	378	const byte_count = math.mul(usize, @sizeOf(T), new_n) catch return Error.OutOfMemory;
287	379	// Note: can't set shrunk memory to undefined as memory shouldn't be modified on realloc failure
288		if (self.rawResize(old_byte_slice, log2a(Slice.alignment), byte_count, return_address)) {
289		const new_bytes: []align(Slice.alignment) u8 = @alignCast(old_byte_slice.ptr[0..byte_count]);
	380	if (self.rawRemap(old_byte_slice, .fromByteUnits(Slice.alignment), byte_count, return_address)) \|p\| {
	381	const new_bytes: []align(Slice.alignment) u8 = @alignCast(p[0..byte_count]);
290	382	return mem.bytesAsSlice(T, new_bytes);
291	383	}
292	384
293		const new_mem = self.rawAlloc(byte_count, log2a(Slice.alignment), return_address) orelse
	385	const new_mem = self.rawAlloc(byte_count, .fromByteUnits(Slice.alignment), return_address) orelse
294	386	return error.OutOfMemory;
295	387	const copy_len = @min(byte_count, old_byte_slice.len);
296	388	@memcpy(new_mem[0..copy_len], old_byte_slice[0..copy_len]);
297		// TODO https://github.com/ziglang/zig/issues/4298
298	389	@memset(old_byte_slice, undefined);
299		self.rawFree(old_byte_slice, log2a(Slice.alignment), return_address);
	390	self.rawFree(old_byte_slice, .fromByteUnits(Slice.alignment), return_address);
300	391
301	392	const new_bytes: []align(Slice.alignment) u8 = @alignCast(new_mem[0..byte_count]);
302	393	return mem.bytesAsSlice(T, new_bytes);
...	...	@@ -311,9 +402,8 @@ pub fn free(self: Allocator, memory: anytype) void {
311	402	const bytes_len = bytes.len + if (Slice.sentinel() != null) @sizeOf(Slice.child) else 0;
312	403	if (bytes_len == 0) return;
313	404	const non_const_ptr = @constCast(bytes.ptr);
314		// TODO: https://github.com/ziglang/zig/issues/4298
315	405	@memset(non_const_ptr[0..bytes_len], undefined);
316		self.rawFree(non_const_ptr[0..bytes_len], log2a(Slice.alignment), @returnAddress());
	406	self.rawFree(non_const_ptr[0..bytes_len], .fromByteUnits(Slice.alignment), @returnAddress());
317	407	}
318	408
319	409	/// Copies `m` to newly allocated memory. Caller owns the memory.
...	...	@@ -330,17 +420,3 @@ pub fn dupeZ(allocator: Allocator, comptime T: type, m: []const T) Error![:0]T {
330	420	new_buf[m.len] = 0;
331	421	return new_buf[0..m.len :0];
332	422	}
333
334		/// TODO replace callsites with `@log2` after this proposal is implemented:
335		/// https://github.com/ziglang/zig/issues/13642
336		inline fn log2a(x: anytype) switch (@typeInfo(@TypeOf(x))) {
337		.int => math.Log2Int(@TypeOf(x)),
338		.comptime_int => comptime_int,
339		else => @compileError("int please"),
340		} {
341		switch (@typeInfo(@TypeOf(x))) {
342		.int => return math.log2_int(@TypeOf(x), x),
343		.comptime_int => return math.log2(x),
344		else => @compileError("bad"),
345		}
346		}

lib/std/os/linux.zig+18-4

...	...	@@ -305,6 +305,13 @@ pub const MAP = switch (native_arch) {
305	305	else => @compileError("missing std.os.linux.MAP constants for this architecture"),
306	306	};
307	307
	308	pub const MREMAP = packed struct(u32) {
	309	MAYMOVE: bool = false,
	310	FIXED: bool = false,
	311	DONTUNMAP: bool = false,
	312	_: u29 = 0,
	313	};
	314
308	315	pub const O = switch (native_arch) {
309	316	.x86_64 => packed struct(u32) {
310	317	ACCMODE: ACCMODE = .RDONLY,
...	...	@@ -892,10 +899,6 @@ pub fn umount2(special: [*:0]const u8, flags: u32) usize {
892	899
893	900	pub fn mmap(address: ?[*]u8, length: usize, prot: usize, flags: MAP, fd: i32, offset: i64) usize {
894	901	if (@hasField(SYS, "mmap2")) {
895		// Make sure the offset is also specified in multiples of page size
896		if ((offset & (MMAP2_UNIT - 1)) != 0)
897		return @bitCast(-@as(isize, @intFromEnum(E.INVAL)));
898
899	902	return syscall6(
900	903	.mmap2,
901	904	@intFromPtr(address),
...	...	@@ -934,6 +937,17 @@ pub fn mprotect(address: [*]const u8, length: usize, protection: usize) usize {
934	937	return syscall3(.mprotect, @intFromPtr(address), length, protection);
935	938	}
936	939
	940	pub fn mremap(old_addr: ?[]const u8, old_len: usize, new_len: usize, flags: MREMAP, new_addr: ?[]const u8) usize {
	941	return syscall5(
	942	.mremap,
	943	@intFromPtr(old_addr),
	944	old_len,
	945	new_len,
	946	@as(u32, @bitCast(flags)),
	947	@intFromPtr(new_addr),
	948	);
	949	}
	950
937	951	pub const MSF = struct {
938	952	pub const ASYNC = 1;
939	953	pub const INVALIDATE = 2;

lib/std/os/linux/IoUring.zig+8-7

...	...	@@ -8,6 +8,7 @@ const posix = std.posix;
8	8	const linux = std.os.linux;
9	9	const testing = std.testing;
10	10	const is_linux = builtin.os.tag == .linux;
	11	const page_size_min = std.heap.page_size_min;
11	12
12	13	fd: posix.fd_t = -1,
13	14	sq: SubmissionQueue,
...	...	@@ -1341,8 +1342,8 @@ pub const SubmissionQueue = struct {
1341	1342	dropped: *u32,
1342	1343	array: []u32,
1343	1344	sqes: []linux.io_uring_sqe,
1344		mmap: []align(mem.page_size) u8,
1345		mmap_sqes: []align(mem.page_size) u8,
	1345	mmap: []align(page_size_min) u8,
	1346	mmap_sqes: []align(page_size_min) u8,
1346	1347
1347	1348	// We use `sqe_head` and `sqe_tail` in the same way as liburing:
1348	1349	// We increment `sqe_tail` (but not `tail`) for each call to `get_sqe()`.
...	...	@@ -1460,7 +1461,7 @@ pub const BufferGroup = struct {
1460	1461	/// Pointer to the memory shared by the kernel.
1461	1462	/// `buffers_count` of `io_uring_buf` structures are shared by the kernel.
1462	1463	/// First `io_uring_buf` is overlaid by `io_uring_buf_ring` struct.
1463		br: *align(mem.page_size) linux.io_uring_buf_ring,
	1464	br: *align(page_size_min) linux.io_uring_buf_ring,
1464	1465	/// Contiguous block of memory of size (buffers_count * buffer_size).
1465	1466	buffers: []u8,
1466	1467	/// Size of each buffer in buffers.
...	...	@@ -1555,7 +1556,7 @@ pub const BufferGroup = struct {
1555	1556	/// `fd` is IO_Uring.fd for which the provided buffer ring is being registered.
1556	1557	/// `entries` is the number of entries requested in the buffer ring, must be power of 2.
1557	1558	/// `group_id` is the chosen buffer group ID, unique in IO_Uring.
1558		pub fn setup_buf_ring(fd: posix.fd_t, entries: u16, group_id: u16) !*align(mem.page_size) linux.io_uring_buf_ring {
	1559	pub fn setup_buf_ring(fd: posix.fd_t, entries: u16, group_id: u16) !*align(page_size_min) linux.io_uring_buf_ring {
1559	1560	if (entries == 0 or entries > 1 << 15) return error.EntriesNotInRange;
1560	1561	if (!std.math.isPowerOfTwo(entries)) return error.EntriesNotPowerOfTwo;
1561	1562
...	...	@@ -1571,7 +1572,7 @@ pub fn setup_buf_ring(fd: posix.fd_t, entries: u16, group_id: u16) !*align(mem.p
1571	1572	errdefer posix.munmap(mmap);
1572	1573	assert(mmap.len == mmap_size);
1573	1574
1574		const br: *align(mem.page_size) linux.io_uring_buf_ring = @ptrCast(mmap.ptr);
	1575	const br: *align(page_size_min) linux.io_uring_buf_ring = @ptrCast(mmap.ptr);
1575	1576	try register_buf_ring(fd, @intFromPtr(br), entries, group_id);
1576	1577	return br;
1577	1578	}
...	...	@@ -1613,9 +1614,9 @@ fn handle_register_buf_ring_result(res: usize) !void {
1613	1614	}
1614	1615
1615	1616	// Unregisters a previously registered shared buffer ring, returned from io_uring_setup_buf_ring.
1616		pub fn free_buf_ring(fd: posix.fd_t, br: *align(mem.page_size) linux.io_uring_buf_ring, entries: u32, group_id: u16) void {
	1617	pub fn free_buf_ring(fd: posix.fd_t, br: *align(page_size_min) linux.io_uring_buf_ring, entries: u32, group_id: u16) void {
1617	1618	unregister_buf_ring(fd, group_id) catch {};
1618		var mmap: []align(mem.page_size) u8 = undefined;
	1619	var mmap: []align(page_size_min) u8 = undefined;
1619	1620	mmap.ptr = @ptrCast(br);
1620	1621	mmap.len = entries * @sizeOf(linux.io_uring_buf);
1621	1622	posix.munmap(mmap);

lib/std/os/linux/tls.zig+10-9

...	...	@@ -17,6 +17,7 @@ const assert = std.debug.assert;
17	17	const native_arch = @import("builtin").cpu.arch;
18	18	const linux = std.os.linux;
19	19	const posix = std.posix;
	20	const page_size_min = std.heap.page_size_min;
20	21
21	22	/// Represents an ELF TLS variant.
22	23	///
...	...	@@ -484,13 +485,13 @@ pub fn prepareArea(area: []u8) usize {
484	485	};
485	486	}
486	487
487		// The main motivation for the size chosen here is that this is how much ends up being requested for
488		// the thread-local variables of the `std.crypto.random` implementation. I'm not sure why it ends up
489		// being so much; the struct itself is only 64 bytes. I think it has to do with being page-aligned
490		// and LLVM or LLD is not smart enough to lay out the TLS data in a space-conserving way. Anyway, I
491		// think it's fine because it's less than 3 pages of memory, and putting it in the ELF like this is
492		// equivalent to moving the `mmap` call below into the kernel, avoiding syscall overhead.
493		var main_thread_area_buffer: [0x2100]u8 align(mem.page_size) = undefined;
	488	/// The main motivation for the size chosen here is that this is how much ends up being requested for
	489	/// the thread-local variables of the `std.crypto.random` implementation. I'm not sure why it ends up
	490	/// being so much; the struct itself is only 64 bytes. I think it has to do with being page-aligned
	491	/// and LLVM or LLD is not smart enough to lay out the TLS data in a space-conserving way. Anyway, I
	492	/// think it's fine because it's less than 3 pages of memory, and putting it in the ELF like this is
	493	/// equivalent to moving the `mmap` call below into the kernel, avoiding syscall overhead.
	494	var main_thread_area_buffer: [0x2100]u8 align(page_size_min) = undefined;
494	495
495	496	/// Computes the layout of the static TLS area, allocates the area, initializes all of its fields,
496	497	/// and assigns the architecture-specific value to the TP register.
...	...	@@ -503,7 +504,7 @@ pub fn initStatic(phdrs: []elf.Phdr) void {
503	504	const area = blk: {
504	505	// Fast path for the common case where the TLS data is really small, avoid an allocation and
505	506	// use our local buffer.
506		if (area_desc.alignment <= mem.page_size and area_desc.size <= main_thread_area_buffer.len) {
	507	if (area_desc.alignment <= page_size_min and area_desc.size <= main_thread_area_buffer.len) {
507	508	break :blk main_thread_area_buffer[0..area_desc.size];
508	509	}
509	510
...	...	@@ -517,7 +518,7 @@ pub fn initStatic(phdrs: []elf.Phdr) void {
517	518	);
518	519	if (@as(isize, @bitCast(begin_addr)) < 0) @trap();
519	520
520		const area_ptr: [*]align(mem.page_size) u8 = @ptrFromInt(begin_addr);
	521	const area_ptr: [*]align(page_size_min) u8 = @ptrFromInt(begin_addr);
521	522
522	523	// Make sure the slice is correctly aligned.
523	524	const begin_aligned_addr = alignForward(begin_addr, area_desc.alignment);

lib/std/os/plan9.zig+2-2

...	...	@@ -367,8 +367,8 @@ pub fn sbrk(n: usize) usize {
367	367	bloc = @intFromPtr(&ExecData.end);
368	368	bloc_max = @intFromPtr(&ExecData.end);
369	369	}
370		const bl = std.mem.alignForward(usize, bloc, std.mem.page_size);
371		const n_aligned = std.mem.alignForward(usize, n, std.mem.page_size);
	370	const bl = std.mem.alignForward(usize, bloc, std.heap.pageSize());
	371	const n_aligned = std.mem.alignForward(usize, n, std.heap.pageSize());
372	372	if (bl + n_aligned > bloc_max) {
373	373	// we need to allocate
374	374	if (brk_(bl + n_aligned) < 0) return 0;

lib/std/os/windows.zig-12

...	...	@@ -2016,18 +2016,6 @@ pub fn InitOnceExecuteOnce(InitOnce: *INIT_ONCE, InitFn: INIT_ONCE_FN, Parameter
2016	2016	assert(kernel32.InitOnceExecuteOnce(InitOnce, InitFn, Parameter, Context) != 0);
2017	2017	}
2018	2018
2019		pub fn HeapFree(hHeap: HANDLE, dwFlags: DWORD, lpMem: *anyopaque) void {
2020		assert(kernel32.HeapFree(hHeap, dwFlags, lpMem) != 0);
2021		}
2022
2023		pub fn HeapDestroy(hHeap: HANDLE) void {
2024		assert(kernel32.HeapDestroy(hHeap) != 0);
2025		}
2026
2027		pub fn LocalFree(hMem: HLOCAL) void {
2028		assert(kernel32.LocalFree(hMem) == null);
2029		}
2030
2031	2019	pub const SetFileTimeError = error{Unexpected};
2032	2020
2033	2021	pub fn SetFileTime(

lib/std/os/windows/kernel32.zig+4-12

...	...	@@ -42,6 +42,7 @@ const WCHAR = windows.WCHAR;
42	42	const WIN32_FIND_DATAW = windows.WIN32_FIND_DATAW;
43	43	const Win32Error = windows.Win32Error;
44	44	const WORD = windows.WORD;
	45	const SYSTEM_INFO = windows.SYSTEM_INFO;
45	46
46	47	// I/O - Filesystem
47	48
...	...	@@ -527,11 +528,6 @@ pub extern "kernel32" fn HeapCreate(
527	528	dwMaximumSize: SIZE_T,
528	529	) callconv(.winapi) ?HANDLE;
529	530
530		// TODO: Wrapper around RtlDestroyHeap (BOOLEAN -> BOOL).
531		pub extern "kernel32" fn HeapDestroy(
532		hHeap: HANDLE,
533		) callconv(.winapi) BOOL;
534
535	531	// TODO: Forwarder to RtlReAllocateHeap.
536	532	pub extern "kernel32" fn HeapReAlloc(
537	533	hHeap: HANDLE,
...	...	@@ -584,10 +580,6 @@ pub extern "kernel32" fn VirtualQuery(
584	580	dwLength: SIZE_T,
585	581	) callconv(.winapi) SIZE_T;
586	582
587		pub extern "kernel32" fn LocalFree(
588		hMem: HLOCAL,
589		) callconv(.winapi) ?HLOCAL;
590
591	583	// TODO: Getter for peb.ProcessHeap
592	584	pub extern "kernel32" fn GetProcessHeap() callconv(.winapi) ?HANDLE;
593	585
...	...	@@ -667,6 +659,6 @@ pub extern "kernel32" fn SetLastError(
667	659	// TODO:
668	660	// Wrapper around KUSER_SHARED_DATA.SystemTime.
669	661	// Much better to use NtQuerySystemTime or NtQuerySystemTimePrecise for guaranteed 0.1ns precision.
670		pub extern "kernel32" fn GetSystemTimeAsFileTime(
671		lpSystemTimeAsFileTime: *FILETIME,
672		) callconv(.winapi) void;
	662	pub extern "kernel32" fn GetSystemTimeAsFileTime(lpSystemTimeAsFileTime: *FILETIME) callconv(.winapi) void;
	663
	664	pub extern "kernel32" fn GetSystemInfo(lpSystemInfo: *SYSTEM_INFO) callconv(.winapi) void;

lib/std/posix.zig+45-9

...	...	@@ -24,6 +24,7 @@ const maxInt = std.math.maxInt;
24	24	const cast = std.math.cast;
25	25	const assert = std.debug.assert;
26	26	const native_os = builtin.os.tag;
	27	const page_size_min = std.heap.page_size_min;
27	28
28	29	test {
29	30	_ = @import("posix/test.zig");
...	...	@@ -82,6 +83,7 @@ pub const MAP = system.MAP;
82	83	pub const MAX_ADDR_LEN = system.MAX_ADDR_LEN;
83	84	pub const MFD = system.MFD;
84	85	pub const MMAP2_UNIT = system.MMAP2_UNIT;
	86	pub const MREMAP = system.MREMAP;
85	87	pub const MSF = system.MSF;
86	88	pub const MSG = system.MSG;
87	89	pub const NAME_MAX = system.NAME_MAX;
...	...	@@ -4694,7 +4696,7 @@ pub const MProtectError = error{
4694	4696	OutOfMemory,
4695	4697	} \|\| UnexpectedError;
4696	4698
4697		pub fn mprotect(memory: []align(mem.page_size) u8, protection: u32) MProtectError!void {
	4699	pub fn mprotect(memory: []align(page_size_min) u8, protection: u32) MProtectError!void {
4698	4700	if (native_os == .windows) {
4699	4701	const win_prot: windows.DWORD = switch (@as(u3, @truncate(protection))) {
4700	4702	0b000 => windows.PAGE_NOACCESS,
...	...	@@ -4759,21 +4761,21 @@ pub const MMapError = error{
4759	4761	/// * SIGSEGV - Attempted write into a region mapped as read-only.
4760	4762	/// * SIGBUS - Attempted access to a portion of the buffer that does not correspond to the file
4761	4763	pub fn mmap(
4762		ptr: ?[*]align(mem.page_size) u8,
	4764	ptr: ?[*]align(page_size_min) u8,
4763	4765	length: usize,
4764	4766	prot: u32,
4765	4767	flags: system.MAP,
4766	4768	fd: fd_t,
4767	4769	offset: u64,
4768		) MMapError![]align(mem.page_size) u8 {
	4770	) MMapError![]align(page_size_min) u8 {
4769	4771	const mmap_sym = if (lfs64_abi) system.mmap64 else system.mmap;
4770	4772	const rc = mmap_sym(ptr, length, prot, @bitCast(flags), fd, @bitCast(offset));
4771	4773	const err: E = if (builtin.link_libc) blk: {
4772		if (rc != std.c.MAP_FAILED) return @as([*]align(mem.page_size) u8, @ptrCast(@alignCast(rc)))[0..length];
	4774	if (rc != std.c.MAP_FAILED) return @as([*]align(page_size_min) u8, @ptrCast(@alignCast(rc)))[0..length];
4773	4775	break :blk @enumFromInt(system._errno().*);
4774	4776	} else blk: {
4775	4777	const err = errno(rc);
4776		if (err == .SUCCESS) return @as([*]align(mem.page_size) u8, @ptrFromInt(rc))[0..length];
	4778	if (err == .SUCCESS) return @as([*]align(page_size_min) u8, @ptrFromInt(rc))[0..length];
4777	4779	break :blk err;
4778	4780	};
4779	4781	switch (err) {
...	...	@@ -4799,7 +4801,7 @@ pub fn mmap(
4799	4801	/// Zig's munmap function does not, for two reasons:
4800	4802	/// * It violates the Zig principle that resource deallocation must succeed.
4801	4803	/// * The Windows function, VirtualFree, has this restriction.
4802		pub fn munmap(memory: []align(mem.page_size) const u8) void {
	4804	pub fn munmap(memory: []align(page_size_min) const u8) void {
4803	4805	switch (errno(system.munmap(memory.ptr, memory.len))) {
4804	4806	.SUCCESS => return,
4805	4807	.INVAL => unreachable, // Invalid parameters.
...	...	@@ -4808,12 +4810,46 @@ pub fn munmap(memory: []align(mem.page_size) const u8) void {
4808	4810	}
4809	4811	}
4810	4812
	4813	pub const MRemapError = error{
	4814	LockedMemoryLimitExceeded,
	4815	/// Either a bug in the calling code, or the operating system abused the
	4816	/// EINVAL error code.
	4817	InvalidSyscallParameters,
	4818	OutOfMemory,
	4819	} \|\| UnexpectedError;
	4820
	4821	pub fn mremap(
	4822	old_address: ?[*]align(page_size_min) u8,
	4823	old_len: usize,
	4824	new_len: usize,
	4825	flags: system.MREMAP,
	4826	new_address: ?[*]align(page_size_min) u8,
	4827	) MRemapError![]align(page_size_min) u8 {
	4828	const rc = system.mremap(old_address, old_len, new_len, flags, new_address);
	4829	const err: E = if (builtin.link_libc) blk: {
	4830	if (rc != std.c.MAP_FAILED) return @as([*]align(page_size_min) u8, @ptrCast(@alignCast(rc)))[0..new_len];
	4831	break :blk @enumFromInt(system._errno().*);
	4832	} else blk: {
	4833	const err = errno(rc);
	4834	if (err == .SUCCESS) return @as([*]align(page_size_min) u8, @ptrFromInt(rc))[0..new_len];
	4835	break :blk err;
	4836	};
	4837	switch (err) {
	4838	.SUCCESS => unreachable,
	4839	.AGAIN => return error.LockedMemoryLimitExceeded,
	4840	.INVAL => return error.InvalidSyscallParameters,
	4841	.NOMEM => return error.OutOfMemory,
	4842	.FAULT => unreachable,
	4843	else => return unexpectedErrno(err),
	4844	}
	4845	}
	4846
4811	4847	pub const MSyncError = error{
4812	4848	UnmappedMemory,
4813	4849	PermissionDenied,
4814	4850	} \|\| UnexpectedError;
4815	4851
4816		pub fn msync(memory: []align(mem.page_size) u8, flags: i32) MSyncError!void {
	4852	pub fn msync(memory: []align(page_size_min) u8, flags: i32) MSyncError!void {
4817	4853	switch (errno(system.msync(memory.ptr, memory.len, flags))) {
4818	4854	.SUCCESS => return,
4819	4855	.PERM => return error.PermissionDenied,
...	...	@@ -7135,7 +7171,7 @@ pub const MincoreError = error{
7135	7171	} \|\| UnexpectedError;
7136	7172
7137	7173	/// Determine whether pages are resident in memory.
7138		pub fn mincore(ptr: []align(mem.page_size) u8, length: usize, vec: []u8) MincoreError!void {
	7174	pub fn mincore(ptr: []align(page_size_min) u8, length: usize, vec: []u8) MincoreError!void {
7139	7175	return switch (errno(system.mincore(ptr, length, vec))) {
7140	7176	.SUCCESS => {},
7141	7177	.AGAIN => error.SystemResources,
...	...	@@ -7181,7 +7217,7 @@ pub const MadviseError = error{
7181	7217
7182	7218	/// Give advice about use of memory.
7183	7219	/// This syscall is optional and is sometimes configured to be disabled.
7184		pub fn madvise(ptr: [*]align(mem.page_size) u8, length: usize, advice: u32) MadviseError!void {
	7220	pub fn madvise(ptr: [*]align(page_size_min) u8, length: usize, advice: u32) MadviseError!void {
7185	7221	switch (errno(system.madvise(ptr, length, advice))) {
7186	7222	.SUCCESS => return,
7187	7223	.PERM => return error.PermissionDenied,

lib/std/process.zig+1-1

...	...	@@ -1560,7 +1560,7 @@ pub fn posixGetUserInfo(name: []const u8) !UserInfo {
1560	1560	ReadGroupId,
1561	1561	};
1562	1562
1563		var buf: [std.mem.page_size]u8 = undefined;
	1563	var buf: [std.heap.page_size_min]u8 = undefined;
1564	1564	var name_index: usize = 0;
1565	1565	var state = State.Start;
1566	1566	var uid: posix.uid_t = 0;

lib/std/start.zig+1-1

...	...	@@ -576,7 +576,7 @@ fn expandStackSize(phdrs: []elf.Phdr) void {
576	576	switch (phdr.p_type) {
577	577	elf.PT_GNU_STACK => {
578	578	if (phdr.p_memsz == 0) break;
579		assert(phdr.p_memsz % std.mem.page_size == 0);
	579	assert(phdr.p_memsz % std.heap.page_size_min == 0);
580	580
581	581	// Silently fail if we are unable to get limits.
582	582	const limits = std.posix.getrlimit(.STACK) catch break;

lib/std/std.zig+7

...	...	@@ -119,6 +119,13 @@ pub const Options = struct {
119	119	args: anytype,
120	120	) void = log.defaultLog,
121	121
	122	/// Overrides `std.heap.page_size_min`.
	123	page_size_min: ?usize = null,
	124	/// Overrides `std.heap.page_size_max`.
	125	page_size_max: ?usize = null,
	126	/// Overrides default implementation for determining OS page size at runtime.
	127	queryPageSize: fn () usize = heap.defaultQueryPageSize,
	128
122	129	fmt_max_depth: usize = fmt.default_max_depth,
123	130
124	131	cryptoRandomSeed: fn (buffer: []u8) void = @import("crypto/tlcsprng.zig").defaultRandomSeed,

lib/std/testing.zig+15-9

...	...	@@ -7,21 +7,27 @@ const math = std.math;
7	7	/// Initialized on startup. Read-only after that.
8	8	pub var random_seed: u32 = 0;
9	9
10		pub const FailingAllocator = @import("testing/failing_allocator.zig").FailingAllocator;
	10	pub const FailingAllocator = @import("testing/FailingAllocator.zig");
	11	pub const failing_allocator = failing_allocator_instance.allocator();
	12	var failing_allocator_instance = FailingAllocator.init(base_allocator_instance.allocator(), .{
	13	.fail_index = 0,
	14	});
	15	var base_allocator_instance = std.heap.FixedBufferAllocator.init("");
11	16
12	17	/// This should only be used in temporary test programs.
13	18	pub const allocator = allocator_instance.allocator();
14		pub var allocator_instance: std.heap.GeneralPurposeAllocator(.{}) = b: {
15		if (!builtin.is_test)
16		@compileError("Cannot use testing allocator outside of test block");
	19	pub var allocator_instance: std.heap.GeneralPurposeAllocator(.{
	20	.stack_trace_frames = if (std.debug.sys_can_stack_trace) 10 else 0,
	21	.resize_stack_traces = true,
	22	// A unique value so that when a default-constructed
	23	// GeneralPurposeAllocator is incorrectly passed to testing allocator, or
	24	// vice versa, panic occurs.
	25	.canary = @truncate(0x2731e675c3a701ba),
	26	}) = b: {
	27	if (!builtin.is_test) @compileError("testing allocator used when not testing");
17	28	break :b .init;
18	29	};
19	30
20		pub const failing_allocator = failing_allocator_instance.allocator();
21		pub var failing_allocator_instance = FailingAllocator.init(base_allocator_instance.allocator(), .{ .fail_index = 0 });
22
23		pub var base_allocator_instance = std.heap.FixedBufferAllocator.init("");
24
25	31	/// TODO https://github.com/ziglang/zig/issues/5738
26	32	pub var log_level = std.log.Level.warn;
27	33

lib/std/testing/FailingAllocator.zig created+161

...	...	@@ -0,0 +1,161 @@
	1	//! Allocator that fails after N allocations, useful for making sure out of
	2	//! memory conditions are handled correctly.
	3	//!
	4	//! To use this, first initialize it and get an allocator with
	5	//!
	6	//! `const failing_allocator = &FailingAllocator.init(<allocator>,
	7	//! <config>).allocator;`
	8	//!
	9	//! Then use `failing_allocator` anywhere you would have used a
	10	//! different allocator.
	11	const std = @import("../std.zig");
	12	const mem = std.mem;
	13	const FailingAllocator = @This();
	14
	15	alloc_index: usize,
	16	resize_index: usize,
	17	internal_allocator: mem.Allocator,
	18	allocated_bytes: usize,
	19	freed_bytes: usize,
	20	allocations: usize,
	21	deallocations: usize,
	22	stack_addresses: [num_stack_frames]usize,
	23	has_induced_failure: bool,
	24	fail_index: usize,
	25	resize_fail_index: usize,
	26
	27	const num_stack_frames = if (std.debug.sys_can_stack_trace) 16 else 0;
	28
	29	pub const Config = struct {
	30	/// The number of successful allocations you can expect from this allocator.
	31	/// The next allocation will fail. For example, with `fail_index` equal to
	32	/// 2, the following test will pass:
	33	///
	34	/// var a = try failing_alloc.create(i32);
	35	/// var b = try failing_alloc.create(i32);
	36	/// testing.expectError(error.OutOfMemory, failing_alloc.create(i32));
	37	fail_index: usize = std.math.maxInt(usize),
	38
	39	/// Number of successful resizes to expect from this allocator. The next resize will fail.
	40	resize_fail_index: usize = std.math.maxInt(usize),
	41	};
	42
	43	pub fn init(internal_allocator: mem.Allocator, config: Config) FailingAllocator {
	44	return FailingAllocator{
	45	.internal_allocator = internal_allocator,
	46	.alloc_index = 0,
	47	.resize_index = 0,
	48	.allocated_bytes = 0,
	49	.freed_bytes = 0,
	50	.allocations = 0,
	51	.deallocations = 0,
	52	.stack_addresses = undefined,
	53	.has_induced_failure = false,
	54	.fail_index = config.fail_index,
	55	.resize_fail_index = config.resize_fail_index,
	56	};
	57	}
	58
	59	pub fn allocator(self: *FailingAllocator) mem.Allocator {
	60	return .{
	61	.ptr = self,
	62	.vtable = &.{
	63	.alloc = alloc,
	64	.resize = resize,
	65	.remap = remap,
	66	.free = free,
	67	},
	68	};
	69	}
	70
	71	fn alloc(
	72	ctx: *anyopaque,
	73	len: usize,
	74	alignment: mem.Alignment,
	75	return_address: usize,
	76	) ?[*]u8 {
	77	const self: *FailingAllocator = @ptrCast(@alignCast(ctx));
	78	if (self.alloc_index == self.fail_index) {
	79	if (!self.has_induced_failure) {
	80	@memset(&self.stack_addresses, 0);
	81	var stack_trace = std.builtin.StackTrace{
	82	.instruction_addresses = &self.stack_addresses,
	83	.index = 0,
	84	};
	85	std.debug.captureStackTrace(return_address, &stack_trace);
	86	self.has_induced_failure = true;
	87	}
	88	return null;
	89	}
	90	const result = self.internal_allocator.rawAlloc(len, alignment, return_address) orelse
	91	return null;
	92	self.allocated_bytes += len;
	93	self.allocations += 1;
	94	self.alloc_index += 1;
	95	return result;
	96	}
	97
	98	fn resize(
	99	ctx: *anyopaque,
	100	memory: []u8,
	101	alignment: mem.Alignment,
	102	new_len: usize,
	103	ra: usize,
	104	) bool {
	105	const self: *FailingAllocator = @ptrCast(@alignCast(ctx));
	106	if (self.resize_index == self.resize_fail_index)
	107	return false;
	108	if (!self.internal_allocator.rawResize(memory, alignment, new_len, ra))
	109	return false;
	110	if (new_len < memory.len) {
	111	self.freed_bytes += memory.len - new_len;
	112	} else {
	113	self.allocated_bytes += new_len - memory.len;
	114	}
	115	self.resize_index += 1;
	116	return true;
	117	}
	118
	119	fn remap(
	120	ctx: *anyopaque,
	121	memory: []u8,
	122	alignment: mem.Alignment,
	123	new_len: usize,
	124	ra: usize,
	125	) ?[*]u8 {
	126	const self: *FailingAllocator = @ptrCast(@alignCast(ctx));
	127	if (self.resize_index == self.resize_fail_index) return null;
	128	const new_ptr = self.internal_allocator.rawRemap(memory, alignment, new_len, ra) orelse return null;
	129	if (new_len < memory.len) {
	130	self.freed_bytes += memory.len - new_len;
	131	} else {
	132	self.allocated_bytes += new_len - memory.len;
	133	}
	134	self.resize_index += 1;
	135	return new_ptr;
	136	}
	137
	138	fn free(
	139	ctx: *anyopaque,
	140	old_mem: []u8,
	141	alignment: mem.Alignment,
	142	ra: usize,
	143	) void {
	144	const self: *FailingAllocator = @ptrCast(@alignCast(ctx));
	145	self.internal_allocator.rawFree(old_mem, alignment, ra);
	146	self.deallocations += 1;
	147	self.freed_bytes += old_mem.len;
	148	}
	149
	150	/// Only valid once `has_induced_failure == true`
	151	pub fn getStackTrace(self: *FailingAllocator) std.builtin.StackTrace {
	152	std.debug.assert(self.has_induced_failure);
	153	var len: usize = 0;
	154	while (len < self.stack_addresses.len and self.stack_addresses[len] != 0) {
	155	len += 1;
	156	}
	157	return .{
	158	.instruction_addresses = &self.stack_addresses,
	159	.index = len,
	160	};
	161	}

lib/std/testing/failing_allocator.zig deleted-142

...	...	@@ -1,142 +0,0 @@
1		const std = @import("../std.zig");
2		const mem = std.mem;
3
4		pub const Config = struct {
5		/// The number of successful allocations you can expect from this allocator.
6		/// The next allocation will fail. For example, with `fail_index` equal to
7		/// 2, the following test will pass:
8		///
9		/// var a = try failing_alloc.create(i32);
10		/// var b = try failing_alloc.create(i32);
11		/// testing.expectError(error.OutOfMemory, failing_alloc.create(i32));
12		fail_index: usize = std.math.maxInt(usize),
13
14		/// Number of successful resizes to expect from this allocator. The next resize will fail.
15		resize_fail_index: usize = std.math.maxInt(usize),
16		};
17
18		/// Allocator that fails after N allocations, useful for making sure out of
19		/// memory conditions are handled correctly.
20		///
21		/// To use this, first initialize it and get an allocator with
22		///
23		/// `const failing_allocator = &FailingAllocator.init(<allocator>,
24		/// <config>).allocator;`
25		///
26		/// Then use `failing_allocator` anywhere you would have used a
27		/// different allocator.
28		pub const FailingAllocator = struct {
29		alloc_index: usize,
30		resize_index: usize,
31		internal_allocator: mem.Allocator,
32		allocated_bytes: usize,
33		freed_bytes: usize,
34		allocations: usize,
35		deallocations: usize,
36		stack_addresses: [num_stack_frames]usize,
37		has_induced_failure: bool,
38		fail_index: usize,
39		resize_fail_index: usize,
40
41		const num_stack_frames = if (std.debug.sys_can_stack_trace) 16 else 0;
42
43		pub fn init(internal_allocator: mem.Allocator, config: Config) FailingAllocator {
44		return FailingAllocator{
45		.internal_allocator = internal_allocator,
46		.alloc_index = 0,
47		.resize_index = 0,
48		.allocated_bytes = 0,
49		.freed_bytes = 0,
50		.allocations = 0,
51		.deallocations = 0,
52		.stack_addresses = undefined,
53		.has_induced_failure = false,
54		.fail_index = config.fail_index,
55		.resize_fail_index = config.resize_fail_index,
56		};
57		}
58
59		pub fn allocator(self: *FailingAllocator) mem.Allocator {
60		return .{
61		.ptr = self,
62		.vtable = &.{
63		.alloc = alloc,
64		.resize = resize,
65		.free = free,
66		},
67		};
68		}
69
70		fn alloc(
71		ctx: *anyopaque,
72		len: usize,
73		log2_ptr_align: u8,
74		return_address: usize,
75		) ?[*]u8 {
76		const self: *FailingAllocator = @ptrCast(@alignCast(ctx));
77		if (self.alloc_index == self.fail_index) {
78		if (!self.has_induced_failure) {
79		@memset(&self.stack_addresses, 0);
80		var stack_trace = std.builtin.StackTrace{
81		.instruction_addresses = &self.stack_addresses,
82		.index = 0,
83		};
84		std.debug.captureStackTrace(return_address, &stack_trace);
85		self.has_induced_failure = true;
86		}
87		return null;
88		}
89		const result = self.internal_allocator.rawAlloc(len, log2_ptr_align, return_address) orelse
90		return null;
91		self.allocated_bytes += len;
92		self.allocations += 1;
93		self.alloc_index += 1;
94		return result;
95		}
96
97		fn resize(
98		ctx: *anyopaque,
99		old_mem: []u8,
100		log2_old_align: u8,
101		new_len: usize,
102		ra: usize,
103		) bool {
104		const self: *FailingAllocator = @ptrCast(@alignCast(ctx));
105		if (self.resize_index == self.resize_fail_index)
106		return false;
107		if (!self.internal_allocator.rawResize(old_mem, log2_old_align, new_len, ra))
108		return false;
109		if (new_len < old_mem.len) {
110		self.freed_bytes += old_mem.len - new_len;
111		} else {
112		self.allocated_bytes += new_len - old_mem.len;
113		}
114		self.resize_index += 1;
115		return true;
116		}
117
118		fn free(
119		ctx: *anyopaque,
120		old_mem: []u8,
121		log2_old_align: u8,
122		ra: usize,
123		) void {
124		const self: *FailingAllocator = @ptrCast(@alignCast(ctx));
125		self.internal_allocator.rawFree(old_mem, log2_old_align, ra);
126		self.deallocations += 1;
127		self.freed_bytes += old_mem.len;
128		}
129
130		/// Only valid once `has_induced_failure == true`
131		pub fn getStackTrace(self: *FailingAllocator) std.builtin.StackTrace {
132		std.debug.assert(self.has_induced_failure);
133		var len: usize = 0;
134		while (len < self.stack_addresses.len and self.stack_addresses[len] != 0) {
135		len += 1;
136		}
137		return .{
138		.instruction_addresses = &self.stack_addresses,
139		.index = len,
140		};
141		}
142		};

lib/std/zip.zig+1-1

...	...	@@ -162,7 +162,7 @@ pub fn decompress(
162	162	var total_uncompressed: u64 = 0;
163	163	switch (method) {
164	164	.store => {
165		var buf: [std.mem.page_size]u8 = undefined;
	165	var buf: [4096]u8 = undefined;
166	166	while (true) {
167	167	const len = try reader.read(&buf);
168	168	if (len == 0) break;

src/Package/Fetch.zig+1-1

...	...	@@ -1249,7 +1249,7 @@ fn unzip(f: *Fetch, out_dir: fs.Dir, reader: anytype) RunError!UnpackResult {
1249	1249	.{@errorName(err)},
1250	1250	));
1251	1251	defer zip_file.close();
1252		var buf: [std.mem.page_size]u8 = undefined;
	1252	var buf: [4096]u8 = undefined;
1253	1253	while (true) {
1254	1254	const len = reader.readAll(&buf) catch \|err\| return f.fail(f.location_tok, try eb.printString(
1255	1255	"read zip stream failed: {s}",

test/compare_output.zig-43

...	...	@@ -493,49 +493,6 @@ pub fn addCases(cases: *tests.CompareOutputContext) void {
493	493	\\
494	494	);
495	495
496		// It is required to override the log function in order to print to stdout instead of stderr
497		cases.add("std.heap.LoggingAllocator logs to std.log",
498		\\const std = @import("std");
499		\\
500		\\pub const std_options: std.Options = .{
501		\\ .log_level = .debug,
502		\\ .logFn = log,
503		\\};
504		\\
505		\\pub fn main() !void {
506		\\ var allocator_buf: [10]u8 = undefined;
507		\\ const fba = std.heap.FixedBufferAllocator.init(&allocator_buf);
508		\\ var fba_wrapped = std.mem.validationWrap(fba);
509		\\ var logging_allocator = std.heap.loggingAllocator(fba_wrapped.allocator());
510		\\ const allocator = logging_allocator.allocator();
511		\\
512		\\ var a = try allocator.alloc(u8, 10);
513		\\ try std.testing.expect(allocator.resize(a, 5));
514		\\ a = a[0..5];
515		\\ try std.testing.expect(a.len == 5);
516		\\ try std.testing.expect(!allocator.resize(a, 20));
517		\\ allocator.free(a);
518		\\}
519		\\
520		\\pub fn log(
521		\\ comptime level: std.log.Level,
522		\\ comptime scope: @TypeOf(.EnumLiteral),
523		\\ comptime format: []const u8,
524		\\ args: anytype,
525		\\) void {
526		\\ const level_txt = comptime level.asText();
527		\\ const prefix2 = if (scope == .default) ": " else "(" ++ @tagName(scope) ++ "): ";
528		\\ const stdout = std.io.getStdOut().writer();
529		\\ nosuspend stdout.print(level_txt ++ prefix2 ++ format ++ "\n", args) catch return;
530		\\}
531		,
532		\\debug: alloc - success - len: 10, ptr_align: 0
533		\\debug: shrink - success - 10 to 5, buf_align: 0
534		\\error: expand - failure - 5 to 20, buf_align: 0
535		\\debug: free - len: 5
536		\\
537		);
538
539	496	cases.add("valid carriage return example", "const io = @import(\"std\").io;\r\n" ++ // Testing CRLF line endings are valid
540	497	"\r\n" ++
541	498	"pub \r fn main() void {\r\n" ++ // Testing isolated carriage return as whitespace is valid