Commit 7eeef5fb2b9dc78679f4091e2a8173d07968b3e5 | zig

author	Andrew Kelley <andrew@ziglang.org> 2025-02-03 19:55:09-08:00
committer	Andrew Kelley <andrew@ziglang.org> 2025-02-06 14:23:23-08:00
log	7eeef5fb2b9dc78679f4091e2a8173d07968b3e5
tree	a5edee99fe9ea1e5c241ec0e7b21268d65a8ff8b
parent	dd2fa4f75d3d2b1214fde22081f0b88850d1b55d

std.mem.Allocator: introduce `remap` function to the interface

This one changes the size of an allocation, allowing it to be relocated.
However, the implementation will still return `null` if it would be
equivalent to

new = alloc
memcpy(new, old)
free(old)

Mainly this prepares for taking advantage of `mremap` which I thought
would be a bigger deal but apparently is only available on Linux. Still,
we should use it on Linux.

7 files changed, 389 insertions(+), 217 deletions(-)

lib
- std

lib/std/array_list.zig+18-19

...	...	@@ -105,21 +105,19 @@ pub fn ArrayListAligned(comptime T: type, comptime alignment: ?u29) type {
105	105	return result;
106	106	}
107	107
108		/// The caller owns the returned memory. Empties this ArrayList,
109		/// Its capacity is cleared, making deinit() safe but unnecessary to call.
	108	/// The caller owns the returned memory. Empties this ArrayList.
	109	/// Its capacity is cleared, making `deinit` safe but unnecessary to call.
110	110	pub fn toOwnedSlice(self: *Self) Allocator.Error!Slice {
111	111	const allocator = self.allocator;
112	112
113	113	const old_memory = self.allocatedSlice();
114		if (allocator.resize(old_memory, self.items.len)) {
115		const result = self.items;
	114	if (allocator.remap(old_memory, self.items.len)) \|new_items\| {
116	115	self.* = init(allocator);
117		return result;
	116	return new_items;
118	117	}
119	118
120	119	const new_memory = try allocator.alignedAlloc(T, alignment, self.items.len);
121	120	@memcpy(new_memory, self.items);
122		@memset(self.items, undefined);
123	121	self.clearAndFree();
124	122	return new_memory;
125	123	}
...	...	@@ -185,8 +183,9 @@ pub fn ArrayListAligned(comptime T: type, comptime alignment: ?u29) type {
185	183	// extra capacity.
186	184	const new_capacity = growCapacity(self.capacity, new_len);
187	185	const old_memory = self.allocatedSlice();
188		if (self.allocator.resize(old_memory, new_capacity)) {
189		self.capacity = new_capacity;
	186	if (self.allocator.remap(old_memory, new_capacity)) \|new_memory\| {
	187	self.items.ptr = new_memory.ptr;
	188	self.capacity = new_memory.len;
190	189	return addManyAtAssumeCapacity(self, index, count);
191	190	}
192	191
...	...	@@ -468,8 +467,9 @@ pub fn ArrayListAligned(comptime T: type, comptime alignment: ?u29) type {
468	467	// the allocator implementation would pointlessly copy our
469	468	// extra capacity.
470	469	const old_memory = self.allocatedSlice();
471		if (self.allocator.resize(old_memory, new_capacity)) {
472		self.capacity = new_capacity;
	470	if (self.allocator.remap(old_memory, new_capacity)) \|new_memory\| {
	471	self.items.ptr = new_memory.ptr;
	472	self.capacity = new_memory.len;
473	473	} else {
474	474	const new_memory = try self.allocator.alignedAlloc(T, alignment, new_capacity);
475	475	@memcpy(new_memory[0..self.items.len], self.items);
...	...	@@ -707,15 +707,13 @@ pub fn ArrayListAlignedUnmanaged(comptime T: type, comptime alignment: ?u29) typ
707	707	/// Its capacity is cleared, making deinit() safe but unnecessary to call.
708	708	pub fn toOwnedSlice(self: *Self, allocator: Allocator) Allocator.Error!Slice {
709	709	const old_memory = self.allocatedSlice();
710		if (allocator.resize(old_memory, self.items.len)) {
711		const result = self.items;
	710	if (allocator.remap(old_memory, self.items.len)) \|new_items\| {
712	711	self.* = .empty;
713		return result;
	712	return new_items;
714	713	}
715	714
716	715	const new_memory = try allocator.alignedAlloc(T, alignment, self.items.len);
717	716	@memcpy(new_memory, self.items);
718		@memset(self.items, undefined);
719	717	self.clearAndFree(allocator);
720	718	return new_memory;
721	719	}
...	...	@@ -1031,9 +1029,9 @@ pub fn ArrayListAlignedUnmanaged(comptime T: type, comptime alignment: ?u29) typ
1031	1029	}
1032	1030
1033	1031	const old_memory = self.allocatedSlice();
1034		if (allocator.resize(old_memory, new_len)) {
1035		self.capacity = new_len;
1036		self.items.len = new_len;
	1032	if (allocator.remap(old_memory, new_len)) \|new_items\| {
	1033	self.capacity = new_items.len;
	1034	self.items = new_items;
1037	1035	return;
1038	1036	}
1039	1037
...	...	@@ -1099,8 +1097,9 @@ pub fn ArrayListAlignedUnmanaged(comptime T: type, comptime alignment: ?u29) typ
1099	1097	// the allocator implementation would pointlessly copy our
1100	1098	// extra capacity.
1101	1099	const old_memory = self.allocatedSlice();
1102		if (allocator.resize(old_memory, new_capacity)) {
1103		self.capacity = new_capacity;
	1100	if (allocator.remap(old_memory, new_capacity)) \|new_memory\| {
	1101	self.items.ptr = new_memory.ptr;
	1102	self.capacity = new_memory.len;
1104	1103	} else {
1105	1104	const new_memory = try allocator.alignedAlloc(T, alignment, new_capacity);
1106	1105	@memcpy(new_memory[0..self.items.len], self.items);

lib/std/heap/FixedBufferAllocator.zig+21-9

...	...	@@ -9,7 +9,7 @@ end_index: usize,
9	9	buffer: []u8,
10	10
11	11	pub fn init(buffer: []u8) FixedBufferAllocator {
12		return FixedBufferAllocator{
	12	return .{
13	13	.buffer = buffer,
14	14	.end_index = 0,
15	15	};
...	...	@@ -22,6 +22,7 @@ pub fn allocator(self: *FixedBufferAllocator) Allocator {
22	22	.vtable = &.{
23	23	.alloc = alloc,
24	24	.resize = resize,
	25	.remap = remap,
25	26	.free = free,
26	27	},
27	28	};
...	...	@@ -36,6 +37,7 @@ pub fn threadSafeAllocator(self: *FixedBufferAllocator) Allocator {
36	37	.vtable = &.{
37	38	.alloc = threadSafeAlloc,
38	39	.resize = Allocator.noResize,
	40	.remap = Allocator.noRemap,
39	41	.free = Allocator.noFree,
40	42	},
41	43	};
...	...	@@ -57,10 +59,10 @@ pub fn isLastAllocation(self: *FixedBufferAllocator, buf: []u8) bool {
57	59	return buf.ptr + buf.len == self.buffer.ptr + self.end_index;
58	60	}
59	61
60		pub fn alloc(ctx: anyopaque, n: usize, log2_ptr_align: u8, ra: usize) ?[]u8 {
	62	pub fn alloc(ctx: anyopaque, n: usize, alignment: mem.Alignment, ra: usize) ?[]u8 {
61	63	const self: *FixedBufferAllocator = @ptrCast(@alignCast(ctx));
62	64	_ = ra;
63		const ptr_align = @as(usize, 1) << @as(Allocator.Log2Align, @intCast(log2_ptr_align));
	65	const ptr_align = alignment.toByteUnits();
64	66	const adjust_off = mem.alignPointerOffset(self.buffer.ptr + self.end_index, ptr_align) orelse return null;
65	67	const adjusted_index = self.end_index + adjust_off;
66	68	const new_end_index = adjusted_index + n;
...	...	@@ -72,12 +74,12 @@ pub fn alloc(ctx: anyopaque, n: usize, log2_ptr_align: u8, ra: usize) ?[]u8 {
72	74	pub fn resize(
73	75	ctx: *anyopaque,
74	76	buf: []u8,
75		log2_buf_align: u8,
	77	alignment: mem.Alignment,
76	78	new_size: usize,
77	79	return_address: usize,
78	80	) bool {
79	81	const self: *FixedBufferAllocator = @ptrCast(@alignCast(ctx));
80		_ = log2_buf_align;
	82	_ = alignment;
81	83	_ = return_address;
82	84	assert(@inComptime() or self.ownsSlice(buf));
83	85
...	...	@@ -99,14 +101,24 @@ pub fn resize(
99	101	return true;
100	102	}
101	103
	104	pub fn remap(
	105	context: *anyopaque,
	106	memory: []u8,
	107	alignment: mem.Alignment,
	108	new_len: usize,
	109	return_address: usize,
	110	) ?[*]u8 {
	111	return if (resize(context, memory, alignment, new_len, return_address)) memory.ptr else null;
	112	}
	113
102	114	pub fn free(
103	115	ctx: *anyopaque,
104	116	buf: []u8,
105		log2_buf_align: u8,
	117	alignment: mem.Alignment,
106	118	return_address: usize,
107	119	) void {
108	120	const self: *FixedBufferAllocator = @ptrCast(@alignCast(ctx));
109		_ = log2_buf_align;
	121	_ = alignment;
110	122	_ = return_address;
111	123	assert(@inComptime() or self.ownsSlice(buf));
112	124
...	...	@@ -115,10 +127,10 @@ pub fn free(
115	127	}
116	128	}
117	129
118		fn threadSafeAlloc(ctx: anyopaque, n: usize, log2_ptr_align: u8, ra: usize) ?[]u8 {
	130	fn threadSafeAlloc(ctx: anyopaque, n: usize, alignment: mem.Alignment, ra: usize) ?[]u8 {
119	131	const self: *FixedBufferAllocator = @ptrCast(@alignCast(ctx));
120	132	_ = ra;
121		const ptr_align = @as(usize, 1) << @as(Allocator.Log2Align, @intCast(log2_ptr_align));
	133	const ptr_align = alignment.toByteUnits();
122	134	var end_index = @atomicLoad(usize, &self.end_index, .seq_cst);
123	135	while (true) {
124	136	const adjust_off = mem.alignPointerOffset(self.buffer.ptr + end_index, ptr_align) orelse return null;

lib/std/heap/PageAllocator.zig+60-39

...	...	@@ -12,18 +12,18 @@ const page_size_min = std.heap.page_size_min;
12	12	pub const vtable: Allocator.VTable = .{
13	13	.alloc = alloc,
14	14	.resize = resize,
	15	.remap = remap,
15	16	.free = free,
16	17	};
17	18
18		fn alloc(context: anyopaque, n: usize, log2_align: u8, ra: usize) ?[]u8 {
19		const requested_alignment: mem.Alignment = @enumFromInt(log2_align);
	19	fn alloc(context: anyopaque, n: usize, alignment: mem.Alignment, ra: usize) ?[]u8 {
20	20	_ = context;
21	21	_ = ra;
22	22	assert(n > 0);
23	23
24	24	const page_size = std.heap.pageSize();
25	25	if (n >= maxInt(usize) - page_size) return null;
26		const alignment_bytes = requested_alignment.toByteUnits();
	26	const alignment_bytes = alignment.toByteUnits();
27	27
28	28	if (native_os == .windows) {
29	29	// According to official documentation, VirtualAlloc aligns to page
...	...	@@ -103,22 +103,52 @@ fn alloc(context: anyopaque, n: usize, log2_align: u8, ra: usize) ?[]u8 {
103	103
104	104	fn resize(
105	105	context: *anyopaque,
106		buf_unaligned: []u8,
107		log2_buf_align: u8,
108		new_size: usize,
	106	memory: []u8,
	107	alignment: mem.Alignment,
	108	new_len: usize,
109	109	return_address: usize,
110	110	) bool {
111	111	_ = context;
112		_ = log2_buf_align;
	112	_ = alignment;
113	113	_ = return_address;
	114	return realloc(memory, new_len, false) != null;
	115	}
	116
	117	pub fn remap(
	118	context: *anyopaque,
	119	memory: []u8,
	120	alignment: mem.Alignment,
	121	new_len: usize,
	122	return_address: usize,
	123	) ?[*]u8 {
	124	_ = context;
	125	_ = alignment;
	126	_ = return_address;
	127	return realloc(memory, new_len, true);
	128	}
	129
	130	fn free(context: *anyopaque, slice: []u8, alignment: mem.Alignment, return_address: usize) void {
	131	_ = context;
	132	_ = alignment;
	133	_ = return_address;
	134
	135	if (native_os == .windows) {
	136	windows.VirtualFree(slice.ptr, 0, windows.MEM_RELEASE);
	137	} else {
	138	const buf_aligned_len = mem.alignForward(usize, slice.len, std.heap.pageSize());
	139	posix.munmap(@alignCast(slice.ptr[0..buf_aligned_len]));
	140	}
	141	}
	142
	143	fn realloc(memory: []u8, new_len: usize, may_move: bool) ?[*]u8 {
114	144	const page_size = std.heap.pageSize();
115		const new_size_aligned = mem.alignForward(usize, new_size, page_size);
	145	const new_size_aligned = mem.alignForward(usize, new_len, page_size);
116	146
117	147	if (native_os == .windows) {
118		if (new_size <= buf_unaligned.len) {
119		const base_addr = @intFromPtr(buf_unaligned.ptr);
120		const old_addr_end = base_addr + buf_unaligned.len;
121		const new_addr_end = mem.alignForward(usize, base_addr + new_size, page_size);
	148	if (new_len <= memory.len) {
	149	const base_addr = @intFromPtr(memory.ptr);
	150	const old_addr_end = base_addr + memory.len;
	151	const new_addr_end = mem.alignForward(usize, base_addr + new_len, page_size);
122	152	if (old_addr_end > new_addr_end) {
123	153	// For shrinking that is not releasing, we will only decommit
124	154	// the pages not needed anymore.
...	...	@@ -128,40 +158,31 @@ fn resize(
128	158	windows.MEM_DECOMMIT,
129	159	);
130	160	}
131		return true;
	161	return memory.ptr;
132	162	}
133		const old_size_aligned = mem.alignForward(usize, buf_unaligned.len, page_size);
	163	const old_size_aligned = mem.alignForward(usize, memory.len, page_size);
134	164	if (new_size_aligned <= old_size_aligned) {
135		return true;
	165	return memory.ptr;
136	166	}
137		return false;
	167	return null;
138	168	}
139	169
140		const buf_aligned_len = mem.alignForward(usize, buf_unaligned.len, page_size);
141		if (new_size_aligned == buf_aligned_len)
142		return true;
	170	const page_aligned_len = mem.alignForward(usize, memory.len, page_size);
	171	if (new_size_aligned == page_aligned_len)
	172	return memory.ptr;
143	173
144		if (new_size_aligned < buf_aligned_len) {
145		const ptr = buf_unaligned.ptr + new_size_aligned;
146		// TODO: if the next_mmap_addr_hint is within the unmapped range, update it
147		posix.munmap(@alignCast(ptr[0 .. buf_aligned_len - new_size_aligned]));
148		return true;
	174	const mremap_available = false; // native_os == .linux;
	175	if (mremap_available) {
	176	// TODO: if the next_mmap_addr_hint is within the remapped range, update it
	177	return posix.mremap(memory, new_len, .{ .MAYMOVE = may_move }, null) catch return null;
149	178	}
150	179
151		// TODO: call mremap
152		// TODO: if the next_mmap_addr_hint is within the remapped range, update it
153		return false;
154		}
155
156		fn free(context: *anyopaque, slice: []u8, log2_buf_align: u8, return_address: usize) void {
157		_ = context;
158		_ = log2_buf_align;
159		_ = return_address;
160
161		if (native_os == .windows) {
162		windows.VirtualFree(slice.ptr, 0, windows.MEM_RELEASE);
163		} else {
164		const buf_aligned_len = mem.alignForward(usize, slice.len, std.heap.pageSize());
165		posix.munmap(@alignCast(slice.ptr[0..buf_aligned_len]));
	180	if (new_size_aligned < page_aligned_len) {
	181	const ptr = memory.ptr + new_size_aligned;
	182	// TODO: if the next_mmap_addr_hint is within the unmapped range, update it
	183	posix.munmap(@alignCast(ptr[0 .. page_aligned_len - new_size_aligned]));
	184	return memory.ptr;
166	185	}
	186
	187	return null;
167	188	}

lib/std/heap/arena_allocator.zig+25-17

...	...	@@ -29,12 +29,14 @@ pub const ArenaAllocator = struct {
29	29	.vtable = &.{
30	30	.alloc = alloc,
31	31	.resize = resize,
	32	.remap = remap,
32	33	.free = free,
33	34	},
34	35	};
35	36	}
36	37
37	38	const BufNode = std.SinglyLinkedList(usize).Node;
	39	const BufNode_alignment: mem.Alignment = .fromByteUnits(@alignOf(BufNode));
38	40
39	41	pub fn init(child_allocator: Allocator) ArenaAllocator {
40	42	return (State{}).promote(child_allocator);
...	...	@@ -47,9 +49,8 @@ pub const ArenaAllocator = struct {
47	49	while (it) \|node\| {
48	50	// this has to occur before the free because the free frees node
49	51	const next_it = node.next;
50		const align_bits = std.math.log2_int(usize, @alignOf(BufNode));
51	52	const alloc_buf = @as([*]u8, @ptrCast(node))[0..node.data];
52		self.child_allocator.rawFree(alloc_buf, align_bits, @returnAddress());
	53	self.child_allocator.rawFree(alloc_buf, BufNode_alignment, @returnAddress());
53	54	it = next_it;
54	55	}
55	56	}
...	...	@@ -120,7 +121,6 @@ pub const ArenaAllocator = struct {
120	121	return true;
121	122	}
122	123	const total_size = requested_capacity + @sizeOf(BufNode);
123		const align_bits = std.math.log2_int(usize, @alignOf(BufNode));
124	124	// Free all nodes except for the last one
125	125	var it = self.state.buffer_list.first;
126	126	const maybe_first_node = while (it) \|node\| {
...	...	@@ -129,7 +129,7 @@ pub const ArenaAllocator = struct {
129	129	if (next_it == null)
130	130	break node;
131	131	const alloc_buf = @as([*]u8, @ptrCast(node))[0..node.data];
132		self.child_allocator.rawFree(alloc_buf, align_bits, @returnAddress());
	132	self.child_allocator.rawFree(alloc_buf, BufNode_alignment, @returnAddress());
133	133	it = next_it;
134	134	} else null;
135	135	std.debug.assert(maybe_first_node == null or maybe_first_node.?.next == null);
...	...	@@ -141,16 +141,16 @@ pub const ArenaAllocator = struct {
141	141	if (first_node.data == total_size)
142	142	return true;
143	143	const first_alloc_buf = @as([*]u8, @ptrCast(first_node))[0..first_node.data];
144		if (self.child_allocator.rawResize(first_alloc_buf, align_bits, total_size, @returnAddress())) {
	144	if (self.child_allocator.rawResize(first_alloc_buf, BufNode_alignment, total_size, @returnAddress())) {
145	145	// successful resize
146	146	first_node.data = total_size;
147	147	} else {
148	148	// manual realloc
149		const new_ptr = self.child_allocator.rawAlloc(total_size, align_bits, @returnAddress()) orelse {
	149	const new_ptr = self.child_allocator.rawAlloc(total_size, BufNode_alignment, @returnAddress()) orelse {
150	150	// we failed to preheat the arena properly, signal this to the user.
151	151	return false;
152	152	};
153		self.child_allocator.rawFree(first_alloc_buf, align_bits, @returnAddress());
	153	self.child_allocator.rawFree(first_alloc_buf, BufNode_alignment, @returnAddress());
154	154	const node: *BufNode = @ptrCast(@alignCast(new_ptr));
155	155	node.* = .{ .data = total_size };
156	156	self.state.buffer_list.first = node;
...	...	@@ -163,8 +163,7 @@ pub const ArenaAllocator = struct {
163	163	const actual_min_size = minimum_size + (@sizeOf(BufNode) + 16);
164	164	const big_enough_len = prev_len + actual_min_size;
165	165	const len = big_enough_len + big_enough_len / 2;
166		const log2_align = comptime std.math.log2_int(usize, @alignOf(BufNode));
167		const ptr = self.child_allocator.rawAlloc(len, log2_align, @returnAddress()) orelse
	166	const ptr = self.child_allocator.rawAlloc(len, BufNode_alignment, @returnAddress()) orelse
168	167	return null;
169	168	const buf_node: *BufNode = @ptrCast(@alignCast(ptr));
170	169	buf_node.* = .{ .data = len };
...	...	@@ -173,11 +172,11 @@ pub const ArenaAllocator = struct {
173	172	return buf_node;
174	173	}
175	174
176		fn alloc(ctx: anyopaque, n: usize, log2_ptr_align: u8, ra: usize) ?[]u8 {
	175	fn alloc(ctx: anyopaque, n: usize, alignment: mem.Alignment, ra: usize) ?[]u8 {
177	176	const self: *ArenaAllocator = @ptrCast(@alignCast(ctx));
178	177	_ = ra;
179	178
180		const ptr_align = @as(usize, 1) << @as(Allocator.Log2Align, @intCast(log2_ptr_align));
	179	const ptr_align = alignment.toByteUnits();
181	180	var cur_node = if (self.state.buffer_list.first) \|first_node\|
182	181	first_node
183	182	else
...	...	@@ -197,8 +196,7 @@ pub const ArenaAllocator = struct {
197	196	}
198	197
199	198	const bigger_buf_size = @sizeOf(BufNode) + new_end_index;
200		const log2_align = comptime std.math.log2_int(usize, @alignOf(BufNode));
201		if (self.child_allocator.rawResize(cur_alloc_buf, log2_align, bigger_buf_size, @returnAddress())) {
	199	if (self.child_allocator.rawResize(cur_alloc_buf, BufNode_alignment, bigger_buf_size, @returnAddress())) {
202	200	cur_node.data = bigger_buf_size;
203	201	} else {
204	202	// Allocate a new node if that's not possible
...	...	@@ -207,9 +205,9 @@ pub const ArenaAllocator = struct {
207	205	}
208	206	}
209	207
210		fn resize(ctx: *anyopaque, buf: []u8, log2_buf_align: u8, new_len: usize, ret_addr: usize) bool {
	208	fn resize(ctx: *anyopaque, buf: []u8, alignment: mem.Alignment, new_len: usize, ret_addr: usize) bool {
211	209	const self: *ArenaAllocator = @ptrCast(@alignCast(ctx));
212		_ = log2_buf_align;
	210	_ = alignment;
213	211	_ = ret_addr;
214	212
215	213	const cur_node = self.state.buffer_list.first orelse return false;
...	...	@@ -231,8 +229,18 @@ pub const ArenaAllocator = struct {
231	229	}
232	230	}
233	231
234		fn free(ctx: *anyopaque, buf: []u8, log2_buf_align: u8, ret_addr: usize) void {
235		_ = log2_buf_align;
	232	fn remap(
	233	context: *anyopaque,
	234	memory: []u8,
	235	alignment: mem.Alignment,
	236	new_len: usize,
	237	return_address: usize,
	238	) ?[*]u8 {
	239	return if (resize(context, memory, alignment, new_len, return_address)) memory.ptr else null;
	240	}
	241
	242	fn free(ctx: *anyopaque, buf: []u8, alignment: mem.Alignment, ret_addr: usize) void {
	243	_ = alignment;
236	244	_ = ret_addr;
237	245
238	246	const self: *ArenaAllocator = @ptrCast(@alignCast(ctx));

lib/std/heap/general_purpose_allocator.zig+86-58

...	...	@@ -226,7 +226,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
226	226	requested_size: if (config.enable_memory_limit) usize else void,
227	227	stack_addresses: [trace_n][stack_n]usize,
228	228	freed: if (config.retain_metadata) bool else void,
229		log2_ptr_align: if (config.never_unmap and config.retain_metadata) u8 else void,
	229	alignment: if (config.never_unmap and config.retain_metadata) mem.Alignment else void,
230	230
231	231	const trace_n = if (config.retain_metadata) traces_per_slot else 1;
232	232
...	...	@@ -281,11 +281,11 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
281	281	return sizes[0..slot_count];
282	282	}
283	283
284		fn log2PtrAligns(bucket: *BucketHeader, size_class: usize) []u8 {
	284	fn log2PtrAligns(bucket: *BucketHeader, size_class: usize) []mem.Alignment {
285	285	if (!config.safety) @compileError("requested size is only stored when safety is enabled");
286	286	const aligns_ptr = @as([*]u8, @ptrCast(bucket)) + bucketAlignsStart(size_class);
287	287	const slot_count = @divExact(page_size, size_class);
288		return aligns_ptr[0..slot_count];
	288	return @ptrCast(aligns_ptr[0..slot_count]);
289	289	}
290	290
291	291	fn stackTracePtr(
...	...	@@ -326,6 +326,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
326	326	.vtable = &.{
327	327	.alloc = alloc,
328	328	.resize = resize,
	329	.remap = remap,
329	330	.free = free,
330	331	},
331	332	};
...	...	@@ -455,7 +456,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
455	456	var it = self.large_allocations.iterator();
456	457	while (it.next()) \|large\| {
457	458	if (large.value_ptr.freed) {
458		self.backing_allocator.rawFree(large.value_ptr.bytes, large.value_ptr.log2_ptr_align, @returnAddress());
	459	self.backing_allocator.rawFree(large.value_ptr.bytes, large.value_ptr.alignment, @returnAddress());
459	460	}
460	461	}
461	462	}
...	...	@@ -583,10 +584,11 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
583	584	fn resizeLarge(
584	585	self: *Self,
585	586	old_mem: []u8,
586		log2_old_align: u8,
	587	alignment: mem.Alignment,
587	588	new_size: usize,
588	589	ret_addr: usize,
589		) bool {
	590	may_move: bool,
	591	) ?[*]u8 {
590	592	const entry = self.large_allocations.getEntry(@intFromPtr(old_mem.ptr)) orelse {
591	593	if (config.safety) {
592	594	@panic("Invalid free");
...	...	@@ -628,30 +630,37 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
628	630	if (config.enable_memory_limit) {
629	631	const new_req_bytes = prev_req_bytes + new_size - entry.value_ptr.requested_size;
630	632	if (new_req_bytes > prev_req_bytes and new_req_bytes > self.requested_memory_limit) {
631		return false;
	633	return null;
632	634	}
633	635	self.total_requested_bytes = new_req_bytes;
634	636	}
635	637
636		if (!self.backing_allocator.rawResize(old_mem, log2_old_align, new_size, ret_addr)) {
	638	const opt_resized_ptr = if (may_move)
	639	self.backing_allocator.rawRemap(old_mem, alignment, new_size, ret_addr)
	640	else if (self.backing_allocator.rawResize(old_mem, alignment, new_size, ret_addr))
	641	old_mem.ptr
	642	else
	643	null;
	644
	645	const resized_ptr = opt_resized_ptr orelse {
637	646	if (config.enable_memory_limit) {
638	647	self.total_requested_bytes = prev_req_bytes;
639	648	}
640		return false;
641		}
	649	return null;
	650	};
642	651
643	652	if (config.enable_memory_limit) {
644	653	entry.value_ptr.requested_size = new_size;
645	654	}
646	655
647	656	if (config.verbose_log) {
648		log.info("large resize {d} bytes at {*} to {d}", .{
649		old_mem.len, old_mem.ptr, new_size,
	657	log.info("large resize {d} bytes at {} to {d} at {}", .{
	658	old_mem.len, old_mem.ptr, new_size, resized_ptr,
650	659	});
651	660	}
652		entry.value_ptr.bytes = old_mem.ptr[0..new_size];
	661	entry.value_ptr.bytes = resized_ptr[0..new_size];
653	662	entry.value_ptr.captureStackTrace(ret_addr, .alloc);
654		return true;
	663	return resized_ptr;
655	664	}
656	665
657	666	/// This function assumes the object is in the large object storage regardless
...	...	@@ -659,7 +668,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
659	668	fn freeLarge(
660	669	self: *Self,
661	670	old_mem: []u8,
662		log2_old_align: u8,
	671	alignment: mem.Alignment,
663	672	ret_addr: usize,
664	673	) void {
665	674	const entry = self.large_allocations.getEntry(@intFromPtr(old_mem.ptr)) orelse {
...	...	@@ -695,7 +704,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
695	704	}
696	705
697	706	if (!config.never_unmap) {
698		self.backing_allocator.rawFree(old_mem, log2_old_align, ret_addr);
	707	self.backing_allocator.rawFree(old_mem, alignment, ret_addr);
699	708	}
700	709
701	710	if (config.enable_memory_limit) {
...	...	@@ -719,22 +728,42 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
719	728	}
720	729
721	730	fn resize(
722		ctx: *anyopaque,
	731	context: *anyopaque,
	732	memory: []u8,
	733	alignment: mem.Alignment,
	734	new_len: usize,
	735	return_address: usize,
	736	) bool {
	737	return realloc(context, memory, alignment, new_len, return_address, false) != null;
	738	}
	739
	740	fn remap(
	741	context: *anyopaque,
	742	memory: []u8,
	743	alignment: mem.Alignment,
	744	new_len: usize,
	745	return_address: usize,
	746	) ?[*]u8 {
	747	return realloc(context, memory, alignment, new_len, return_address, true);
	748	}
	749
	750	fn realloc(
	751	context: *anyopaque,
723	752	old_mem: []u8,
724		log2_old_align_u8: u8,
725		new_size: usize,
	753	alignment: mem.Alignment,
	754	new_len: usize,
726	755	ret_addr: usize,
727		) bool {
728		const self: *Self = @ptrCast(@alignCast(ctx));
729		const log2_old_align = @as(Allocator.Log2Align, @intCast(log2_old_align_u8));
	756	may_move: bool,
	757	) ?[*]u8 {
	758	const self: *Self = @ptrCast(@alignCast(context));
730	759	self.mutex.lock();
731	760	defer self.mutex.unlock();
732	761
733	762	assert(old_mem.len != 0);
734	763
735		const aligned_size = @max(old_mem.len, @as(usize, 1) << log2_old_align);
	764	const aligned_size = @max(old_mem.len, alignment.toByteUnits());
736	765	if (aligned_size > largest_bucket_object_size) {
737		return self.resizeLarge(old_mem, log2_old_align, new_size, ret_addr);
	766	return self.resizeLarge(old_mem, alignment, new_len, ret_addr, may_move);
738	767	}
739	768	const size_class_hint = math.ceilPowerOfTwoAssert(usize, aligned_size);
740	769
...	...	@@ -758,7 +787,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
758	787	}
759	788	}
760	789	}
761		return self.resizeLarge(old_mem, log2_old_align, new_size, ret_addr);
	790	return self.resizeLarge(old_mem, alignment, new_len, ret_addr, may_move);
762	791	};
763	792	const byte_offset = @intFromPtr(old_mem.ptr) - @intFromPtr(bucket.page);
764	793	const slot_index = @as(SlotIndex, @intCast(byte_offset / size_class));
...	...	@@ -779,8 +808,8 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
779	808	if (config.safety) {
780	809	const requested_size = bucket.requestedSizes(size_class)[slot_index];
781	810	if (requested_size == 0) @panic("Invalid free");
782		const log2_ptr_align = bucket.log2PtrAligns(size_class)[slot_index];
783		if (old_mem.len != requested_size or log2_old_align != log2_ptr_align) {
	811	const slot_alignment = bucket.log2PtrAligns(size_class)[slot_index];
	812	if (old_mem.len != requested_size or alignment != slot_alignment) {
784	813	var addresses: [stack_n]usize = [1]usize{0} ** stack_n;
785	814	var free_stack_trace = StackTrace{
786	815	.instruction_addresses = &addresses,
...	...	@@ -795,10 +824,10 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
795	824	free_stack_trace,
796	825	});
797	826	}
798		if (log2_old_align != log2_ptr_align) {
	827	if (alignment != slot_alignment) {
799	828	log.err("Allocation alignment {d} does not match resize alignment {d}. Allocation: {} Resize: {}", .{
800		@as(usize, 1) << @as(math.Log2Int(usize), @intCast(log2_ptr_align)),
801		@as(usize, 1) << @as(math.Log2Int(usize), @intCast(log2_old_align)),
	829	slot_alignment.toByteUnits(),
	830	alignment.toByteUnits(),
802	831	bucketStackTrace(bucket, size_class, slot_index, .alloc),
803	832	free_stack_trace,
804	833	});
...	...	@@ -807,52 +836,51 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
807	836	}
808	837	const prev_req_bytes = self.total_requested_bytes;
809	838	if (config.enable_memory_limit) {
810		const new_req_bytes = prev_req_bytes + new_size - old_mem.len;
	839	const new_req_bytes = prev_req_bytes + new_len - old_mem.len;
811	840	if (new_req_bytes > prev_req_bytes and new_req_bytes > self.requested_memory_limit) {
812		return false;
	841	return null;
813	842	}
814	843	self.total_requested_bytes = new_req_bytes;
815	844	}
816	845
817		const new_aligned_size = @max(new_size, @as(usize, 1) << log2_old_align);
	846	const new_aligned_size = @max(new_len, alignment.toByteUnits());
818	847	const new_size_class = math.ceilPowerOfTwoAssert(usize, new_aligned_size);
819	848	if (new_size_class <= size_class) {
820		if (old_mem.len > new_size) {
821		@memset(old_mem[new_size..], undefined);
	849	if (old_mem.len > new_len) {
	850	@memset(old_mem[new_len..], undefined);
822	851	}
823	852	if (config.verbose_log) {
824	853	log.info("small resize {d} bytes at {*} to {d}", .{
825		old_mem.len, old_mem.ptr, new_size,
	854	old_mem.len, old_mem.ptr, new_len,
826	855	});
827	856	}
828	857	if (config.safety) {
829		bucket.requestedSizes(size_class)[slot_index] = @intCast(new_size);
	858	bucket.requestedSizes(size_class)[slot_index] = @intCast(new_len);
830	859	}
831		return true;
	860	return old_mem.ptr;
832	861	}
833	862
834	863	if (config.enable_memory_limit) {
835	864	self.total_requested_bytes = prev_req_bytes;
836	865	}
837		return false;
	866	return null;
838	867	}
839	868
840	869	fn free(
841	870	ctx: *anyopaque,
842	871	old_mem: []u8,
843		log2_old_align_u8: u8,
	872	alignment: mem.Alignment,
844	873	ret_addr: usize,
845	874	) void {
846	875	const self: *Self = @ptrCast(@alignCast(ctx));
847		const log2_old_align = @as(Allocator.Log2Align, @intCast(log2_old_align_u8));
848	876	self.mutex.lock();
849	877	defer self.mutex.unlock();
850	878
851	879	assert(old_mem.len != 0);
852	880
853		const aligned_size = @max(old_mem.len, @as(usize, 1) << log2_old_align);
	881	const aligned_size = @max(old_mem.len, alignment.toByteUnits());
854	882	if (aligned_size > largest_bucket_object_size) {
855		self.freeLarge(old_mem, log2_old_align, ret_addr);
	883	self.freeLarge(old_mem, alignment, ret_addr);
856	884	return;
857	885	}
858	886	const size_class_hint = math.ceilPowerOfTwoAssert(usize, aligned_size);
...	...	@@ -877,7 +905,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
877	905	}
878	906	}
879	907	}
880		self.freeLarge(old_mem, log2_old_align, ret_addr);
	908	self.freeLarge(old_mem, alignment, ret_addr);
881	909	return;
882	910	};
883	911	const byte_offset = @intFromPtr(old_mem.ptr) - @intFromPtr(bucket.page);
...	...	@@ -900,8 +928,8 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
900	928	if (config.safety) {
901	929	const requested_size = bucket.requestedSizes(size_class)[slot_index];
902	930	if (requested_size == 0) @panic("Invalid free");
903		const log2_ptr_align = bucket.log2PtrAligns(size_class)[slot_index];
904		if (old_mem.len != requested_size or log2_old_align != log2_ptr_align) {
	931	const slot_alignment = bucket.log2PtrAligns(size_class)[slot_index];
	932	if (old_mem.len != requested_size or alignment != slot_alignment) {
905	933	var addresses: [stack_n]usize = [1]usize{0} ** stack_n;
906	934	var free_stack_trace = StackTrace{
907	935	.instruction_addresses = &addresses,
...	...	@@ -916,10 +944,10 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
916	944	free_stack_trace,
917	945	});
918	946	}
919		if (log2_old_align != log2_ptr_align) {
	947	if (alignment != slot_alignment) {
920	948	log.err("Allocation alignment {d} does not match free alignment {d}. Allocation: {} Free: {}", .{
921		@as(usize, 1) << @as(math.Log2Int(usize), @intCast(log2_ptr_align)),
922		@as(usize, 1) << @as(math.Log2Int(usize), @intCast(log2_old_align)),
	949	slot_alignment.toByteUnits(),
	950	alignment.toByteUnits(),
923	951	bucketStackTrace(bucket, size_class, slot_index, .alloc),
924	952	free_stack_trace,
925	953	});
...	...	@@ -981,24 +1009,24 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
981	1009	return true;
982	1010	}
983	1011
984		fn alloc(ctx: anyopaque, len: usize, log2_ptr_align: u8, ret_addr: usize) ?[]u8 {
	1012	fn alloc(ctx: anyopaque, len: usize, alignment: mem.Alignment, ret_addr: usize) ?[]u8 {
985	1013	const self: *Self = @ptrCast(@alignCast(ctx));
986	1014	self.mutex.lock();
987	1015	defer self.mutex.unlock();
988	1016	if (!self.isAllocationAllowed(len)) return null;
989		return allocInner(self, len, @as(Allocator.Log2Align, @intCast(log2_ptr_align)), ret_addr) catch return null;
	1017	return allocInner(self, len, alignment, ret_addr) catch return null;
990	1018	}
991	1019
992	1020	fn allocInner(
993	1021	self: *Self,
994	1022	len: usize,
995		log2_ptr_align: Allocator.Log2Align,
	1023	alignment: mem.Alignment,
996	1024	ret_addr: usize,
997	1025	) Allocator.Error![*]u8 {
998		const new_aligned_size = @max(len, @as(usize, 1) << @as(Allocator.Log2Align, @intCast(log2_ptr_align)));
	1026	const new_aligned_size = @max(len, alignment.toByteUnits());
999	1027	if (new_aligned_size > largest_bucket_object_size) {
1000	1028	try self.large_allocations.ensureUnusedCapacity(self.backing_allocator, 1);
1001		const ptr = self.backing_allocator.rawAlloc(len, log2_ptr_align, ret_addr) orelse
	1029	const ptr = self.backing_allocator.rawAlloc(len, alignment, ret_addr) orelse
1002	1030	return error.OutOfMemory;
1003	1031	const slice = ptr[0..len];
1004	1032
...	...	@@ -1016,7 +1044,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
1016	1044	if (config.retain_metadata) {
1017	1045	gop.value_ptr.freed = false;
1018	1046	if (config.never_unmap) {
1019		gop.value_ptr.log2_ptr_align = log2_ptr_align;
	1047	gop.value_ptr.alignment = alignment;
1020	1048	}
1021	1049	}
1022	1050
...	...	@@ -1030,7 +1058,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
1030	1058	const slot = try self.allocSlot(new_size_class, ret_addr);
1031	1059	if (config.safety) {
1032	1060	slot.bucket.requestedSizes(new_size_class)[slot.slot_index] = @intCast(len);
1033		slot.bucket.log2PtrAligns(new_size_class)[slot.slot_index] = log2_ptr_align;
	1061	slot.bucket.log2PtrAligns(new_size_class)[slot.slot_index] = alignment;
1034	1062	}
1035	1063	if (config.verbose_log) {
1036	1064	log.info("small alloc {d} bytes at {*}", .{ len, slot.ptr });
...	...	@@ -1150,7 +1178,7 @@ test "realloc" {
1150	1178	}
1151	1179
1152	1180	test "shrink" {
1153		var gpa = GeneralPurposeAllocator(test_config){};
	1181	var gpa: GeneralPurposeAllocator(test_config) = .{};
1154	1182	defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak");
1155	1183	const allocator = gpa.allocator();
1156	1184
...	...	@@ -1214,7 +1242,7 @@ test "realloc small object to large object" {
1214	1242	}
1215	1243
1216	1244	test "shrink large object to large object" {
1217		var gpa = GeneralPurposeAllocator(test_config){};
	1245	var gpa: GeneralPurposeAllocator(test_config) = .{};
1218	1246	defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak");
1219	1247	const allocator = gpa.allocator();
1220	1248

lib/std/mem/Allocator.zig+149-65

...	...	@@ -6,19 +6,21 @@ const math = std.math;
6	6	const mem = std.mem;
7	7	const Allocator = @This();
8	8	const builtin = @import("builtin");
	9	const Alignment = std.mem.Alignment;
9	10
10	11	pub const Error = error{OutOfMemory};
11	12	pub const Log2Align = math.Log2Int(usize);
12	13
13	14	/// The type erased pointer to the allocator implementation.
14		/// Any comparison of this field may result in illegal behavior, since it may be set to
15		/// `undefined` in cases where the allocator implementation does not have any associated
16		/// state.
	15	///
	16	/// Any comparison of this field may result in illegal behavior, since it may
	17	/// be set to `undefined` in cases where the allocator implementation does not
	18	/// have any associated state.
17	19	ptr: *anyopaque,
18	20	vtable: *const VTable,
19	21
20	22	pub const VTable = struct {
21		/// Allocate exactly `len` bytes aligned to `1 << ptr_align`, or return `null`
	23	/// Allocate exactly `len` bytes aligned to `alignment`, or return `null`
22	24	/// indicating the allocation failed.
23	25	///
24	26	/// `ret_addr` is optionally provided as the first return address of the
...	...	@@ -27,12 +29,14 @@ pub const VTable = struct {
27	29	///
28	30	/// The returned slice of memory must have been `@memset` to `undefined`
29	31	/// by the allocator implementation.
30		alloc: const fn (ctx: anyopaque, len: usize, ptr_align: u8, ret_addr: usize) ?[*]u8,
	32	alloc: const fn (anyopaque, len: usize, alignment: Alignment, ret_addr: usize) ?[*]u8,
31	33
32		/// Attempt to expand or shrink memory in place. `buf.len` must equal the
33		/// length requested from the most recent successful call to `alloc` or
34		/// `resize`. `buf_align` must equal the same value that was passed as the
35		/// `ptr_align` parameter to the original `alloc` call.
	34	/// Attempt to expand or shrink memory in place.
	35	///
	36	/// `memory.len` must equal the length requested from the most recent
	37	/// successful call to `alloc` or `resize`. `alignment` must equal the same
	38	/// value that was passed as the `alignment` parameter to the original
	39	/// `alloc` call.
36	40	///
37	41	/// A result of `true` indicates the resize was successful and the
38	42	/// allocation now has the same address but a size of `new_len`. `false`
...	...	@@ -44,72 +48,114 @@ pub const VTable = struct {
44	48	/// `ret_addr` is optionally provided as the first return address of the
45	49	/// allocation call stack. If the value is `0` it means no return address
46	50	/// has been provided.
47		resize: const fn (ctx: anyopaque, buf: []u8, buf_align: u8, new_len: usize, ret_addr: usize) bool,
	51	resize: const fn (anyopaque, memory: []u8, alignment: Alignment, new_len: usize, ret_addr: usize) bool,
48	52
49		/// Free and invalidate a buffer.
	53	/// Attempt to expand or shrink memory, allowing relocation.
	54	///
	55	/// `memory.len` must equal the length requested from the most recent
	56	/// successful call to `alloc` or `resize`. `alignment` must equal the same
	57	/// value that was passed as the `alignment` parameter to the original
	58	/// `alloc` call.
	59	///
	60	/// A non-`null` return value indicates the resize was successful. The
	61	/// allocation may have same address, or may have been relocated. In either
	62	/// case, the allocation now has size of `new_len`. A `null` return value
	63	/// indicates that the resize would be equivalent to allocating new memory,
	64	/// copying the bytes from the old memory, and then freeing the old memory.
	65	/// In such case, it is more efficient for the caller to perform the copy.
50	66	///
51		/// `buf.len` must equal the most recent length returned by `alloc` or
	67	/// `new_len` must be greater than zero.
	68	///
	69	/// `ret_addr` is optionally provided as the first return address of the
	70	/// allocation call stack. If the value is `0` it means no return address
	71	/// has been provided.
	72	remap: const fn (anyopaque, memory: []u8, alignment: Alignment, new_len: usize, ret_addr: usize) ?[*]u8,
	73
	74	/// Free and invalidate a region of memory.
	75	///
	76	/// `memory.len` must equal the most recent length returned by `alloc` or
52	77	/// given to a successful `resize` call.
53	78	///
54		/// `buf_align` must equal the same value that was passed as the
55		/// `ptr_align` parameter to the original `alloc` call.
	79	/// `alignment` must equal the same value that was passed as the
	80	/// `alignment` parameter to the original `alloc` call.
56	81	///
57	82	/// `ret_addr` is optionally provided as the first return address of the
58	83	/// allocation call stack. If the value is `0` it means no return address
59	84	/// has been provided.
60		free: const fn (ctx: anyopaque, buf: []u8, buf_align: u8, ret_addr: usize) void,
	85	free: const fn (anyopaque, memory: []u8, alignment: Alignment, ret_addr: usize) void,
61	86	};
62	87
63	88	pub fn noResize(
64	89	self: *anyopaque,
65		buf: []u8,
66		log2_buf_align: u8,
	90	memory: []u8,
	91	alignment: Alignment,
67	92	new_len: usize,
68	93	ret_addr: usize,
69	94	) bool {
70	95	_ = self;
71		_ = buf;
72		_ = log2_buf_align;
	96	_ = memory;
	97	_ = alignment;
73	98	_ = new_len;
74	99	_ = ret_addr;
75	100	return false;
76	101	}
77	102
	103	pub fn noRemap(
	104	self: *anyopaque,
	105	memory: []u8,
	106	alignment: Alignment,
	107	new_len: usize,
	108	ret_addr: usize,
	109	) ?[*]u8 {
	110	_ = self;
	111	_ = memory;
	112	_ = alignment;
	113	_ = new_len;
	114	_ = ret_addr;
	115	return null;
	116	}
	117
78	118	pub fn noFree(
79	119	self: *anyopaque,
80		buf: []u8,
81		log2_buf_align: u8,
	120	memory: []u8,
	121	alignment: Alignment,
82	122	ret_addr: usize,
83	123	) void {
84	124	_ = self;
85		_ = buf;
86		_ = log2_buf_align;
	125	_ = memory;
	126	_ = alignment;
87	127	_ = ret_addr;
88	128	}
89	129
90	130	/// This function is not intended to be called except from within the
91	131	/// implementation of an Allocator
92		pub inline fn rawAlloc(self: Allocator, len: usize, ptr_align: u8, ret_addr: usize) ?[*]u8 {
93		return self.vtable.alloc(self.ptr, len, ptr_align, ret_addr);
	132	pub inline fn rawAlloc(a: Allocator, len: usize, alignment: Alignment, ret_addr: usize) ?[*]u8 {
	133	return a.vtable.alloc(a.ptr, len, alignment, ret_addr);
94	134	}
95	135
96	136	/// This function is not intended to be called except from within the
97		/// implementation of an Allocator
98		pub inline fn rawResize(self: Allocator, buf: []u8, log2_buf_align: u8, new_len: usize, ret_addr: usize) bool {
99		return self.vtable.resize(self.ptr, buf, log2_buf_align, new_len, ret_addr);
	137	/// implementation of an Allocator.
	138	pub inline fn rawResize(a: Allocator, memory: []u8, alignment: Alignment, new_len: usize, ret_addr: usize) bool {
	139	return a.vtable.resize(a.ptr, memory, alignment, new_len, ret_addr);
	140	}
	141
	142	/// This function is not intended to be called except from within the
	143	/// implementation of an Allocator.
	144	pub inline fn rawRemap(a: Allocator, memory: []u8, alignment: Alignment, new_len: usize, ret_addr: usize) ?[*]u8 {
	145	return a.vtable.remap(a.ptr, memory, alignment, new_len, ret_addr);
100	146	}
101	147
102	148	/// This function is not intended to be called except from within the
103	149	/// implementation of an Allocator
104		pub inline fn rawFree(self: Allocator, buf: []u8, log2_buf_align: u8, ret_addr: usize) void {
105		return self.vtable.free(self.ptr, buf, log2_buf_align, ret_addr);
	150	pub inline fn rawFree(a: Allocator, memory: []u8, alignment: Alignment, ret_addr: usize) void {
	151	return a.vtable.free(a.ptr, memory, alignment, ret_addr);
106	152	}
107	153
108	154	/// Returns a pointer to undefined memory.
109	155	/// Call `destroy` with the result to free the memory.
110		pub fn create(self: Allocator, comptime T: type) Error!*T {
	156	pub fn create(a: Allocator, comptime T: type) Error!*T {
111	157	if (@sizeOf(T) == 0) return @as(*T, @ptrFromInt(math.maxInt(usize)));
112		const ptr: *T = @ptrCast(try self.allocBytesWithAlignment(@alignOf(T), @sizeOf(T), @returnAddress()));
	158	const ptr: *T = @ptrCast(try a.allocBytesWithAlignment(@alignOf(T), @sizeOf(T), @returnAddress()));
113	159	return ptr;
114	160	}
115	161
...	...	@@ -121,7 +167,7 @@ pub fn destroy(self: Allocator, ptr: anytype) void {
121	167	const T = info.child;
122	168	if (@sizeOf(T) == 0) return;
123	169	const non_const_ptr = @as([*]u8, @ptrCast(@constCast(ptr)));
124		self.rawFree(non_const_ptr[0..@sizeOf(T)], log2a(info.alignment), @returnAddress());
	170	self.rawFree(non_const_ptr[0..@sizeOf(T)], .fromByteUnits(info.alignment), @returnAddress());
125	171	}
126	172
127	173	/// Allocates an array of `n` items of type `T` and sets all the
...	...	@@ -224,36 +270,88 @@ fn allocBytesWithAlignment(self: Allocator, comptime alignment: u29, byte_count:
224	270	return @as([*]align(alignment) u8, @ptrFromInt(ptr));
225	271	}
226	272
227		const byte_ptr = self.rawAlloc(byte_count, log2a(alignment), return_address) orelse return Error.OutOfMemory;
	273	const byte_ptr = self.rawAlloc(byte_count, .fromByteUnits(alignment), return_address) orelse return Error.OutOfMemory;
228	274	// TODO: https://github.com/ziglang/zig/issues/4298
229	275	@memset(byte_ptr[0..byte_count], undefined);
230	276	return @alignCast(byte_ptr);
231	277	}
232	278
233		/// Requests to modify the size of an allocation. It is guaranteed to not move
234		/// the pointer, however the allocator implementation may refuse the resize
235		/// request by returning `false`.
236		pub fn resize(self: Allocator, old_mem: anytype, new_n: usize) bool {
237		const Slice = @typeInfo(@TypeOf(old_mem)).pointer;
	279	/// Request to modify the size of an allocation.
	280	///
	281	/// It is guaranteed to not move the pointer, however the allocator
	282	/// implementation may refuse the resize request by returning `false`.
	283	///
	284	/// `allocation` may be an empty slice, in which case a new allocation is made.
	285	///
	286	/// `new_len` may be zero, in which case the allocation is freed.
	287	pub fn resize(self: Allocator, allocation: anytype, new_len: usize) bool {
	288	const Slice = @typeInfo(@TypeOf(allocation)).pointer;
238	289	const T = Slice.child;
239		if (new_n == 0) {
240		self.free(old_mem);
	290	const alignment = Slice.alignment;
	291	if (new_len == 0) {
	292	self.free(allocation);
241	293	return true;
242	294	}
243		if (old_mem.len == 0) {
	295	if (allocation.len == 0) {
244	296	return false;
245	297	}
246		const old_byte_slice = mem.sliceAsBytes(old_mem);
	298	const old_memory = mem.sliceAsBytes(allocation);
	299	// I would like to use saturating multiplication here, but LLVM cannot lower it
	300	// on WebAssembly: https://github.com/ziglang/zig/issues/9660
	301	//const new_len_bytes = new_len *\| @sizeOf(T);
	302	const new_len_bytes = math.mul(usize, @sizeOf(T), new_len) catch return false;
	303	return self.rawResize(old_memory, .fromByteUnits(alignment), new_len_bytes, @returnAddress());
	304	}
	305
	306	/// Request to modify the size of an allocation, allowing relocation.
	307	///
	308	/// A non-`null` return value indicates the resize was successful. The
	309	/// allocation may have same address, or may have been relocated. In either
	310	/// case, the allocation now has size of `new_len`. A `null` return value
	311	/// indicates that the resize would be equivalent to allocating new memory,
	312	/// copying the bytes from the old memory, and then freeing the old memory.
	313	/// In such case, it is more efficient for the caller to perform those
	314	/// operations.
	315	///
	316	/// `allocation` may be an empty slice, in which case a new allocation is made.
	317	///
	318	/// `new_len` may be zero, in which case the allocation is freed.
	319	pub fn remap(self: Allocator, allocation: anytype, new_len: usize) t: {
	320	const Slice = @typeInfo(@TypeOf(allocation)).pointer;
	321	break :t ?[]align(Slice.alignment) Slice.child;
	322	} {
	323	const Slice = @typeInfo(@TypeOf(allocation)).pointer;
	324	const T = Slice.child;
	325	const alignment = Slice.alignment;
	326	if (new_len == 0) {
	327	self.free(allocation);
	328	return allocation[0..0];
	329	}
	330	if (allocation.len == 0) {
	331	return null;
	332	}
	333	const old_memory = mem.sliceAsBytes(allocation);
247	334	// I would like to use saturating multiplication here, but LLVM cannot lower it
248	335	// on WebAssembly: https://github.com/ziglang/zig/issues/9660
249		//const new_byte_count = new_n *\| @sizeOf(T);
250		const new_byte_count = math.mul(usize, @sizeOf(T), new_n) catch return false;
251		return self.rawResize(old_byte_slice, log2a(Slice.alignment), new_byte_count, @returnAddress());
	336	//const new_len_bytes = new_len *\| @sizeOf(T);
	337	const new_len_bytes = math.mul(usize, @sizeOf(T), new_len) catch return null;
	338	const new_ptr = self.rawRemap(old_memory, .fromByteUnits(alignment), new_len_bytes, @returnAddress()) orelse return null;
	339	const new_memory: []align(alignment) u8 = @alignCast(new_ptr[0..new_len_bytes]);
	340	return mem.bytesAsSlice(T, new_memory);
252	341	}
253	342
254	343	/// This function requests a new byte size for an existing allocation, which
255	344	/// can be larger, smaller, or the same size as the old memory allocation.
	345	///
256	346	/// If `new_n` is 0, this is the same as `free` and it always succeeds.
	347	///
	348	/// `old_mem` may have length zero, which makes a new allocation.
	349	///
	350	/// This function only fails on out-of-memory conditions, unlike:
	351	/// * `remap` which returns `null` when the `Allocator` implementation cannot
	352	/// do the realloc more efficiently than the caller
	353	/// * `resize` which returns `false` when the `Allocator` implementation cannot
	354	/// change the size without relocating the allocation.
257	355	pub fn realloc(self: Allocator, old_mem: anytype, new_n: usize) t: {
258	356	const Slice = @typeInfo(@TypeOf(old_mem)).pointer;
259	357	break :t Error![]align(Slice.alignment) Slice.child;
...	...	@@ -284,18 +382,18 @@ pub fn reallocAdvanced(
284	382	const old_byte_slice = mem.sliceAsBytes(old_mem);
285	383	const byte_count = math.mul(usize, @sizeOf(T), new_n) catch return Error.OutOfMemory;
286	384	// Note: can't set shrunk memory to undefined as memory shouldn't be modified on realloc failure
287		if (self.rawResize(old_byte_slice, log2a(Slice.alignment), byte_count, return_address)) {
288		const new_bytes: []align(Slice.alignment) u8 = @alignCast(old_byte_slice.ptr[0..byte_count]);
	385	if (self.rawRemap(old_byte_slice, .fromByteUnits(Slice.alignment), byte_count, return_address)) \|p\| {
	386	const new_bytes: []align(Slice.alignment) u8 = @alignCast(p[0..byte_count]);
289	387	return mem.bytesAsSlice(T, new_bytes);
290	388	}
291	389
292		const new_mem = self.rawAlloc(byte_count, log2a(Slice.alignment), return_address) orelse
	390	const new_mem = self.rawAlloc(byte_count, .fromByteUnits(Slice.alignment), return_address) orelse
293	391	return error.OutOfMemory;
294	392	const copy_len = @min(byte_count, old_byte_slice.len);
295	393	@memcpy(new_mem[0..copy_len], old_byte_slice[0..copy_len]);
296	394	// TODO https://github.com/ziglang/zig/issues/4298
297	395	@memset(old_byte_slice, undefined);
298		self.rawFree(old_byte_slice, log2a(Slice.alignment), return_address);
	396	self.rawFree(old_byte_slice, .fromByteUnits(Slice.alignment), return_address);
299	397
300	398	const new_bytes: []align(Slice.alignment) u8 = @alignCast(new_mem[0..byte_count]);
301	399	return mem.bytesAsSlice(T, new_bytes);
...	...	@@ -312,7 +410,7 @@ pub fn free(self: Allocator, memory: anytype) void {
312	410	const non_const_ptr = @constCast(bytes.ptr);
313	411	// TODO: https://github.com/ziglang/zig/issues/4298
314	412	@memset(non_const_ptr[0..bytes_len], undefined);
315		self.rawFree(non_const_ptr[0..bytes_len], log2a(Slice.alignment), @returnAddress());
	413	self.rawFree(non_const_ptr[0..bytes_len], .fromByteUnits(Slice.alignment), @returnAddress());
316	414	}
317	415
318	416	/// Copies `m` to newly allocated memory. Caller owns the memory.
...	...	@@ -329,17 +427,3 @@ pub fn dupeZ(allocator: Allocator, comptime T: type, m: []const T) Error![:0]T {
329	427	new_buf[m.len] = 0;
330	428	return new_buf[0..m.len :0];
331	429	}
332
333		/// TODO replace callsites with `@log2` after this proposal is implemented:
334		/// https://github.com/ziglang/zig/issues/13642
335		inline fn log2a(x: anytype) switch (@typeInfo(@TypeOf(x))) {
336		.int => math.Log2Int(@TypeOf(x)),
337		.comptime_int => comptime_int,
338		else => @compileError("int please"),
339		} {
340		switch (@typeInfo(@TypeOf(x))) {
341		.int => return math.log2_int(@TypeOf(x), x),
342		.comptime_int => return math.log2(x),
343		else => @compileError("bad"),
344		}
345		}

lib/std/testing/failing_allocator.zig+30-10

...	...	@@ -62,6 +62,7 @@ pub const FailingAllocator = struct {
62	62	.vtable = &.{
63	63	.alloc = alloc,
64	64	.resize = resize,
	65	.remap = remap,
65	66	.free = free,
66	67	},
67	68	};
...	...	@@ -70,7 +71,7 @@ pub const FailingAllocator = struct {
70	71	fn alloc(
71	72	ctx: *anyopaque,
72	73	len: usize,
73		log2_ptr_align: u8,
	74	alignment: mem.Alignment,
74	75	return_address: usize,
75	76	) ?[*]u8 {
76	77	const self: *FailingAllocator = @ptrCast(@alignCast(ctx));
...	...	@@ -86,7 +87,7 @@ pub const FailingAllocator = struct {
86	87	}
87	88	return null;
88	89	}
89		const result = self.internal_allocator.rawAlloc(len, log2_ptr_align, return_address) orelse
	90	const result = self.internal_allocator.rawAlloc(len, alignment, return_address) orelse
90	91	return null;
91	92	self.allocated_bytes += len;
92	93	self.allocations += 1;
...	...	@@ -96,33 +97,52 @@ pub const FailingAllocator = struct {
96	97
97	98	fn resize(
98	99	ctx: *anyopaque,
99		old_mem: []u8,
100		log2_old_align: u8,
	100	memory: []u8,
	101	alignment: mem.Alignment,
101	102	new_len: usize,
102	103	ra: usize,
103	104	) bool {
104	105	const self: *FailingAllocator = @ptrCast(@alignCast(ctx));
105	106	if (self.resize_index == self.resize_fail_index)
106	107	return false;
107		if (!self.internal_allocator.rawResize(old_mem, log2_old_align, new_len, ra))
	108	if (!self.internal_allocator.rawResize(memory, alignment, new_len, ra))
108	109	return false;
109		if (new_len < old_mem.len) {
110		self.freed_bytes += old_mem.len - new_len;
	110	if (new_len < memory.len) {
	111	self.freed_bytes += memory.len - new_len;
111	112	} else {
112		self.allocated_bytes += new_len - old_mem.len;
	113	self.allocated_bytes += new_len - memory.len;
113	114	}
114	115	self.resize_index += 1;
115	116	return true;
116	117	}
117	118
	119	fn remap(
	120	ctx: *anyopaque,
	121	memory: []u8,
	122	alignment: mem.Alignment,
	123	new_len: usize,
	124	ra: usize,
	125	) ?[*]u8 {
	126	const self: *FailingAllocator = @ptrCast(@alignCast(ctx));
	127	if (self.resize_index == self.resize_fail_index) return null;
	128	const new_ptr = self.internal_allocator.rawRemap(memory, alignment, new_len, ra) orelse return null;
	129	if (new_len < memory.len) {
	130	self.freed_bytes += memory.len - new_len;
	131	} else {
	132	self.allocated_bytes += new_len - memory.len;
	133	}
	134	self.resize_index += 1;
	135	return new_ptr;
	136	}
	137
118	138	fn free(
119	139	ctx: *anyopaque,
120	140	old_mem: []u8,
121		log2_old_align: u8,
	141	alignment: mem.Alignment,
122	142	ra: usize,
123	143	) void {
124	144	const self: *FailingAllocator = @ptrCast(@alignCast(ctx));
125		self.internal_allocator.rawFree(old_mem, log2_old_align, ra);
	145	self.internal_allocator.rawFree(old_mem, alignment, ra);
126	146	self.deallocations += 1;
127	147	self.freed_bytes += old_mem.len;
128	148	}