Commit 91f41bdc7095635b752af0ca3aee0a41ce864f8c | zig

author	Andrew Kelley <andrew@ziglang.org> 2025-01-30 18:21:54-08:00
committer	Andrew Kelley <andrew@ziglang.org> 2025-02-06 14:23:23-08:00
log	91f41bdc7095635b752af0ca3aee0a41ce864f8c
tree	677a487ea33489462a3575d33d5567b0a9e40504
parent	5c638845390fed5cf1c504ac013a4f75c3b7c690

std.heap.PageAllocator: restore high alignment functionality

This allocator now supports alignments greater than page size, with the
same implementation as it used before.

This is a partial revert of ceb0a632cfd6a4eada6bd27bf6a3754e95dcac86.

It looks like VirtualAlloc2 has better solutions to this problem,
including features such as MEM_RESERVE_PLACEHOLDER and MEM_LARGE_PAGES.
This possibility can be investigated as a follow-up task.

1 files changed, 67 insertions(+), 17 deletions(-)

lib
- std
  - heap
    PageAllocator.zig

lib/std/heap/PageAllocator.zig+67-17

...	...	@@ -15,51 +15,100 @@ pub const vtable: Allocator.VTable = .{
15	15	.free = free,
16	16	};
17	17
18		fn alloc(_: anyopaque, n: usize, log2_align: u8, ra: usize) ?[]u8 {
	18	fn alloc(context: anyopaque, n: usize, log2_align: u8, ra: usize) ?[]u8 {
	19	const requested_alignment: mem.Alignment = @enumFromInt(log2_align);
	20	_ = context;
19	21	_ = ra;
20		_ = log2_align;
21	22	assert(n > 0);
22	23
	24	const page_size = std.heap.pageSize();
	25	if (n >= maxInt(usize) - page_size) return null;
	26	const alignment_bytes = requested_alignment.toByteUnits();
	27
23	28	if (native_os == .windows) {
	29	// According to official documentation, VirtualAlloc aligns to page
	30	// boundary, however, empirically it reserves pages on a 64K boundary.
	31	// Since it is very likely the requested alignment will be honored,
	32	// this logic first tries a call with exactly the size requested,
	33	// before falling back to the loop below.
	34	// https://devblogs.microsoft.com/oldnewthing/?p=42223
24	35	const addr = windows.VirtualAlloc(
25	36	null,
26
27	37	// VirtualAlloc will round the length to a multiple of page size.
28		// VirtualAlloc docs: If the lpAddress parameter is NULL, this value is rounded up to the next page boundary
	38	// "If the lpAddress parameter is NULL, this value is rounded up to
	39	// the next page boundary".
29	40	n,
30
31	41	windows.MEM_COMMIT \| windows.MEM_RESERVE,
32	42	windows.PAGE_READWRITE,
33	43	) catch return null;
34		return @ptrCast(addr);
35		}
36	44
37		const page_size = std.heap.pageSize();
38		if (n >= maxInt(usize) - page_size) return null;
	45	if (mem.isAligned(@intFromPtr(addr), alignment_bytes))
	46	return @ptrCast(addr);
	47
	48	// Fallback: reserve a range of memory large enough to find a
	49	// sufficiently aligned address, then free the entire range and
	50	// immediately allocate the desired subset. Another thread may have won
	51	// the race to map the target range, in which case a retry is needed.
	52	windows.VirtualFree(addr, 0, windows.MEM_RELEASE);
	53
	54	const overalloc_len = n + alignment_bytes - page_size;
	55	const aligned_len = mem.alignForward(usize, n, page_size);
	56
	57	while (true) {
	58	const reserved_addr = windows.VirtualAlloc(
	59	null,
	60	overalloc_len,
	61	windows.MEM_RESERVE,
	62	windows.PAGE_NOACCESS,
	63	) catch return null;
	64	const aligned_addr = mem.alignForward(usize, @intFromPtr(reserved_addr), alignment_bytes);
	65	windows.VirtualFree(reserved_addr, 0, windows.MEM_RELEASE);
	66	const ptr = windows.VirtualAlloc(
	67	@ptrFromInt(aligned_addr),
	68	aligned_len,
	69	windows.MEM_COMMIT \| windows.MEM_RESERVE,
	70	windows.PAGE_READWRITE,
	71	) catch continue;
	72	return @ptrCast(ptr);
	73	}
	74	}
39	75
40	76	const aligned_len = mem.alignForward(usize, n, page_size);
	77	const max_drop_len = alignment_bytes - @min(alignment_bytes, page_size);
	78	const overalloc_len = if (max_drop_len <= aligned_len - n)
	79	aligned_len
	80	else
	81	mem.alignForward(usize, aligned_len + max_drop_len, page_size);
41	82	const hint = @atomicLoad(@TypeOf(std.heap.next_mmap_addr_hint), &std.heap.next_mmap_addr_hint, .unordered);
42	83	const slice = posix.mmap(
43	84	hint,
44		aligned_len,
	85	overalloc_len,
45	86	posix.PROT.READ \| posix.PROT.WRITE,
46	87	.{ .TYPE = .PRIVATE, .ANONYMOUS = true },
47	88	-1,
48	89	0,
49	90	) catch return null;
50		assert(mem.isAligned(@intFromPtr(slice.ptr), page_size_min));
51		const new_hint: [*]align(std.heap.page_size_min) u8 = @alignCast(slice.ptr + aligned_len);
	91	const result_ptr = mem.alignPointer(slice.ptr, alignment_bytes) orelse return null;
	92	// Unmap the extra bytes that were only requested in order to guarantee
	93	// that the range of memory we were provided had a proper alignment in it
	94	// somewhere. The extra bytes could be at the beginning, or end, or both.
	95	const drop_len = result_ptr - slice.ptr;
	96	if (drop_len != 0) posix.munmap(slice[0..drop_len]);
	97	const remaining_len = overalloc_len - drop_len;
	98	if (remaining_len > aligned_len) posix.munmap(@alignCast(result_ptr[aligned_len..remaining_len]));
	99	const new_hint: [*]align(page_size_min) u8 = @alignCast(result_ptr + aligned_len);
52	100	_ = @cmpxchgStrong(@TypeOf(std.heap.next_mmap_addr_hint), &std.heap.next_mmap_addr_hint, hint, new_hint, .monotonic, .monotonic);
53		return slice.ptr;
	101	return result_ptr;
54	102	}
55	103
56	104	fn resize(
57		_: *anyopaque,
	105	context: *anyopaque,
58	106	buf_unaligned: []u8,
59	107	log2_buf_align: u8,
60	108	new_size: usize,
61	109	return_address: usize,
62	110	) bool {
	111	_ = context;
63	112	_ = log2_buf_align;
64	113	_ = return_address;
65	114	const page_size = std.heap.pageSize();
...	...	@@ -71,8 +120,8 @@ fn resize(
71	120	const old_addr_end = base_addr + buf_unaligned.len;
72	121	const new_addr_end = mem.alignForward(usize, base_addr + new_size, page_size);
73	122	if (old_addr_end > new_addr_end) {
74		// For shrinking that is not releasing, we will only
75		// decommit the pages not needed anymore.
	123	// For shrinking that is not releasing, we will only decommit
	124	// the pages not needed anymore.
76	125	windows.VirtualFree(
77	126	@as(*anyopaque, @ptrFromInt(new_addr_end)),
78	127	old_addr_end - new_addr_end,
...	...	@@ -104,7 +153,8 @@ fn resize(
104	153	return false;
105	154	}
106	155
107		fn free(_: *anyopaque, slice: []u8, log2_buf_align: u8, return_address: usize) void {
	156	fn free(context: *anyopaque, slice: []u8, log2_buf_align: u8, return_address: usize) void {
	157	_ = context;
108	158	_ = log2_buf_align;
109	159	_ = return_address;
110	160