Commit dc9648f868ed8ad08f040753767c03976bbcf3b7 | zig

author	Jonathan Marler <johnnymarler@gmail.com> 2020-04-17 14:15:36-06:00
committer	Jonathan Marler <johnnymarler@gmail.com> 2020-06-26 13:34:48-06:00
log	dc9648f868ed8ad08f040753767c03976bbcf3b7
tree	56025ae541deb21028ae9d96124bad9226987896
parent	129a4fb251f8eab22eacf219fbf81006baec3251

new allocator interface

10 files changed, 624 insertions(+), 461 deletions(-)

lib
- std
  - array_list.zig
  - c.zig
  - heap.zig
  - heap
    arena_allocator.zig
    logging_allocator.zig
  - mem.zig
  - os
    windows
    bits.zig
  - testing.zig
  - testing
    failing_allocator.zig
    leak_count_allocator.zig

lib/std/array_list.zig+4-2

...	...	@@ -219,7 +219,8 @@ pub fn ArrayListAligned(comptime T: type, comptime alignment: ?u29) type {
219	219	if (better_capacity >= new_capacity) break;
220	220	}
221	221
222		const new_memory = try self.allocator.realloc(self.allocatedSlice(), better_capacity);
	222	const new_memory = try self.allocator.reallocAtLeast(self.allocatedSlice(), better_capacity);
	223	assert(new_memory.len >= better_capacity);
223	224	self.items.ptr = new_memory.ptr;
224	225	self.capacity = new_memory.len;
225	226	}
...	...	@@ -441,7 +442,8 @@ pub fn ArrayListAlignedUnmanaged(comptime T: type, comptime alignment: ?u29) typ
441	442	if (better_capacity >= new_capacity) break;
442	443	}
443	444
444		const new_memory = try allocator.realloc(self.allocatedSlice(), better_capacity);
	445	const new_memory = try allocator.reallocAtLeast(self.allocatedSlice(), better_capacity);
	446	assert(new_memory.len >= better_capacity);
445	447	self.items.ptr = new_memory.ptr;
446	448	self.capacity = new_memory.len;
447	449	}

lib/std/c.zig+11

...	...	@@ -232,6 +232,17 @@ pub extern "c" fn setuid(uid: c_uint) c_int;
232	232
233	233	pub extern "c" fn aligned_alloc(alignment: usize, size: usize) ?*c_void;
234	234	pub extern "c" fn malloc(usize) ?*c_void;
	235
	236	pub usingnamespace switch (builtin.os.tag) {
	237	.linux, .freebsd, .kfreebsd, .netbsd, .openbsd => struct {
	238	pub extern "c" fn malloc_usable_size(?*const c_void) usize;
	239	},
	240	.macosx, .ios, .watchos, .tvos => struct {
	241	pub extern "c" fn malloc_size(?*const c_void) usize;
	242	},
	243	else => struct {},
	244	};
	245
235	246	pub extern "c" fn realloc(?c_void, usize) ?c_void;
236	247	pub extern "c" fn free(*c_void) void;
237	248	pub extern "c" fn posix_memalign(memptr: **c_void, alignment: usize, size: usize) c_int;

lib/std/heap.zig+271-323

...	...	@@ -15,48 +15,88 @@ pub const ArenaAllocator = @import("heap/arena_allocator.zig").ArenaAllocator;
15	15
16	16	const Allocator = mem.Allocator;
17	17
18		pub const c_allocator = &c_allocator_state;
	18	usingnamespace if (comptime @hasDecl(c, "malloc_size")) struct {
	19	pub const supports_malloc_size = true;
	20	pub const malloc_size = c.malloc_size;
	21	} else if (comptime @hasDecl(c, "malloc_usable_size")) struct {
	22	pub const supports_malloc_size = true;
	23	pub const malloc_size = c.malloc_usable_size;
	24	} else struct {
	25	pub const supports_malloc_size = false;
	26	};
	27
	28	pub const c_allocator = mem.getAllocatorPtr(&c_allocator_state);
19	29	var c_allocator_state = Allocator{
20		.reallocFn = cRealloc,
21		.shrinkFn = cShrink,
	30	.allocFn = cAlloc,
	31	.resizeFn = cResize,
22	32	};
23	33
24		fn cRealloc(self: *Allocator, old_mem: []u8, old_align: u29, new_size: usize, new_align: u29) ![]u8 {
25		assert(new_align <= @alignOf(c_longdouble));
26		const old_ptr = if (old_mem.len == 0) null else @ptrCast(*c_void, old_mem.ptr);
27		const buf = c.realloc(old_ptr, new_size) orelse return error.OutOfMemory;
28		return @ptrCast([*]u8, buf)[0..new_size];
	34	fn cAlloc(self: *Allocator, len: usize, ptr_align: u29, len_align: u29) Allocator.Error![]u8 {
	35	assert(ptr_align <= @alignOf(c_longdouble));
	36	const ptr = @ptrCast([*]u8, c.malloc(len) orelse return error.OutOfMemory);
	37	if (len_align == 0) {
	38	return ptr[0..len];
	39	}
	40	const full_len = init: {
	41	if (comptime supports_malloc_size) {
	42	const s = malloc_size(ptr);
	43	assert(s >= len);
	44	break :init s;
	45	}
	46	break :init len;
	47	};
	48	return ptr[0..mem.alignBackwardAnyAlign(full_len, len_align)];
29	49	}
30	50
31		fn cShrink(self: *Allocator, old_mem: []u8, old_align: u29, new_size: usize, new_align: u29) []u8 {
32		const old_ptr = @ptrCast(*c_void, old_mem.ptr);
33		const buf = c.realloc(old_ptr, new_size) orelse return old_mem[0..new_size];
34		return @ptrCast([*]u8, buf)[0..new_size];
	51	fn cResize(self: *Allocator, buf: []u8, new_len: usize, len_align: u29) Allocator.Error!usize {
	52	if (new_len == 0) {
	53	c.free(buf.ptr);
	54	return 0;
	55	}
	56	if (new_len <= buf.len) {
	57	return mem.alignAllocLen(buf.len, new_len, len_align);
	58	}
	59	if (comptime supports_malloc_size) {
	60	const full_len = malloc_size(buf.ptr);
	61	if (new_len <= full_len) {
	62	return mem.alignAllocLen(full_len, new_len, len_align);
	63	}
	64	}
	65	// TODO: could we still use realloc? are there any cases where we can guarantee that realloc won't move memory?
	66	return error.OutOfMemory;
35	67	}
36	68
37	69	/// This allocator makes a syscall directly for every allocation and free.
38	70	/// Thread-safe and lock-free.
39	71	pub const page_allocator = if (std.Target.current.isWasm())
40		&wasm_page_allocator_state
	72	mem.getAllocatorPtr(&wasm_page_allocator_state)
41	73	else if (std.Target.current.os.tag == .freestanding)
42	74	root.os.heap.page_allocator
43	75	else
44		&page_allocator_state;
	76	mem.getAllocatorPtr(&page_allocator_state);
45	77
46	78	var page_allocator_state = Allocator{
47		.reallocFn = PageAllocator.realloc,
48		.shrinkFn = PageAllocator.shrink,
	79	.allocFn = PageAllocator.alloc,
	80	.resizeFn = PageAllocator.resize,
49	81	};
50	82	var wasm_page_allocator_state = Allocator{
51		.reallocFn = WasmPageAllocator.realloc,
52		.shrinkFn = WasmPageAllocator.shrink,
	83	.allocFn = WasmPageAllocator.alloc,
	84	.resizeFn = WasmPageAllocator.resize,
53	85	};
54	86
55	87	pub const direct_allocator = @compileError("deprecated; use std.heap.page_allocator");
56	88
	89	/// Verifies that the adjusted length will still map to the full length
	90	pub fn alignPageAllocLen(full_len: usize, len: usize, len_align: u29) usize {
	91	const aligned_len = mem.alignAllocLen(full_len, len, len_align);
	92	assert(mem.alignForward(aligned_len, mem.page_size) == full_len);
	93	return aligned_len;
	94	}
	95
57	96	const PageAllocator = struct {
58		fn alloc(allocator: *Allocator, n: usize, alignment: u29) error{OutOfMemory}![]u8 {
59		if (n == 0) return &[0]u8{};
	97	fn alloc(allocator: *Allocator, n: usize, alignment: u29, len_align: u29) error{OutOfMemory}![]u8 {
	98	assert(n > 0);
	99	const alignedLen = mem.alignForward(n, mem.page_size);
60	100
61	101	if (builtin.os.tag == .windows) {
62	102	const w = os.windows;
...	...	@@ -68,21 +108,21 @@ const PageAllocator = struct {
68	108	// see https://devblogs.microsoft.com/oldnewthing/?p=42223
69	109	const addr = w.VirtualAlloc(
70	110	null,
71		n,
	111	alignedLen,
72	112	w.MEM_COMMIT \| w.MEM_RESERVE,
73	113	w.PAGE_READWRITE,
74	114	) catch return error.OutOfMemory;
75	115
76	116	// If the allocation is sufficiently aligned, use it.
77	117	if (@ptrToInt(addr) & (alignment - 1) == 0) {
78		return @ptrCast([*]u8, addr)[0..n];
	118	return @ptrCast([*]u8, addr)[0..alignPageAllocLen(alignedLen, n, len_align)];
79	119	}
80	120
81	121	// If it wasn't, actually do an explicitely aligned allocation.
82	122	w.VirtualFree(addr, 0, w.MEM_RELEASE);
83		const alloc_size = n + alignment;
	123	const alloc_size = n + alignment - mem.page_size;
84	124
85		const final_addr = while (true) {
	125	while (true) {
86	126	// Reserve a range of memory large enough to find a sufficiently
87	127	// aligned address.
88	128	const reserved_addr = w.VirtualAlloc(
...	...	@@ -102,48 +142,50 @@ const PageAllocator = struct {
102	142	// until it succeeds.
103	143	const ptr = w.VirtualAlloc(
104	144	@intToPtr(*c_void, aligned_addr),
105		n,
	145	alignedLen,
106	146	w.MEM_COMMIT \| w.MEM_RESERVE,
107	147	w.PAGE_READWRITE,
108	148	) catch continue;
109	149
110		return @ptrCast([*]u8, ptr)[0..n];
111		};
112
113		return @ptrCast([*]u8, final_addr)[0..n];
	150	return @ptrCast([*]u8, ptr)[0..alignPageAllocLen(alignedLen, n, len_align)];
	151	}
114	152	}
115	153
116		const alloc_size = if (alignment <= mem.page_size) n else n + alignment;
	154	const maxDropLen = alignment - std.math.min(alignment, mem.page_size);
	155	const allocLen = if (maxDropLen <= alignedLen - n) alignedLen
	156	else mem.alignForward(alignedLen + maxDropLen, mem.page_size);
117	157	const slice = os.mmap(
118	158	null,
119		mem.alignForward(alloc_size, mem.page_size),
	159	allocLen,
120	160	os.PROT_READ \| os.PROT_WRITE,
121	161	os.MAP_PRIVATE \| os.MAP_ANONYMOUS,
122	162	-1,
123	163	0,
124	164	) catch return error.OutOfMemory;
125		if (alloc_size == n) return slice[0..n];
	165	assert(mem.isAligned(@ptrToInt(slice.ptr), mem.page_size));
126	166
127	167	const aligned_addr = mem.alignForward(@ptrToInt(slice.ptr), alignment);
128	168
129	169	// Unmap the extra bytes that were only requested in order to guarantee
130	170	// that the range of memory we were provided had a proper alignment in
131	171	// it somewhere. The extra bytes could be at the beginning, or end, or both.
132		const unused_start_len = aligned_addr - @ptrToInt(slice.ptr);
133		if (unused_start_len != 0) {
134		os.munmap(slice[0..unused_start_len]);
	172	const dropLen = aligned_addr - @ptrToInt(slice.ptr);
	173	if (dropLen != 0) {
	174	os.munmap(slice[0..dropLen]);
135	175	}
136		const aligned_end_addr = mem.alignForward(aligned_addr + n, mem.page_size);
137		const unused_end_len = @ptrToInt(slice.ptr) + slice.len - aligned_end_addr;
138		if (unused_end_len != 0) {
139		os.munmap(@intToPtr([*]align(mem.page_size) u8, aligned_end_addr)[0..unused_end_len]);
	176
	177	// Unmap extra pages
	178	const alignedBufferLen = allocLen - dropLen;
	179	if (alignedBufferLen > alignedLen) {
	180	os.munmap(@alignCast(mem.page_size, @intToPtr([*]u8, aligned_addr))[alignedLen..alignedBufferLen]);
140	181	}
141	182
142		return @intToPtr([*]u8, aligned_addr)[0..n];
	183	return @intToPtr([*]u8, aligned_addr)[0..alignPageAllocLen(alignedLen, n, len_align)];
143	184	}
144	185
145		fn shrink(allocator: *Allocator, old_mem_unaligned: []u8, old_align: u29, new_size: usize, new_align: u29) []u8 {
146		const old_mem = @alignCast(mem.page_size, old_mem_unaligned);
	186	fn resize(allocator: *Allocator, buf_unaligned: []u8, new_size: usize, len_align: u29) Allocator.Error!usize {
	187	const new_size_aligned = mem.alignForward(new_size, mem.page_size);
	188
147	189	if (builtin.os.tag == .windows) {
148	190	const w = os.windows;
149	191	if (new_size == 0) {
...	...	@@ -153,100 +195,45 @@ const PageAllocator = struct {
153	195	// is reserved in the initial allocation call to VirtualAlloc."
154	196	// So we can only use MEM_RELEASE when actually releasing the
155	197	// whole allocation.
156		w.VirtualFree(old_mem.ptr, 0, w.MEM_RELEASE);
157		} else {
158		const base_addr = @ptrToInt(old_mem.ptr);
159		const old_addr_end = base_addr + old_mem.len;
160		const new_addr_end = base_addr + new_size;
161		const new_addr_end_rounded = mem.alignForward(new_addr_end, mem.page_size);
162		if (old_addr_end > new_addr_end_rounded) {
	198	w.VirtualFree(buf_unaligned.ptr, 0, w.MEM_RELEASE);
	199	return 0;
	200	}
	201	if (new_size < buf_unaligned.len) {
	202	const base_addr = @ptrToInt(buf_unaligned.ptr);
	203	const old_addr_end = base_addr + buf_unaligned.len;
	204	const new_addr_end = mem.alignForward(base_addr + new_size, mem.page_size);
	205	if (old_addr_end > new_addr_end) {
163	206	// For shrinking that is not releasing, we will only
164	207	// decommit the pages not needed anymore.
165	208	w.VirtualFree(
166		@intToPtr(*c_void, new_addr_end_rounded),
167		old_addr_end - new_addr_end_rounded,
	209	@intToPtr(*c_void, new_addr_end),
	210	old_addr_end - new_addr_end,
168	211	w.MEM_DECOMMIT,
169	212	);
170	213	}
	214	return alignPageAllocLen(new_size_aligned, new_size, len_align);
171	215	}
172		return old_mem[0..new_size];
173		}
174		const base_addr = @ptrToInt(old_mem.ptr);
175		const old_addr_end = base_addr + old_mem.len;
176		const new_addr_end = base_addr + new_size;
177		const new_addr_end_rounded = mem.alignForward(new_addr_end, mem.page_size);
178		if (old_addr_end > new_addr_end_rounded) {
179		const ptr = @intToPtr([*]align(mem.page_size) u8, new_addr_end_rounded);
180		os.munmap(ptr[0 .. old_addr_end - new_addr_end_rounded]);
181		}
182		return old_mem[0..new_size];
183		}
184
185		fn realloc(allocator: *Allocator, old_mem_unaligned: []u8, old_align: u29, new_size: usize, new_align: u29) ![]u8 {
186		const old_mem = @alignCast(mem.page_size, old_mem_unaligned);
187		if (builtin.os.tag == .windows) {
188		if (old_mem.len == 0) {
189		return alloc(allocator, new_size, new_align);
	216	if (new_size == buf_unaligned.len) {
	217	return alignPageAllocLen(new_size_aligned, new_size, len_align);
190	218	}
	219	// new_size > buf_unaligned.len not implemented
	220	return error.OutOfMemory;
	221	}
191	222
192		if (new_size <= old_mem.len and new_align <= old_align) {
193		return shrink(allocator, old_mem, old_align, new_size, new_align);
194		}
195
196		const w = os.windows;
197		const base_addr = @ptrToInt(old_mem.ptr);
198
199		if (new_align > old_align and base_addr & (new_align - 1) != 0) {
200		// Current allocation doesn't satisfy the new alignment.
201		// For now we'll do a new one no matter what, but maybe
202		// there is something smarter to do instead.
203		const result = try alloc(allocator, new_size, new_align);
204		assert(old_mem.len != 0);
205		@memcpy(result.ptr, old_mem.ptr, std.math.min(old_mem.len, result.len));
206		w.VirtualFree(old_mem.ptr, 0, w.MEM_RELEASE);
207
208		return result;
209		}
210
211		const old_addr_end = base_addr + old_mem.len;
212		const old_addr_end_rounded = mem.alignForward(old_addr_end, mem.page_size);
213		const new_addr_end = base_addr + new_size;
214		const new_addr_end_rounded = mem.alignForward(new_addr_end, mem.page_size);
215		if (new_addr_end_rounded == old_addr_end_rounded) {
216		// The reallocation fits in the already allocated pages.
217		return @ptrCast([*]u8, old_mem.ptr)[0..new_size];
218		}
219		assert(new_addr_end_rounded > old_addr_end_rounded);
220
221		// We need to commit new pages.
222		const additional_size = new_addr_end - old_addr_end_rounded;
223		const realloc_addr = w.kernel32.VirtualAlloc(
224		@intToPtr(*c_void, old_addr_end_rounded),
225		additional_size,
226		w.MEM_COMMIT \| w.MEM_RESERVE,
227		w.PAGE_READWRITE,
228		) orelse {
229		// Committing new pages at the end of the existing allocation
230		// failed, we need to try a new one.
231		const new_alloc_mem = try alloc(allocator, new_size, new_align);
232		@memcpy(new_alloc_mem.ptr, old_mem.ptr, old_mem.len);
233		w.VirtualFree(old_mem.ptr, 0, w.MEM_RELEASE);
234
235		return new_alloc_mem;
236		};
	223	const buf_aligned_len = mem.alignForward(buf_unaligned.len, mem.page_size);
	224	if (new_size_aligned == buf_aligned_len)
	225	return alignPageAllocLen(new_size_aligned, new_size, len_align);
237	226
238		assert(@ptrToInt(realloc_addr) == old_addr_end_rounded);
239		return @ptrCast([*]u8, old_mem.ptr)[0..new_size];
240		}
241		if (new_size <= old_mem.len and new_align <= old_align) {
242		return shrink(allocator, old_mem, old_align, new_size, new_align);
243		}
244		const result = try alloc(allocator, new_size, new_align);
245		if (old_mem.len != 0) {
246		@memcpy(result.ptr, old_mem.ptr, std.math.min(old_mem.len, result.len));
247		os.munmap(old_mem);
	227	if (new_size_aligned < buf_aligned_len) {
	228	const ptr = @intToPtr([*]align(mem.page_size) u8, @ptrToInt(buf_unaligned.ptr) + new_size_aligned);
	229	os.munmap(ptr[0 .. buf_aligned_len - new_size_aligned]);
	230	if (new_size_aligned == 0)
	231	return 0;
	232	return alignPageAllocLen(new_size_aligned, new_size, len_align);
248	233	}
249		return result;
	234
	235	// TODO: call mremap
	236	return error.OutOfMemory;
250	237	}
251	238	};
252	239
...	...	@@ -338,16 +325,24 @@ const WasmPageAllocator = struct {
338	325	}
339	326
340	327	fn nPages(memsize: usize) usize {
341		return std.mem.alignForward(memsize, std.mem.page_size) / std.mem.page_size;
	328	return mem.alignForward(memsize, mem.page_size) / mem.page_size;
342	329	}
343	330
344		fn alloc(allocator: *Allocator, page_count: usize, alignment: u29) error{OutOfMemory}!usize {
345		var idx = conventional.useRecycled(page_count);
346		if (idx != FreeBlock.not_found) {
347		return idx;
	331	fn alloc(allocator: *Allocator, len: usize, alignment: u29, len_align: u29) error{OutOfMemory}![]u8 {
	332	const page_count = nPages(len);
	333	const page_idx = try allocPages(page_count);
	334	return @intToPtr([]u8, page_idx mem.page_size)
	335	[0..alignPageAllocLen(page_count * mem.page_size, len, len_align)];
	336	}
	337	fn allocPages(page_count: usize) !usize {
	338	{
	339	const idx = conventional.useRecycled(page_count);
	340	if (idx != FreeBlock.not_found) {
	341	return idx;
	342	}
348	343	}
349	344
350		idx = extended.useRecycled(page_count);
	345	const idx = extended.useRecycled(page_count);
351	346	if (idx != FreeBlock.not_found) {
352	347	return idx + extendedOffset();
353	348	}
...	...	@@ -360,51 +355,36 @@ const WasmPageAllocator = struct {
360	355	return @intCast(usize, prev_page_count);
361	356	}
362	357
363		pub fn realloc(allocator: *Allocator, old_mem: []u8, old_align: u29, new_size: usize, new_align: u29) Allocator.Error![]u8 {
364		if (new_align > std.mem.page_size) {
365		return error.OutOfMemory;
	358	fn freePages(start: usize, end: usize) void {
	359	if (start < extendedOffset()) {
	360	conventional.recycle(start, std.math.min(extendedOffset(), end) - start);
366	361	}
367
368		if (nPages(new_size) == nPages(old_mem.len)) {
369		return old_mem.ptr[0..new_size];
370		} else if (new_size < old_mem.len) {
371		return shrink(allocator, old_mem, old_align, new_size, new_align);
372		} else {
373		const page_idx = try alloc(allocator, nPages(new_size), new_align);
374		const new_mem = @intToPtr([]u8, page_idx std.mem.page_size)[0..new_size];
375		std.mem.copy(u8, new_mem, old_mem);
376		_ = shrink(allocator, old_mem, old_align, 0, 0);
377		return new_mem;
	362	if (end > extendedOffset()) {
	363	var new_end = end;
	364	if (!extended.isInitialized()) {
	365	// Steal the last page from the memory currently being recycled
	366	// TODO: would it be better if we use the first page instead?
	367	new_end -= 1;
	368
	369	extended.data = @intToPtr([]u128, new_end mem.page_size)[0 .. mem.page_size / @sizeOf(u128)];
	370	// Since this is the first page being freed and we consume it, assume nothing is free.
	371	mem.set(u128, extended.data, PageStatus.none_free);
	372	}
	373	const clamped_start = std.math.max(extendedOffset(), start);
	374	extended.recycle(clamped_start - extendedOffset(), new_end - clamped_start);
378	375	}
379	376	}
380	377
381		pub fn shrink(allocator: *Allocator, old_mem: []u8, old_align: u29, new_size: usize, new_align: u29) []u8 {
382		@setCold(true);
383		const free_start = nPages(@ptrToInt(old_mem.ptr) + new_size);
384		var free_end = nPages(@ptrToInt(old_mem.ptr) + old_mem.len);
385
386		if (free_end > free_start) {
387		if (free_start < extendedOffset()) {
388		const clamped_end = std.math.min(extendedOffset(), free_end);
389		conventional.recycle(free_start, clamped_end - free_start);
390		}
391
392		if (free_end > extendedOffset()) {
393		if (!extended.isInitialized()) {
394		// Steal the last page from the memory currently being recycled
395		// TODO: would it be better if we use the first page instead?
396		free_end -= 1;
397
398		extended.data = @intToPtr([]u128, free_end std.mem.page_size)[0 .. std.mem.page_size / @sizeOf(u128)];
399		// Since this is the first page being freed and we consume it, assume nothing is free.
400		std.mem.set(u128, extended.data, PageStatus.none_free);
401		}
402		const clamped_start = std.math.max(extendedOffset(), free_start);
403		extended.recycle(clamped_start - extendedOffset(), free_end - clamped_start);
404		}
	378	fn resize(allocator: *Allocator, buf: []u8, new_len: usize, len_align: u29) error{OutOfMemory}!usize {
	379	const aligned_len = mem.alignForward(buf.len, mem.page_size);
	380	if (new_len > aligned_len) return error.OutOfMemory;
	381	const current_n = nPages(aligned_len);
	382	const new_n = nPages(new_len);
	383	if (new_n != current_n) {
	384	const base = nPages(@ptrToInt(buf.ptr));
	385	freePages(base + new_n, base + current_n);
405	386	}
406
407		return old_mem[0..new_size];
	387	return if (new_len == 0) 0 else alignPageAllocLen(new_n * mem.page_size, new_len, len_align);
408	388	}
409	389	};
410	390
...	...	@@ -418,8 +398,8 @@ pub const HeapAllocator = switch (builtin.os.tag) {
418	398	pub fn init() HeapAllocator {
419	399	return HeapAllocator{
420	400	.allocator = Allocator{
421		.reallocFn = realloc,
422		.shrinkFn = shrink,
	401	.allocFn = alloc,
	402	.resizeFn = resize,
423	403	},
424	404	.heap_handle = null,
425	405	};
...	...	@@ -431,11 +411,14 @@ pub const HeapAllocator = switch (builtin.os.tag) {
431	411	}
432	412	}
433	413
434		fn alloc(allocator: *Allocator, n: usize, alignment: u29) error{OutOfMemory}![]u8 {
	414	fn getRecordPtr(buf: []u8) *align(1) usize {
	415	return @intToPtr(*align(1) usize, @ptrToInt(buf.ptr) + buf.len);
	416	}
	417
	418	fn alloc(allocator: *Allocator, n: usize, ptr_align: u29, len_align: u29) error{OutOfMemory}![]u8 {
435	419	const self = @fieldParentPtr(HeapAllocator, "allocator", allocator);
436		if (n == 0) return &[0]u8{};
437	420
438		const amt = n + alignment + @sizeOf(usize);
	421	const amt = n + ptr_align - 1 + @sizeOf(usize);
439	422	const optional_heap_handle = @atomicLoad(?HeapHandle, &self.heap_handle, builtin.AtomicOrder.SeqCst);
440	423	const heap_handle = optional_heap_handle orelse blk: {
441	424	const options = if (builtin.single_threaded) os.windows.HEAP_NO_SERIALIZE else 0;
...	...	@@ -446,66 +429,60 @@ pub const HeapAllocator = switch (builtin.os.tag) {
446	429	};
447	430	const ptr = os.windows.kernel32.HeapAlloc(heap_handle, 0, amt) orelse return error.OutOfMemory;
448	431	const root_addr = @ptrToInt(ptr);
449		const adjusted_addr = mem.alignForward(root_addr, alignment);
450		const record_addr = adjusted_addr + n;
451		@intToPtr(align(1) usize, record_addr). = root_addr;
452		return @intToPtr([*]u8, adjusted_addr)[0..n];
453		}
454
455		fn shrink(allocator: *Allocator, old_mem: []u8, old_align: u29, new_size: usize, new_align: u29) []u8 {
456		return realloc(allocator, old_mem, old_align, new_size, new_align) catch {
457		const old_adjusted_addr = @ptrToInt(old_mem.ptr);
458		const old_record_addr = old_adjusted_addr + old_mem.len;
459		const root_addr = @intToPtr(align(1) usize, old_record_addr).;
460		const old_ptr = @intToPtr(*c_void, root_addr);
461		const new_record_addr = old_record_addr - new_size + old_mem.len;
462		@intToPtr(align(1) usize, new_record_addr). = root_addr;
463		return old_mem[0..new_size];
	432	const aligned_addr = mem.alignForward(root_addr, ptr_align);
	433	const return_len = init: {
	434	if (len_align == 0) break :init n;
	435	const full_len = os.windows.kernel32.HeapSize(heap_handle, 0, ptr);
	436	assert(full_len != std.math.maxInt(usize));
	437	assert(full_len >= amt);
	438	break :init mem.alignBackwardAnyAlign(full_len - (aligned_addr - root_addr), len_align);
464	439	};
	440	const buf = @intToPtr([*]u8, aligned_addr)[0..return_len];
	441	getRecordPtr(buf).* = root_addr;
	442	return buf;
465	443	}
466	444
467		fn realloc(allocator: *Allocator, old_mem: []u8, old_align: u29, new_size: usize, new_align: u29) ![]u8 {
468		if (old_mem.len == 0) return alloc(allocator, new_size, new_align);
469
	445	fn resize(allocator: *Allocator, buf: []u8, new_size: usize, len_align: u29) error{OutOfMemory}!usize {
470	446	const self = @fieldParentPtr(HeapAllocator, "allocator", allocator);
471		const old_adjusted_addr = @ptrToInt(old_mem.ptr);
472		const old_record_addr = old_adjusted_addr + old_mem.len;
473		const root_addr = @intToPtr(align(1) usize, old_record_addr).;
474		const old_ptr = @intToPtr(*c_void, root_addr);
475
476	447	if (new_size == 0) {
477		os.windows.HeapFree(self.heap_handle.?, 0, old_ptr);
478		return old_mem[0..0];
	448	os.windows.HeapFree(self.heap_handle.?, 0, @intToPtr(c_void ,getRecordPtr(buf).));
	449	return 0;
479	450	}
480	451
481		const amt = new_size + new_align + @sizeOf(usize);
	452	const root_addr = getRecordPtr(buf).*;
	453	const align_offset = @ptrToInt(buf.ptr) - root_addr;
	454	const amt = align_offset + new_size + @sizeOf(usize);
482	455	const new_ptr = os.windows.kernel32.HeapReAlloc(
483	456	self.heap_handle.?,
484		0,
485		old_ptr,
	457	os.windows.HEAP_REALLOC_IN_PLACE_ONLY,
	458	@intToPtr(*c_void, root_addr),
486	459	amt,
487	460	) orelse return error.OutOfMemory;
488		const offset = old_adjusted_addr - root_addr;
489		const new_root_addr = @ptrToInt(new_ptr);
490		var new_adjusted_addr = new_root_addr + offset;
491		const offset_is_valid = new_adjusted_addr + new_size + @sizeOf(usize) <= new_root_addr + amt;
492		const offset_is_aligned = new_adjusted_addr % new_align == 0;
493		if (!offset_is_valid or !offset_is_aligned) {
494		// If HeapReAlloc didn't happen to move the memory to the new alignment,
495		// or the memory starting at the old offset would be outside of the new allocation,
496		// then we need to copy the memory to a valid aligned address and use that
497		const new_aligned_addr = mem.alignForward(new_root_addr, new_align);
498		@memcpy(@intToPtr([]u8, new_aligned_addr), @intToPtr([]u8, new_adjusted_addr), std.math.min(old_mem.len, new_size));
499		new_adjusted_addr = new_aligned_addr;
500		}
501		const new_record_addr = new_adjusted_addr + new_size;
502		@intToPtr(align(1) usize, new_record_addr). = new_root_addr;
503		return @intToPtr([*]u8, new_adjusted_addr)[0..new_size];
	461	assert(new_ptr == @intToPtr(*c_void, root_addr));
	462	const return_len = init: {
	463	if (len_align == 0) break :init new_size;
	464	const full_len = os.windows.kernel32.HeapSize(self.heap_handle.?, 0, new_ptr);
	465	assert(full_len != std.math.maxInt(usize));
	466	assert(full_len >= amt);
	467	break :init mem.alignBackwardAnyAlign(full_len - align_offset, len_align);
	468	};
	469	getRecordPtr(buf.ptr[0..return_len]).* = root_addr;
	470	return return_len;
504	471	}
505	472	},
506	473	else => @compileError("Unsupported OS"),
507	474	};
508	475
	476	fn sliceContainsPtr(container: []u8, ptr: [*]u8) bool {
	477	return @ptrToInt(ptr) >= @ptrToInt(container.ptr) and
	478	@ptrToInt(ptr) < (@ptrToInt(container.ptr) + container.len);
	479	}
	480
	481	fn sliceContainsSlice(container: []u8, slice: []u8) bool {
	482	return @ptrToInt(slice.ptr) >= @ptrToInt(container.ptr) and
	483	(@ptrToInt(slice.ptr) + slice.len) <= (@ptrToInt(container.ptr) + container.len);
	484	}
	485
509	486	pub const FixedBufferAllocator = struct {
510	487	allocator: Allocator,
511	488	end_index: usize,
...	...	@@ -514,19 +491,33 @@ pub const FixedBufferAllocator = struct {
514	491	pub fn init(buffer: []u8) FixedBufferAllocator {
515	492	return FixedBufferAllocator{
516	493	.allocator = Allocator{
517		.reallocFn = realloc,
518		.shrinkFn = shrink,
	494	.allocFn = alloc,
	495	.resizeFn = resize,
519	496	},
520	497	.buffer = buffer,
521	498	.end_index = 0,
522	499	};
523	500	}
524	501
525		fn alloc(allocator: *Allocator, n: usize, alignment: u29) ![]u8 {
	502	pub fn ownsPtr(self: FixedBufferAllocator, ptr: []u8) bool {
	503	return sliceContainsPtr(self.buffer, ptr);
	504	}
	505
	506	pub fn ownsSlice(self: *FixedBufferAllocator, slice: []u8) bool {
	507	return sliceContainsSlice(self.buffer, slice);
	508	}
	509
	510	// NOTE: this will not work in all cases, if the last allocation had an adjusted_index
	511	// then we won't be able to determine what the last allocation was. This is because
	512	// the alignForward operation done in alloc is not reverisible.
	513	pub fn isLastAllocation(self: *FixedBufferAllocator, buf: []u8) bool {
	514	return buf.ptr + buf.len == self.buffer.ptr + self.end_index;
	515	}
	516
	517	fn alloc(allocator: *Allocator, n: usize, ptr_align: u29, len_align: u29) ![]u8 {
526	518	const self = @fieldParentPtr(FixedBufferAllocator, "allocator", allocator);
527		const addr = @ptrToInt(self.buffer.ptr) + self.end_index;
528		const adjusted_addr = mem.alignForward(addr, alignment);
529		const adjusted_index = self.end_index + (adjusted_addr - addr);
	519	const aligned_addr = mem.alignForward(@ptrToInt(self.buffer.ptr) + self.end_index, ptr_align);
	520	const adjusted_index = aligned_addr - @ptrToInt(self.buffer.ptr);
530	521	const new_end_index = adjusted_index + n;
531	522	if (new_end_index > self.buffer.len) {
532	523	return error.OutOfMemory;
...	...	@@ -534,33 +525,32 @@ pub const FixedBufferAllocator = struct {
534	525	const result = self.buffer[adjusted_index..new_end_index];
535	526	self.end_index = new_end_index;
536	527
537		return result;
	528	return result[0..mem.alignAllocLen(result.len, n, len_align)];
538	529	}
539	530
540		fn realloc(allocator: *Allocator, old_mem: []u8, old_align: u29, new_size: usize, new_align: u29) ![]u8 {
	531	fn resize(allocator: *Allocator, buf: []u8, new_size: usize, len_align: u29) Allocator.Error!usize {
541	532	const self = @fieldParentPtr(FixedBufferAllocator, "allocator", allocator);
542		assert(old_mem.len <= self.end_index);
543		if (old_mem.ptr == self.buffer.ptr + self.end_index - old_mem.len and
544		mem.alignForward(@ptrToInt(old_mem.ptr), new_align) == @ptrToInt(old_mem.ptr))
545		{
546		const start_index = self.end_index - old_mem.len;
547		const new_end_index = start_index + new_size;
548		if (new_end_index > self.buffer.len) return error.OutOfMemory;
549		const result = self.buffer[start_index..new_end_index];
550		self.end_index = new_end_index;
551		return result;
552		} else if (new_size <= old_mem.len and new_align <= old_align) {
553		// We can't do anything with the memory, so tell the client to keep it.
554		return error.OutOfMemory;
555		} else {
556		const result = try alloc(allocator, new_size, new_align);
557		@memcpy(result.ptr, old_mem.ptr, std.math.min(old_mem.len, result.len));
558		return result;
	533	assert(self.ownsSlice(buf)); // sanity check
	534
	535	if (!self.isLastAllocation(buf)) {
	536	if (new_size > buf.len)
	537	return error.OutOfMemory;
	538	return if (new_size == 0) 0 else mem.alignAllocLen(buf.len, new_size, len_align);
559	539	}
560		}
561	540
562		fn shrink(allocator: *Allocator, old_mem: []u8, old_align: u29, new_size: usize, new_align: u29) []u8 {
563		return old_mem[0..new_size];
	541	if (new_size <= buf.len) {
	542	const sub = buf.len - new_size;
	543	self.end_index -= sub;
	544	return if (new_size == 0) 0 else mem.alignAllocLen(buf.len - sub, new_size, len_align);
	545	}
	546
	547	var add = new_size - buf.len;
	548	if (add + self.end_index > self.buffer.len) {
	549	//add = self.buffer.len - self.end_index;
	550	return error.OutOfMemory;
	551	}
	552	self.end_index += add;
	553	return mem.alignAllocLen(buf.len + add, new_size, len_align);
564	554	}
565	555
566	556	pub fn reset(self: *FixedBufferAllocator) void {
...	...	@@ -581,20 +571,20 @@ pub const ThreadSafeFixedBufferAllocator = blk: {
581	571	pub fn init(buffer: []u8) ThreadSafeFixedBufferAllocator {
582	572	return ThreadSafeFixedBufferAllocator{
583	573	.allocator = Allocator{
584		.reallocFn = realloc,
585		.shrinkFn = shrink,
	574	.allocFn = alloc,
	575	.resizeFn = Allocator.noResize,
586	576	},
587	577	.buffer = buffer,
588	578	.end_index = 0,
589	579	};
590	580	}
591	581
592		fn alloc(allocator: *Allocator, n: usize, alignment: u29) ![]u8 {
	582	fn alloc(allocator: *Allocator, n: usize, ptr_align: u29, len_align: u29) ![]u8 {
593	583	const self = @fieldParentPtr(ThreadSafeFixedBufferAllocator, "allocator", allocator);
594	584	var end_index = @atomicLoad(usize, &self.end_index, builtin.AtomicOrder.SeqCst);
595	585	while (true) {
596	586	const addr = @ptrToInt(self.buffer.ptr) + end_index;
597		const adjusted_addr = mem.alignForward(addr, alignment);
	587	const adjusted_addr = mem.alignForward(addr, ptr_align);
598	588	const adjusted_index = end_index + (adjusted_addr - addr);
599	589	const new_end_index = adjusted_index + n;
600	590	if (new_end_index > self.buffer.len) {
...	...	@@ -604,21 +594,6 @@ pub const ThreadSafeFixedBufferAllocator = blk: {
604	594	}
605	595	}
606	596
607		fn realloc(allocator: *Allocator, old_mem: []u8, old_align: u29, new_size: usize, new_align: u29) ![]u8 {
608		if (new_size <= old_mem.len and new_align <= old_align) {
609		// We can't do anything useful with the memory, tell the client to keep it.
610		return error.OutOfMemory;
611		} else {
612		const result = try alloc(allocator, new_size, new_align);
613		@memcpy(result.ptr, old_mem.ptr, std.math.min(old_mem.len, result.len));
614		return result;
615		}
616		}
617
618		fn shrink(allocator: *Allocator, old_mem: []u8, old_align: u29, new_size: usize, new_align: u29) []u8 {
619		return old_mem[0..new_size];
620		}
621
622	597	pub fn reset(self: *ThreadSafeFixedBufferAllocator) void {
623	598	self.end_index = 0;
624	599	}
...	...	@@ -632,8 +607,8 @@ pub fn stackFallback(comptime size: usize, fallback_allocator: *Allocator) Stack
632	607	.fallback_allocator = fallback_allocator,
633	608	.fixed_buffer_allocator = undefined,
634	609	.allocator = Allocator{
635		.reallocFn = StackFallbackAllocator(size).realloc,
636		.shrinkFn = StackFallbackAllocator(size).shrink,
	610	.allocFn = StackFallbackAllocator(size).realloc,
	611	.resizeFn = StackFallbackAllocator(size).resize,
637	612	},
638	613	};
639	614	}
...	...	@@ -652,58 +627,19 @@ pub fn StackFallbackAllocator(comptime size: usize) type {
652	627	return &self.allocator;
653	628	}
654	629
655		fn realloc(allocator: *Allocator, old_mem: []u8, old_align: u29, new_size: usize, new_align: u29) ![]u8 {
	630	fn alloc(allocator: Allocator, len: usize, ptr_align: u29, len_align: u29) error{OutOfMemory}![]u8 {
656	631	const self = @fieldParentPtr(Self, "allocator", allocator);
657		const in_buffer = @ptrToInt(old_mem.ptr) >= @ptrToInt(&self.buffer) and
658		@ptrToInt(old_mem.ptr) < @ptrToInt(&self.buffer) + self.buffer.len;
659		if (in_buffer) {
660		return FixedBufferAllocator.realloc(
661		&self.fixed_buffer_allocator.allocator,
662		old_mem,
663		old_align,
664		new_size,
665		new_align,
666		) catch {
667		const result = try self.fallback_allocator.reallocFn(
668		self.fallback_allocator,
669		&[0]u8{},
670		undefined,
671		new_size,
672		new_align,
673		);
674		mem.copy(u8, result, old_mem);
675		return result;
676		};
677		}
678		return self.fallback_allocator.reallocFn(
679		self.fallback_allocator,
680		old_mem,
681		old_align,
682		new_size,
683		new_align,
684		);
	632	return FixedBufferAllocator.alloc(&self.fixed_buffer_allocator, len, ptr_align) catch
	633	return fallback_allocator.alloc(len, ptr_align);
685	634	}
686	635
687		fn shrink(allocator: *Allocator, old_mem: []u8, old_align: u29, new_size: usize, new_align: u29) []u8 {
	636	fn resize(self: *Allocator, buf: []u8, new_len: usize, len_align: u29) error{OutOfMemory}!void {
688	637	const self = @fieldParentPtr(Self, "allocator", allocator);
689		const in_buffer = @ptrToInt(old_mem.ptr) >= @ptrToInt(&self.buffer) and
690		@ptrToInt(old_mem.ptr) < @ptrToInt(&self.buffer) + self.buffer.len;
691		if (in_buffer) {
692		return FixedBufferAllocator.shrink(
693		&self.fixed_buffer_allocator.allocator,
694		old_mem,
695		old_align,
696		new_size,
697		new_align,
698		);
	638	if (self.fixed_buffer_allocator.ownsPtr(buf.ptr)) {
	639	try self.fixed_buffer_allocator.callResizeFn(buf, new_len);
	640	} else {
	641	try self.fallback_allocator.callResizeFn(buf, new_len);
699	642	}
700		return self.fallback_allocator.shrinkFn(
701		self.fallback_allocator,
702		old_mem,
703		old_align,
704		new_size,
705		new_align,
706		);
707	643	}
708	644	};
709	645	}
...	...	@@ -718,8 +654,8 @@ test "c_allocator" {
718	654
719	655	test "WasmPageAllocator internals" {
720	656	if (comptime std.Target.current.isWasm()) {
721		const conventional_memsize = WasmPageAllocator.conventional.totalPages() * std.mem.page_size;
722		const initial = try page_allocator.alloc(u8, std.mem.page_size);
	657	const conventional_memsize = WasmPageAllocator.conventional.totalPages() * mem.page_size;
	658	const initial = try page_allocator.alloc(u8, mem.page_size);
723	659	std.debug.assert(@ptrToInt(initial.ptr) < conventional_memsize); // If this isn't conventional, the rest of these tests don't make sense. Also we have a serious memory leak in the test suite.
724	660
725	661	var inplace = try page_allocator.realloc(initial, 1);
...	...	@@ -799,7 +735,7 @@ test "ArenaAllocator" {
799	735
800	736	var test_fixed_buffer_allocator_memory: [800000 * @sizeOf(u64)]u8 = undefined;
801	737	test "FixedBufferAllocator" {
802		var fixed_buffer_allocator = FixedBufferAllocator.init(test_fixed_buffer_allocator_memory[0..]);
	738	var fixed_buffer_allocator = mem.sanityWrap(FixedBufferAllocator.init(test_fixed_buffer_allocator_memory[0..]));
803	739
804	740	try testAllocator(&fixed_buffer_allocator.allocator);
805	741	try testAllocatorAligned(&fixed_buffer_allocator.allocator, 16);
...	...	@@ -865,7 +801,10 @@ test "ThreadSafeFixedBufferAllocator" {
865	801	try testAllocatorAlignedShrink(&fixed_buffer_allocator.allocator);
866	802	}
867	803
868		fn testAllocator(allocator: *mem.Allocator) !void {
	804	fn testAllocator(base_allocator: *mem.Allocator) !void {
	805	var sanityAllocator = mem.sanityWrap(base_allocator);
	806	const allocator = &sanityAllocator.allocator;
	807
869	808	var slice = try allocator.alloc(*i32, 100);
870	809	testing.expect(slice.len == 100);
871	810	for (slice) \|*item, i\| {
...	...	@@ -893,7 +832,10 @@ fn testAllocator(allocator: *mem.Allocator) !void {
893	832	allocator.free(slice);
894	833	}
895	834
896		fn testAllocatorAligned(allocator: *mem.Allocator, comptime alignment: u29) !void {
	835	fn testAllocatorAligned(base_allocator: *mem.Allocator, comptime alignment: u29) !void {
	836	var sanityAllocator = mem.sanityWrap(base_allocator);
	837	const allocator = &sanityAllocator.allocator;
	838
897	839	// initial
898	840	var slice = try allocator.alignedAlloc(u8, alignment, 10);
899	841	testing.expect(slice.len == 10);
...	...	@@ -917,7 +859,10 @@ fn testAllocatorAligned(allocator: *mem.Allocator, comptime alignment: u29) !voi
917	859	testing.expect(slice.len == 0);
918	860	}
919	861
920		fn testAllocatorLargeAlignment(allocator: *mem.Allocator) mem.Allocator.Error!void {
	862	fn testAllocatorLargeAlignment(base_allocator: *mem.Allocator) mem.Allocator.Error!void {
	863	var sanityAllocator = mem.sanityWrap(base_allocator);
	864	const allocator = &sanityAllocator.allocator;
	865
921	866	//Maybe a platform's page_size is actually the same as or
922	867	// very near usize?
923	868	if (mem.page_size << 2 > maxInt(usize)) return;
...	...	@@ -946,7 +891,10 @@ fn testAllocatorLargeAlignment(allocator: *mem.Allocator) mem.Allocator.Error!vo
946	891	allocator.free(slice);
947	892	}
948	893
949		fn testAllocatorAlignedShrink(allocator: *mem.Allocator) mem.Allocator.Error!void {
	894	fn testAllocatorAlignedShrink(base_allocator: *mem.Allocator) mem.Allocator.Error!void {
	895	var sanityAllocator = mem.sanityWrap(base_allocator);
	896	const allocator = &sanityAllocator.allocator;
	897
950	898	var debug_buffer: [1000]u8 = undefined;
951	899	const debug_allocator = &FixedBufferAllocator.init(&debug_buffer).allocator;
952	900

lib/std/heap/arena_allocator.zig+7-22

...	...	@@ -20,8 +20,8 @@ pub const ArenaAllocator = struct {
20	20	pub fn promote(self: State, child_allocator: *Allocator) ArenaAllocator {
21	21	return .{
22	22	.allocator = Allocator{
23		.reallocFn = realloc,
24		.shrinkFn = shrink,
	23	.allocFn = alloc,
	24	.resizeFn = Allocator.noResize,
25	25	},
26	26	.child_allocator = child_allocator,
27	27	.state = self,
...	...	@@ -61,38 +61,23 @@ pub const ArenaAllocator = struct {
61	61	return buf_node;
62	62	}
63	63
64		fn alloc(allocator: *Allocator, n: usize, alignment: u29) ![]u8 {
	64	fn alloc(allocator: *Allocator, n: usize, ptr_align: u29, len_align: u29) ![]u8 {
65	65	const self = @fieldParentPtr(ArenaAllocator, "allocator", allocator);
66	66
67		var cur_node = if (self.state.buffer_list.first) \|first_node\| first_node else try self.createNode(0, n + alignment);
	67	var cur_node = if (self.state.buffer_list.first) \|first_node\| first_node else try self.createNode(0, n + ptr_align);
68	68	while (true) {
69	69	const cur_buf = cur_node.data[@sizeOf(BufNode)..];
70	70	const addr = @ptrToInt(cur_buf.ptr) + self.state.end_index;
71		const adjusted_addr = mem.alignForward(addr, alignment);
	71	const adjusted_addr = mem.alignForward(addr, ptr_align);
72	72	const adjusted_index = self.state.end_index + (adjusted_addr - addr);
73	73	const new_end_index = adjusted_index + n;
74	74	if (new_end_index > cur_buf.len) {
75		cur_node = try self.createNode(cur_buf.len, n + alignment);
	75	cur_node = try self.createNode(cur_buf.len, n + ptr_align);
76	76	continue;
77	77	}
78	78	const result = cur_buf[adjusted_index..new_end_index];
79	79	self.state.end_index = new_end_index;
80		return result;
	80	return result[0..mem.alignAllocLen(result.len, n, len_align)];
81	81	}
82	82	}
83
84		fn realloc(allocator: *Allocator, old_mem: []u8, old_align: u29, new_size: usize, new_align: u29) ![]u8 {
85		if (new_size <= old_mem.len and new_align <= new_size) {
86		// We can't do anything with the memory, so tell the client to keep it.
87		return error.OutOfMemory;
88		} else {
89		const result = try alloc(allocator, new_size, new_align);
90		@memcpy(result.ptr, old_mem.ptr, std.math.min(old_mem.len, result.len));
91		return result;
92		}
93		}
94
95		fn shrink(allocator: *Allocator, old_mem: []u8, old_align: u29, new_size: usize, new_align: u29) []u8 {
96		return old_mem[0..new_size];
97		}
98	83	};

lib/std/heap/logging_allocator.zig+37-24

...	...	@@ -15,39 +15,45 @@ pub fn LoggingAllocator(comptime OutStreamType: type) type {
15	15	pub fn init(parent_allocator: *Allocator, out_stream: OutStreamType) Self {
16	16	return Self{
17	17	.allocator = Allocator{
18		.reallocFn = realloc,
19		.shrinkFn = shrink,
	18	.allocFn = alloc,
	19	.resizeFn = resize,
20	20	},
21	21	.parent_allocator = parent_allocator,
22	22	.out_stream = out_stream,
23	23	};
24	24	}
25	25
26		fn realloc(allocator: *Allocator, old_mem: []u8, old_align: u29, new_size: usize, new_align: u29) ![]u8 {
	26	fn alloc(allocator: *Allocator, len: usize, ptr_align: u29, len_align: u29) error{OutOfMemory}![]u8 {
27	27	const self = @fieldParentPtr(Self, "allocator", allocator);
28		if (old_mem.len == 0) {
29		self.out_stream.print("allocation of {} ", .{new_size}) catch {};
30		} else {
31		self.out_stream.print("resize from {} to {} ", .{ old_mem.len, new_size }) catch {};
32		}
33		const result = self.parent_allocator.reallocFn(self.parent_allocator, old_mem, old_align, new_size, new_align);
	28	self.out_stream.print("alloc : {}", .{len}) catch {};
	29	const result = self.parent_allocator.callAllocFn(len, ptr_align, len_align);
34	30	if (result) \|buff\| {
35		self.out_stream.print("success!\n", .{}) catch {};
	31	self.out_stream.print(" success!\n", .{}) catch {};
36	32	} else \|err\| {
37		self.out_stream.print("failure!\n", .{}) catch {};
	33	self.out_stream.print(" failure!\n", .{}) catch {};
38	34	}
39	35	return result;
40	36	}
41	37
42		fn shrink(allocator: *Allocator, old_mem: []u8, old_align: u29, new_size: usize, new_align: u29) []u8 {
	38	fn resize(allocator: *Allocator, buf: []u8, new_len: usize, len_align: u29) error{OutOfMemory}!usize {
43	39	const self = @fieldParentPtr(Self, "allocator", allocator);
44		const result = self.parent_allocator.shrinkFn(self.parent_allocator, old_mem, old_align, new_size, new_align);
45		if (new_size == 0) {
46		self.out_stream.print("free of {} bytes success!\n", .{old_mem.len}) catch {};
	40	if (new_len == 0) {
	41	self.out_stream.print("free : {}\n", .{buf.len}) catch {};
	42	} else if (new_len <= buf.len) {
	43	self.out_stream.print("shrink: {} to {}\n", .{buf.len, new_len}) catch {};
47	44	} else {
48		self.out_stream.print("shrink from {} bytes to {} bytes success!\n", .{ old_mem.len, new_size }) catch {};
	45	self.out_stream.print("expand: {} to {}", .{ buf.len, new_len }) catch {};
	46	}
	47	if (self.parent_allocator.callResizeFn(buf, new_len, len_align)) \|resized_len\| {
	48	if (new_len > buf.len) {
	49	self.out_stream.print(" success!\n", .{}) catch {};
	50	}
	51	return resized_len;
	52	} else \|e\| {
	53	std.debug.assert(new_len > buf.len);
	54	self.out_stream.print(" failure!\n", .{}) catch {};
	55	return e;
49	56	}
50		return result;
51	57	}
52	58	};
53	59	}
...	...	@@ -60,17 +66,24 @@ pub fn loggingAllocator(
60	66	}
61	67
62	68	test "LoggingAllocator" {
63		var buf: [255]u8 = undefined;
64		var fbs = std.io.fixedBufferStream(&buf);
	69	var log_buf: [255]u8 = undefined;
	70	var fbs = std.io.fixedBufferStream(&log_buf);
65	71
66		const allocator = &loggingAllocator(std.testing.allocator, fbs.outStream()).allocator;
	72	var allocator_buf: [10]u8 = undefined;
	73	var fixedBufferAllocator = std.mem.sanityWrap(std.heap.FixedBufferAllocator.init(&allocator_buf));
	74	const allocator = &loggingAllocator(&fixedBufferAllocator.allocator, fbs.outStream()).allocator;
67	75
68		const ptr = try allocator.alloc(u8, 10);
69		allocator.free(ptr);
	76	var a = try allocator.alloc(u8, 10);
	77	a.len = allocator.shrinkBytes(a, 5, 0);
	78	std.debug.assert(a.len == 5);
	79	std.testing.expectError(error.OutOfMemory, allocator.callResizeFn(a, 20, 0));
	80	allocator.free(a);
70	81
71	82	std.testing.expectEqualSlices(u8,
72		\\allocation of 10 success!
73		\\free of 10 bytes success!
	83	\\alloc : 10 success!
	84	\\shrink: 10 to 5
	85	\\expand: 5 to 20 failure!
	86	\\free : 5
74	87	\\
75	88	, fbs.getWritten());
76	89	}

lib/std/mem.zig+263-56

...	...	@@ -16,6 +16,52 @@ pub const page_size = switch (builtin.arch) {
16	16	pub const Allocator = struct {
17	17	pub const Error = error{OutOfMemory};
18	18
	19	/// Attempt to allocate at least `len` bytes aligned to `ptr_align`.
	20	///
	21	/// If `len_align` is `0`, then the length returned MUST be exactly `len` bytes,
	22	/// otherwise, the length must be aligned to `len_align`.
	23	///
	24	/// `len` must be greater than or equal to `len_align` and must be aligned by `len_align`.
	25	allocFn: fn (self: *Allocator, len: usize, ptr_align: u29, len_align: u29) Error![]u8,
	26
	27	/// Attempt to expand or shrink memory in place. `buf.len` must equal the most recent
	28	/// length returned by `allocFn` or `resizeFn`.
	29	///
	30	/// Passing a `new_len` of 0 frees and invalidates the buffer such that it can no
	31	/// longer be passed to `resizeFn`.
	32	///
	33	/// error.OutOfMemory can only be returned if `new_len` is greater than `buf.len`.
	34	/// If `buf` cannot be expanded to accomodate `new_len`, then the allocation MUST be
	35	/// unmodified and error.OutOfMemory MUST be returned.
	36	///
	37	/// If `len_align` is `0`, then the length returned MUST be exactly `len` bytes,
	38	/// otherwise, the length must be aligned to `len_align`.
	39	///
	40	/// `new_len` must be greater than or equal to `len_align` and must be aligned by `len_align`.
	41	resizeFn: fn (self: *Allocator, buf: []u8, new_len: usize, len_align: u29) Error!usize,
	42
	43	pub fn callAllocFn(self: *Allocator, new_len: usize, alignment: u29, len_align: u29) Error![]u8 {
	44	return self.allocFn(self, new_len, alignment, len_align);
	45	}
	46
	47	pub fn callResizeFn(self: *Allocator, buf: []u8, new_len: usize, len_align: u29) Error!usize {
	48	return self.resizeFn(self, buf, new_len, len_align);
	49	}
	50
	51	/// Set to resizeFn if in-place resize is not supported.
	52	pub fn noResize(self: *Allocator, buf: []u8, new_len: usize, len_align: u29) Error!usize {
	53	if (new_len > buf.len)
	54	return error.OutOfMemory;
	55	return new_len;
	56	}
	57
	58	/// Call `resizeFn`, but caller guarantees that `new_len` <= `buf.len` meaning
	59	/// error.OutOfMemory should be impossible.
	60	pub fn shrinkBytes(self: *Allocator, buf: []u8, new_len: usize, len_align: u29) usize {
	61	assert(new_len <= buf.len);
	62	return self.callResizeFn(buf, new_len, len_align) catch unreachable;
	63	}
	64
19	65	/// Realloc is used to modify the size or alignment of an existing allocation,
20	66	/// as well as to provide the allocator with an opportunity to move an allocation
21	67	/// to a better location.
...	...	@@ -24,7 +70,7 @@ pub const Allocator = struct {
24	70	/// When the size/alignment is less than or equal to the previous allocation,
25	71	/// this function returns `error.OutOfMemory` when the allocator decides the client
26	72	/// would be better off keeping the extra alignment/size. Clients will call
27		/// `shrinkFn` when they require the allocator to track a new alignment/size,
	73	/// `callResizeFn` when they require the allocator to track a new alignment/size,
28	74	/// and so this function should only return success when the allocator considers
29	75	/// the reallocation desirable from the allocator's perspective.
30	76	/// As an example, `std.ArrayList` tracks a "capacity", and therefore can handle
...	...	@@ -37,16 +83,15 @@ pub const Allocator = struct {
37	83	/// as `old_mem` was when `reallocFn` is called. The bytes of
38	84	/// `return_value[old_mem.len..]` have undefined values.
39	85	/// The returned slice must have its pointer aligned at least to `new_alignment` bytes.
40		reallocFn: fn (
	86	fn reallocBytes(
41	87	self: *Allocator,
42	88	/// Guaranteed to be the same as what was returned from most recent call to
43		/// `reallocFn` or `shrinkFn`.
	89	/// `allocFn` or `resizeFn`.
44	90	/// If `old_mem.len == 0` then this is a new allocation and `new_byte_count`
45	91	/// is guaranteed to be >= 1.
46	92	old_mem: []u8,
47	93	/// If `old_mem.len == 0` then this is `undefined`, otherwise:
48		/// Guaranteed to be the same as what was returned from most recent call to
49		/// `reallocFn` or `shrinkFn`.
	94	/// Guaranteed to be the same as what was passed to `allocFn`.
50	95	/// Guaranteed to be >= 1.
51	96	/// Guaranteed to be a power of 2.
52	97	old_alignment: u29,
...	...	@@ -57,23 +102,49 @@ pub const Allocator = struct {
57	102	/// Guaranteed to be a power of 2.
58	103	/// Returned slice's pointer must have this alignment.
59	104	new_alignment: u29,
60		) Error![]u8,
	105	/// 0 indicates the length of the slice returned MUST match `new_byte_count` exactly
	106	/// non-zero means the length of the returned slice must be aligned by `len_align`
	107	/// `new_len` must be aligned by `len_align`
	108	len_align: u29,
	109	) Error![]u8 {
	110	if (old_mem.len == 0) {
	111	const new_mem = try self.callAllocFn(new_byte_count, new_alignment, len_align);
	112	@memset(new_mem.ptr, undefined, new_byte_count);
	113	return new_mem;
	114	}
61	115
62		/// This function deallocates memory. It must succeed.
63		shrinkFn: fn (
64		self: *Allocator,
65		/// Guaranteed to be the same as what was returned from most recent call to
66		/// `reallocFn` or `shrinkFn`.
67		old_mem: []u8,
68		/// Guaranteed to be the same as what was returned from most recent call to
69		/// `reallocFn` or `shrinkFn`.
70		old_alignment: u29,
71		/// Guaranteed to be less than or equal to `old_mem.len`.
72		new_byte_count: usize,
73		/// If `new_byte_count == 0` then this is `undefined`, otherwise:
74		/// Guaranteed to be less than or equal to `old_alignment`.
75		new_alignment: u29,
76		) []u8,
	116	if (isAligned(@ptrToInt(old_mem.ptr), new_alignment)) {
	117	if (new_byte_count <= old_mem.len) {
	118	const shrunk_len = self.shrinkBytes(old_mem, new_byte_count, len_align);
	119	if (shrunk_len < old_mem.len) {
	120	@memset(old_mem.ptr + shrunk_len, undefined, old_mem.len - shrunk_len);
	121	}
	122	return old_mem.ptr[0..shrunk_len];
	123	}
	124	if (self.callResizeFn(old_mem, new_byte_count, len_align)) \|resized_len\| {
	125	assert(resized_len >= new_byte_count);
	126	@memset(old_mem.ptr + new_byte_count, undefined, resized_len - new_byte_count);
	127	return old_mem.ptr[0..resized_len];
	128	} else \|_\| { }
	129	}
	130	if (new_byte_count <= old_mem.len and new_alignment <= old_alignment) {
	131	return error.OutOfMemory;
	132	}
	133	return self.moveBytes(old_mem, new_byte_count, new_alignment, len_align);
	134	}
	135
	136	/// Move the given memory to a new location in the given allocator to accomodate a new
	137	/// size and alignment.
	138	fn moveBytes(self: *Allocator, old_mem: []u8, new_len: usize, new_alignment: u29, len_align: u29) Error![]u8 {
	139	assert(old_mem.len > 0);
	140	assert(new_len > 0);
	141	const new_mem = try self.callAllocFn(new_len, new_alignment, len_align);
	142	@memcpy(new_mem.ptr, old_mem.ptr, std.math.min(new_len, old_mem.len));
	143	// DISABLED TO AVOID BUGS IN TRANSLATE C
	144	//@memset(old_mem.ptr, undefined, old_mem.len);
	145	_ = self.shrinkBytes(old_mem, 0, 0);
	146	return new_mem;
	147	}
77	148
78	149	/// Returns a pointer to undefined memory.
79	150	/// Call `destroy` with the result to free the memory.
...	...	@@ -89,8 +160,7 @@ pub const Allocator = struct {
89	160	const T = @TypeOf(ptr).Child;
90	161	if (@sizeOf(T) == 0) return;
91	162	const non_const_ptr = @intToPtr([*]u8, @ptrToInt(ptr));
92		const shrink_result = self.shrinkFn(self, non_const_ptr[0..@sizeOf(T)], @alignOf(T), 0, 1);
93		assert(shrink_result.len == 0);
	163	_ = self.shrinkBytes(non_const_ptr[0..@sizeOf(T)], 0, 0);
94	164	}
95	165
96	166	/// Allocates an array of `n` items of type `T` and sets all the
...	...	@@ -150,9 +220,21 @@ pub const Allocator = struct {
150	220	/// null means naturally aligned
151	221	comptime alignment: ?u29,
152	222	n: usize,
	223	) Error![]align(alignment orelse @alignOf(T)) T {
	224	return self.alignedAlloc2(T, alignment, n, .exact);
	225	}
	226
	227	const Exact = enum {exact,atLeast};
	228	pub fn alignedAlloc2(
	229	self: *Allocator,
	230	comptime T: type,
	231	/// null means naturally aligned
	232	comptime alignment: ?u29,
	233	n: usize,
	234	exact: Exact,
153	235	) Error![]align(alignment orelse @alignOf(T)) T {
154	236	const a = if (alignment) \|a\| blk: {
155		if (a == @alignOf(T)) return alignedAlloc(self, T, null, n);
	237	if (a == @alignOf(T)) return alignedAlloc2(self, T, null, n, exact);
156	238	break :blk a;
157	239	} else @alignOf(T);
158	240
...	...	@@ -161,15 +243,16 @@ pub const Allocator = struct {
161	243	}
162	244
163	245	const byte_count = math.mul(usize, @sizeOf(T), n) catch return Error.OutOfMemory;
164		const byte_slice = try self.reallocFn(self, &[0]u8{}, undefined, byte_count, a);
165		assert(byte_slice.len == byte_count);
	246	// TODO The `if (alignment == null)` blocks are workarounds for zig not being able to
	247	// access certain type information about T without creating a circular dependency in async
	248	// functions that heap-allocate their own frame with @Frame(func).
	249	const sizeOfT = if (alignment == null) @intCast(u29, @divExact(byte_count, n)) else @sizeOf(T);
	250	const byte_slice = try self.callAllocFn(byte_count, a, if (exact == .exact) @as(u29, 0) else sizeOfT);
	251	assert(if (exact == .exact) byte_slice.len == byte_count else byte_slice.len >= byte_count);
166	252	@memset(byte_slice.ptr, undefined, byte_slice.len);
167	253	if (alignment == null) {
168		// TODO This is a workaround for zig not being able to successfully do
169		// @bytesToSlice(T, @alignCast(a, byte_slice)) without resolving alignment of T,
170		// which causes a circular dependency in async functions which try to heap-allocate
171		// their own frame with @Frame(func).
172		return @intToPtr([*]T, @ptrToInt(byte_slice.ptr))[0..n];
	254	// This if block is a workaround (see comment above)
	255	return @intToPtr([*]T, @ptrToInt(byte_slice.ptr))[0..@divExact(byte_slice.len, @sizeOf(T))];
173	256	} else {
174	257	return mem.bytesAsSlice(T, @alignCast(a, byte_slice));
175	258	}
...	...	@@ -190,7 +273,15 @@ pub const Allocator = struct {
190	273	break :t Error![]align(Slice.alignment) Slice.child;
191	274	} {
192	275	const old_alignment = @typeInfo(@TypeOf(old_mem)).Pointer.alignment;
193		return self.alignedRealloc(old_mem, old_alignment, new_n);
	276	return self.alignedRealloc2(old_mem, old_alignment, new_n, .exact);
	277	}
	278
	279	pub fn reallocAtLeast(self: *Allocator, old_mem: var, new_n: usize) t: {
	280	const Slice = @typeInfo(@TypeOf(old_mem)).Pointer;
	281	break :t Error![]align(Slice.alignment) Slice.child;
	282	} {
	283	const old_alignment = @typeInfo(@TypeOf(old_mem)).Pointer.alignment;
	284	return self.alignedRealloc2(old_mem, old_alignment, new_n, .atLeast);
194	285	}
195	286
196	287	/// This is the same as `realloc`, except caller may additionally request
...	...	@@ -201,11 +292,24 @@ pub const Allocator = struct {
201	292	old_mem: var,
202	293	comptime new_alignment: u29,
203	294	new_n: usize,
	295	) Error![]align(new_alignment) @typeInfo(@TypeOf(old_mem)).Pointer.child {
	296	return self.alignedRealloc2(old_mem, new_alignment, new_n, .exact);
	297	}
	298
	299	/// This is the same as `realloc`, except caller may additionally request
	300	/// a new alignment, which can be larger, smaller, or the same as the old
	301	/// allocation.
	302	pub fn alignedRealloc2(
	303	self: *Allocator,
	304	old_mem: var,
	305	comptime new_alignment: u29,
	306	new_n: usize,
	307	exact: Exact,
204	308	) Error![]align(new_alignment) @typeInfo(@TypeOf(old_mem)).Pointer.child {
205	309	const Slice = @typeInfo(@TypeOf(old_mem)).Pointer;
206	310	const T = Slice.child;
207	311	if (old_mem.len == 0) {
208		return self.alignedAlloc(T, new_alignment, new_n);
	312	return self.alignedAlloc2(T, new_alignment, new_n, exact);
209	313	}
210	314	if (new_n == 0) {
211	315	self.free(old_mem);
...	...	@@ -215,12 +319,9 @@ pub const Allocator = struct {
215	319	const old_byte_slice = mem.sliceAsBytes(old_mem);
216	320	const byte_count = math.mul(usize, @sizeOf(T), new_n) catch return Error.OutOfMemory;
217	321	// Note: can't set shrunk memory to undefined as memory shouldn't be modified on realloc failure
218		const byte_slice = try self.reallocFn(self, old_byte_slice, Slice.alignment, byte_count, new_alignment);
219		assert(byte_slice.len == byte_count);
220		if (new_n > old_mem.len) {
221		@memset(byte_slice.ptr + old_byte_slice.len, undefined, byte_slice.len - old_byte_slice.len);
222		}
223		return mem.bytesAsSlice(T, @alignCast(new_alignment, byte_slice));
	322	const new_byte_slice = try self.reallocBytes(old_byte_slice, Slice.alignment, byte_count, new_alignment,
	323	if (exact == .exact) @as(u29, 0) else @sizeOf(T));
	324	return mem.bytesAsSlice(T, @alignCast(new_alignment, new_byte_slice));
224	325	}
225	326
226	327	/// Prefer calling realloc to shrink if you can tolerate failure, such as
...	...	@@ -248,12 +349,9 @@ pub const Allocator = struct {
248	349	const Slice = @typeInfo(@TypeOf(old_mem)).Pointer;
249	350	const T = Slice.child;
250	351
251		if (new_n == 0) {
252		self.free(old_mem);
253		return old_mem[0..0];
254		}
255
256		assert(new_n <= old_mem.len);
	352	if (new_n == old_mem.len)
	353	return old_mem;
	354	assert(new_n < old_mem.len);
257	355	assert(new_alignment <= Slice.alignment);
258	356
259	357	// Here we skip the overflow checking on the multiplication because
...	...	@@ -262,9 +360,8 @@ pub const Allocator = struct {
262	360
263	361	const old_byte_slice = mem.sliceAsBytes(old_mem);
264	362	@memset(old_byte_slice.ptr + byte_count, undefined, old_byte_slice.len - byte_count);
265		const byte_slice = self.shrinkFn(self, old_byte_slice, Slice.alignment, byte_count, new_alignment);
266		assert(byte_slice.len == byte_count);
267		return mem.bytesAsSlice(T, @alignCast(new_alignment, byte_slice));
	363	_ = self.shrinkBytes(old_byte_slice, byte_count, 0);
	364	return old_mem[0..new_n];
268	365	}
269	366
270	367	/// Free an array allocated with `alloc`. To free a single item,
...	...	@@ -276,8 +373,7 @@ pub const Allocator = struct {
276	373	if (bytes_len == 0) return;
277	374	const non_const_ptr = @intToPtr([*]u8, @ptrToInt(bytes.ptr));
278	375	@memset(non_const_ptr, undefined, bytes_len);
279		const shrink_result = self.shrinkFn(self, non_const_ptr[0..bytes_len], Slice.alignment, 0, 1);
280		assert(shrink_result.len == 0);
	376	_ = self.shrinkBytes(non_const_ptr[0..bytes_len], 0, 0);
281	377	}
282	378
283	379	/// Copies `m` to newly allocated memory. Caller owns the memory.
...	...	@@ -296,16 +392,107 @@ pub const Allocator = struct {
296	392	}
297	393	};
298	394
	395	/// Given a pointer to an allocator, return the *Allocator for it. `allocatorStatePtr` can
	396	/// either be a `*Allocator`, in which case it is returned as-is, otherwise, the address of
	397	/// the `allocator` field is returned.
	398	pub fn getAllocatorPtr(allocatorStatePtr: var) *Allocator {
	399	// allocator must be a pointer or else this function will return a copy of the allocator which
	400	// is not what this is for
	401	const T = @TypeOf(allocatorStatePtr);
	402	switch (@typeInfo(T)) {
	403	.Pointer => {},
	404	else => @compileError("getAllocatorPtr expects a pointer to an allocator but got: " ++ @typeName(T)),
	405	}
	406	if (T == *Allocator)
	407	return allocatorStatePtr;
	408	return &allocatorStatePtr.allocator;
	409	}
	410
	411	/// Detects and asserts if the std.mem.Allocator interface is violated
	412	pub fn SanityAllocator(comptime T: type) type { return struct {
	413	const Self = @This();
	414	allocator: Allocator,
	415	underlying_allocator: T,
	416	pub fn init(allocator: T) @This() {
	417	return .{
	418	.allocator = .{
	419	.allocFn = alloc,
	420	.resizeFn = resize,
	421	},
	422	.underlying_allocator = allocator,
	423	};
	424	}
	425	fn getUnderlyingAllocatorPtr(self: @This()) Allocator {
	426	if (T == *Allocator) return self.underlying_allocator;
	427	return getAllocatorPtr(&self.underlying_allocator);
	428	}
	429	pub fn alloc(allocator: *Allocator, n: usize, ptr_align: u29, len_align: u29) Allocator.Error![]u8 {
	430	assert(n > 0);
	431	assert(mem.isValidAlign(ptr_align));
	432	if (len_align != 0) {
	433	assert(mem.isAlignedAnyAlign(n, len_align));
	434	assert(n >= len_align);
	435	}
	436
	437	const self = @fieldParentPtr(@This(), "allocator", allocator);
	438	const result = try self.getUnderlyingAllocatorPtr().callAllocFn(n, ptr_align, len_align);
	439	if (len_align == 0) {
	440	assert(result.len == n);
	441	} else {
	442	assert(result.len >= n);
	443	assert(mem.isAlignedAnyAlign(result.len, len_align));
	444	}
	445	return result;
	446	}
	447	pub fn resize(allocator: *Allocator, buf: []u8, new_len: usize, len_align: u29) Allocator.Error!usize {
	448	assert(buf.len > 0);
	449	if (len_align != 0) {
	450	assert(mem.isAlignedAnyAlign(new_len, len_align));
	451	assert(new_len >= len_align);
	452	}
	453	const self = @fieldParentPtr(@This(), "allocator", allocator);
	454	const result = try self.getUnderlyingAllocatorPtr().callResizeFn(buf, new_len, len_align);
	455	if (len_align == 0) {
	456	assert(result == new_len);
	457	} else {
	458	assert(result >= new_len);
	459	assert(mem.isAlignedAnyAlign(result, len_align));
	460	}
	461	return result;
	462	}
	463	pub usingnamespace if (T == *Allocator or !@hasDecl(T, "reset")) struct {} else struct {
	464	pub fn reset(self: *Self) void {
	465	self.underlying_allocator.reset();
	466	}
	467	};
	468	};}
	469
	470	pub fn sanityWrap(allocator: var) SanityAllocator(@TypeOf(allocator)) {
	471	return SanityAllocator(@TypeOf(allocator)).init(allocator);
	472	}
	473
	474	/// An allocator helper function. Adjusts an allocation length satisfy `len_align`.
	475	/// `full_len` should be the full capacity of the allocation which may be greater
	476	/// than the `len` that was requsted. This function should only be used by allocators
	477	/// that are unaffected by `len_align`.
	478	pub fn alignAllocLen(full_len: usize, alloc_len: usize, len_align: u29) usize {
	479	assert(alloc_len > 0);
	480	assert(alloc_len >= len_align);
	481	assert(full_len >= alloc_len);
	482	if (len_align == 0)
	483	return alloc_len;
	484	const adjusted = alignBackwardAnyAlign(full_len, len_align);
	485	assert(adjusted >= alloc_len);
	486	return adjusted;
	487	}
	488
299	489	var failAllocator = Allocator{
300		.reallocFn = failAllocatorRealloc,
301		.shrinkFn = failAllocatorShrink,
	490	.allocFn = failAllocatorAlloc,
	491	.resizeFn = Allocator.noResize,
302	492	};
303		fn failAllocatorRealloc(self: *Allocator, old_mem: []u8, old_align: u29, new_size: usize, new_align: u29) ![]u8 {
	493	fn failAllocatorAlloc(self: *Allocator, n: usize, alignment: u29, len_align: u29) Allocator.Error![]u8 {
304	494	return error.OutOfMemory;
305	495	}
306		fn failAllocatorShrink(self: *Allocator, old_mem: []u8, old_align: u29, new_size: usize, new_align: u29) []u8 {
307		@panic("failAllocatorShrink should never be called because it cannot allocate");
308		}
309	496
310	497	test "mem.Allocator basics" {
311	498	testing.expectError(error.OutOfMemory, failAllocator.alloc(u8, 1));
...	...	@@ -2190,6 +2377,13 @@ test "alignForward" {
2190	2377	testing.expect(alignForward(17, 8) == 24);
2191	2378	}
2192	2379
	2380	pub fn alignBackwardAnyAlign(i: usize, alignment: usize) usize {
	2381	if (@popCount(usize, alignment) == 1)
	2382	return alignBackward(i, alignment);
	2383	assert(alignment != 0);
	2384	return i - @mod(i, alignment);
	2385	}
	2386
2193	2387	/// Round an address up to the previous aligned address
2194	2388	/// The alignment must be a power of 2 and greater than 0.
2195	2389	pub fn alignBackward(addr: usize, alignment: usize) usize {
...	...	@@ -2206,6 +2400,19 @@ pub fn alignBackwardGeneric(comptime T: type, addr: T, alignment: T) T {
2206	2400	return addr & ~(alignment - 1);
2207	2401	}
2208	2402
	2403	/// Returns whether `alignment` is a valid alignment, meaning it is
	2404	/// a positive power of 2.
	2405	pub fn isValidAlign(alignment: u29) bool {
	2406	return @popCount(u29, alignment) == 1;
	2407	}
	2408
	2409	pub fn isAlignedAnyAlign(i: usize, alignment: usize) bool {
	2410	if (@popCount(usize, alignment) == 1)
	2411	return isAligned(i, alignment);
	2412	assert(alignment != 0);
	2413	return 0 == @mod(i, alignment);
	2414	}
	2415
2209	2416	/// Given an address and an alignment, return true if the address is a multiple of the alignment
2210	2417	/// The alignment must be a power of 2 and greater than 0.
2211	2418	pub fn isAligned(addr: usize, alignment: usize) bool {

lib/std/os/windows/bits.zig+1

...	...	@@ -593,6 +593,7 @@ pub const FILE_CURRENT = 1;
593	593	pub const FILE_END = 2;
594	594
595	595	pub const HEAP_CREATE_ENABLE_EXECUTE = 0x00040000;
	596	pub const HEAP_REALLOC_IN_PLACE_ONLY = 0x00000010;
596	597	pub const HEAP_GENERATE_EXCEPTIONS = 0x00000004;
597	598	pub const HEAP_NO_SERIALIZE = 0x00000001;
598	599

lib/std/testing.zig+1-1

...	...	@@ -11,7 +11,7 @@ pub var allocator_instance = LeakCountAllocator.init(&base_allocator_instance.al
11	11	pub const failing_allocator = &failing_allocator_instance.allocator;
12	12	pub var failing_allocator_instance = FailingAllocator.init(&base_allocator_instance.allocator, 0);
13	13
14		pub var base_allocator_instance = std.heap.ThreadSafeFixedBufferAllocator.init(allocator_mem[0..]);
	14	pub var base_allocator_instance = std.mem.sanityWrap(std.heap.ThreadSafeFixedBufferAllocator.init(allocator_mem[0..]));
15	15	var allocator_mem: [2 * 1024 * 1024]u8 = undefined;
16	16
17	17	/// This function is intended to be used only in tests. It prints diagnostics to stderr

lib/std/testing/failing_allocator.zig+18-23

...	...	@@ -39,43 +39,38 @@ pub const FailingAllocator = struct {
39	39	.allocations = 0,
40	40	.deallocations = 0,
41	41	.allocator = mem.Allocator{
42		.reallocFn = realloc,
43		.shrinkFn = shrink,
	42	.allocFn = alloc,
	43	.resizeFn = resize,
44	44	},
45	45	};
46	46	}
47	47
48		fn realloc(allocator: *mem.Allocator, old_mem: []u8, old_align: u29, new_size: usize, new_align: u29) ![]u8 {
	48	fn alloc(allocator: *std.mem.Allocator, len: usize, ptr_align: u29, len_align: u29) error{OutOfMemory}![]u8 {
49	49	const self = @fieldParentPtr(FailingAllocator, "allocator", allocator);
50	50	if (self.index == self.fail_index) {
51	51	return error.OutOfMemory;
52	52	}
53		const result = try self.internal_allocator.reallocFn(
54		self.internal_allocator,
55		old_mem,
56		old_align,
57		new_size,
58		new_align,
59		);
60		if (new_size < old_mem.len) {
61		self.freed_bytes += old_mem.len - new_size;
62		if (new_size == 0)
63		self.deallocations += 1;
64		} else if (new_size > old_mem.len) {
65		self.allocated_bytes += new_size - old_mem.len;
66		if (old_mem.len == 0)
67		self.allocations += 1;
68		}
	53	const result = try self.internal_allocator.callAllocFn(len, ptr_align, len_align);
	54	self.allocated_bytes += result.len;
	55	self.allocations += 1;
69	56	self.index += 1;
70	57	return result;
71	58	}
72	59
73		fn shrink(allocator: *mem.Allocator, old_mem: []u8, old_align: u29, new_size: usize, new_align: u29) []u8 {
	60	fn resize(allocator: *std.mem.Allocator, old_mem: []u8, new_len: usize, len_align: u29) error{OutOfMemory}!usize {
74	61	const self = @fieldParentPtr(FailingAllocator, "allocator", allocator);
75		const r = self.internal_allocator.shrinkFn(self.internal_allocator, old_mem, old_align, new_size, new_align);
76		self.freed_bytes += old_mem.len - r.len;
77		if (new_size == 0)
	62	const r = self.internal_allocator.callResizeFn(old_mem, new_len, len_align) catch \|e\| {
	63	std.debug.assert(new_len > old_mem.len);
	64	return e;
	65	};
	66	if (new_len == 0) {
78	67	self.deallocations += 1;
	68	self.freed_bytes += old_mem.len;
	69	} else if (r < old_mem.len) {
	70	self.freed_bytes += old_mem.len - r;
	71	} else {
	72	self.allocated_bytes += r - old_mem.len;
	73	}
79	74	return r;
80	75	}
81	76	};

lib/std/testing/leak_count_allocator.zig+11-10

...	...	@@ -14,23 +14,21 @@ pub const LeakCountAllocator = struct {
14	14	return .{
15	15	.count = 0,
16	16	.allocator = .{
17		.reallocFn = realloc,
18		.shrinkFn = shrink,
	17	.allocFn = alloc,
	18	.resizeFn = resize,
19	19	},
20	20	.internal_allocator = allocator,
21	21	};
22	22	}
23	23
24		fn realloc(allocator: *std.mem.Allocator, old_mem: []u8, old_align: u29, new_size: usize, new_align: u29) ![]u8 {
	24	fn alloc(allocator: *std.mem.Allocator, len: usize, ptr_align: u29, len_align: u29) error{OutOfMemory}![]u8 {
25	25	const self = @fieldParentPtr(LeakCountAllocator, "allocator", allocator);
26		var data = try self.internal_allocator.reallocFn(self.internal_allocator, old_mem, old_align, new_size, new_align);
27		if (old_mem.len == 0) {
28		self.count += 1;
29		}
30		return data;
	26	const ptr = try self.internal_allocator.callAllocFn(len, ptr_align, len_align);
	27	self.count += 1;
	28	return ptr;
31	29	}
32	30
33		fn shrink(allocator: *std.mem.Allocator, old_mem: []u8, old_align: u29, new_size: usize, new_align: u29) []u8 {
	31	fn resize(allocator: *std.mem.Allocator, old_mem: []u8, new_size: usize, len_align: u29) error{OutOfMemory}!usize {
34	32	const self = @fieldParentPtr(LeakCountAllocator, "allocator", allocator);
35	33	if (new_size == 0) {
36	34	if (self.count == 0) {
...	...	@@ -38,7 +36,10 @@ pub const LeakCountAllocator = struct {
38	36	}
39	37	self.count -= 1;
40	38	}
41		return self.internal_allocator.shrinkFn(self.internal_allocator, old_mem, old_align, new_size, new_align);
	39	return self.internal_allocator.callResizeFn(old_mem, new_size, len_align) catch \|e\| {
	40	std.debug.assert(new_size > old_mem.len);
	41	return e;
	42	};
42	43	}
43	44
44	45	pub fn validate(self: LeakCountAllocator) !void {