Commit cc17f84cccc540143f3fd19fe32218478d4a0c6f | zig

author	Andrew Kelley <andrew@ziglang.org> 2020-08-07 22:35:15-07:00
committer	Andrew Kelley <andrew@ziglang.org> 2020-08-07 22:45:45-07:00
log	cc17f84cccc540143f3fd19fe32218478d4a0c6f
tree	7cfd3d3c9e5a9bd17347b4ec7093aadf464b74de
parent	30bace66d4336ebc9f27d68e5807a80ab3b68d67

std: introduce GeneralPurposeAllocator

`std.GeneralPurposeAllocator` is now available. It is a function that
takes a configuration struct (with default field values) and returns an
allocator. There is a detailed description of this allocator in the
doc comments at the top of the new file.

The main feature of this allocator is that it is *safe*. It
prevents double-free, use-after-free, and detects leaks.

Some deprecation compile errors are removed.

The Allocator interface gains `old_align` as a new parameter to
`resizeFn`. This is useful to quickly look up allocations.

`std.heap.page_allocator` is improved to use mmap address hints to avoid
obtaining the same virtual address pages when unmapping and mapping
pages. The new general purpose allocator uses the page allocator as its
backing allocator by default.

`std.testing.allocator` is replaced with usage of this new allocator,
which does leak checking, and so the LeakCheckAllocator is retired.

stage1 is improved so that the `@typeInfo` of a pointer has a lazy value
for the alignment of the child type, to avoid false dependency loops
when dealing with pointers to async function frames.

The `std.mem.Allocator` interface is refactored to be in its own file.

`std.Mutex` now exposes the dummy mutex with `std.Mutex.Dummy`.

This allocator is great for debug mode, however it needs some work to
have better performance in release modes. The next step will be setting
up a series of tests in ziglang/gotta-go-fast and then making
improvements to the implementation.

14 files changed, 1497 insertions(+), 551 deletions(-)

lib
- std
  - array_list.zig
  - debug.zig
  - heap.zig
  - heap
    arena_allocator.zig
    general_purpose_allocator.zig
    logging_allocator.zig
  - mem.zig
  - mem
    Allocator.zig
  - mutex.zig
  - special
    test_runner.zig
  - testing.zig
  - testing
    failing_allocator.zig
    leak_count_allocator.zig
src
- ir.cpp

lib/std/array_list.zig+1

...	...	@@ -263,6 +263,7 @@ pub fn ArrayListAligned(comptime T: type, comptime alignment: ?u29) type {
263	263	if (better_capacity >= new_capacity) break;
264	264	}
265	265
	266	// TODO This can be optimized to avoid needlessly copying undefined memory.
266	267	const new_memory = try self.allocator.reallocAtLeast(self.allocatedSlice(), better_capacity);
267	268	self.items.ptr = new_memory.ptr;
268	269	self.capacity = new_memory.len;

lib/std/debug.zig-3

...	...	@@ -19,9 +19,6 @@ const windows = std.os.windows;
19	19
20	20	pub const leb = @import("debug/leb128.zig");
21	21
22		pub const global_allocator = @compileError("Please switch to std.testing.allocator.");
23		pub const failing_allocator = @compileError("Please switch to std.testing.failing_allocator.");
24
25	22	pub const runtime_safety = switch (builtin.mode) {
26	23	.Debug, .ReleaseSafe => true,
27	24	.ReleaseFast, .ReleaseSmall => false,

lib/std/heap.zig+33-21

...	...	@@ -12,6 +12,7 @@ const maxInt = std.math.maxInt;
12	12	pub const LoggingAllocator = @import("heap/logging_allocator.zig").LoggingAllocator;
13	13	pub const loggingAllocator = @import("heap/logging_allocator.zig").loggingAllocator;
14	14	pub const ArenaAllocator = @import("heap/arena_allocator.zig").ArenaAllocator;
	15	pub const GeneralPurposeAllocator = @import("heap/general_purpose_allocator.zig").GeneralPurposeAllocator;
15	16
16	17	const Allocator = mem.Allocator;
17	18
...	...	@@ -53,7 +54,7 @@ fn cAlloc(self: *Allocator, len: usize, ptr_align: u29, len_align: u29) Allocato
53	54	return ptr[0..mem.alignBackwardAnyAlign(full_len, len_align)];
54	55	}
55	56
56		fn cResize(self: *Allocator, buf: []u8, new_len: usize, len_align: u29) Allocator.Error!usize {
	57	fn cResize(self: *Allocator, buf: []u8, old_align: u29, new_len: usize, len_align: u29) Allocator.Error!usize {
57	58	if (new_len == 0) {
58	59	c.free(buf.ptr);
59	60	return 0;
...	...	@@ -88,8 +89,6 @@ var wasm_page_allocator_state = Allocator{
88	89	.resizeFn = WasmPageAllocator.resize,
89	90	};
90	91
91		pub const direct_allocator = @compileError("deprecated; use std.heap.page_allocator");
92
93	92	/// Verifies that the adjusted length will still map to the full length
94	93	pub fn alignPageAllocLen(full_len: usize, len: usize, len_align: u29) usize {
95	94	const aligned_len = mem.alignAllocLen(full_len, len, len_align);
...	...	@@ -97,10 +96,13 @@ pub fn alignPageAllocLen(full_len: usize, len: usize, len_align: u29) usize {
97	96	return aligned_len;
98	97	}
99	98
	99	/// TODO Utilize this on Windows.
	100	pub var next_mmap_addr_hint: ?[*]align(mem.page_size) u8 = null;
	101
100	102	const PageAllocator = struct {
101	103	fn alloc(allocator: *Allocator, n: usize, alignment: u29, len_align: u29) error{OutOfMemory}![]u8 {
102	104	assert(n > 0);
103		const alignedLen = mem.alignForward(n, mem.page_size);
	105	const aligned_len = mem.alignForward(n, mem.page_size);
104	106
105	107	if (builtin.os.tag == .windows) {
106	108	const w = os.windows;
...	...	@@ -112,14 +114,14 @@ const PageAllocator = struct {
112	114	// see https://devblogs.microsoft.com/oldnewthing/?p=42223
113	115	const addr = w.VirtualAlloc(
114	116	null,
115		alignedLen,
	117	aligned_len,
116	118	w.MEM_COMMIT \| w.MEM_RESERVE,
117	119	w.PAGE_READWRITE,
118	120	) catch return error.OutOfMemory;
119	121
120	122	// If the allocation is sufficiently aligned, use it.
121	123	if (@ptrToInt(addr) & (alignment - 1) == 0) {
122		return @ptrCast([*]u8, addr)[0..alignPageAllocLen(alignedLen, n, len_align)];
	124	return @ptrCast([*]u8, addr)[0..alignPageAllocLen(aligned_len, n, len_align)];
123	125	}
124	126
125	127	// If it wasn't, actually do an explicitely aligned allocation.
...	...	@@ -146,20 +148,24 @@ const PageAllocator = struct {
146	148	// until it succeeds.
147	149	const ptr = w.VirtualAlloc(
148	150	@intToPtr(*c_void, aligned_addr),
149		alignedLen,
	151	aligned_len,
150	152	w.MEM_COMMIT \| w.MEM_RESERVE,
151	153	w.PAGE_READWRITE,
152	154	) catch continue;
153	155
154		return @ptrCast([*]u8, ptr)[0..alignPageAllocLen(alignedLen, n, len_align)];
	156	return @ptrCast([*]u8, ptr)[0..alignPageAllocLen(aligned_len, n, len_align)];
155	157	}
156	158	}
157	159
158		const maxDropLen = alignment - std.math.min(alignment, mem.page_size);
159		const allocLen = if (maxDropLen <= alignedLen - n) alignedLen else mem.alignForward(alignedLen + maxDropLen, mem.page_size);
	160	const max_drop_len = alignment - std.math.min(alignment, mem.page_size);
	161	const alloc_len = if (max_drop_len <= aligned_len - n)
	162	aligned_len
	163	else
	164	mem.alignForward(aligned_len + max_drop_len, mem.page_size);
	165	const hint = @atomicLoad(@TypeOf(next_mmap_addr_hint), &next_mmap_addr_hint, .Unordered);
160	166	const slice = os.mmap(
161		null,
162		allocLen,
	167	hint,
	168	alloc_len,
163	169	os.PROT_READ \| os.PROT_WRITE,
164	170	os.MAP_PRIVATE \| os.MAP_ANONYMOUS,
165	171	-1,
...	...	@@ -168,25 +174,29 @@ const PageAllocator = struct {
168	174	assert(mem.isAligned(@ptrToInt(slice.ptr), mem.page_size));
169	175
170	176	const aligned_addr = mem.alignForward(@ptrToInt(slice.ptr), alignment);
	177	const result_ptr = @alignCast(mem.page_size, @intToPtr([*]u8, aligned_addr));
171	178
172	179	// Unmap the extra bytes that were only requested in order to guarantee
173	180	// that the range of memory we were provided had a proper alignment in
174	181	// it somewhere. The extra bytes could be at the beginning, or end, or both.
175		const dropLen = aligned_addr - @ptrToInt(slice.ptr);
176		if (dropLen != 0) {
177		os.munmap(slice[0..dropLen]);
	182	const drop_len = aligned_addr - @ptrToInt(slice.ptr);
	183	if (drop_len != 0) {
	184	os.munmap(slice[0..drop_len]);
178	185	}
179	186
180	187	// Unmap extra pages
181		const alignedBufferLen = allocLen - dropLen;
182		if (alignedBufferLen > alignedLen) {
183		os.munmap(@alignCast(mem.page_size, @intToPtr([*]u8, aligned_addr))[alignedLen..alignedBufferLen]);
	188	const aligned_buffer_len = alloc_len - drop_len;
	189	if (aligned_buffer_len > aligned_len) {
	190	os.munmap(result_ptr[aligned_len..aligned_buffer_len]);
184	191	}
185	192
186		return @intToPtr([*]u8, aligned_addr)[0..alignPageAllocLen(alignedLen, n, len_align)];
	193	const new_hint = @alignCast(mem.page_size, result_ptr + aligned_len);
	194	_ = @cmpxchgStrong(@TypeOf(next_mmap_addr_hint), &next_mmap_addr_hint, hint, new_hint, .Monotonic, .Monotonic);
	195
	196	return result_ptr[0..alignPageAllocLen(aligned_len, n, len_align)];
187	197	}
188	198
189		fn resize(allocator: *Allocator, buf_unaligned: []u8, new_size: usize, len_align: u29) Allocator.Error!usize {
	199	fn resize(allocator: *Allocator, buf_unaligned: []u8, buf_align: u29, new_size: usize, len_align: u29) Allocator.Error!usize {
190	200	const new_size_aligned = mem.alignForward(new_size, mem.page_size);
191	201
192	202	if (builtin.os.tag == .windows) {
...	...	@@ -229,6 +239,7 @@ const PageAllocator = struct {
229	239
230	240	if (new_size_aligned < buf_aligned_len) {
231	241	const ptr = @intToPtr([*]align(mem.page_size) u8, @ptrToInt(buf_unaligned.ptr) + new_size_aligned);
	242	// TODO: if the next_mmap_addr_hint is within the unmapped range, update it
232	243	os.munmap(ptr[0 .. buf_aligned_len - new_size_aligned]);
233	244	if (new_size_aligned == 0)
234	245	return 0;
...	...	@@ -236,6 +247,7 @@ const PageAllocator = struct {
236	247	}
237	248
238	249	// TODO: call mremap
	250	// TODO: if the next_mmap_addr_hint is within the remapped range, update it
239	251	return error.OutOfMemory;
240	252	}
241	253	};
...	...	@@ -538,7 +550,7 @@ pub const FixedBufferAllocator = struct {
538	550	return result;
539	551	}
540	552
541		fn resize(allocator: *Allocator, buf: []u8, new_size: usize, len_align: u29) Allocator.Error!usize {
	553	fn resize(allocator: *Allocator, buf: []u8, buf_align: u29, new_size: usize, len_align: u29) Allocator.Error!usize {
542	554	const self = @fieldParentPtr(FixedBufferAllocator, "allocator", allocator);
543	555	assert(self.ownsSlice(buf)); // sanity check
544	556

lib/std/heap/arena_allocator.zig+1-1

...	...	@@ -49,7 +49,7 @@ pub const ArenaAllocator = struct {
49	49	const actual_min_size = minimum_size + (@sizeOf(BufNode) + 16);
50	50	const big_enough_len = prev_len + actual_min_size;
51	51	const len = big_enough_len + big_enough_len / 2;
52		const buf = try self.child_allocator.callAllocFn(len, @alignOf(BufNode), 1);
	52	const buf = try self.child_allocator.allocFn(self.child_allocator, len, @alignOf(BufNode), 1);
53	53	const buf_node = @ptrCast(*BufNode, @alignCast(@alignOf(BufNode), buf.ptr));
54	54	buf_node.* = BufNode{
55	55	.data = buf,

lib/std/heap/general_purpose_allocator.zig created+920

...	...	@@ -0,0 +1,920 @@
	1	//! # General Purpose Allocator
	2	//!
	3	//! ## Design Priorities
	4	//!
	5	//! ### `OptimizationMode.debug` and `OptimizationMode.release_safe`:
	6	//!
	7	//! * Detect double free, and print stack trace of:
	8	//! - Where it was first allocated
	9	//! - Where it was freed the first time
	10	//! - Where it was freed the second time
	11	//!
	12	//! * Detect leaks and print stack trace of:
	13	//! - Where it was allocated
	14	//!
	15	//! * When a page of memory is no longer needed, give it back to resident memory
	16	//! as soon as possible, so that it causes page faults when used.
	17	//!
	18	//! * Do not re-use memory slots, so that memory safety is upheld. For small
	19	//! allocations, this is handled here; for larger ones it is handled in the
	20	//! backing allocator (by default `std.heap.page_allocator`).
	21	//!
	22	//! * Make pointer math errors unlikely to harm memory from
	23	//! unrelated allocations.
	24	//!
	25	//! * It's OK for these mechanisms to cost some extra overhead bytes.
	26	//!
	27	//! * It's OK for performance cost for these mechanisms.
	28	//!
	29	//! * Rogue memory writes should not harm the allocator's state.
	30	//!
	31	//! * Cross platform. Operates based on a backing allocator which makes it work
	32	//! everywhere, even freestanding.
	33	//!
	34	//! * Compile-time configuration.
	35	//!
	36	//! ### `OptimizationMode.release_fast` (note: not much work has gone into this use case yet):
	37	//!
	38	//! * Low fragmentation is primary concern
	39	//! * Performance of worst-case latency is secondary concern
	40	//! * Performance of average-case latency is next
	41	//! * Finally, having freed memory unmapped, and pointer math errors unlikely to
	42	//! harm memory from unrelated allocations are nice-to-haves.
	43	//!
	44	//! ### `OptimizationMode.release_small` (note: not much work has gone into this use case yet):
	45	//!
	46	//! * Small binary code size of the executable is the primary concern.
	47	//! * Next, defer to the `.release_fast` priority list.
	48	//!
	49	//! ## Basic Design:
	50	//!
	51	//! Small allocations are divided into buckets:
	52	//!
	53	//! ```
	54	//! index obj_size
	55	//! 0 1
	56	//! 1 2
	57	//! 2 4
	58	//! 3 8
	59	//! 4 16
	60	//! 5 32
	61	//! 6 64
	62	//! 7 128
	63	//! 8 256
	64	//! 9 512
	65	//! 10 1024
	66	//! 11 2048
	67	//! ```
	68	//!
	69	//! The main allocator state has an array of all the "current" buckets for each
	70	//! size class. Each slot in the array can be null, meaning the bucket for that
	71	//! size class is not allocated. When the first object is allocated for a given
	72	//! size class, it allocates 1 page of memory from the OS. This page is
	73	//! divided into "slots" - one per allocated object. Along with the page of memory
	74	//! for object slots, as many pages as necessary are allocated to store the
	75	//! BucketHeader, followed by "used bits", and two stack traces for each slot
	76	//! (allocation trace and free trace).
	77	//!
	78	//! The "used bits" are 1 bit per slot representing whether the slot is used.
	79	//! Allocations use the data to iterate to find a free slot. Frees assert that the
	80	//! corresponding bit is 1 and set it to 0.
	81	//!
	82	//! Buckets have prev and next pointers. When there is only one bucket for a given
	83	//! size class, both prev and next point to itself. When all slots of a bucket are
	84	//! used, a new bucket is allocated, and enters the doubly linked list. The main
	85	//! allocator state tracks the "current" bucket for each size class. Leak detection
	86	//! currently only checks the current bucket.
	87	//!
	88	//! Resizing detects if the size class is unchanged or smaller, in which case the same
	89	//! pointer is returned unmodified. If a larger size class is required,
	90	//! `error.OutOfMemory` is returned.
	91	//!
	92	//! Large objects are allocated directly using the backing allocator and their metadata is stored
	93	//! in a `std.HashMap` using the backing allocator.
	94
	95	const std = @import("std");
	96	const math = std.math;
	97	const assert = std.debug.assert;
	98	const Allocator = std.mem.Allocator;
	99	const page_size = std.mem.page_size;
	100	const StackTrace = std.builtin.StackTrace;
	101
	102	/// Integer type for pointing to slots in a small allocation
	103	const SlotIndex = std.meta.Int(false, math.log2(page_size) + 1);
	104
	105	pub const Config = struct {
	106	/// Number of stack frames to capture.
	107	stack_trace_frames: usize = if (std.debug.runtime_safety) @as(usize, 6) else @as(usize, 0),
	108
	109	/// If true, the allocator will have two fields:
	110	/// * `total_requested_bytes` which tracks the total allocated bytes of memory requested.
	111	/// * `requested_memory_limit` which causes allocations to return `error.OutOfMemory`
	112	/// when the `total_requested_bytes` exceeds this limit.
	113	/// If false, these fields will be `void`.
	114	enable_memory_limit: bool = false,
	115
	116	/// Whether to enable safety checks.
	117	safety: bool = std.debug.runtime_safety,
	118
	119	/// Whether the allocator may be used simultaneously from multiple threads.
	120	thread_safe: bool = !std.builtin.single_threaded,
	121	};
	122
	123	pub fn GeneralPurposeAllocator(comptime config: Config) type {
	124	return struct {
	125	allocator: Allocator = Allocator{
	126	.allocFn = alloc,
	127	.resizeFn = resize,
	128	},
	129	backing_allocator: *Allocator = std.heap.page_allocator,
	130	buckets: [small_bucket_count]?BucketHeader = [1]?BucketHeader{null} ** small_bucket_count,
	131	large_allocations: LargeAllocTable = .{},
	132
	133	total_requested_bytes: @TypeOf(total_requested_bytes_init) = total_requested_bytes_init,
	134	requested_memory_limit: @TypeOf(requested_memory_limit_init) = requested_memory_limit_init,
	135
	136	mutex: @TypeOf(mutex_init) = mutex_init,
	137
	138	const Self = @This();
	139
	140	const total_requested_bytes_init = if (config.enable_memory_limit) @as(usize, 0) else {};
	141	const requested_memory_limit_init = if (config.enable_memory_limit) @as(usize, math.maxInt(usize)) else {};
	142
	143	const mutex_init = if (config.thread_safe) std.Mutex.init() else std.Mutex.Dummy.init();
	144
	145	const stack_n = config.stack_trace_frames;
	146	const one_trace_size = @sizeOf(usize) * stack_n;
	147	const traces_per_slot = 2;
	148
	149	pub const Error = std.mem.Allocator.Error;
	150
	151	const small_bucket_count = math.log2(page_size);
	152	const largest_bucket_object_size = 1 << (small_bucket_count - 1);
	153
	154	const LargeAlloc = struct {
	155	bytes: []u8,
	156	stack_addresses: [stack_n]usize,
	157
	158	fn dumpStackTrace(self: *LargeAlloc) void {
	159	var len: usize = 0;
	160	while (len < stack_n and self.stack_addresses[len] != 0) {
	161	len += 1;
	162	}
	163	const stack_trace = StackTrace{
	164	.instruction_addresses = &self.stack_addresses,
	165	.index = len,
	166	};
	167	std.debug.dumpStackTrace(stack_trace);
	168	}
	169	};
	170	const LargeAllocTable = std.HashMapUnmanaged(usize, LargeAlloc, hash_addr, eql_addr, false);
	171
	172	// Bucket: In memory, in order:
	173	// * BucketHeader
	174	// * bucket_used_bits: [N]u8, // 1 bit for every slot; 1 byte for every 8 slots
	175	// * stack_trace_addresses: [N]usize, // traces_per_slot for every allocation
	176
	177	const BucketHeader = struct {
	178	prev: *BucketHeader,
	179	next: *BucketHeader,
	180	page: [*]align(page_size) u8,
	181	alloc_cursor: SlotIndex,
	182	used_count: SlotIndex,
	183
	184	fn usedBits(bucket: BucketHeader, index: usize) u8 {
	185	return @intToPtr(*u8, @ptrToInt(bucket) + @sizeOf(BucketHeader) + index);
	186	}
	187
	188	fn stackTracePtr(
	189	bucket: *BucketHeader,
	190	size_class: usize,
	191	slot_index: SlotIndex,
	192	trace_kind: TraceKind,
	193	) *[stack_n]usize {
	194	const start_ptr = @ptrCast([*]u8, bucket) + bucketStackFramesStart(size_class);
	195	const addr = start_ptr + one_trace_size * traces_per_slot * slot_index +
	196	@enumToInt(trace_kind) * @as(usize, one_trace_size);
	197	return @ptrCast(*[stack_n]usize, @alignCast(@alignOf(usize), addr));
	198	}
	199
	200	fn captureStackTrace(
	201	bucket: *BucketHeader,
	202	return_address: usize,
	203	size_class: usize,
	204	slot_index: SlotIndex,
	205	trace_kind: TraceKind,
	206	) void {
	207	// Initialize them to 0. When determining the count we must look
	208	// for non zero addresses.
	209	const stack_addresses = bucket.stackTracePtr(size_class, slot_index, trace_kind);
	210	collectStackTrace(return_address, stack_addresses);
	211	}
	212	};
	213
	214	fn bucketStackTrace(
	215	bucket: *BucketHeader,
	216	size_class: usize,
	217	slot_index: SlotIndex,
	218	trace_kind: TraceKind,
	219	) StackTrace {
	220	const stack_addresses = bucket.stackTracePtr(size_class, slot_index, trace_kind);
	221	var len: usize = 0;
	222	while (len < stack_n and stack_addresses[len] != 0) {
	223	len += 1;
	224	}
	225	return StackTrace{
	226	.instruction_addresses = stack_addresses,
	227	.index = len,
	228	};
	229	}
	230
	231	fn bucketStackFramesStart(size_class: usize) usize {
	232	return std.mem.alignForward(
	233	@sizeOf(BucketHeader) + usedBitsCount(size_class),
	234	@alignOf(usize),
	235	);
	236	}
	237
	238	fn bucketSize(size_class: usize) usize {
	239	const slot_count = @divExact(page_size, size_class);
	240	return bucketStackFramesStart(size_class) + one_trace_size * traces_per_slot * slot_count;
	241	}
	242
	243	fn usedBitsCount(size_class: usize) usize {
	244	const slot_count = @divExact(page_size, size_class);
	245	if (slot_count < 8) return 1;
	246	return @divExact(slot_count, 8);
	247	}
	248
	249	fn detectLeaksInBucket(
	250	bucket: *BucketHeader,
	251	size_class: usize,
	252	used_bits_count: usize,
	253	) void {
	254	var used_bits_byte: usize = 0;
	255	while (used_bits_byte < used_bits_count) : (used_bits_byte += 1) {
	256	const used_byte = bucket.usedBits(used_bits_byte).*;
	257	if (used_byte != 0) {
	258	var bit_index: u3 = 0;
	259	while (true) : (bit_index += 1) {
	260	const is_used = @truncate(u1, used_byte >> bit_index) != 0;
	261	if (is_used) {
	262	std.debug.print("\nMemory leak detected:\n", .{});
	263	const slot_index = @intCast(SlotIndex, used_bits_byte * 8 + bit_index);
	264	const stack_trace = bucketStackTrace(
	265	bucket,
	266	size_class,
	267	slot_index,
	268	.alloc,
	269	);
	270	std.debug.dumpStackTrace(stack_trace);
	271	}
	272	if (bit_index == math.maxInt(u3))
	273	break;
	274	}
	275	}
	276	}
	277	}
	278
	279	pub fn deinit(self: *Self) void {
	280	for (self.buckets) \|optional_bucket, bucket_i\| {
	281	const first_bucket = optional_bucket orelse continue;
	282	const size_class = @as(usize, 1) << @intCast(u6, bucket_i);
	283	const used_bits_count = usedBitsCount(size_class);
	284	var bucket = first_bucket;
	285	while (true) {
	286	detectLeaksInBucket(bucket, size_class, used_bits_count);
	287	bucket = bucket.next;
	288	if (bucket == first_bucket)
	289	break;
	290	}
	291	}
	292	for (self.large_allocations.items()) \|*large_alloc\| {
	293	std.debug.print("\nMemory leak detected:\n", .{});
	294	large_alloc.value.dumpStackTrace();
	295	}
	296	self.large_allocations.deinit(self.backing_allocator);
	297	self.* = undefined;
	298	}
	299
	300	fn collectStackTrace(first_trace_addr: usize, addresses: *[stack_n]usize) void {
	301	std.mem.set(usize, addresses, 0);
	302	var stack_trace = StackTrace{
	303	.instruction_addresses = addresses,
	304	.index = 0,
	305	};
	306	std.debug.captureStackTrace(first_trace_addr, &stack_trace);
	307	}
	308
	309	fn allocSlot(self: Self, size_class: usize, trace_addr: usize) Error![]u8 {
	310	const bucket_index = math.log2(size_class);
	311	const first_bucket = self.buckets[bucket_index] orelse try self.createBucket(
	312	size_class,
	313	bucket_index,
	314	);
	315	var bucket = first_bucket;
	316	const slot_count = @divExact(page_size, size_class);
	317	while (bucket.alloc_cursor == slot_count) {
	318	const prev_bucket = bucket;
	319	bucket = prev_bucket.next;
	320	if (bucket == first_bucket) {
	321	// make a new one
	322	bucket = try self.createBucket(size_class, bucket_index);
	323	bucket.prev = prev_bucket;
	324	bucket.next = prev_bucket.next;
	325	prev_bucket.next = bucket;
	326	bucket.next.prev = bucket;
	327	}
	328	}
	329	// change the allocator's current bucket to be this one
	330	self.buckets[bucket_index] = bucket;
	331
	332	const slot_index = bucket.alloc_cursor;
	333	bucket.alloc_cursor += 1;
	334
	335	var used_bits_byte = bucket.usedBits(slot_index / 8);
	336	const used_bit_index: u3 = @intCast(u3, slot_index % 8); // TODO cast should be unnecessary
	337	used_bits_byte.* \|= (@as(u8, 1) << used_bit_index);
	338	bucket.used_count += 1;
	339	bucket.captureStackTrace(trace_addr, size_class, slot_index, .alloc);
	340	return bucket.page + slot_index * size_class;
	341	}
	342
	343	fn searchBucket(
	344	self: *Self,
	345	bucket_index: usize,
	346	addr: usize,
	347	) ?*BucketHeader {
	348	const first_bucket = self.buckets[bucket_index] orelse return null;
	349	var bucket = first_bucket;
	350	while (true) {
	351	const in_bucket_range = (addr >= @ptrToInt(bucket.page) and
	352	addr < @ptrToInt(bucket.page) + page_size);
	353	if (in_bucket_range) return bucket;
	354	bucket = bucket.prev;
	355	if (bucket == first_bucket) {
	356	return null;
	357	}
	358	self.buckets[bucket_index] = bucket;
	359	}
	360	}
	361
	362	fn freeSlot(
	363	self: *Self,
	364	bucket: *BucketHeader,
	365	bucket_index: usize,
	366	size_class: usize,
	367	slot_index: SlotIndex,
	368	used_byte: *u8,
	369	used_bit_index: u3,
	370	trace_addr: usize,
	371	) void {
	372	// Capture stack trace to be the "first free", in case a double free happens.
	373	bucket.captureStackTrace(@returnAddress(), size_class, slot_index, .free);
	374
	375	used_byte.* &= ~(@as(u8, 1) << used_bit_index);
	376	bucket.used_count -= 1;
	377	if (bucket.used_count == 0) {
	378	if (bucket.next == bucket) {
	379	// it's the only bucket and therefore the current one
	380	self.buckets[bucket_index] = null;
	381	} else {
	382	bucket.next.prev = bucket.prev;
	383	bucket.prev.next = bucket.next;
	384	self.buckets[bucket_index] = bucket.prev;
	385	}
	386	self.backing_allocator.free(bucket.page[0..page_size]);
	387	const bucket_size = bucketSize(size_class);
	388	const bucket_slice = @ptrCast([*]align(@alignOf(BucketHeader)) u8, bucket)[0..bucket_size];
	389	self.backing_allocator.free(bucket_slice);
	390	} else {
	391	// TODO Set the slot data to undefined.
	392	// Related: https://github.com/ziglang/zig/issues/4298
	393	}
	394	}
	395
	396	/// This function assumes the object is in the large object storage regardless
	397	/// of the parameters.
	398	fn resizeLarge(
	399	self: *Self,
	400	old_mem: []u8,
	401	old_align: u29,
	402	new_size: usize,
	403	len_align: u29,
	404	return_addr: usize,
	405	) Error!usize {
	406	const entry = self.large_allocations.getEntry(@ptrToInt(old_mem.ptr)) orelse {
	407	if (config.safety) {
	408	@panic("Invalid free");
	409	} else {
	410	unreachable;
	411	}
	412	};
	413
	414	if (config.safety and old_mem.len != entry.value.bytes.len) {
	415	std.debug.print("\nAllocation size {} bytes does not match free size {}. Allocated here:\n", .{
	416	entry.value.bytes.len,
	417	old_mem.len,
	418	});
	419	entry.value.dumpStackTrace();
	420
	421	@panic("\nFree here:");
	422	}
	423
	424	const result_len = try self.backing_allocator.resizeFn(self.backing_allocator, old_mem, old_align, new_size, len_align);
	425
	426	if (result_len == 0) {
	427	self.large_allocations.removeAssertDiscard(@ptrToInt(old_mem.ptr));
	428	return 0;
	429	}
	430
	431	entry.value.bytes = old_mem.ptr[0..result_len];
	432	collectStackTrace(return_addr, &entry.value.stack_addresses);
	433	return result_len;
	434	}
	435
	436	pub fn setRequestedMemoryLimit(self: *Self, limit: usize) void {
	437	self.requested_memory_limit = limit;
	438	}
	439
	440	fn resize(
	441	allocator: *Allocator,
	442	old_mem: []u8,
	443	old_align: u29,
	444	new_size: usize,
	445	len_align: u29,
	446	) Error!usize {
	447	const self = @fieldParentPtr(Self, "allocator", allocator);
	448
	449	const held = self.mutex.acquire();
	450	defer held.release();
	451
	452	const prev_req_bytes = self.total_requested_bytes;
	453	if (config.enable_memory_limit) {
	454	const new_req_bytes = prev_req_bytes + new_size - old_mem.len;
	455	if (new_req_bytes > prev_req_bytes and new_req_bytes > self.requested_memory_limit) {
	456	return error.OutOfMemory;
	457	}
	458	self.total_requested_bytes = new_req_bytes;
	459	}
	460	errdefer if (config.enable_memory_limit) {
	461	self.total_requested_bytes = prev_req_bytes;
	462	};
	463
	464	assert(old_mem.len != 0);
	465
	466	const aligned_size = math.max(old_mem.len, old_align);
	467	if (aligned_size > largest_bucket_object_size) {
	468	return self.resizeLarge(old_mem, old_align, new_size, len_align, @returnAddress());
	469	}
	470	const size_class_hint = up_to_nearest_power_of_2(usize, aligned_size);
	471
	472	var bucket_index = math.log2(size_class_hint);
	473	var size_class: usize = size_class_hint;
	474	const bucket = while (bucket_index < small_bucket_count) : (bucket_index += 1) {
	475	if (self.searchBucket(bucket_index, @ptrToInt(old_mem.ptr))) \|bucket\| {
	476	break bucket;
	477	}
	478	size_class *= 2;
	479	} else {
	480	return self.resizeLarge(old_mem, old_align, new_size, len_align, @returnAddress());
	481	};
	482	const byte_offset = @ptrToInt(old_mem.ptr) - @ptrToInt(bucket.page);
	483	const slot_index = @intCast(SlotIndex, byte_offset / size_class);
	484	const used_byte_index = slot_index / 8;
	485	const used_bit_index = @intCast(u3, slot_index % 8);
	486	const used_byte = bucket.usedBits(used_byte_index);
	487	const is_used = @truncate(u1, used_byte.* >> used_bit_index) != 0;
	488	if (!is_used) {
	489	if (config.safety) {
	490	// print allocation stack trace
	491	std.debug.print("\nDouble free detected, allocated here:\n", .{});
	492	const alloc_stack_trace = bucketStackTrace(bucket, size_class, slot_index, .alloc);
	493	std.debug.dumpStackTrace(alloc_stack_trace);
	494	std.debug.print("\nFirst free here:\n", .{});
	495	const free_stack_trace = bucketStackTrace(bucket, size_class, slot_index, .free);
	496	std.debug.dumpStackTrace(free_stack_trace);
	497	@panic("\nSecond free here:");
	498	} else {
	499	unreachable;
	500	}
	501	}
	502	if (new_size == 0) {
	503	self.freeSlot(bucket, bucket_index, size_class, slot_index, used_byte, used_bit_index, @returnAddress());
	504	return @as(usize, 0);
	505	}
	506	const new_aligned_size = math.max(new_size, old_align);
	507	const new_size_class = up_to_nearest_power_of_2(usize, new_aligned_size);
	508	if (new_size_class <= size_class) {
	509	return new_size;
	510	}
	511	return error.OutOfMemory;
	512	}
	513
	514	fn alloc(allocator: *Allocator, len: usize, ptr_align: u29, len_align: u29) Error![]u8 {
	515	const self = @fieldParentPtr(Self, "allocator", allocator);
	516
	517	const held = self.mutex.acquire();
	518	defer held.release();
	519
	520	const prev_req_bytes = self.total_requested_bytes;
	521	if (config.enable_memory_limit) {
	522	const new_req_bytes = prev_req_bytes + len;
	523	if (new_req_bytes > self.requested_memory_limit) {
	524	return error.OutOfMemory;
	525	}
	526	self.total_requested_bytes = new_req_bytes;
	527	}
	528	errdefer if (config.enable_memory_limit) {
	529	self.total_requested_bytes = prev_req_bytes;
	530	};
	531
	532	const new_aligned_size = math.max(len, ptr_align);
	533	if (new_aligned_size > largest_bucket_object_size) {
	534	try self.large_allocations.ensureCapacity(
	535	self.backing_allocator,
	536	self.large_allocations.entries.items.len + 1,
	537	);
	538
	539	const slice = try self.backing_allocator.allocFn(self.backing_allocator, len, ptr_align, len_align);
	540
	541	const gop = self.large_allocations.getOrPutAssumeCapacity(@ptrToInt(slice.ptr));
	542	assert(!gop.found_existing); // This would mean the kernel double-mapped pages.
	543	gop.entry.value.bytes = slice;
	544	collectStackTrace(@returnAddress(), &gop.entry.value.stack_addresses);
	545
	546	return slice;
	547	} else {
	548	const new_size_class = up_to_nearest_power_of_2(usize, new_aligned_size);
	549	const ptr = try self.allocSlot(new_size_class, @returnAddress());
	550	return ptr[0..len];
	551	}
	552	}
	553
	554	fn createBucket(self: Self, size_class: usize, bucket_index: usize) Error!BucketHeader {
	555	const page = try self.backing_allocator.allocAdvanced(u8, page_size, page_size, .exact);
	556	errdefer self.backing_allocator.free(page);
	557
	558	const bucket_size = bucketSize(size_class);
	559	const bucket_bytes = try self.backing_allocator.allocAdvanced(u8, @alignOf(BucketHeader), bucket_size, .exact);
	560	const ptr = @ptrCast(*BucketHeader, bucket_bytes.ptr);
	561	ptr.* = BucketHeader{
	562	.prev = ptr,
	563	.next = ptr,
	564	.page = page.ptr,
	565	.alloc_cursor = 0,
	566	.used_count = 0,
	567	};
	568	self.buckets[bucket_index] = ptr;
	569	// Set the used bits to all zeroes
	570	@memset(@as(*[1]u8, ptr.usedBits(0)), 0, usedBitsCount(size_class));
	571	return ptr;
	572	}
	573	};
	574	}
	575
	576	const TraceKind = enum {
	577	alloc,
	578	free,
	579	};
	580
	581	fn up_to_nearest_power_of_2(comptime T: type, n: T) T {
	582	var power: T = 1;
	583	while (power < n)
	584	power *= 2;
	585	return power;
	586	}
	587
	588	fn hash_addr(addr: usize) u32 {
	589	if (@sizeOf(usize) == @sizeOf(u32))
	590	return addr;
	591	comptime assert(@sizeOf(usize) == 8);
	592	return @intCast(u32, addr >> 32) ^ @truncate(u32, addr);
	593	}
	594
	595	fn eql_addr(a: usize, b: usize) bool {
	596	return a == b;
	597	}
	598
	599	const test_config = Config{};
	600
	601	test "small allocations - free in same order" {
	602	var gpda = GeneralPurposeAllocator(test_config){};
	603	defer gpda.deinit();
	604	const allocator = &gpda.allocator;
	605
	606	var list = std.ArrayList(*u64).init(std.testing.allocator);
	607	defer list.deinit();
	608
	609	var i: usize = 0;
	610	while (i < 513) : (i += 1) {
	611	const ptr = try allocator.create(u64);
	612	try list.append(ptr);
	613	}
	614
	615	for (list.items) \|ptr\| {
	616	allocator.destroy(ptr);
	617	}
	618	}
	619
	620	test "small allocations - free in reverse order" {
	621	var gpda = GeneralPurposeAllocator(test_config){};
	622	defer gpda.deinit();
	623	const allocator = &gpda.allocator;
	624
	625	var list = std.ArrayList(*u64).init(std.testing.allocator);
	626	defer list.deinit();
	627
	628	var i: usize = 0;
	629	while (i < 513) : (i += 1) {
	630	const ptr = try allocator.create(u64);
	631	try list.append(ptr);
	632	}
	633
	634	while (list.popOrNull()) \|ptr\| {
	635	allocator.destroy(ptr);
	636	}
	637	}
	638
	639	test "large allocations" {
	640	var gpda = GeneralPurposeAllocator(test_config){};
	641	defer gpda.deinit();
	642	const allocator = &gpda.allocator;
	643
	644	const ptr1 = try allocator.alloc(u64, 42768);
	645	const ptr2 = try allocator.alloc(u64, 52768);
	646	allocator.free(ptr1);
	647	const ptr3 = try allocator.alloc(u64, 62768);
	648	allocator.free(ptr3);
	649	allocator.free(ptr2);
	650	}
	651
	652	test "realloc" {
	653	var gpda = GeneralPurposeAllocator(test_config){};
	654	defer gpda.deinit();
	655	const allocator = &gpda.allocator;
	656
	657	var slice = try allocator.alignedAlloc(u8, @alignOf(u32), 1);
	658	defer allocator.free(slice);
	659	slice[0] = 0x12;
	660
	661	// This reallocation should keep its pointer address.
	662	const old_slice = slice;
	663	slice = try allocator.realloc(slice, 2);
	664	assert(old_slice.ptr == slice.ptr);
	665	assert(slice[0] == 0x12);
	666	slice[1] = 0x34;
	667
	668	// This requires upgrading to a larger size class
	669	slice = try allocator.realloc(slice, 17);
	670	assert(slice[0] == 0x12);
	671	assert(slice[1] == 0x34);
	672	}
	673
	674	test "shrink" {
	675	var gpda = GeneralPurposeAllocator(test_config){};
	676	defer gpda.deinit();
	677	const allocator = &gpda.allocator;
	678
	679	var slice = try allocator.alloc(u8, 20);
	680	defer allocator.free(slice);
	681
	682	std.mem.set(u8, slice, 0x11);
	683
	684	slice = allocator.shrink(slice, 17);
	685
	686	for (slice) \|b\| {
	687	assert(b == 0x11);
	688	}
	689
	690	slice = allocator.shrink(slice, 16);
	691
	692	for (slice) \|b\| {
	693	assert(b == 0x11);
	694	}
	695	}
	696
	697	test "large object - grow" {
	698	var gpda = GeneralPurposeAllocator(test_config){};
	699	defer gpda.deinit();
	700	const allocator = &gpda.allocator;
	701
	702	var slice1 = try allocator.alloc(u8, page_size * 2 - 20);
	703	defer allocator.free(slice1);
	704
	705	var old = slice1;
	706	slice1 = try allocator.realloc(slice1, page_size * 2 - 10);
	707	assert(slice1.ptr == old.ptr);
	708
	709	slice1 = try allocator.realloc(slice1, page_size * 2);
	710	assert(slice1.ptr == old.ptr);
	711
	712	slice1 = try allocator.realloc(slice1, page_size * 2 + 1);
	713	}
	714
	715	test "realloc small object to large object" {
	716	var gpda = GeneralPurposeAllocator(test_config){};
	717	defer gpda.deinit();
	718	const allocator = &gpda.allocator;
	719
	720	var slice = try allocator.alloc(u8, 70);
	721	defer allocator.free(slice);
	722	slice[0] = 0x12;
	723	slice[60] = 0x34;
	724
	725	// This requires upgrading to a large object
	726	const large_object_size = page_size * 2 + 50;
	727	slice = try allocator.realloc(slice, large_object_size);
	728	assert(slice[0] == 0x12);
	729	assert(slice[60] == 0x34);
	730	}
	731
	732	test "shrink large object to large object" {
	733	var gpda = GeneralPurposeAllocator(test_config){};
	734	defer gpda.deinit();
	735	const allocator = &gpda.allocator;
	736
	737	var slice = try allocator.alloc(u8, page_size * 2 + 50);
	738	defer allocator.free(slice);
	739	slice[0] = 0x12;
	740	slice[60] = 0x34;
	741
	742	slice = try allocator.resize(slice, page_size * 2 + 1);
	743	assert(slice[0] == 0x12);
	744	assert(slice[60] == 0x34);
	745
	746	slice = allocator.shrink(slice, page_size * 2 + 1);
	747	assert(slice[0] == 0x12);
	748	assert(slice[60] == 0x34);
	749
	750	slice = try allocator.realloc(slice, page_size * 2);
	751	assert(slice[0] == 0x12);
	752	assert(slice[60] == 0x34);
	753	}
	754
	755	test "shrink large object to large object with larger alignment" {
	756	var gpda = GeneralPurposeAllocator(test_config){};
	757	defer gpda.deinit();
	758	const allocator = &gpda.allocator;
	759
	760	var debug_buffer: [1000]u8 = undefined;
	761	const debug_allocator = &std.heap.FixedBufferAllocator.init(&debug_buffer).allocator;
	762
	763	const alloc_size = page_size * 2 + 50;
	764	var slice = try allocator.alignedAlloc(u8, 16, alloc_size);
	765	defer allocator.free(slice);
	766
	767	var stuff_to_free = std.ArrayList([]align(16) u8).init(debug_allocator);
	768	while (isAligned(@ptrToInt(slice.ptr), page_size * 2)) {
	769	try stuff_to_free.append(slice);
	770	slice = try allocator.alignedAlloc(u8, 16, alloc_size);
	771	}
	772	while (stuff_to_free.popOrNull()) \|item\| {
	773	allocator.free(item);
	774	}
	775	slice[0] = 0x12;
	776	slice[60] = 0x34;
	777
	778	slice = try allocator.alignedRealloc(slice, page_size * 2, alloc_size / 2);
	779	assert(slice[0] == 0x12);
	780	assert(slice[60] == 0x34);
	781	}
	782
	783	test "realloc large object to small object" {
	784	var gpda = GeneralPurposeAllocator(test_config){};
	785	defer gpda.deinit();
	786	const allocator = &gpda.allocator;
	787
	788	var slice = try allocator.alloc(u8, page_size * 2 + 50);
	789	defer allocator.free(slice);
	790	slice[0] = 0x12;
	791	slice[16] = 0x34;
	792
	793	slice = try allocator.realloc(slice, 19);
	794	assert(slice[0] == 0x12);
	795	assert(slice[16] == 0x34);
	796	}
	797
	798	test "non-page-allocator backing allocator" {
	799	var gpda = GeneralPurposeAllocator(.{}){ .backing_allocator = std.testing.allocator };
	800	defer gpda.deinit();
	801	const allocator = &gpda.allocator;
	802
	803	const ptr = try allocator.create(i32);
	804	defer allocator.destroy(ptr);
	805	}
	806
	807	test "realloc large object to larger alignment" {
	808	var gpda = GeneralPurposeAllocator(test_config){};
	809	defer gpda.deinit();
	810	const allocator = &gpda.allocator;
	811
	812	var debug_buffer: [1000]u8 = undefined;
	813	const debug_allocator = &std.heap.FixedBufferAllocator.init(&debug_buffer).allocator;
	814
	815	var slice = try allocator.alignedAlloc(u8, 16, page_size * 2 + 50);
	816	defer allocator.free(slice);
	817
	818	var stuff_to_free = std.ArrayList([]align(16) u8).init(debug_allocator);
	819	while (isAligned(@ptrToInt(slice.ptr), page_size * 2)) {
	820	try stuff_to_free.append(slice);
	821	slice = try allocator.alignedAlloc(u8, 16, page_size * 2 + 50);
	822	}
	823	while (stuff_to_free.popOrNull()) \|item\| {
	824	allocator.free(item);
	825	}
	826	slice[0] = 0x12;
	827	slice[16] = 0x34;
	828
	829	slice = try allocator.alignedRealloc(slice, 32, page_size * 2 + 100);
	830	assert(slice[0] == 0x12);
	831	assert(slice[16] == 0x34);
	832
	833	slice = try allocator.alignedRealloc(slice, 32, page_size * 2 + 25);
	834	assert(slice[0] == 0x12);
	835	assert(slice[16] == 0x34);
	836
	837	slice = try allocator.alignedRealloc(slice, page_size * 2, page_size * 2 + 100);
	838	assert(slice[0] == 0x12);
	839	assert(slice[16] == 0x34);
	840	}
	841
	842	fn isAligned(addr: usize, alignment: usize) bool {
	843	// 000010000 // example addr
	844	// 000001111 // subtract 1
	845	// 111110000 // binary not
	846	const aligned_addr = (addr & ~(alignment - 1));
	847	return aligned_addr == addr;
	848	}
	849
	850	test "isAligned works" {
	851	assert(isAligned(0, 4));
	852	assert(isAligned(1, 1));
	853	assert(isAligned(2, 1));
	854	assert(isAligned(2, 2));
	855	assert(!isAligned(2, 4));
	856	assert(isAligned(3, 1));
	857	assert(!isAligned(3, 2));
	858	assert(!isAligned(3, 4));
	859	assert(isAligned(4, 4));
	860	assert(isAligned(4, 2));
	861	assert(isAligned(4, 1));
	862	assert(!isAligned(4, 8));
	863	assert(!isAligned(4, 16));
	864	}
	865
	866	test "large object shrinks to small but allocation fails during shrink" {
	867	var failing_allocator = std.testing.FailingAllocator.init(std.heap.page_allocator, 3);
	868	var gpda = GeneralPurposeAllocator(.{}){ .backing_allocator = &failing_allocator.allocator };
	869	defer gpda.deinit();
	870	const allocator = &gpda.allocator;
	871
	872	var slice = try allocator.alloc(u8, page_size * 2 + 50);
	873	defer allocator.free(slice);
	874	slice[0] = 0x12;
	875	slice[3] = 0x34;
	876
	877	// Next allocation will fail in the backing allocator of the GeneralPurposeAllocator
	878
	879	slice = allocator.shrink(slice, 4);
	880	assert(slice[0] == 0x12);
	881	assert(slice[3] == 0x34);
	882	}
	883
	884	test "objects of size 1024 and 2048" {
	885	var gpda = GeneralPurposeAllocator(test_config){};
	886	defer gpda.deinit();
	887	const allocator = &gpda.allocator;
	888
	889	const slice = try allocator.alloc(u8, 1025);
	890	const slice2 = try allocator.alloc(u8, 3000);
	891
	892	allocator.free(slice);
	893	allocator.free(slice2);
	894	}
	895
	896	test "setting a memory cap" {
	897	var gpda = GeneralPurposeAllocator(.{ .enable_memory_limit = true }){};
	898	defer gpda.deinit();
	899	const allocator = &gpda.allocator;
	900
	901	gpda.setRequestedMemoryLimit(1010);
	902
	903	const small = try allocator.create(i32);
	904	assert(gpda.total_requested_bytes == 4);
	905
	906	const big = try allocator.alloc(u8, 1000);
	907	assert(gpda.total_requested_bytes == 1004);
	908
	909	std.testing.expectError(error.OutOfMemory, allocator.create(u64));
	910
	911	allocator.destroy(small);
	912	assert(gpda.total_requested_bytes == 1000);
	913
	914	allocator.free(big);
	915	assert(gpda.total_requested_bytes == 0);
	916
	917	const exact = try allocator.alloc(u8, 1010);
	918	assert(gpda.total_requested_bytes == 1010);
	919	allocator.free(exact);
	920	}

lib/std/heap/logging_allocator.zig+11-5

...	...	@@ -26,7 +26,7 @@ pub fn LoggingAllocator(comptime OutStreamType: type) type {
26	26	fn alloc(allocator: *Allocator, len: usize, ptr_align: u29, len_align: u29) error{OutOfMemory}![]u8 {
27	27	const self = @fieldParentPtr(Self, "allocator", allocator);
28	28	self.out_stream.print("alloc : {}", .{len}) catch {};
29		const result = self.parent_allocator.callAllocFn(len, ptr_align, len_align);
	29	const result = self.parent_allocator.allocFn(self.parent_allocator, len, ptr_align, len_align);
30	30	if (result) \|buff\| {
31	31	self.out_stream.print(" success!\n", .{}) catch {};
32	32	} else \|err\| {
...	...	@@ -35,7 +35,13 @@ pub fn LoggingAllocator(comptime OutStreamType: type) type {
35	35	return result;
36	36	}
37	37
38		fn resize(allocator: *Allocator, buf: []u8, new_len: usize, len_align: u29) error{OutOfMemory}!usize {
	38	fn resize(
	39	allocator: *Allocator,
	40	buf: []u8,
	41	buf_align: u29,
	42	new_len: usize,
	43	len_align: u29,
	44	) error{OutOfMemory}!usize {
39	45	const self = @fieldParentPtr(Self, "allocator", allocator);
40	46	if (new_len == 0) {
41	47	self.out_stream.print("free : {}\n", .{buf.len}) catch {};
...	...	@@ -44,7 +50,7 @@ pub fn LoggingAllocator(comptime OutStreamType: type) type {
44	50	} else {
45	51	self.out_stream.print("expand: {} to {}", .{ buf.len, new_len }) catch {};
46	52	}
47		if (self.parent_allocator.callResizeFn(buf, new_len, len_align)) \|resized_len\| {
	53	if (self.parent_allocator.resizeFn(self.parent_allocator, buf, buf_align, new_len, len_align)) \|resized_len\| {
48	54	if (new_len > buf.len) {
49	55	self.out_stream.print(" success!\n", .{}) catch {};
50	56	}
...	...	@@ -74,9 +80,9 @@ test "LoggingAllocator" {
74	80	const allocator = &loggingAllocator(&fixedBufferAllocator.allocator, fbs.outStream()).allocator;
75	81
76	82	var a = try allocator.alloc(u8, 10);
77		a.len = allocator.shrinkBytes(a, 5, 0);
	83	a.len = allocator.shrinkBytes(a, 1, 5, 0);
78	84	std.debug.assert(a.len == 5);
79		std.testing.expectError(error.OutOfMemory, allocator.callResizeFn(a, 20, 0));
	85	std.testing.expectError(error.OutOfMemory, allocator.resizeFn(allocator, a, 1, 20, 0));
80	86	allocator.free(a);
81	87
82	88	std.testing.expectEqualSlices(u8,

lib/std/mem.zig+13-383

...	...	@@ -8,391 +8,13 @@ const meta = std.meta;
8	8	const trait = meta.trait;
9	9	const testing = std.testing;
10	10
11		// https://github.com/ziglang/zig/issues/2564
	11	/// https://github.com/ziglang/zig/issues/2564
12	12	pub const page_size = switch (builtin.arch) {
13	13	.wasm32, .wasm64 => 64 * 1024,
14	14	else => 4 * 1024,
15	15	};
16	16
17		pub const Allocator = struct {
18		pub const Error = error{OutOfMemory};
19
20		/// Attempt to allocate at least `len` bytes aligned to `ptr_align`.
21		///
22		/// If `len_align` is `0`, then the length returned MUST be exactly `len` bytes,
23		/// otherwise, the length must be aligned to `len_align`.
24		///
25		/// `len` must be greater than or equal to `len_align` and must be aligned by `len_align`.
26		allocFn: fn (self: *Allocator, len: usize, ptr_align: u29, len_align: u29) Error![]u8,
27
28		/// Attempt to expand or shrink memory in place. `buf.len` must equal the most recent
29		/// length returned by `allocFn` or `resizeFn`.
30		///
31		/// Passing a `new_len` of 0 frees and invalidates the buffer such that it can no
32		/// longer be passed to `resizeFn`.
33		///
34		/// error.OutOfMemory can only be returned if `new_len` is greater than `buf.len`.
35		/// If `buf` cannot be expanded to accomodate `new_len`, then the allocation MUST be
36		/// unmodified and error.OutOfMemory MUST be returned.
37		///
38		/// If `len_align` is `0`, then the length returned MUST be exactly `len` bytes,
39		/// otherwise, the length must be aligned to `len_align`.
40		///
41		/// `new_len` must be greater than or equal to `len_align` and must be aligned by `len_align`.
42		resizeFn: fn (self: *Allocator, buf: []u8, new_len: usize, len_align: u29) Error!usize,
43
44		pub fn callAllocFn(self: *Allocator, new_len: usize, alignment: u29, len_align: u29) Error![]u8 {
45		return self.allocFn(self, new_len, alignment, len_align);
46		}
47
48		pub fn callResizeFn(self: *Allocator, buf: []u8, new_len: usize, len_align: u29) Error!usize {
49		return self.resizeFn(self, buf, new_len, len_align);
50		}
51
52		/// Set to resizeFn if in-place resize is not supported.
53		pub fn noResize(self: *Allocator, buf: []u8, new_len: usize, len_align: u29) Error!usize {
54		if (new_len > buf.len)
55		return error.OutOfMemory;
56		return new_len;
57		}
58
59		/// Call `resizeFn`, but caller guarantees that `new_len` <= `buf.len` meaning
60		/// error.OutOfMemory should be impossible.
61		pub fn shrinkBytes(self: *Allocator, buf: []u8, new_len: usize, len_align: u29) usize {
62		assert(new_len <= buf.len);
63		return self.callResizeFn(buf, new_len, len_align) catch unreachable;
64		}
65
66		/// Realloc is used to modify the size or alignment of an existing allocation,
67		/// as well as to provide the allocator with an opportunity to move an allocation
68		/// to a better location.
69		/// When the size/alignment is greater than the previous allocation, this function
70		/// returns `error.OutOfMemory` when the requested new allocation could not be granted.
71		/// When the size/alignment is less than or equal to the previous allocation,
72		/// this function returns `error.OutOfMemory` when the allocator decides the client
73		/// would be better off keeping the extra alignment/size. Clients will call
74		/// `callResizeFn` when they require the allocator to track a new alignment/size,
75		/// and so this function should only return success when the allocator considers
76		/// the reallocation desirable from the allocator's perspective.
77		/// As an example, `std.ArrayList` tracks a "capacity", and therefore can handle
78		/// reallocation failure, even when `new_n` <= `old_mem.len`. A `FixedBufferAllocator`
79		/// would always return `error.OutOfMemory` for `reallocFn` when the size/alignment
80		/// is less than or equal to the old allocation, because it cannot reclaim the memory,
81		/// and thus the `std.ArrayList` would be better off retaining its capacity.
82		/// When `reallocFn` returns,
83		/// `return_value[0..min(old_mem.len, new_byte_count)]` must be the same
84		/// as `old_mem` was when `reallocFn` is called. The bytes of
85		/// `return_value[old_mem.len..]` have undefined values.
86		/// The returned slice must have its pointer aligned at least to `new_alignment` bytes.
87		fn reallocBytes(
88		self: *Allocator,
89		/// Guaranteed to be the same as what was returned from most recent call to
90		/// `allocFn` or `resizeFn`.
91		/// If `old_mem.len == 0` then this is a new allocation and `new_byte_count`
92		/// is guaranteed to be >= 1.
93		old_mem: []u8,
94		/// If `old_mem.len == 0` then this is `undefined`, otherwise:
95		/// Guaranteed to be the same as what was passed to `allocFn`.
96		/// Guaranteed to be >= 1.
97		/// Guaranteed to be a power of 2.
98		old_alignment: u29,
99		/// If `new_byte_count` is 0 then this is a free and it is guaranteed that
100		/// `old_mem.len != 0`.
101		new_byte_count: usize,
102		/// Guaranteed to be >= 1.
103		/// Guaranteed to be a power of 2.
104		/// Returned slice's pointer must have this alignment.
105		new_alignment: u29,
106		/// 0 indicates the length of the slice returned MUST match `new_byte_count` exactly
107		/// non-zero means the length of the returned slice must be aligned by `len_align`
108		/// `new_len` must be aligned by `len_align`
109		len_align: u29,
110		) Error![]u8 {
111		if (old_mem.len == 0) {
112		const new_mem = try self.callAllocFn(new_byte_count, new_alignment, len_align);
113		@memset(new_mem.ptr, undefined, new_byte_count);
114		return new_mem;
115		}
116
117		if (isAligned(@ptrToInt(old_mem.ptr), new_alignment)) {
118		if (new_byte_count <= old_mem.len) {
119		const shrunk_len = self.shrinkBytes(old_mem, new_byte_count, len_align);
120		return old_mem.ptr[0..shrunk_len];
121		}
122		if (self.callResizeFn(old_mem, new_byte_count, len_align)) \|resized_len\| {
123		assert(resized_len >= new_byte_count);
124		@memset(old_mem.ptr + new_byte_count, undefined, resized_len - new_byte_count);
125		return old_mem.ptr[0..resized_len];
126		} else \|_\| {}
127		}
128		if (new_byte_count <= old_mem.len and new_alignment <= old_alignment) {
129		return error.OutOfMemory;
130		}
131		return self.moveBytes(old_mem, new_byte_count, new_alignment, len_align);
132		}
133
134		/// Move the given memory to a new location in the given allocator to accomodate a new
135		/// size and alignment.
136		fn moveBytes(self: *Allocator, old_mem: []u8, new_len: usize, new_alignment: u29, len_align: u29) Error![]u8 {
137		assert(old_mem.len > 0);
138		assert(new_len > 0);
139		const new_mem = try self.callAllocFn(new_len, new_alignment, len_align);
140		@memcpy(new_mem.ptr, old_mem.ptr, std.math.min(new_len, old_mem.len));
141		// DISABLED TO AVOID BUGS IN TRANSLATE C
142		// use './zig build test-translate-c' to reproduce, some of the symbols in the
143		// generated C code will be a sequence of 0xaa (the undefined value), meaning
144		// it is printing data that has been freed
145		//@memset(old_mem.ptr, undefined, old_mem.len);
146		_ = self.shrinkBytes(old_mem, 0, 0);
147		return new_mem;
148		}
149
150		/// Returns a pointer to undefined memory.
151		/// Call `destroy` with the result to free the memory.
152		pub fn create(self: Allocator, comptime T: type) Error!T {
153		if (@sizeOf(T) == 0) return &(T{});
154		const slice = try self.alloc(T, 1);
155		return &slice[0];
156		}
157
158		/// `ptr` should be the return value of `create`, or otherwise
159		/// have the same address and alignment property.
160		pub fn destroy(self: *Allocator, ptr: anytype) void {
161		const T = @TypeOf(ptr).Child;
162		if (@sizeOf(T) == 0) return;
163		const non_const_ptr = @intToPtr([*]u8, @ptrToInt(ptr));
164		_ = self.shrinkBytes(non_const_ptr[0..@sizeOf(T)], 0, 0);
165		}
166
167		/// Allocates an array of `n` items of type `T` and sets all the
168		/// items to `undefined`. Depending on the Allocator
169		/// implementation, it may be required to call `free` once the
170		/// memory is no longer needed, to avoid a resource leak. If the
171		/// `Allocator` implementation is unknown, then correct code will
172		/// call `free` when done.
173		///
174		/// For allocating a single item, see `create`.
175		pub fn alloc(self: *Allocator, comptime T: type, n: usize) Error![]T {
176		return self.alignedAlloc(T, null, n);
177		}
178
179		pub fn allocWithOptions(
180		self: *Allocator,
181		comptime Elem: type,
182		n: usize,
183		/// null means naturally aligned
184		comptime optional_alignment: ?u29,
185		comptime optional_sentinel: ?Elem,
186		) Error!AllocWithOptionsPayload(Elem, optional_alignment, optional_sentinel) {
187		if (optional_sentinel) \|sentinel\| {
188		const ptr = try self.alignedAlloc(Elem, optional_alignment, n + 1);
189		ptr[n] = sentinel;
190		return ptr[0..n :sentinel];
191		} else {
192		return self.alignedAlloc(Elem, optional_alignment, n);
193		}
194		}
195
196		fn AllocWithOptionsPayload(comptime Elem: type, comptime alignment: ?u29, comptime sentinel: ?Elem) type {
197		if (sentinel) \|s\| {
198		return [:s]align(alignment orelse @alignOf(Elem)) Elem;
199		} else {
200		return []align(alignment orelse @alignOf(Elem)) Elem;
201		}
202		}
203
204		/// Allocates an array of `n + 1` items of type `T` and sets the first `n`
205		/// items to `undefined` and the last item to `sentinel`. Depending on the
206		/// Allocator implementation, it may be required to call `free` once the
207		/// memory is no longer needed, to avoid a resource leak. If the
208		/// `Allocator` implementation is unknown, then correct code will
209		/// call `free` when done.
210		///
211		/// For allocating a single item, see `create`.
212		///
213		/// Deprecated; use `allocWithOptions`.
214		pub fn allocSentinel(self: *Allocator, comptime Elem: type, n: usize, comptime sentinel: Elem) Error![:sentinel]Elem {
215		return self.allocWithOptions(Elem, n, null, sentinel);
216		}
217
218		/// Deprecated: use `allocAdvanced`
219		pub fn alignedAlloc(
220		self: *Allocator,
221		comptime T: type,
222		/// null means naturally aligned
223		comptime alignment: ?u29,
224		n: usize,
225		) Error![]align(alignment orelse @alignOf(T)) T {
226		return self.allocAdvanced(T, alignment, n, .exact);
227		}
228
229		const Exact = enum { exact, at_least };
230		pub fn allocAdvanced(
231		self: *Allocator,
232		comptime T: type,
233		/// null means naturally aligned
234		comptime alignment: ?u29,
235		n: usize,
236		exact: Exact,
237		) Error![]align(alignment orelse @alignOf(T)) T {
238		const a = if (alignment) \|a\| blk: {
239		if (a == @alignOf(T)) return allocAdvanced(self, T, null, n, exact);
240		break :blk a;
241		} else @alignOf(T);
242
243		if (n == 0) {
244		return @as([*]align(a) T, undefined)[0..0];
245		}
246
247		const byte_count = math.mul(usize, @sizeOf(T), n) catch return Error.OutOfMemory;
248		// TODO The `if (alignment == null)` blocks are workarounds for zig not being able to
249		// access certain type information about T without creating a circular dependency in async
250		// functions that heap-allocate their own frame with @Frame(func).
251		const sizeOfT = if (alignment == null) @intCast(u29, @divExact(byte_count, n)) else @sizeOf(T);
252		const byte_slice = try self.callAllocFn(byte_count, a, if (exact == .exact) @as(u29, 0) else sizeOfT);
253		switch (exact) {
254		.exact => assert(byte_slice.len == byte_count),
255		.at_least => assert(byte_slice.len >= byte_count),
256		}
257		@memset(byte_slice.ptr, undefined, byte_slice.len);
258		if (alignment == null) {
259		// This if block is a workaround (see comment above)
260		return @intToPtr([*]T, @ptrToInt(byte_slice.ptr))[0..@divExact(byte_slice.len, @sizeOf(T))];
261		} else {
262		return mem.bytesAsSlice(T, @alignCast(a, byte_slice));
263		}
264		}
265
266		/// This function requests a new byte size for an existing allocation,
267		/// which can be larger, smaller, or the same size as the old memory
268		/// allocation.
269		/// This function is preferred over `shrink`, because it can fail, even
270		/// when shrinking. This gives the allocator a chance to perform a
271		/// cheap shrink operation if possible, or otherwise return OutOfMemory,
272		/// indicating that the caller should keep their capacity, for example
273		/// in `std.ArrayList.shrink`.
274		/// If you need guaranteed success, call `shrink`.
275		/// If `new_n` is 0, this is the same as `free` and it always succeeds.
276		pub fn realloc(self: *Allocator, old_mem: anytype, new_n: usize) t: {
277		const Slice = @typeInfo(@TypeOf(old_mem)).Pointer;
278		break :t Error![]align(Slice.alignment) Slice.child;
279		} {
280		const old_alignment = @typeInfo(@TypeOf(old_mem)).Pointer.alignment;
281		return self.reallocAdvanced(old_mem, old_alignment, new_n, .exact);
282		}
283
284		pub fn reallocAtLeast(self: *Allocator, old_mem: anytype, new_n: usize) t: {
285		const Slice = @typeInfo(@TypeOf(old_mem)).Pointer;
286		break :t Error![]align(Slice.alignment) Slice.child;
287		} {
288		const old_alignment = @typeInfo(@TypeOf(old_mem)).Pointer.alignment;
289		return self.reallocAdvanced(old_mem, old_alignment, new_n, .at_least);
290		}
291
292		// Deprecated: use `reallocAdvanced`
293		pub fn alignedRealloc(
294		self: *Allocator,
295		old_mem: anytype,
296		comptime new_alignment: u29,
297		new_n: usize,
298		) Error![]align(new_alignment) @typeInfo(@TypeOf(old_mem)).Pointer.child {
299		return self.reallocAdvanced(old_mem, new_alignment, new_n, .exact);
300		}
301
302		/// This is the same as `realloc`, except caller may additionally request
303		/// a new alignment, which can be larger, smaller, or the same as the old
304		/// allocation.
305		pub fn reallocAdvanced(
306		self: *Allocator,
307		old_mem: anytype,
308		comptime new_alignment: u29,
309		new_n: usize,
310		exact: Exact,
311		) Error![]align(new_alignment) @typeInfo(@TypeOf(old_mem)).Pointer.child {
312		const Slice = @typeInfo(@TypeOf(old_mem)).Pointer;
313		const T = Slice.child;
314		if (old_mem.len == 0) {
315		return self.allocAdvanced(T, new_alignment, new_n, exact);
316		}
317		if (new_n == 0) {
318		self.free(old_mem);
319		return @as([*]align(new_alignment) T, undefined)[0..0];
320		}
321
322		const old_byte_slice = mem.sliceAsBytes(old_mem);
323		const byte_count = math.mul(usize, @sizeOf(T), new_n) catch return Error.OutOfMemory;
324		// Note: can't set shrunk memory to undefined as memory shouldn't be modified on realloc failure
325		const new_byte_slice = try self.reallocBytes(old_byte_slice, Slice.alignment, byte_count, new_alignment, if (exact == .exact) @as(u29, 0) else @sizeOf(T));
326		return mem.bytesAsSlice(T, @alignCast(new_alignment, new_byte_slice));
327		}
328
329		/// Prefer calling realloc to shrink if you can tolerate failure, such as
330		/// in an ArrayList data structure with a storage capacity.
331		/// Shrink always succeeds, and `new_n` must be <= `old_mem.len`.
332		/// Returned slice has same alignment as old_mem.
333		/// Shrinking to 0 is the same as calling `free`.
334		pub fn shrink(self: *Allocator, old_mem: anytype, new_n: usize) t: {
335		const Slice = @typeInfo(@TypeOf(old_mem)).Pointer;
336		break :t []align(Slice.alignment) Slice.child;
337		} {
338		const old_alignment = @typeInfo(@TypeOf(old_mem)).Pointer.alignment;
339		return self.alignedShrink(old_mem, old_alignment, new_n);
340		}
341
342		/// This is the same as `shrink`, except caller may additionally request
343		/// a new alignment, which must be smaller or the same as the old
344		/// allocation.
345		pub fn alignedShrink(
346		self: *Allocator,
347		old_mem: anytype,
348		comptime new_alignment: u29,
349		new_n: usize,
350		) []align(new_alignment) @typeInfo(@TypeOf(old_mem)).Pointer.child {
351		const Slice = @typeInfo(@TypeOf(old_mem)).Pointer;
352		const T = Slice.child;
353
354		if (new_n == old_mem.len)
355		return old_mem;
356		assert(new_n < old_mem.len);
357		assert(new_alignment <= Slice.alignment);
358
359		// Here we skip the overflow checking on the multiplication because
360		// new_n <= old_mem.len and the multiplication didn't overflow for that operation.
361		const byte_count = @sizeOf(T) * new_n;
362
363		const old_byte_slice = mem.sliceAsBytes(old_mem);
364		@memset(old_byte_slice.ptr + byte_count, undefined, old_byte_slice.len - byte_count);
365		_ = self.shrinkBytes(old_byte_slice, byte_count, 0);
366		return old_mem[0..new_n];
367		}
368
369		/// Free an array allocated with `alloc`. To free a single item,
370		/// see `destroy`.
371		pub fn free(self: *Allocator, memory: anytype) void {
372		const Slice = @typeInfo(@TypeOf(memory)).Pointer;
373		const bytes = mem.sliceAsBytes(memory);
374		const bytes_len = bytes.len + if (Slice.sentinel != null) @sizeOf(Slice.child) else 0;
375		if (bytes_len == 0) return;
376		const non_const_ptr = @intToPtr([*]u8, @ptrToInt(bytes.ptr));
377		@memset(non_const_ptr, undefined, bytes_len);
378		_ = self.shrinkBytes(non_const_ptr[0..bytes_len], 0, 0);
379		}
380
381		/// Copies `m` to newly allocated memory. Caller owns the memory.
382		pub fn dupe(allocator: *Allocator, comptime T: type, m: []const T) ![]T {
383		const new_buf = try allocator.alloc(T, m.len);
384		copy(T, new_buf, m);
385		return new_buf;
386		}
387
388		/// Copies `m` to newly allocated memory, with a null-terminated element. Caller owns the memory.
389		pub fn dupeZ(allocator: *Allocator, comptime T: type, m: []const T) ![:0]T {
390		const new_buf = try allocator.alloc(T, m.len + 1);
391		copy(T, new_buf, m);
392		new_buf[m.len] = 0;
393		return new_buf[0..m.len :0];
394		}
395		};
	17	pub const Allocator = @import("mem/Allocator.zig");
396	18
397	19	/// Detects and asserts if the std.mem.Allocator interface is violated by the caller
398	20	/// or the allocator.
...	...	@@ -424,7 +46,8 @@ pub fn ValidationAllocator(comptime T: type) type {
424	46	}
425	47
426	48	const self = @fieldParentPtr(@This(), "allocator", allocator);
427		const result = try self.getUnderlyingAllocatorPtr().callAllocFn(n, ptr_align, len_align);
	49	const underlying = self.getUnderlyingAllocatorPtr();
	50	const result = try underlying.allocFn(underlying, n, ptr_align, len_align);
428	51	assert(mem.isAligned(@ptrToInt(result.ptr), ptr_align));
429	52	if (len_align == 0) {
430	53	assert(result.len == n);
...	...	@@ -434,14 +57,21 @@ pub fn ValidationAllocator(comptime T: type) type {
434	57	}
435	58	return result;
436	59	}
437		pub fn resize(allocator: *Allocator, buf: []u8, new_len: usize, len_align: u29) Allocator.Error!usize {
	60	pub fn resize(
	61	allocator: *Allocator,
	62	buf: []u8,
	63	buf_align: u29,
	64	new_len: usize,
	65	len_align: u29,
	66	) Allocator.Error!usize {
438	67	assert(buf.len > 0);
439	68	if (len_align != 0) {
440	69	assert(mem.isAlignedAnyAlign(new_len, len_align));
441	70	assert(new_len >= len_align);
442	71	}
443	72	const self = @fieldParentPtr(@This(), "allocator", allocator);
444		const result = try self.getUnderlyingAllocatorPtr().callResizeFn(buf, new_len, len_align);
	73	const underlying = self.getUnderlyingAllocatorPtr();
	74	const result = try underlying.resizeFn(underlying, buf, buf_align, new_len, len_align);
445	75	if (len_align == 0) {
446	76	assert(result == new_len);
447	77	} else {

lib/std/mem/Allocator.zig created+410

...	...	@@ -0,0 +1,410 @@
	1	//! The standard memory allocation interface.
	2
	3	const std = @import("../std.zig");
	4	const assert = std.debug.assert;
	5	const math = std.math;
	6	const mem = std.mem;
	7	const Allocator = @This();
	8
	9	pub const Error = error{OutOfMemory};
	10
	11	/// Attempt to allocate at least `len` bytes aligned to `ptr_align`.
	12	///
	13	/// If `len_align` is `0`, then the length returned MUST be exactly `len` bytes,
	14	/// otherwise, the length must be aligned to `len_align`.
	15	///
	16	/// `len` must be greater than or equal to `len_align` and must be aligned by `len_align`.
	17	allocFn: fn (self: *Allocator, len: usize, ptr_align: u29, len_align: u29) Error![]u8,
	18
	19	/// Attempt to expand or shrink memory in place. `buf.len` must equal the most recent
	20	/// length returned by `allocFn` or `resizeFn`. `buf_align` must equal the same value
	21	/// that was passed as the `ptr_align` parameter to the original `allocFn` call.
	22	///
	23	/// Passing a `new_len` of 0 frees and invalidates the buffer such that it can no
	24	/// longer be passed to `resizeFn`.
	25	///
	26	/// error.OutOfMemory can only be returned if `new_len` is greater than `buf.len`.
	27	/// If `buf` cannot be expanded to accomodate `new_len`, then the allocation MUST be
	28	/// unmodified and error.OutOfMemory MUST be returned.
	29	///
	30	/// If `len_align` is `0`, then the length returned MUST be exactly `len` bytes,
	31	/// otherwise, the length must be aligned to `len_align`. Note that `len_align` does not
	32	/// provide a way to modify the alignment of a pointer. Rather it provides an API for
	33	/// accepting more bytes of memory from the allocator than requested.
	34	///
	35	/// `new_len` must be greater than or equal to `len_align` and must be aligned by `len_align`.
	36	resizeFn: fn (self: *Allocator, buf: []u8, buf_align: u29, new_len: usize, len_align: u29) Error!usize,
	37
	38	/// Set to resizeFn if in-place resize is not supported.
	39	pub fn noResize(self: *Allocator, buf: []u8, buf_align: u29, new_len: usize, len_align: u29) Error!usize {
	40	if (new_len > buf.len)
	41	return error.OutOfMemory;
	42	return new_len;
	43	}
	44
	45	/// Call `resizeFn`, but caller guarantees that `new_len` <= `buf.len` meaning
	46	/// error.OutOfMemory should be impossible.
	47	pub fn shrinkBytes(self: *Allocator, buf: []u8, buf_align: u29, new_len: usize, len_align: u29) usize {
	48	assert(new_len <= buf.len);
	49	return self.resizeFn(self, buf, buf_align, new_len, len_align) catch unreachable;
	50	}
	51
	52	/// Realloc is used to modify the size or alignment of an existing allocation,
	53	/// as well as to provide the allocator with an opportunity to move an allocation
	54	/// to a better location.
	55	/// When the size/alignment is greater than the previous allocation, this function
	56	/// returns `error.OutOfMemory` when the requested new allocation could not be granted.
	57	/// When the size/alignment is less than or equal to the previous allocation,
	58	/// this function returns `error.OutOfMemory` when the allocator decides the client
	59	/// would be better off keeping the extra alignment/size. Clients will call
	60	/// `resizeFn` when they require the allocator to track a new alignment/size,
	61	/// and so this function should only return success when the allocator considers
	62	/// the reallocation desirable from the allocator's perspective.
	63	/// As an example, `std.ArrayList` tracks a "capacity", and therefore can handle
	64	/// reallocation failure, even when `new_n` <= `old_mem.len`. A `FixedBufferAllocator`
	65	/// would always return `error.OutOfMemory` for `reallocFn` when the size/alignment
	66	/// is less than or equal to the old allocation, because it cannot reclaim the memory,
	67	/// and thus the `std.ArrayList` would be better off retaining its capacity.
	68	/// When `reallocFn` returns,
	69	/// `return_value[0..min(old_mem.len, new_byte_count)]` must be the same
	70	/// as `old_mem` was when `reallocFn` is called. The bytes of
	71	/// `return_value[old_mem.len..]` have undefined values.
	72	/// The returned slice must have its pointer aligned at least to `new_alignment` bytes.
	73	fn reallocBytes(
	74	self: *Allocator,
	75	/// Guaranteed to be the same as what was returned from most recent call to
	76	/// `allocFn` or `resizeFn`.
	77	/// If `old_mem.len == 0` then this is a new allocation and `new_byte_count`
	78	/// is guaranteed to be >= 1.
	79	old_mem: []u8,
	80	/// If `old_mem.len == 0` then this is `undefined`, otherwise:
	81	/// Guaranteed to be the same as what was passed to `allocFn`.
	82	/// Guaranteed to be >= 1.
	83	/// Guaranteed to be a power of 2.
	84	old_alignment: u29,
	85	/// If `new_byte_count` is 0 then this is a free and it is guaranteed that
	86	/// `old_mem.len != 0`.
	87	new_byte_count: usize,
	88	/// Guaranteed to be >= 1.
	89	/// Guaranteed to be a power of 2.
	90	/// Returned slice's pointer must have this alignment.
	91	new_alignment: u29,
	92	/// 0 indicates the length of the slice returned MUST match `new_byte_count` exactly
	93	/// non-zero means the length of the returned slice must be aligned by `len_align`
	94	/// `new_len` must be aligned by `len_align`
	95	len_align: u29,
	96	) Error![]u8 {
	97	if (old_mem.len == 0) {
	98	const new_mem = try self.allocFn(self, new_byte_count, new_alignment, len_align);
	99	// TODO: https://github.com/ziglang/zig/issues/4298
	100	@memset(new_mem.ptr, undefined, new_byte_count);
	101	return new_mem;
	102	}
	103
	104	if (mem.isAligned(@ptrToInt(old_mem.ptr), new_alignment)) {
	105	if (new_byte_count <= old_mem.len) {
	106	const shrunk_len = self.shrinkBytes(old_mem, old_alignment, new_byte_count, len_align);
	107	return old_mem.ptr[0..shrunk_len];
	108	}
	109	if (self.resizeFn(self, old_mem, old_alignment, new_byte_count, len_align)) \|resized_len\| {
	110	assert(resized_len >= new_byte_count);
	111	// TODO: https://github.com/ziglang/zig/issues/4298
	112	@memset(old_mem.ptr + new_byte_count, undefined, resized_len - new_byte_count);
	113	return old_mem.ptr[0..resized_len];
	114	} else \|_\| {}
	115	}
	116	if (new_byte_count <= old_mem.len and new_alignment <= old_alignment) {
	117	return error.OutOfMemory;
	118	}
	119	return self.moveBytes(old_mem, old_alignment, new_byte_count, new_alignment, len_align);
	120	}
	121
	122	/// Move the given memory to a new location in the given allocator to accomodate a new
	123	/// size and alignment.
	124	fn moveBytes(
	125	self: *Allocator,
	126	old_mem: []u8,
	127	old_align: u29,
	128	new_len: usize,
	129	new_alignment: u29,
	130	len_align: u29,
	131	) Error![]u8 {
	132	assert(old_mem.len > 0);
	133	assert(new_len > 0);
	134	const new_mem = try self.allocFn(self, new_len, new_alignment, len_align);
	135	@memcpy(new_mem.ptr, old_mem.ptr, math.min(new_len, old_mem.len));
	136	// TODO DISABLED TO AVOID BUGS IN TRANSLATE C
	137	// TODO see also https://github.com/ziglang/zig/issues/4298
	138	// use './zig build test-translate-c' to reproduce, some of the symbols in the
	139	// generated C code will be a sequence of 0xaa (the undefined value), meaning
	140	// it is printing data that has been freed
	141	//@memset(old_mem.ptr, undefined, old_mem.len);
	142	_ = self.shrinkBytes(old_mem, old_align, 0, 0);
	143	return new_mem;
	144	}
	145
	146	/// Returns a pointer to undefined memory.
	147	/// Call `destroy` with the result to free the memory.
	148	pub fn create(self: Allocator, comptime T: type) Error!T {
	149	if (@sizeOf(T) == 0) return &(T{});
	150	const slice = try self.alloc(T, 1);
	151	return &slice[0];
	152	}
	153
	154	/// `ptr` should be the return value of `create`, or otherwise
	155	/// have the same address and alignment property.
	156	pub fn destroy(self: *Allocator, ptr: anytype) void {
	157	const T = @TypeOf(ptr).Child;
	158	if (@sizeOf(T) == 0) return;
	159	const non_const_ptr = @intToPtr([*]u8, @ptrToInt(ptr));
	160	const ptr_align = @typeInfo(@TypeOf(ptr)).Pointer.alignment;
	161	_ = self.shrinkBytes(non_const_ptr[0..@sizeOf(T)], ptr_align, 0, 0);
	162	}
	163
	164	/// Allocates an array of `n` items of type `T` and sets all the
	165	/// items to `undefined`. Depending on the Allocator
	166	/// implementation, it may be required to call `free` once the
	167	/// memory is no longer needed, to avoid a resource leak. If the
	168	/// `Allocator` implementation is unknown, then correct code will
	169	/// call `free` when done.
	170	///
	171	/// For allocating a single item, see `create`.
	172	pub fn alloc(self: *Allocator, comptime T: type, n: usize) Error![]T {
	173	return self.alignedAlloc(T, null, n);
	174	}
	175
	176	pub fn allocWithOptions(
	177	self: *Allocator,
	178	comptime Elem: type,
	179	n: usize,
	180	/// null means naturally aligned
	181	comptime optional_alignment: ?u29,
	182	comptime optional_sentinel: ?Elem,
	183	) Error!AllocWithOptionsPayload(Elem, optional_alignment, optional_sentinel) {
	184	if (optional_sentinel) \|sentinel\| {
	185	const ptr = try self.alignedAlloc(Elem, optional_alignment, n + 1);
	186	ptr[n] = sentinel;
	187	return ptr[0..n :sentinel];
	188	} else {
	189	return self.alignedAlloc(Elem, optional_alignment, n);
	190	}
	191	}
	192
	193	fn AllocWithOptionsPayload(comptime Elem: type, comptime alignment: ?u29, comptime sentinel: ?Elem) type {
	194	if (sentinel) \|s\| {
	195	return [:s]align(alignment orelse @alignOf(Elem)) Elem;
	196	} else {
	197	return []align(alignment orelse @alignOf(Elem)) Elem;
	198	}
	199	}
	200
	201	/// Allocates an array of `n + 1` items of type `T` and sets the first `n`
	202	/// items to `undefined` and the last item to `sentinel`. Depending on the
	203	/// Allocator implementation, it may be required to call `free` once the
	204	/// memory is no longer needed, to avoid a resource leak. If the
	205	/// `Allocator` implementation is unknown, then correct code will
	206	/// call `free` when done.
	207	///
	208	/// For allocating a single item, see `create`.
	209	///
	210	/// Deprecated; use `allocWithOptions`.
	211	pub fn allocSentinel(self: *Allocator, comptime Elem: type, n: usize, comptime sentinel: Elem) Error![:sentinel]Elem {
	212	return self.allocWithOptions(Elem, n, null, sentinel);
	213	}
	214
	215	/// Deprecated: use `allocAdvanced`
	216	pub fn alignedAlloc(
	217	self: *Allocator,
	218	comptime T: type,
	219	/// null means naturally aligned
	220	comptime alignment: ?u29,
	221	n: usize,
	222	) Error![]align(alignment orelse @alignOf(T)) T {
	223	return self.allocAdvanced(T, alignment, n, .exact);
	224	}
	225
	226	const Exact = enum { exact, at_least };
	227	pub fn allocAdvanced(
	228	self: *Allocator,
	229	comptime T: type,
	230	/// null means naturally aligned
	231	comptime alignment: ?u29,
	232	n: usize,
	233	exact: Exact,
	234	) Error![]align(alignment orelse @alignOf(T)) T {
	235	const a = if (alignment) \|a\| blk: {
	236	if (a == @alignOf(T)) return allocAdvanced(self, T, null, n, exact);
	237	break :blk a;
	238	} else @alignOf(T);
	239
	240	if (n == 0) {
	241	return @as([*]align(a) T, undefined)[0..0];
	242	}
	243
	244	const byte_count = math.mul(usize, @sizeOf(T), n) catch return Error.OutOfMemory;
	245	// TODO The `if (alignment == null)` blocks are workarounds for zig not being able to
	246	// access certain type information about T without creating a circular dependency in async
	247	// functions that heap-allocate their own frame with @Frame(func).
	248	const sizeOfT = if (alignment == null) @intCast(u29, @divExact(byte_count, n)) else @sizeOf(T);
	249	const byte_slice = try self.allocFn(self, byte_count, a, if (exact == .exact) @as(u29, 0) else sizeOfT);
	250	switch (exact) {
	251	.exact => assert(byte_slice.len == byte_count),
	252	.at_least => assert(byte_slice.len >= byte_count),
	253	}
	254	// TODO: https://github.com/ziglang/zig/issues/4298
	255	@memset(byte_slice.ptr, undefined, byte_slice.len);
	256	if (alignment == null) {
	257	// This if block is a workaround (see comment above)
	258	return @intToPtr([*]T, @ptrToInt(byte_slice.ptr))[0..@divExact(byte_slice.len, @sizeOf(T))];
	259	} else {
	260	return mem.bytesAsSlice(T, @alignCast(a, byte_slice));
	261	}
	262	}
	263
	264	/// Increases or decreases the size of an allocation. It is guaranteed to not move the pointer.
	265	pub fn resize(self: *Allocator, old_mem: anytype, new_n: usize) Error!@TypeOf(old_mem) {
	266	const Slice = @typeInfo(@TypeOf(old_mem)).Pointer;
	267	const T = Slice.child;
	268	if (new_n == 0) {
	269	self.free(old_mem);
	270	return &[0]T{};
	271	}
	272	const old_byte_slice = mem.sliceAsBytes(old_mem);
	273	const new_byte_count = math.mul(usize, @sizeOf(T), new_n) catch return Error.OutOfMemory;
	274	const rc = try self.resizeFn(self, old_byte_slice, Slice.alignment, new_byte_count, 0);
	275	assert(rc == new_byte_count);
	276	const new_byte_slice = old_mem.ptr[0..new_byte_count];
	277	return mem.bytesAsSlice(T, new_byte_slice);
	278	}
	279
	280	/// This function requests a new byte size for an existing allocation,
	281	/// which can be larger, smaller, or the same size as the old memory
	282	/// allocation.
	283	/// This function is preferred over `shrink`, because it can fail, even
	284	/// when shrinking. This gives the allocator a chance to perform a
	285	/// cheap shrink operation if possible, or otherwise return OutOfMemory,
	286	/// indicating that the caller should keep their capacity, for example
	287	/// in `std.ArrayList.shrink`.
	288	/// If you need guaranteed success, call `shrink`.
	289	/// If `new_n` is 0, this is the same as `free` and it always succeeds.
	290	pub fn realloc(self: *Allocator, old_mem: anytype, new_n: usize) t: {
	291	const Slice = @typeInfo(@TypeOf(old_mem)).Pointer;
	292	break :t Error![]align(Slice.alignment) Slice.child;
	293	} {
	294	const old_alignment = @typeInfo(@TypeOf(old_mem)).Pointer.alignment;
	295	return self.reallocAdvanced(old_mem, old_alignment, new_n, .exact);
	296	}
	297
	298	pub fn reallocAtLeast(self: *Allocator, old_mem: anytype, new_n: usize) t: {
	299	const Slice = @typeInfo(@TypeOf(old_mem)).Pointer;
	300	break :t Error![]align(Slice.alignment) Slice.child;
	301	} {
	302	const old_alignment = @typeInfo(@TypeOf(old_mem)).Pointer.alignment;
	303	return self.reallocAdvanced(old_mem, old_alignment, new_n, .at_least);
	304	}
	305
	306	// Deprecated: use `reallocAdvanced`
	307	pub fn alignedRealloc(
	308	self: *Allocator,
	309	old_mem: anytype,
	310	comptime new_alignment: u29,
	311	new_n: usize,
	312	) Error![]align(new_alignment) @typeInfo(@TypeOf(old_mem)).Pointer.child {
	313	return self.reallocAdvanced(old_mem, new_alignment, new_n, .exact);
	314	}
	315
	316	/// This is the same as `realloc`, except caller may additionally request
	317	/// a new alignment, which can be larger, smaller, or the same as the old
	318	/// allocation.
	319	pub fn reallocAdvanced(
	320	self: *Allocator,
	321	old_mem: anytype,
	322	comptime new_alignment: u29,
	323	new_n: usize,
	324	exact: Exact,
	325	) Error![]align(new_alignment) @typeInfo(@TypeOf(old_mem)).Pointer.child {
	326	const Slice = @typeInfo(@TypeOf(old_mem)).Pointer;
	327	const T = Slice.child;
	328	if (old_mem.len == 0) {
	329	return self.allocAdvanced(T, new_alignment, new_n, exact);
	330	}
	331	if (new_n == 0) {
	332	self.free(old_mem);
	333	return @as([*]align(new_alignment) T, undefined)[0..0];
	334	}
	335
	336	const old_byte_slice = mem.sliceAsBytes(old_mem);
	337	const byte_count = math.mul(usize, @sizeOf(T), new_n) catch return Error.OutOfMemory;
	338	// Note: can't set shrunk memory to undefined as memory shouldn't be modified on realloc failure
	339	const new_byte_slice = try self.reallocBytes(old_byte_slice, Slice.alignment, byte_count, new_alignment, if (exact == .exact) @as(u29, 0) else @sizeOf(T));
	340	return mem.bytesAsSlice(T, @alignCast(new_alignment, new_byte_slice));
	341	}
	342
	343	/// Prefer calling realloc to shrink if you can tolerate failure, such as
	344	/// in an ArrayList data structure with a storage capacity.
	345	/// Shrink always succeeds, and `new_n` must be <= `old_mem.len`.
	346	/// Returned slice has same alignment as old_mem.
	347	/// Shrinking to 0 is the same as calling `free`.
	348	pub fn shrink(self: *Allocator, old_mem: anytype, new_n: usize) t: {
	349	const Slice = @typeInfo(@TypeOf(old_mem)).Pointer;
	350	break :t []align(Slice.alignment) Slice.child;
	351	} {
	352	const old_alignment = @typeInfo(@TypeOf(old_mem)).Pointer.alignment;
	353	return self.alignedShrink(old_mem, old_alignment, new_n);
	354	}
	355
	356	/// This is the same as `shrink`, except caller may additionally request
	357	/// a new alignment, which must be smaller or the same as the old
	358	/// allocation.
	359	pub fn alignedShrink(
	360	self: *Allocator,
	361	old_mem: anytype,
	362	comptime new_alignment: u29,
	363	new_n: usize,
	364	) []align(new_alignment) @typeInfo(@TypeOf(old_mem)).Pointer.child {
	365	const Slice = @typeInfo(@TypeOf(old_mem)).Pointer;
	366	const T = Slice.child;
	367
	368	if (new_n == old_mem.len)
	369	return old_mem;
	370	assert(new_n < old_mem.len);
	371	assert(new_alignment <= Slice.alignment);
	372
	373	// Here we skip the overflow checking on the multiplication because
	374	// new_n <= old_mem.len and the multiplication didn't overflow for that operation.
	375	const byte_count = @sizeOf(T) * new_n;
	376
	377	const old_byte_slice = mem.sliceAsBytes(old_mem);
	378	// TODO: https://github.com/ziglang/zig/issues/4298
	379	@memset(old_byte_slice.ptr + byte_count, undefined, old_byte_slice.len - byte_count);
	380	_ = self.shrinkBytes(old_byte_slice, Slice.alignment, byte_count, 0);
	381	return old_mem[0..new_n];
	382	}
	383
	384	/// Free an array allocated with `alloc`. To free a single item,
	385	/// see `destroy`.
	386	pub fn free(self: *Allocator, memory: anytype) void {
	387	const Slice = @typeInfo(@TypeOf(memory)).Pointer;
	388	const bytes = mem.sliceAsBytes(memory);
	389	const bytes_len = bytes.len + if (Slice.sentinel != null) @sizeOf(Slice.child) else 0;
	390	if (bytes_len == 0) return;
	391	const non_const_ptr = @intToPtr([*]u8, @ptrToInt(bytes.ptr));
	392	// TODO: https://github.com/ziglang/zig/issues/4298
	393	@memset(non_const_ptr, undefined, bytes_len);
	394	_ = self.shrinkBytes(non_const_ptr[0..bytes_len], Slice.alignment, 0, 0);
	395	}
	396
	397	/// Copies `m` to newly allocated memory. Caller owns the memory.
	398	pub fn dupe(allocator: *Allocator, comptime T: type, m: []const T) ![]T {
	399	const new_buf = try allocator.alloc(T, m.len);
	400	mem.copy(T, new_buf, m);
	401	return new_buf;
	402	}
	403
	404	/// Copies `m` to newly allocated memory, with a null-terminated element. Caller owns the memory.
	405	pub fn dupeZ(allocator: *Allocator, comptime T: type, m: []const T) ![:0]T {
	406	const new_buf = try allocator.alloc(T, m.len + 1);
	407	mem.copy(T, new_buf, m);
	408	new_buf[m.len] = 0;
	409	return new_buf[0..m.len :0];
	410	}

lib/std/mutex.zig+51-43

...	...	@@ -30,49 +30,7 @@ const ResetEvent = std.ResetEvent;
30	30	/// // ... lock not acquired
31	31	/// }
32	32	pub const Mutex = if (builtin.single_threaded)
33		struct {
34		lock: @TypeOf(lock_init),
35
36		const lock_init = if (std.debug.runtime_safety) false else {};
37
38		pub const Held = struct {
39		mutex: *Mutex,
40
41		pub fn release(self: Held) void {
42		if (std.debug.runtime_safety) {
43		self.mutex.lock = false;
44		}
45		}
46		};
47
48		/// Create a new mutex in unlocked state.
49		pub fn init() Mutex {
50		return Mutex{ .lock = lock_init };
51		}
52
53		/// Free a mutex created with init. Calling this while the
54		/// mutex is held is illegal behavior.
55		pub fn deinit(self: *Mutex) void {
56		self.* = undefined;
57		}
58
59		/// Try to acquire the mutex without blocking. Returns null if
60		/// the mutex is unavailable. Otherwise returns Held. Call
61		/// release on Held.
62		pub fn tryAcquire(self: *Mutex) ?Held {
63		if (std.debug.runtime_safety) {
64		if (self.lock) return null;
65		self.lock = true;
66		}
67		return Held{ .mutex = self };
68		}
69
70		/// Acquire the mutex. Will deadlock if the mutex is already
71		/// held by the calling thread.
72		pub fn acquire(self: *Mutex) Held {
73		return self.tryAcquire() orelse @panic("deadlock detected");
74		}
75		}
	33	Dummy
76	34	else if (builtin.os.tag == .windows)
77	35	// https://locklessinc.com/articles/keyed_events/
78	36	extern union {
...	...	@@ -82,6 +40,8 @@ else if (builtin.os.tag == .windows)
82	40	const WAKE = 1 << 8;
83	41	const WAIT = 1 << 9;
84	42
	43	pub const Dummy = Dummy;
	44
85	45	pub fn init() Mutex {
86	46	return Mutex{ .waiters = 0 };
87	47	}
...	...	@@ -166,6 +126,8 @@ else if (builtin.link_libc or builtin.os.tag == .linux)
166	126	struct {
167	127	state: usize,
168	128
	129	pub const Dummy = Dummy;
	130
169	131	/// number of times to spin trying to acquire the lock.
170	132	/// https://webkit.org/blog/6161/locking-in-webkit/
171	133	const SPIN_COUNT = 40;
...	...	@@ -298,6 +260,52 @@ else if (builtin.link_libc or builtin.os.tag == .linux)
298	260	else
299	261	SpinLock;
300	262
	263	/// This has the sematics as `Mutex`, however it does not actually do any
	264	/// synchronization. Operations are safety-checked no-ops.
	265	pub const Dummy = struct {
	266	lock: @TypeOf(lock_init),
	267
	268	const lock_init = if (std.debug.runtime_safety) false else {};
	269
	270	pub const Held = struct {
	271	mutex: *Dummy,
	272
	273	pub fn release(self: Held) void {
	274	if (std.debug.runtime_safety) {
	275	self.mutex.lock = false;
	276	}
	277	}
	278	};
	279
	280	/// Create a new mutex in unlocked state.
	281	pub fn init() Dummy {
	282	return Dummy{ .lock = lock_init };
	283	}
	284
	285	/// Free a mutex created with init. Calling this while the
	286	/// mutex is held is illegal behavior.
	287	pub fn deinit(self: *Dummy) void {
	288	self.* = undefined;
	289	}
	290
	291	/// Try to acquire the mutex without blocking. Returns null if
	292	/// the mutex is unavailable. Otherwise returns Held. Call
	293	/// release on Held.
	294	pub fn tryAcquire(self: *Dummy) ?Held {
	295	if (std.debug.runtime_safety) {
	296	if (self.lock) return null;
	297	self.lock = true;
	298	}
	299	return Held{ .mutex = self };
	300	}
	301
	302	/// Acquire the mutex. Will deadlock if the mutex is already
	303	/// held by the calling thread.
	304	pub fn acquire(self: *Dummy) Held {
	305	return self.tryAcquire() orelse @panic("deadlock detected");
	306	}
	307	};
	308
301	309	const TestContext = struct {
302	310	mutex: *Mutex,
303	311	data: i128,

lib/std/special/test_runner.zig+8-11

...	...	@@ -20,14 +20,15 @@ pub fn main() anyerror!void {
20	20	async_frame_buffer = &[_]u8{};
21	21
22	22	for (test_fn_list) \|test_fn, i\| {
23		std.testing.base_allocator_instance.reset();
	23	std.testing.allocator_instance = std.heap.GeneralPurposeAllocator(.{}){};
	24	defer std.testing.allocator_instance.deinit();
24	25	std.testing.log_level = .warn;
25	26
26	27	var test_node = root_node.start(test_fn.name, null);
27	28	test_node.activate();
28	29	progress.refresh();
29	30	if (progress.terminal == null) {
30		std.debug.warn("{}/{} {}...", .{ i + 1, test_fn_list.len, test_fn.name });
	31	std.debug.print("{}/{} {}...", .{ i + 1, test_fn_list.len, test_fn.name });
31	32	}
32	33	const result = if (test_fn.async_frame_size) \|size\| switch (io_mode) {
33	34	.evented => blk: {
...	...	@@ -42,24 +43,20 @@ pub fn main() anyerror!void {
42	43	skip_count += 1;
43	44	test_node.end();
44	45	progress.log("{}...SKIP (async test)\n", .{test_fn.name});
45		if (progress.terminal == null) std.debug.warn("SKIP (async test)\n", .{});
	46	if (progress.terminal == null) std.debug.print("SKIP (async test)\n", .{});
46	47	continue;
47	48	},
48	49	} else test_fn.func();
49	50	if (result) \|_\| {
50	51	ok_count += 1;
51	52	test_node.end();
52		std.testing.allocator_instance.validate() catch \|err\| switch (err) {
53		error.Leak => std.debug.panic("", .{}),
54		else => std.debug.panic("error.{}", .{@errorName(err)}),
55		};
56		if (progress.terminal == null) std.debug.warn("OK\n", .{});
	53	if (progress.terminal == null) std.debug.print("OK\n", .{});
57	54	} else \|err\| switch (err) {
58	55	error.SkipZigTest => {
59	56	skip_count += 1;
60	57	test_node.end();
61	58	progress.log("{}...SKIP\n", .{test_fn.name});
62		if (progress.terminal == null) std.debug.warn("SKIP\n", .{});
	59	if (progress.terminal == null) std.debug.print("SKIP\n", .{});
63	60	},
64	61	else => {
65	62	progress.log("", .{});
...	...	@@ -69,9 +66,9 @@ pub fn main() anyerror!void {
69	66	}
70	67	root_node.end();
71	68	if (ok_count == test_fn_list.len) {
72		std.debug.warn("All {} tests passed.\n", .{ok_count});
	69	std.debug.print("All {} tests passed.\n", .{ok_count});
73	70	} else {
74		std.debug.warn("{} passed; {} skipped.\n", .{ ok_count, skip_count });
	71	std.debug.print("{} passed; {} skipped.\n", .{ ok_count, skip_count });
75	72	}
76	73	}
77	74

lib/std/testing.zig+14-16

...	...	@@ -1,18 +1,16 @@
1	1	const std = @import("std.zig");
2		const warn = std.debug.warn;
	2	const print = std.debug.print;
3	3
4		pub const LeakCountAllocator = @import("testing/leak_count_allocator.zig").LeakCountAllocator;
5	4	pub const FailingAllocator = @import("testing/failing_allocator.zig").FailingAllocator;
6	5
7	6	/// This should only be used in temporary test programs.
8	7	pub const allocator = &allocator_instance.allocator;
9		pub var allocator_instance = LeakCountAllocator.init(&base_allocator_instance.allocator);
	8	pub var allocator_instance: std.heap.GeneralPurposeAllocator(.{}) = undefined;
10	9
11	10	pub const failing_allocator = &failing_allocator_instance.allocator;
12	11	pub var failing_allocator_instance = FailingAllocator.init(&base_allocator_instance.allocator, 0);
13	12
14		pub var base_allocator_instance = std.mem.validationWrap(std.heap.ThreadSafeFixedBufferAllocator.init(allocator_mem[0..]));
15		var allocator_mem: [2 * 1024 * 1024]u8 = undefined;
	13	pub var base_allocator_instance = std.heap.FixedBufferAllocator.init("");
16	14
17	15	/// TODO https://github.com/ziglang/zig/issues/5738
18	16	pub var log_level = std.log.Level.warn;
...	...	@@ -326,22 +324,22 @@ test "expectEqual vector" {
326	324
327	325	pub fn expectEqualStrings(expected: []const u8, actual: []const u8) void {
328	326	if (std.mem.indexOfDiff(u8, actual, expected)) \|diff_index\| {
329		warn("\n====== expected this output: =========\n", .{});
	327	print("\n====== expected this output: =========\n", .{});
330	328	printWithVisibleNewlines(expected);
331		warn("\n======== instead found this: =========\n", .{});
	329	print("\n======== instead found this: =========\n", .{});
332	330	printWithVisibleNewlines(actual);
333		warn("\n======================================\n", .{});
	331	print("\n======================================\n", .{});
334	332
335	333	var diff_line_number: usize = 1;
336	334	for (expected[0..diff_index]) \|value\| {
337	335	if (value == '\n') diff_line_number += 1;
338	336	}
339		warn("First difference occurs on line {}:\n", .{diff_line_number});
	337	print("First difference occurs on line {}:\n", .{diff_line_number});
340	338
341		warn("expected:\n", .{});
	339	print("expected:\n", .{});
342	340	printIndicatorLine(expected, diff_index);
343	341
344		warn("found:\n", .{});
	342	print("found:\n", .{});
345	343	printIndicatorLine(actual, diff_index);
346	344
347	345	@panic("test failure");
...	...	@@ -362,9 +360,9 @@ fn printIndicatorLine(source: []const u8, indicator_index: usize) void {
362	360	{
363	361	var i: usize = line_begin_index;
364	362	while (i < indicator_index) : (i += 1)
365		warn(" ", .{});
	363	print(" ", .{});
366	364	}
367		warn("^\n", .{});
	365	print("^\n", .{});
368	366	}
369	367
370	368	fn printWithVisibleNewlines(source: []const u8) void {
...	...	@@ -372,15 +370,15 @@ fn printWithVisibleNewlines(source: []const u8) void {
372	370	while (std.mem.indexOf(u8, source[i..], "\n")) \|nl\| : (i += nl + 1) {
373	371	printLine(source[i .. i + nl]);
374	372	}
375		warn("{}␃\n", .{source[i..]}); // End of Text symbol (ETX)
	373	print("{}␃\n", .{source[i..]}); // End of Text symbol (ETX)
376	374	}
377	375
378	376	fn printLine(line: []const u8) void {
379	377	if (line.len != 0) switch (line[line.len - 1]) {
380		' ', '\t' => warn("{}⏎\n", .{line}), // Carriage return symbol,
	378	' ', '\t' => print("{}⏎\n", .{line}), // Carriage return symbol,
381	379	else => {},
382	380	};
383		warn("{}\n", .{line});
	381	print("{}\n", .{line});
384	382	}
385	383
386	384	test "" {

lib/std/testing/failing_allocator.zig+9-3

...	...	@@ -50,16 +50,22 @@ pub const FailingAllocator = struct {
50	50	if (self.index == self.fail_index) {
51	51	return error.OutOfMemory;
52	52	}
53		const result = try self.internal_allocator.callAllocFn(len, ptr_align, len_align);
	53	const result = try self.internal_allocator.allocFn(self.internal_allocator, len, ptr_align, len_align);
54	54	self.allocated_bytes += result.len;
55	55	self.allocations += 1;
56	56	self.index += 1;
57	57	return result;
58	58	}
59	59
60		fn resize(allocator: *std.mem.Allocator, old_mem: []u8, new_len: usize, len_align: u29) error{OutOfMemory}!usize {
	60	fn resize(
	61	allocator: *std.mem.Allocator,
	62	old_mem: []u8,
	63	old_align: u29,
	64	new_len: usize,
	65	len_align: u29,
	66	) error{OutOfMemory}!usize {
61	67	const self = @fieldParentPtr(FailingAllocator, "allocator", allocator);
62		const r = self.internal_allocator.callResizeFn(old_mem, new_len, len_align) catch \|e\| {
	68	const r = self.internal_allocator.resizeFn(self.internal_allocator, old_mem, old_align, new_len, len_align) catch \|e\| {
63	69	std.debug.assert(new_len > old_mem.len);
64	70	return e;
65	71	};

lib/std/testing/leak_count_allocator.zig deleted-51

...	...	@@ -1,51 +0,0 @@
1		const std = @import("../std.zig");
2
3		/// This allocator is used in front of another allocator and counts the numbers of allocs and frees.
4		/// The test runner asserts every alloc has a corresponding free at the end of each test.
5		///
6		/// The detection algorithm is incredibly primitive and only accounts for number of calls.
7		/// This should be replaced by the general purpose debug allocator.
8		pub const LeakCountAllocator = struct {
9		count: usize,
10		allocator: std.mem.Allocator,
11		internal_allocator: *std.mem.Allocator,
12
13		pub fn init(allocator: *std.mem.Allocator) LeakCountAllocator {
14		return .{
15		.count = 0,
16		.allocator = .{
17		.allocFn = alloc,
18		.resizeFn = resize,
19		},
20		.internal_allocator = allocator,
21		};
22		}
23
24		fn alloc(allocator: *std.mem.Allocator, len: usize, ptr_align: u29, len_align: u29) error{OutOfMemory}![]u8 {
25		const self = @fieldParentPtr(LeakCountAllocator, "allocator", allocator);
26		const ptr = try self.internal_allocator.callAllocFn(len, ptr_align, len_align);
27		self.count += 1;
28		return ptr;
29		}
30
31		fn resize(allocator: *std.mem.Allocator, old_mem: []u8, new_size: usize, len_align: u29) error{OutOfMemory}!usize {
32		const self = @fieldParentPtr(LeakCountAllocator, "allocator", allocator);
33		if (new_size == 0) {
34		if (self.count == 0) {
35		std.debug.panic("error - too many calls to free, most likely double free", .{});
36		}
37		self.count -= 1;
38		}
39		return self.internal_allocator.callResizeFn(old_mem, new_size, len_align) catch \|e\| {
40		std.debug.assert(new_size > old_mem.len);
41		return e;
42		};
43		}
44
45		pub fn validate(self: LeakCountAllocator) !void {
46		if (self.count > 0) {
47		std.debug.warn("error - detected leaked allocations without matching free: {}\n", .{self.count});
48		return error.Leak;
49		}
50		}
51		};

src/ir.cpp+26-14

...	...	@@ -25067,12 +25067,12 @@ static PtrLen size_enum_index_to_ptr_len(BuiltinPtrSize size_enum_index) {
25067	25067	zig_unreachable();
25068	25068	}
25069	25069
25070		static ZigValue create_ptr_like_type_info(IrAnalyze ira, ZigType *ptr_type_entry) {
25071		Error err;
	25070	static ZigValue create_ptr_like_type_info(IrAnalyze ira, IrInst source_instr, ZigType ptr_type_entry) {
25072	25071	ZigType *attrs_type;
25073	25072	BuiltinPtrSize size_enum_index;
25074	25073	if (is_slice(ptr_type_entry)) {
25075		attrs_type = ptr_type_entry->data.structure.fields[slice_ptr_index]->type_entry;
	25074	TypeStructField *ptr_field = ptr_type_entry->data.structure.fields[slice_ptr_index];
	25075	attrs_type = resolve_struct_field_type(ira->codegen, ptr_field);
25076	25076	size_enum_index = BuiltinPtrSizeSlice;
25077	25077	} else if (ptr_type_entry->id == ZigTypeIdPointer) {
25078	25078	attrs_type = ptr_type_entry;
...	...	@@ -25081,9 +25081,6 @@ static ZigValue create_ptr_like_type_info(IrAnalyze ira, ZigType *ptr_type_ent
25081	25081	zig_unreachable();
25082	25082	}
25083	25083
25084		if ((err = type_resolve(ira->codegen, attrs_type->data.pointer.child_type, ResolveStatusSizeKnown)))
25085		return nullptr;
25086
25087	25084	ZigType *type_info_pointer_type = ir_type_info_get_type(ira, "Pointer", nullptr);
25088	25085	assertNoError(type_resolve(ira->codegen, type_info_pointer_type, ResolveStatusSizeKnown));
25089	25086
...	...	@@ -25114,9 +25111,18 @@ static ZigValue create_ptr_like_type_info(IrAnalyze ira, ZigType *ptr_type_ent
25114	25111	fields[2]->data.x_bool = attrs_type->data.pointer.is_volatile;
25115	25112	// alignment: u32
25116	25113	ensure_field_index(result->type, "alignment", 3);
25117		fields[3]->special = ConstValSpecialStatic;
25118	25114	fields[3]->type = ira->codegen->builtin_types.entry_num_lit_int;
25119		bigint_init_unsigned(&fields[3]->data.x_bigint, get_ptr_align(ira->codegen, attrs_type));
	25115	if (attrs_type->data.pointer.explicit_alignment != 0) {
	25116	fields[3]->special = ConstValSpecialStatic;
	25117	bigint_init_unsigned(&fields[3]->data.x_bigint, attrs_type->data.pointer.explicit_alignment);
	25118	} else {
	25119	LazyValueAlignOf *lazy_align_of = heap::c_allocator.create<LazyValueAlignOf>();
	25120	lazy_align_of->ira = ira; ira_ref(ira);
	25121	fields[3]->special = ConstValSpecialLazy;
	25122	fields[3]->data.x_lazy = &lazy_align_of->base;
	25123	lazy_align_of->base.id = LazyValueIdAlignOf;
	25124	lazy_align_of->target_type = ir_const_type(ira, source_instr, attrs_type->data.pointer.child_type);
	25125	}
25120	25126	// child: type
25121	25127	ensure_field_index(result->type, "child", 4);
25122	25128	fields[4]->special = ConstValSpecialStatic;
...	...	@@ -25130,7 +25136,7 @@ static ZigValue create_ptr_like_type_info(IrAnalyze ira, ZigType *ptr_type_ent
25130	25136	// sentinel: anytype
25131	25137	ensure_field_index(result->type, "sentinel", 6);
25132	25138	fields[6]->special = ConstValSpecialStatic;
25133		if (attrs_type->data.pointer.child_type->id != ZigTypeIdOpaque) {
	25139	if (attrs_type->data.pointer.sentinel != nullptr) {
25134	25140	fields[6]->type = get_optional_type(ira->codegen, attrs_type->data.pointer.child_type);
25135	25141	set_optional_payload(fields[6], attrs_type->data.pointer.sentinel);
25136	25142	} else {
...	...	@@ -25165,9 +25171,6 @@ static Error ir_make_type_info_value(IrAnalyze ira, IrInst source_instr, ZigTy
25165	25171	assert(type_entry != nullptr);
25166	25172	assert(!type_is_invalid(type_entry));
25167	25173
25168		if ((err = type_resolve(ira->codegen, type_entry, ResolveStatusSizeKnown)))
25169		return err;
25170
25171	25174	auto entry = ira->codegen->type_info_cache.maybe_get(type_entry);
25172	25175	if (entry != nullptr) {
25173	25176	*out = entry->value;
...	...	@@ -25231,7 +25234,7 @@ static Error ir_make_type_info_value(IrAnalyze ira, IrInst source_instr, ZigTy
25231	25234	}
25232	25235	case ZigTypeIdPointer:
25233	25236	{
25234		result = create_ptr_like_type_info(ira, type_entry);
	25237	result = create_ptr_like_type_info(ira, source_instr, type_entry);
25235	25238	if (result == nullptr)
25236	25239	return ErrorSemanticAnalyzeFail;
25237	25240	break;
...	...	@@ -25317,6 +25320,9 @@ static Error ir_make_type_info_value(IrAnalyze ira, IrInst source_instr, ZigTy
25317	25320	}
25318	25321	case ZigTypeIdEnum:
25319	25322	{
	25323	if ((err = type_resolve(ira->codegen, type_entry, ResolveStatusSizeKnown)))
	25324	return err;
	25325
25320	25326	result = ira->codegen->pass1_arena->create<ZigValue>();
25321	25327	result->special = ConstValSpecialStatic;
25322	25328	result->type = ir_type_info_get_type(ira, "Enum", nullptr);
...	...	@@ -25455,6 +25461,9 @@ static Error ir_make_type_info_value(IrAnalyze ira, IrInst source_instr, ZigTy
25455	25461	}
25456	25462	case ZigTypeIdUnion:
25457	25463	{
	25464	if ((err = type_resolve(ira->codegen, type_entry, ResolveStatusSizeKnown)))
	25465	return err;
	25466
25458	25467	result = ira->codegen->pass1_arena->create<ZigValue>();
25459	25468	result->special = ConstValSpecialStatic;
25460	25469	result->type = ir_type_info_get_type(ira, "Union", nullptr);
...	...	@@ -25545,12 +25554,15 @@ static Error ir_make_type_info_value(IrAnalyze ira, IrInst source_instr, ZigTy
25545	25554	case ZigTypeIdStruct:
25546	25555	{
25547	25556	if (type_entry->data.structure.special == StructSpecialSlice) {
25548		result = create_ptr_like_type_info(ira, type_entry);
	25557	result = create_ptr_like_type_info(ira, source_instr, type_entry);
25549	25558	if (result == nullptr)
25550	25559	return ErrorSemanticAnalyzeFail;
25551	25560	break;
25552	25561	}
25553	25562
	25563	if ((err = type_resolve(ira->codegen, type_entry, ResolveStatusSizeKnown)))
	25564	return err;
	25565
25554	25566	result = ira->codegen->pass1_arena->create<ZigValue>();
25555	25567	result->special = ConstValSpecialStatic;
25556	25568	result->type = ir_type_info_get_type(ira, "Struct", nullptr);