authorgravatar for leecannon@leecannon.xyzLee Cannon <leecannon@leecannon.xyz> 2021-11-06 00:54:35+00:00
committergravatar for leecannon@leecannon.xyzLee Cannon <leecannon@leecannon.xyz> 2021-11-30 23:32:48+00:00
logf68cda738ad0d3e9bc0f328befad301d9e23756e
treea08f43aa38d630a71f0686dc9bda9181454d1a0b
parent23866b1f81010277b204d6f3f5db23d020a76400
signaturelock-open Commit is signed but in an unrecognized format.

allocgate: split free out from resize

9 files changed, 458 insertions(+), 160 deletions(-)

lib/std/heap.zig+107-34
...@@ -132,10 +132,6 @@ const CAllocator = struct {...@@ -132,10 +132,6 @@ const CAllocator = struct {
132 ) Allocator.Error!usize {132 ) Allocator.Error!usize {
133 _ = buf_align;133 _ = buf_align;
134 _ = return_address;134 _ = return_address;
135 if (new_len == 0) {
136 alignedFree(buf.ptr);
137 return 0;
138 }
139 if (new_len <= buf.len) {135 if (new_len <= buf.len) {
140 return mem.alignAllocLen(buf.len, new_len, len_align);136 return mem.alignAllocLen(buf.len, new_len, len_align);
141 }137 }
...@@ -147,6 +143,17 @@ const CAllocator = struct {...@@ -147,6 +143,17 @@ const CAllocator = struct {
147 }143 }
148 return error.OutOfMemory;144 return error.OutOfMemory;
149 }145 }
146
147 fn free(
148 _: *c_void,
149 buf: []u8,
150 buf_align: u29,
151 return_address: usize,
152 ) void {
153 _ = buf_align;
154 _ = return_address;
155 alignedFree(buf.ptr);
156 }
150};157};
151158
152/// Supports the full Allocator interface, including alignment, and exploiting159/// Supports the full Allocator interface, including alignment, and exploiting
...@@ -159,6 +166,7 @@ pub const c_allocator = Allocator{...@@ -159,6 +166,7 @@ pub const c_allocator = Allocator{
159const c_allocator_vtable = Allocator.VTable{166const c_allocator_vtable = Allocator.VTable{
160 .alloc = CAllocator.alloc,167 .alloc = CAllocator.alloc,
161 .resize = CAllocator.resize,168 .resize = CAllocator.resize,
169 .free = CAllocator.free,
162};170};
163171
164/// Asserts allocations are within `@alignOf(std.c.max_align_t)` and directly calls172/// Asserts allocations are within `@alignOf(std.c.max_align_t)` and directly calls
...@@ -173,6 +181,7 @@ pub const raw_c_allocator = Allocator{...@@ -173,6 +181,7 @@ pub const raw_c_allocator = Allocator{
173const raw_c_allocator_vtable = Allocator.VTable{181const raw_c_allocator_vtable = Allocator.VTable{
174 .alloc = rawCAlloc,182 .alloc = rawCAlloc,
175 .resize = rawCResize,183 .resize = rawCResize,
184 .free = rawCFree,
176};185};
177186
178fn rawCAlloc(187fn rawCAlloc(
...@@ -199,16 +208,23 @@ fn rawCResize(...@@ -199,16 +208,23 @@ fn rawCResize(
199) Allocator.Error!usize {208) Allocator.Error!usize {
200 _ = old_align;209 _ = old_align;
201 _ = ret_addr;210 _ = ret_addr;
202 if (new_len == 0) {
203 c.free(buf.ptr);
204 return 0;
205 }
206 if (new_len <= buf.len) {211 if (new_len <= buf.len) {
207 return mem.alignAllocLen(buf.len, new_len, len_align);212 return mem.alignAllocLen(buf.len, new_len, len_align);
208 }213 }
209 return error.OutOfMemory;214 return error.OutOfMemory;
210}215}
211216
217fn rawCFree(
218 _: *c_void,
219 buf: []u8,
220 old_align: u29,
221 ret_addr: usize,
222) void {
223 _ = old_align;
224 _ = ret_addr;
225 c.free(buf.ptr);
226}
227
212/// This allocator makes a syscall directly for every allocation and free.228/// This allocator makes a syscall directly for every allocation and free.
213/// Thread-safe and lock-free.229/// Thread-safe and lock-free.
214pub const page_allocator = if (builtin.target.isWasm())230pub const page_allocator = if (builtin.target.isWasm())
...@@ -238,6 +254,7 @@ const PageAllocator = struct {...@@ -238,6 +254,7 @@ const PageAllocator = struct {
238 const vtable = Allocator.VTable{254 const vtable = Allocator.VTable{
239 .alloc = alloc,255 .alloc = alloc,
240 .resize = resize,256 .resize = resize,
257 .free = free,
241 };258 };
242259
243 fn alloc(_: *c_void, n: usize, alignment: u29, len_align: u29, ra: usize) error{OutOfMemory}![]u8 {260 fn alloc(_: *c_void, n: usize, alignment: u29, len_align: u29, ra: usize) error{OutOfMemory}![]u8 {
...@@ -351,16 +368,6 @@ const PageAllocator = struct {...@@ -351,16 +368,6 @@ const PageAllocator = struct {
351368
352 if (builtin.os.tag == .windows) {369 if (builtin.os.tag == .windows) {
353 const w = os.windows;370 const w = os.windows;
354 if (new_size == 0) {
355 // From the docs:
356 // "If the dwFreeType parameter is MEM_RELEASE, this parameter
357 // must be 0 (zero). The function frees the entire region that
358 // is reserved in the initial allocation call to VirtualAlloc."
359 // So we can only use MEM_RELEASE when actually releasing the
360 // whole allocation.
361 w.VirtualFree(buf_unaligned.ptr, 0, w.MEM_RELEASE);
362 return 0;
363 }
364 if (new_size <= buf_unaligned.len) {371 if (new_size <= buf_unaligned.len) {
365 const base_addr = @ptrToInt(buf_unaligned.ptr);372 const base_addr = @ptrToInt(buf_unaligned.ptr);
366 const old_addr_end = base_addr + buf_unaligned.len;373 const old_addr_end = base_addr + buf_unaligned.len;
...@@ -391,8 +398,6 @@ const PageAllocator = struct {...@@ -391,8 +398,6 @@ const PageAllocator = struct {
391 const ptr = @alignCast(mem.page_size, buf_unaligned.ptr + new_size_aligned);398 const ptr = @alignCast(mem.page_size, buf_unaligned.ptr + new_size_aligned);
392 // TODO: if the next_mmap_addr_hint is within the unmapped range, update it399 // TODO: if the next_mmap_addr_hint is within the unmapped range, update it
393 os.munmap(ptr[0 .. buf_aligned_len - new_size_aligned]);400 os.munmap(ptr[0 .. buf_aligned_len - new_size_aligned]);
394 if (new_size_aligned == 0)
395 return 0;
396 return alignPageAllocLen(new_size_aligned, new_size, len_align);401 return alignPageAllocLen(new_size_aligned, new_size, len_align);
397 }402 }
398403
...@@ -400,6 +405,19 @@ const PageAllocator = struct {...@@ -400,6 +405,19 @@ const PageAllocator = struct {
400 // TODO: if the next_mmap_addr_hint is within the remapped range, update it405 // TODO: if the next_mmap_addr_hint is within the remapped range, update it
401 return error.OutOfMemory;406 return error.OutOfMemory;
402 }407 }
408
409 fn free(_: *c_void, buf_unaligned: []u8, buf_align: u29, return_address: usize) void {
410 _ = buf_align;
411 _ = return_address;
412
413 if (builtin.os.tag == .windows) {
414 os.windows.VirtualFree(buf_unaligned.ptr, 0, os.windows.MEM_RELEASE);
415 } else {
416 const buf_aligned_len = mem.alignForward(buf_unaligned.len, mem.page_size);
417 const ptr = @alignCast(mem.page_size, buf_unaligned.ptr);
418 os.munmap(ptr[0..buf_aligned_len]);
419 }
420 }
403};421};
404422
405const WasmPageAllocator = struct {423const WasmPageAllocator = struct {
...@@ -412,6 +430,7 @@ const WasmPageAllocator = struct {...@@ -412,6 +430,7 @@ const WasmPageAllocator = struct {
412 const vtable = Allocator.VTable{430 const vtable = Allocator.VTable{
413 .alloc = alloc,431 .alloc = alloc,
414 .resize = resize,432 .resize = resize,
433 .free = free,
415 };434 };
416435
417 const PageStatus = enum(u1) {436 const PageStatus = enum(u1) {
...@@ -571,7 +590,21 @@ const WasmPageAllocator = struct {...@@ -571,7 +590,21 @@ const WasmPageAllocator = struct {
571 const base = nPages(@ptrToInt(buf.ptr));590 const base = nPages(@ptrToInt(buf.ptr));
572 freePages(base + new_n, base + current_n);591 freePages(base + new_n, base + current_n);
573 }592 }
574 return if (new_len == 0) 0 else alignPageAllocLen(new_n * mem.page_size, new_len, len_align);593 return alignPageAllocLen(new_n * mem.page_size, new_len, len_align);
594 }
595
596 fn free(
597 _: *c_void,
598 buf: []u8,
599 buf_align: u29,
600 return_address: usize,
601 ) void {
602 _ = buf_align;
603 _ = return_address;
604 const aligned_len = mem.alignForward(buf.len, mem.page_size);
605 const current_n = nPages(aligned_len);
606 const base = nPages(@ptrToInt(buf.ptr));
607 freePages(base, base + current_n);
575 }608 }
576};609};
577610
...@@ -588,7 +621,7 @@ pub const HeapAllocator = switch (builtin.os.tag) {...@@ -588,7 +621,7 @@ pub const HeapAllocator = switch (builtin.os.tag) {
588 }621 }
589622
590 pub fn allocator(self: *HeapAllocator) Allocator {623 pub fn allocator(self: *HeapAllocator) Allocator {
591 return Allocator.init(self, alloc, resize);624 return Allocator.init(self, alloc, resize, free);
592 }625 }
593626
594 pub fn deinit(self: *HeapAllocator) void {627 pub fn deinit(self: *HeapAllocator) void {
...@@ -644,10 +677,6 @@ pub const HeapAllocator = switch (builtin.os.tag) {...@@ -644,10 +677,6 @@ pub const HeapAllocator = switch (builtin.os.tag) {
644 ) error{OutOfMemory}!usize {677 ) error{OutOfMemory}!usize {
645 _ = buf_align;678 _ = buf_align;
646 _ = return_address;679 _ = return_address;
647 if (new_size == 0) {
648 os.windows.HeapFree(self.heap_handle.?, 0, @intToPtr(*c_void, getRecordPtr(buf).*));
649 return 0;
650 }
651680
652 const root_addr = getRecordPtr(buf).*;681 const root_addr = getRecordPtr(buf).*;
653 const align_offset = @ptrToInt(buf.ptr) - root_addr;682 const align_offset = @ptrToInt(buf.ptr) - root_addr;
...@@ -669,6 +698,17 @@ pub const HeapAllocator = switch (builtin.os.tag) {...@@ -669,6 +698,17 @@ pub const HeapAllocator = switch (builtin.os.tag) {
669 getRecordPtr(buf.ptr[0..return_len]).* = root_addr;698 getRecordPtr(buf.ptr[0..return_len]).* = root_addr;
670 return return_len;699 return return_len;
671 }700 }
701
702 fn free(
703 self: *HeapAllocator,
704 buf: []u8,
705 buf_align: u29,
706 return_address: usize,
707 ) void {
708 _ = buf_align;
709 _ = return_address;
710 os.windows.HeapFree(self.heap_handle.?, 0, @intToPtr(*c_void, getRecordPtr(buf).*));
711 }
672 },712 },
673 else => @compileError("Unsupported OS"),713 else => @compileError("Unsupported OS"),
674};714};
...@@ -696,13 +736,18 @@ pub const FixedBufferAllocator = struct {...@@ -696,13 +736,18 @@ pub const FixedBufferAllocator = struct {
696736
697 /// *WARNING* using this at the same time as the interface returned by `threadSafeAllocator` is not thread safe737 /// *WARNING* using this at the same time as the interface returned by `threadSafeAllocator` is not thread safe
698 pub fn allocator(self: *FixedBufferAllocator) Allocator {738 pub fn allocator(self: *FixedBufferAllocator) Allocator {
699 return Allocator.init(self, alloc, resize);739 return Allocator.init(self, alloc, resize, free);
700 }740 }
701741
702 /// Provides a lock free thread safe `Allocator` interface to the underlying `FixedBufferAllocator`742 /// Provides a lock free thread safe `Allocator` interface to the underlying `FixedBufferAllocator`
703 /// *WARNING* using this at the same time as the interface returned by `getAllocator` is not thread safe743 /// *WARNING* using this at the same time as the interface returned by `getAllocator` is not thread safe
704 pub fn threadSafeAllocator(self: *FixedBufferAllocator) Allocator {744 pub fn threadSafeAllocator(self: *FixedBufferAllocator) Allocator {
705 return Allocator.init(self, threadSafeAlloc, Allocator.NoResize(FixedBufferAllocator).noResize);745 return Allocator.init(
746 self,
747 threadSafeAlloc,
748 Allocator.NoResize(FixedBufferAllocator).noResize,
749 Allocator.NoOpFree(FixedBufferAllocator).noOpFree,
750 );
706 }751 }
707752
708 pub fn ownsPtr(self: *FixedBufferAllocator, ptr: [*]u8) bool {753 pub fn ownsPtr(self: *FixedBufferAllocator, ptr: [*]u8) bool {
...@@ -715,7 +760,7 @@ pub const FixedBufferAllocator = struct {...@@ -715,7 +760,7 @@ pub const FixedBufferAllocator = struct {
715760
716 /// NOTE: this will not work in all cases, if the last allocation had an adjusted_index761 /// NOTE: this will not work in all cases, if the last allocation had an adjusted_index
717 /// then we won't be able to determine what the last allocation was. This is because762 /// then we won't be able to determine what the last allocation was. This is because
718 /// the alignForward operation done in alloc is not reverisible.763 /// the alignForward operation done in alloc is not reversible.
719 pub fn isLastAllocation(self: *FixedBufferAllocator, buf: []u8) bool {764 pub fn isLastAllocation(self: *FixedBufferAllocator, buf: []u8) bool {
720 return buf.ptr + buf.len == self.buffer.ptr + self.end_index;765 return buf.ptr + buf.len == self.buffer.ptr + self.end_index;
721 }766 }
...@@ -751,13 +796,13 @@ pub const FixedBufferAllocator = struct {...@@ -751,13 +796,13 @@ pub const FixedBufferAllocator = struct {
751 if (!self.isLastAllocation(buf)) {796 if (!self.isLastAllocation(buf)) {
752 if (new_size > buf.len)797 if (new_size > buf.len)
753 return error.OutOfMemory;798 return error.OutOfMemory;
754 return if (new_size == 0) 0 else mem.alignAllocLen(buf.len, new_size, len_align);799 return mem.alignAllocLen(buf.len, new_size, len_align);
755 }800 }
756801
757 if (new_size <= buf.len) {802 if (new_size <= buf.len) {
758 const sub = buf.len - new_size;803 const sub = buf.len - new_size;
759 self.end_index -= sub;804 self.end_index -= sub;
760 return if (new_size == 0) 0 else mem.alignAllocLen(buf.len - sub, new_size, len_align);805 return mem.alignAllocLen(buf.len - sub, new_size, len_align);
761 }806 }
762807
763 const add = new_size - buf.len;808 const add = new_size - buf.len;
...@@ -768,6 +813,21 @@ pub const FixedBufferAllocator = struct {...@@ -768,6 +813,21 @@ pub const FixedBufferAllocator = struct {
768 return new_size;813 return new_size;
769 }814 }
770815
816 fn free(
817 self: *FixedBufferAllocator,
818 buf: []u8,
819 buf_align: u29,
820 return_address: usize,
821 ) void {
822 _ = buf_align;
823 _ = return_address;
824 assert(self.ownsSlice(buf)); // sanity check
825
826 if (self.isLastAllocation(buf)) {
827 self.end_index -= buf.len;
828 }
829 }
830
771 fn threadSafeAlloc(self: *FixedBufferAllocator, n: usize, ptr_align: u29, len_align: u29, ra: usize) ![]u8 {831 fn threadSafeAlloc(self: *FixedBufferAllocator, n: usize, ptr_align: u29, len_align: u29, ra: usize) ![]u8 {
772 _ = len_align;832 _ = len_align;
773 _ = ra;833 _ = ra;
...@@ -810,7 +870,7 @@ pub fn StackFallbackAllocator(comptime size: usize) type {...@@ -810,7 +870,7 @@ pub fn StackFallbackAllocator(comptime size: usize) type {
810 /// WARNING: This functions both fetches a `std.mem.Allocator` interface to this allocator *and* resets the internal buffer allocator870 /// WARNING: This functions both fetches a `std.mem.Allocator` interface to this allocator *and* resets the internal buffer allocator
811 pub fn get(self: *Self) Allocator {871 pub fn get(self: *Self) Allocator {
812 self.fixed_buffer_allocator = FixedBufferAllocator.init(self.buffer[0..]);872 self.fixed_buffer_allocator = FixedBufferAllocator.init(self.buffer[0..]);
813 return Allocator.init(self, alloc, resize);873 return Allocator.init(self, alloc, resize, free);
814 }874 }
815875
816 fn alloc(876 fn alloc(
...@@ -821,7 +881,7 @@ pub fn StackFallbackAllocator(comptime size: usize) type {...@@ -821,7 +881,7 @@ pub fn StackFallbackAllocator(comptime size: usize) type {
821 return_address: usize,881 return_address: usize,
822 ) error{OutOfMemory}![]u8 {882 ) error{OutOfMemory}![]u8 {
823 return FixedBufferAllocator.alloc(&self.fixed_buffer_allocator, len, ptr_align, len_align, return_address) catch883 return FixedBufferAllocator.alloc(&self.fixed_buffer_allocator, len, ptr_align, len_align, return_address) catch
824 return self.fallback_allocator.vtable.alloc(self.fallback_allocator.ptr, len, ptr_align, len_align, return_address);884 return self.fallback_allocator.rawAlloc(len, ptr_align, len_align, return_address);
825 }885 }
826886
827 fn resize(887 fn resize(
...@@ -835,7 +895,20 @@ pub fn StackFallbackAllocator(comptime size: usize) type {...@@ -835,7 +895,20 @@ pub fn StackFallbackAllocator(comptime size: usize) type {
835 if (self.fixed_buffer_allocator.ownsPtr(buf.ptr)) {895 if (self.fixed_buffer_allocator.ownsPtr(buf.ptr)) {
836 return FixedBufferAllocator.resize(&self.fixed_buffer_allocator, buf, buf_align, new_len, len_align, return_address);896 return FixedBufferAllocator.resize(&self.fixed_buffer_allocator, buf, buf_align, new_len, len_align, return_address);
837 } else {897 } else {
838 return self.fallback_allocator.vtable.resize(self.fallback_allocator.ptr, buf, buf_align, new_len, len_align, return_address);898 return self.fallback_allocator.rawResize(buf, buf_align, new_len, len_align, return_address);
899 }
900 }
901
902 fn free(
903 self: *Self,
904 buf: []u8,
905 buf_align: u29,
906 return_address: usize,
907 ) void {
908 if (self.fixed_buffer_allocator.ownsPtr(buf.ptr)) {
909 return FixedBufferAllocator.free(&self.fixed_buffer_allocator, buf, buf_align, return_address);
910 } else {
911 return self.fallback_allocator.rawFree(buf, buf_align, return_address);
839 }912 }
840 }913 }
841 };914 };
lib/std/heap/arena_allocator.zig+14-2
...@@ -24,7 +24,7 @@ pub const ArenaAllocator = struct {...@@ -24,7 +24,7 @@ pub const ArenaAllocator = struct {
24 };24 };
2525
26 pub fn allocator(self: *ArenaAllocator) Allocator {26 pub fn allocator(self: *ArenaAllocator) Allocator {
27 return Allocator.init(self, alloc, resize);27 return Allocator.init(self, alloc, resize, free);
28 }28 }
2929
30 const BufNode = std.SinglyLinkedList([]u8).Node;30 const BufNode = std.SinglyLinkedList([]u8).Node;
...@@ -47,7 +47,7 @@ pub const ArenaAllocator = struct {...@@ -47,7 +47,7 @@ pub const ArenaAllocator = struct {
47 const actual_min_size = minimum_size + (@sizeOf(BufNode) + 16);47 const actual_min_size = minimum_size + (@sizeOf(BufNode) + 16);
48 const big_enough_len = prev_len + actual_min_size;48 const big_enough_len = prev_len + actual_min_size;
49 const len = big_enough_len + big_enough_len / 2;49 const len = big_enough_len + big_enough_len / 2;
50 const buf = try self.child_allocator.vtable.alloc(self.child_allocator.ptr, len, @alignOf(BufNode), 1, @returnAddress());50 const buf = try self.child_allocator.rawAlloc(len, @alignOf(BufNode), 1, @returnAddress());
51 const buf_node = @ptrCast(*BufNode, @alignCast(@alignOf(BufNode), buf.ptr));51 const buf_node = @ptrCast(*BufNode, @alignCast(@alignOf(BufNode), buf.ptr));
52 buf_node.* = BufNode{52 buf_node.* = BufNode{
53 .data = buf,53 .data = buf,
...@@ -111,4 +111,16 @@ pub const ArenaAllocator = struct {...@@ -111,4 +111,16 @@ pub const ArenaAllocator = struct {
111 return error.OutOfMemory;111 return error.OutOfMemory;
112 }112 }
113 }113 }
114
115 fn free(self: *ArenaAllocator, buf: []u8, buf_align: u29, ret_addr: usize) void {
116 _ = buf_align;
117 _ = ret_addr;
118
119 const cur_node = self.state.buffer_list.first orelse return;
120 const cur_buf = cur_node.data[@sizeOf(BufNode)..];
121
122 if (@ptrToInt(cur_buf.ptr) + self.state.end_index == @ptrToInt(buf.ptr) + buf.len) {
123 self.state.end_index -= buf.len;
124 }
125 }
114};126};
lib/std/heap/general_purpose_allocator.zig+171-74
...@@ -281,7 +281,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {...@@ -281,7 +281,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
281 };281 };
282282
283 pub fn allocator(self: *Self) Allocator {283 pub fn allocator(self: *Self) Allocator {
284 return Allocator.init(self, alloc, resize);284 return Allocator.init(self, alloc, resize, free);
285 }285 }
286286
287 fn bucketStackTrace(287 fn bucketStackTrace(
...@@ -388,7 +388,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {...@@ -388,7 +388,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
388 var it = self.large_allocations.iterator();388 var it = self.large_allocations.iterator();
389 while (it.next()) |large| {389 while (it.next()) |large| {
390 if (large.value_ptr.freed) {390 if (large.value_ptr.freed) {
391 _ = self.backing_allocator.vtable.resize(self.backing_allocator.ptr, large.value_ptr.bytes, large.value_ptr.ptr_align, 0, 0, @returnAddress()) catch unreachable;391 self.backing_allocator.rawFree(large.value_ptr.bytes, large.value_ptr.ptr_align, @returnAddress());
392 }392 }
393 }393 }
394 }394 }
...@@ -529,9 +529,6 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {...@@ -529,9 +529,6 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
529 if (config.retain_metadata and entry.value_ptr.freed) {529 if (config.retain_metadata and entry.value_ptr.freed) {
530 if (config.safety) {530 if (config.safety) {
531 reportDoubleFree(ret_addr, entry.value_ptr.getStackTrace(.alloc), entry.value_ptr.getStackTrace(.free));531 reportDoubleFree(ret_addr, entry.value_ptr.getStackTrace(.alloc), entry.value_ptr.getStackTrace(.free));
532 // Recoverable if this is a free.
533 if (new_size == 0)
534 return @as(usize, 0);
535 @panic("Unrecoverable double free");532 @panic("Unrecoverable double free");
536 } else {533 } else {
537 unreachable;534 unreachable;
...@@ -555,7 +552,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {...@@ -555,7 +552,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
555552
556 // Do memory limit accounting with requested sizes rather than what backing_allocator returns553 // Do memory limit accounting with requested sizes rather than what backing_allocator returns
557 // because if we want to return error.OutOfMemory, we have to leave allocation untouched, and554 // because if we want to return error.OutOfMemory, we have to leave allocation untouched, and
558 // that is impossible to guarantee after calling backing_allocator.vtable.resize.555 // that is impossible to guarantee after calling backing_allocator.rawResize.
559 const prev_req_bytes = self.total_requested_bytes;556 const prev_req_bytes = self.total_requested_bytes;
560 if (config.enable_memory_limit) {557 if (config.enable_memory_limit) {
561 const new_req_bytes = prev_req_bytes + new_size - entry.value_ptr.requested_size;558 const new_req_bytes = prev_req_bytes + new_size - entry.value_ptr.requested_size;
...@@ -568,29 +565,12 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {...@@ -568,29 +565,12 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
568 self.total_requested_bytes = prev_req_bytes;565 self.total_requested_bytes = prev_req_bytes;
569 };566 };
570567
571 const result_len = if (config.never_unmap and new_size == 0)568 const result_len = try self.backing_allocator.rawResize(old_mem, old_align, new_size, len_align, ret_addr);
572 0
573 else
574 try self.backing_allocator.vtable.resize(self.backing_allocator.ptr, old_mem, old_align, new_size, len_align, ret_addr);
575569
576 if (config.enable_memory_limit) {570 if (config.enable_memory_limit) {
577 entry.value_ptr.requested_size = new_size;571 entry.value_ptr.requested_size = new_size;
578 }572 }
579573
580 if (result_len == 0) {
581 if (config.verbose_log) {
582 log.info("large free {d} bytes at {*}", .{ old_mem.len, old_mem.ptr });
583 }
584
585 if (!config.retain_metadata) {
586 assert(self.large_allocations.remove(@ptrToInt(old_mem.ptr)));
587 } else {
588 entry.value_ptr.freed = true;
589 entry.value_ptr.captureStackTrace(ret_addr, .free);
590 }
591 return 0;
592 }
593
594 if (config.verbose_log) {574 if (config.verbose_log) {
595 log.info("large resize {d} bytes at {*} to {d}", .{575 log.info("large resize {d} bytes at {*} to {d}", .{
596 old_mem.len, old_mem.ptr, new_size,576 old_mem.len, old_mem.ptr, new_size,
...@@ -601,6 +581,64 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {...@@ -601,6 +581,64 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
601 return result_len;581 return result_len;
602 }582 }
603583
584 /// This function assumes the object is in the large object storage regardless
585 /// of the parameters.
586 fn freeLarge(
587 self: *Self,
588 old_mem: []u8,
589 old_align: u29,
590 ret_addr: usize,
591 ) void {
592 _ = old_align;
593
594 const entry = self.large_allocations.getEntry(@ptrToInt(old_mem.ptr)) orelse {
595 if (config.safety) {
596 @panic("Invalid free");
597 } else {
598 unreachable;
599 }
600 };
601
602 if (config.retain_metadata and entry.value_ptr.freed) {
603 if (config.safety) {
604 reportDoubleFree(ret_addr, entry.value_ptr.getStackTrace(.alloc), entry.value_ptr.getStackTrace(.free));
605 return;
606 } else {
607 unreachable;
608 }
609 }
610
611 if (config.safety and old_mem.len != entry.value_ptr.bytes.len) {
612 var addresses: [stack_n]usize = [1]usize{0} ** stack_n;
613 var free_stack_trace = StackTrace{
614 .instruction_addresses = &addresses,
615 .index = 0,
616 };
617 std.debug.captureStackTrace(ret_addr, &free_stack_trace);
618 log.err("Allocation size {d} bytes does not match free size {d}. Allocation: {s} Free: {s}", .{
619 entry.value_ptr.bytes.len,
620 old_mem.len,
621 entry.value_ptr.getStackTrace(.alloc),
622 free_stack_trace,
623 });
624 }
625
626 if (config.enable_memory_limit) {
627 self.total_requested_bytes -= entry.value_ptr.requested_size;
628 }
629
630 if (config.verbose_log) {
631 log.info("large free {d} bytes at {*}", .{ old_mem.len, old_mem.ptr });
632 }
633
634 if (!config.retain_metadata) {
635 assert(self.large_allocations.remove(@ptrToInt(old_mem.ptr)));
636 } else {
637 entry.value_ptr.freed = true;
638 entry.value_ptr.captureStackTrace(ret_addr, .free);
639 }
640 }
641
604 pub fn setRequestedMemoryLimit(self: *Self, limit: usize) void {642 pub fn setRequestedMemoryLimit(self: *Self, limit: usize) void {
605 self.requested_memory_limit = limit;643 self.requested_memory_limit = limit;
606 }644 }
...@@ -656,9 +694,6 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {...@@ -656,9 +694,6 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
656 if (!is_used) {694 if (!is_used) {
657 if (config.safety) {695 if (config.safety) {
658 reportDoubleFree(ret_addr, bucketStackTrace(bucket, size_class, slot_index, .alloc), bucketStackTrace(bucket, size_class, slot_index, .free));696 reportDoubleFree(ret_addr, bucketStackTrace(bucket, size_class, slot_index, .alloc), bucketStackTrace(bucket, size_class, slot_index, .free));
659 // Recoverable if this is a free.
660 if (new_size == 0)
661 return @as(usize, 0);
662 @panic("Unrecoverable double free");697 @panic("Unrecoverable double free");
663 } else {698 } else {
664 unreachable;699 unreachable;
...@@ -678,52 +713,6 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {...@@ -678,52 +713,6 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
678 self.total_requested_bytes = prev_req_bytes;713 self.total_requested_bytes = prev_req_bytes;
679 };714 };
680715
681 if (new_size == 0) {
682 // Capture stack trace to be the "first free", in case a double free happens.
683 bucket.captureStackTrace(ret_addr, size_class, slot_index, .free);
684
685 used_byte.* &= ~(@as(u8, 1) << used_bit_index);
686 bucket.used_count -= 1;
687 if (bucket.used_count == 0) {
688 if (bucket.next == bucket) {
689 // it's the only bucket and therefore the current one
690 self.buckets[bucket_index] = null;
691 } else {
692 bucket.next.prev = bucket.prev;
693 bucket.prev.next = bucket.next;
694 self.buckets[bucket_index] = bucket.prev;
695 }
696 if (!config.never_unmap) {
697 self.backing_allocator.free(bucket.page[0..page_size]);
698 }
699 if (!config.retain_metadata) {
700 self.freeBucket(bucket, size_class);
701 } else {
702 // move alloc_cursor to end so we can tell size_class later
703 const slot_count = @divExact(page_size, size_class);
704 bucket.alloc_cursor = @truncate(SlotIndex, slot_count);
705 if (self.empty_buckets) |prev_bucket| {
706 // empty_buckets is ordered newest to oldest through prev so that if
707 // config.never_unmap is false and backing_allocator reuses freed memory
708 // then searchBuckets will always return the newer, relevant bucket
709 bucket.prev = prev_bucket;
710 bucket.next = prev_bucket.next;
711 prev_bucket.next = bucket;
712 bucket.next.prev = bucket;
713 } else {
714 bucket.prev = bucket;
715 bucket.next = bucket;
716 }
717 self.empty_buckets = bucket;
718 }
719 } else {
720 @memset(old_mem.ptr, undefined, old_mem.len);
721 }
722 if (config.verbose_log) {
723 log.info("small free {d} bytes at {*}", .{ old_mem.len, old_mem.ptr });
724 }
725 return @as(usize, 0);
726 }
727 const new_aligned_size = math.max(new_size, old_align);716 const new_aligned_size = math.max(new_size, old_align);
728 const new_size_class = math.ceilPowerOfTwoAssert(usize, new_aligned_size);717 const new_size_class = math.ceilPowerOfTwoAssert(usize, new_aligned_size);
729 if (new_size_class <= size_class) {718 if (new_size_class <= size_class) {
...@@ -740,6 +729,114 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {...@@ -740,6 +729,114 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
740 return error.OutOfMemory;729 return error.OutOfMemory;
741 }730 }
742731
732 fn free(
733 self: *Self,
734 old_mem: []u8,
735 old_align: u29,
736 ret_addr: usize,
737 ) void {
738 const held = self.mutex.acquire();
739 defer held.release();
740
741 assert(old_mem.len != 0);
742
743 const aligned_size = math.max(old_mem.len, old_align);
744 if (aligned_size > largest_bucket_object_size) {
745 self.freeLarge(old_mem, old_align, ret_addr);
746 return;
747 }
748 const size_class_hint = math.ceilPowerOfTwoAssert(usize, aligned_size);
749
750 var bucket_index = math.log2(size_class_hint);
751 var size_class: usize = size_class_hint;
752 const bucket = while (bucket_index < small_bucket_count) : (bucket_index += 1) {
753 if (searchBucket(self.buckets[bucket_index], @ptrToInt(old_mem.ptr))) |bucket| {
754 // move bucket to head of list to optimize search for nearby allocations
755 self.buckets[bucket_index] = bucket;
756 break bucket;
757 }
758 size_class *= 2;
759 } else blk: {
760 if (config.retain_metadata) {
761 if (!self.large_allocations.contains(@ptrToInt(old_mem.ptr))) {
762 // object not in active buckets or a large allocation, so search empty buckets
763 if (searchBucket(self.empty_buckets, @ptrToInt(old_mem.ptr))) |bucket| {
764 // bucket is empty so is_used below will always be false and we exit there
765 break :blk bucket;
766 } else {
767 @panic("Invalid free");
768 }
769 }
770 }
771 self.freeLarge(old_mem, old_align, ret_addr);
772 return;
773 };
774 const byte_offset = @ptrToInt(old_mem.ptr) - @ptrToInt(bucket.page);
775 const slot_index = @intCast(SlotIndex, byte_offset / size_class);
776 const used_byte_index = slot_index / 8;
777 const used_bit_index = @intCast(u3, slot_index % 8);
778 const used_byte = bucket.usedBits(used_byte_index);
779 const is_used = @truncate(u1, used_byte.* >> used_bit_index) != 0;
780 if (!is_used) {
781 if (config.safety) {
782 reportDoubleFree(ret_addr, bucketStackTrace(bucket, size_class, slot_index, .alloc), bucketStackTrace(bucket, size_class, slot_index, .free));
783 // Recoverable if this is a free.
784 return;
785 } else {
786 unreachable;
787 }
788 }
789
790 // Definitely an in-use small alloc now.
791 if (config.enable_memory_limit) {
792 self.total_requested_bytes -= old_mem.len;
793 }
794
795 // Capture stack trace to be the "first free", in case a double free happens.
796 bucket.captureStackTrace(ret_addr, size_class, slot_index, .free);
797
798 used_byte.* &= ~(@as(u8, 1) << used_bit_index);
799 bucket.used_count -= 1;
800 if (bucket.used_count == 0) {
801 if (bucket.next == bucket) {
802 // it's the only bucket and therefore the current one
803 self.buckets[bucket_index] = null;
804 } else {
805 bucket.next.prev = bucket.prev;
806 bucket.prev.next = bucket.next;
807 self.buckets[bucket_index] = bucket.prev;
808 }
809 if (!config.never_unmap) {
810 self.backing_allocator.free(bucket.page[0..page_size]);
811 }
812 if (!config.retain_metadata) {
813 self.freeBucket(bucket, size_class);
814 } else {
815 // move alloc_cursor to end so we can tell size_class later
816 const slot_count = @divExact(page_size, size_class);
817 bucket.alloc_cursor = @truncate(SlotIndex, slot_count);
818 if (self.empty_buckets) |prev_bucket| {
819 // empty_buckets is ordered newest to oldest through prev so that if
820 // config.never_unmap is false and backing_allocator reuses freed memory
821 // then searchBuckets will always return the newer, relevant bucket
822 bucket.prev = prev_bucket;
823 bucket.next = prev_bucket.next;
824 prev_bucket.next = bucket;
825 bucket.next.prev = bucket;
826 } else {
827 bucket.prev = bucket;
828 bucket.next = bucket;
829 }
830 self.empty_buckets = bucket;
831 }
832 } else {
833 @memset(old_mem.ptr, undefined, old_mem.len);
834 }
835 if (config.verbose_log) {
836 log.info("small free {d} bytes at {*}", .{ old_mem.len, old_mem.ptr });
837 }
838 }
839
743 // Returns true if an allocation of `size` bytes is within the specified840 // Returns true if an allocation of `size` bytes is within the specified
744 // limits if enable_memory_limit is true841 // limits if enable_memory_limit is true
745 fn isAllocationAllowed(self: *Self, size: usize) bool {842 fn isAllocationAllowed(self: *Self, size: usize) bool {
...@@ -764,7 +861,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {...@@ -764,7 +861,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
764 const new_aligned_size = math.max(len, ptr_align);861 const new_aligned_size = math.max(len, ptr_align);
765 if (new_aligned_size > largest_bucket_object_size) {862 if (new_aligned_size > largest_bucket_object_size) {
766 try self.large_allocations.ensureUnusedCapacity(self.backing_allocator, 1);863 try self.large_allocations.ensureUnusedCapacity(self.backing_allocator, 1);
767 const slice = try self.backing_allocator.vtable.alloc(self.backing_allocator.ptr, len, ptr_align, len_align, ret_addr);864 const slice = try self.backing_allocator.rawAlloc(len, ptr_align, len_align, ret_addr);
768865
769 const gop = self.large_allocations.getOrPutAssumeCapacity(@ptrToInt(slice.ptr));866 const gop = self.large_allocations.getOrPutAssumeCapacity(@ptrToInt(slice.ptr));
770 if (config.retain_metadata and !config.never_unmap) {867 if (config.retain_metadata and !config.never_unmap) {
lib/std/heap/log_to_writer_allocator.zig+14-6
...@@ -18,7 +18,7 @@ pub fn LogToWriterAllocator(comptime Writer: type) type {...@@ -18,7 +18,7 @@ pub fn LogToWriterAllocator(comptime Writer: type) type {
18 }18 }
1919
20 pub fn allocator(self: *Self) Allocator {20 pub fn allocator(self: *Self) Allocator {
21 return Allocator.init(self, alloc, resize);21 return Allocator.init(self, alloc, resize, free);
22 }22 }
2323
24 fn alloc(24 fn alloc(
...@@ -29,7 +29,7 @@ pub fn LogToWriterAllocator(comptime Writer: type) type {...@@ -29,7 +29,7 @@ pub fn LogToWriterAllocator(comptime Writer: type) type {
29 ra: usize,29 ra: usize,
30 ) error{OutOfMemory}![]u8 {30 ) error{OutOfMemory}![]u8 {
31 self.writer.print("alloc : {}", .{len}) catch {};31 self.writer.print("alloc : {}", .{len}) catch {};
32 const result = self.parent_allocator.vtable.alloc(self.parent_allocator.ptr, len, ptr_align, len_align, ra);32 const result = self.parent_allocator.rawAlloc(len, ptr_align, len_align, ra);
33 if (result) |_| {33 if (result) |_| {
34 self.writer.print(" success!\n", .{}) catch {};34 self.writer.print(" success!\n", .{}) catch {};
35 } else |_| {35 } else |_| {
...@@ -46,14 +46,12 @@ pub fn LogToWriterAllocator(comptime Writer: type) type {...@@ -46,14 +46,12 @@ pub fn LogToWriterAllocator(comptime Writer: type) type {
46 len_align: u29,46 len_align: u29,
47 ra: usize,47 ra: usize,
48 ) error{OutOfMemory}!usize {48 ) error{OutOfMemory}!usize {
49 if (new_len == 0) {49 if (new_len <= buf.len) {
50 self.writer.print("free : {}\n", .{buf.len}) catch {};
51 } else if (new_len <= buf.len) {
52 self.writer.print("shrink: {} to {}\n", .{ buf.len, new_len }) catch {};50 self.writer.print("shrink: {} to {}\n", .{ buf.len, new_len }) catch {};
53 } else {51 } else {
54 self.writer.print("expand: {} to {}", .{ buf.len, new_len }) catch {};52 self.writer.print("expand: {} to {}", .{ buf.len, new_len }) catch {};
55 }53 }
56 if (self.parent_allocator.vtable.resize(self.parent_allocator.ptr, buf, buf_align, new_len, len_align, ra)) |resized_len| {54 if (self.parent_allocator.rawResize(buf, buf_align, new_len, len_align, ra)) |resized_len| {
57 if (new_len > buf.len) {55 if (new_len > buf.len) {
58 self.writer.print(" success!\n", .{}) catch {};56 self.writer.print(" success!\n", .{}) catch {};
59 }57 }
...@@ -64,6 +62,16 @@ pub fn LogToWriterAllocator(comptime Writer: type) type {...@@ -64,6 +62,16 @@ pub fn LogToWriterAllocator(comptime Writer: type) type {
64 return e;62 return e;
65 }63 }
66 }64 }
65
66 fn free(
67 self: *Self,
68 buf: []u8,
69 buf_align: u29,
70 ra: usize,
71 ) void {
72 self.writer.print("free : {}\n", .{buf.len}) catch {};
73 self.parent_allocator.rawFree(buf, buf_align, ra);
74 }
67 };75 };
68}76}
6977
lib/std/heap/logging_allocator.zig+14-6
...@@ -33,7 +33,7 @@ pub fn ScopedLoggingAllocator(...@@ -33,7 +33,7 @@ pub fn ScopedLoggingAllocator(
33 }33 }
3434
35 pub fn allocator(self: *Self) Allocator {35 pub fn allocator(self: *Self) Allocator {
36 return Allocator.init(self, alloc, resize);36 return Allocator.init(self, alloc, resize, free);
37 }37 }
3838
39 // This function is required as the `std.log.log` function is not public39 // This function is required as the `std.log.log` function is not public
...@@ -53,7 +53,7 @@ pub fn ScopedLoggingAllocator(...@@ -53,7 +53,7 @@ pub fn ScopedLoggingAllocator(
53 len_align: u29,53 len_align: u29,
54 ra: usize,54 ra: usize,
55 ) error{OutOfMemory}![]u8 {55 ) error{OutOfMemory}![]u8 {
56 const result = self.parent_allocator.vtable.alloc(self.parent_allocator.ptr, len, ptr_align, len_align, ra);56 const result = self.parent_allocator.rawAlloc(len, ptr_align, len_align, ra);
57 if (result) |_| {57 if (result) |_| {
58 logHelper(58 logHelper(
59 success_log_level,59 success_log_level,
...@@ -78,10 +78,8 @@ pub fn ScopedLoggingAllocator(...@@ -78,10 +78,8 @@ pub fn ScopedLoggingAllocator(
78 len_align: u29,78 len_align: u29,
79 ra: usize,79 ra: usize,
80 ) error{OutOfMemory}!usize {80 ) error{OutOfMemory}!usize {
81 if (self.parent_allocator.vtable.resize(self.parent_allocator.ptr, buf, buf_align, new_len, len_align, ra)) |resized_len| {81 if (self.parent_allocator.rawResize(buf, buf_align, new_len, len_align, ra)) |resized_len| {
82 if (new_len == 0) {82 if (new_len <= buf.len) {
83 logHelper(success_log_level, "free - success - len: {}", .{buf.len});
84 } else if (new_len <= buf.len) {
85 logHelper(83 logHelper(
86 success_log_level,84 success_log_level,
87 "shrink - success - {} to {}, len_align: {}, buf_align: {}",85 "shrink - success - {} to {}, len_align: {}, buf_align: {}",
...@@ -106,6 +104,16 @@ pub fn ScopedLoggingAllocator(...@@ -106,6 +104,16 @@ pub fn ScopedLoggingAllocator(
106 return err;104 return err;
107 }105 }
108 }106 }
107
108 fn free(
109 self: *Self,
110 buf: []u8,
111 buf_align: u29,
112 ra: usize,
113 ) void {
114 self.parent_allocator.rawFree(buf, buf_align, ra);
115 logHelper(success_log_level, "free - len: {}", .{buf.len});
116 }
109 };117 };
110}118}
111119
lib/std/mem.zig+17-3
...@@ -47,7 +47,7 @@ pub fn ValidationAllocator(comptime T: type) type {...@@ -47,7 +47,7 @@ pub fn ValidationAllocator(comptime T: type) type {
47 }47 }
4848
49 pub fn allocator(self: *Self) Allocator {49 pub fn allocator(self: *Self) Allocator {
50 return Allocator.init(self, alloc, resize);50 return Allocator.init(self, alloc, resize, free);
51 }51 }
5252
53 fn getUnderlyingAllocatorPtr(self: *Self) Allocator {53 fn getUnderlyingAllocatorPtr(self: *Self) Allocator {
...@@ -70,7 +70,7 @@ pub fn ValidationAllocator(comptime T: type) type {...@@ -70,7 +70,7 @@ pub fn ValidationAllocator(comptime T: type) type {
70 }70 }
7171
72 const underlying = self.getUnderlyingAllocatorPtr();72 const underlying = self.getUnderlyingAllocatorPtr();
73 const result = try underlying.vtable.alloc(underlying.ptr, n, ptr_align, len_align, ret_addr);73 const result = try underlying.rawAlloc(n, ptr_align, len_align, ret_addr);
74 assert(mem.isAligned(@ptrToInt(result.ptr), ptr_align));74 assert(mem.isAligned(@ptrToInt(result.ptr), ptr_align));
75 if (len_align == 0) {75 if (len_align == 0) {
76 assert(result.len == n);76 assert(result.len == n);
...@@ -95,7 +95,7 @@ pub fn ValidationAllocator(comptime T: type) type {...@@ -95,7 +95,7 @@ pub fn ValidationAllocator(comptime T: type) type {
95 assert(new_len >= len_align);95 assert(new_len >= len_align);
96 }96 }
97 const underlying = self.getUnderlyingAllocatorPtr();97 const underlying = self.getUnderlyingAllocatorPtr();
98 const result = try underlying.vtable.resize(underlying.ptr, buf, buf_align, new_len, len_align, ret_addr);98 const result = try underlying.rawResize(buf, buf_align, new_len, len_align, ret_addr);
99 if (len_align == 0) {99 if (len_align == 0) {
100 assert(result == new_len);100 assert(result == new_len);
101 } else {101 } else {
...@@ -104,6 +104,19 @@ pub fn ValidationAllocator(comptime T: type) type {...@@ -104,6 +104,19 @@ pub fn ValidationAllocator(comptime T: type) type {
104 }104 }
105 return result;105 return result;
106 }106 }
107
108 pub fn free(
109 self: *Self,
110 buf: []u8,
111 buf_align: u29,
112 ret_addr: usize,
113 ) void {
114 _ = self;
115 _ = buf_align;
116 _ = ret_addr;
117 assert(buf.len > 0);
118 }
119
107 pub usingnamespace if (T == Allocator or !@hasDecl(T, "reset")) struct {} else struct {120 pub usingnamespace if (T == Allocator or !@hasDecl(T, "reset")) struct {} else struct {
108 pub fn reset(self: *Self) void {121 pub fn reset(self: *Self) void {
109 self.underlying_allocator.reset();122 self.underlying_allocator.reset();
...@@ -139,6 +152,7 @@ const fail_allocator = Allocator{...@@ -139,6 +152,7 @@ const fail_allocator = Allocator{
139const failAllocator_vtable = Allocator.VTable{152const failAllocator_vtable = Allocator.VTable{
140 .alloc = failAllocatorAlloc,153 .alloc = failAllocatorAlloc,
141 .resize = Allocator.NoResize(c_void).noResize,154 .resize = Allocator.NoResize(c_void).noResize,
155 .free = Allocator.NoOpFree(c_void).noOpFree,
142};156};
143157
144fn failAllocatorAlloc(_: *c_void, n: usize, alignment: u29, len_align: u29, ra: usize) Allocator.Error![]u8 {158fn failAllocatorAlloc(_: *c_void, n: usize, alignment: u29, len_align: u29, ra: usize) Allocator.Error![]u8 {
lib/std/mem/Allocator.zig+93-21
...@@ -5,6 +5,7 @@ const assert = std.debug.assert;...@@ -5,6 +5,7 @@ const assert = std.debug.assert;
5const math = std.math;5const math = std.math;
6const mem = std.mem;6const mem = std.mem;
7const Allocator = @This();7const Allocator = @This();
8const builtin = @import("builtin");
89
9pub const Error = error{OutOfMemory};10pub const Error = error{OutOfMemory};
1011
...@@ -28,9 +29,6 @@ pub const VTable = struct {...@@ -28,9 +29,6 @@ pub const VTable = struct {
28 /// length returned by `alloc` or `resize`. `buf_align` must equal the same value29 /// length returned by `alloc` or `resize`. `buf_align` must equal the same value
29 /// that was passed as the `ptr_align` parameter to the original `alloc` call.30 /// that was passed as the `ptr_align` parameter to the original `alloc` call.
30 ///31 ///
31 /// Passing a `new_len` of 0 frees and invalidates the buffer such that it can no
32 /// longer be passed to `resize`.
33 ///
34 /// error.OutOfMemory can only be returned if `new_len` is greater than `buf.len`.32 /// error.OutOfMemory can only be returned if `new_len` is greater than `buf.len`.
35 /// If `buf` cannot be expanded to accomodate `new_len`, then the allocation MUST be33 /// If `buf` cannot be expanded to accomodate `new_len`, then the allocation MUST be
36 /// unmodified and error.OutOfMemory MUST be returned.34 /// unmodified and error.OutOfMemory MUST be returned.
...@@ -40,36 +38,54 @@ pub const VTable = struct {...@@ -40,36 +38,54 @@ pub const VTable = struct {
40 /// provide a way to modify the alignment of a pointer. Rather it provides an API for38 /// provide a way to modify the alignment of a pointer. Rather it provides an API for
41 /// accepting more bytes of memory from the allocator than requested.39 /// accepting more bytes of memory from the allocator than requested.
42 ///40 ///
43 /// `new_len` must be greater than or equal to `len_align` and must be aligned by `len_align`.41 /// `new_len` must be greater than zero, greater than or equal to `len_align` and must be aligned by `len_align`.
44 ///42 ///
45 /// `ret_addr` is optionally provided as the first return address of the allocation call stack.43 /// `ret_addr` is optionally provided as the first return address of the allocation call stack.
46 /// If the value is `0` it means no return address has been provided.44 /// If the value is `0` it means no return address has been provided.
47 resize: fn (ptr: *c_void, buf: []u8, buf_align: u29, new_len: usize, len_align: u29, ret_addr: usize) Error!usize,45 resize: fn (ptr: *c_void, buf: []u8, buf_align: u29, new_len: usize, len_align: u29, ret_addr: usize) Error!usize,
46
47 /// Free and invalidate a buffer. `buf.len` must equal the most recent length returned by `alloc` or `resize`.
48 /// `buf_align` must equal the same value that was passed as the `ptr_align` parameter to the original `alloc` call.
49 ///
50 /// `ret_addr` is optionally provided as the first return address of the allocation call stack.
51 /// If the value is `0` it means no return address has been provided.
52 free: fn (ptr: *c_void, buf: []u8, buf_align: u29, ret_addr: usize) void,
48};53};
4954
50pub fn init(55pub fn init(
51 pointer: anytype,56 pointer: anytype,
52 comptime allocFn: fn (ptr: @TypeOf(pointer), len: usize, ptr_align: u29, len_align: u29, ret_addr: usize) Error![]u8,57 comptime allocFn: fn (ptr: @TypeOf(pointer), len: usize, ptr_align: u29, len_align: u29, ret_addr: usize) Error![]u8,
53 comptime resizeFn: fn (ptr: @TypeOf(pointer), buf: []u8, buf_align: u29, new_len: usize, len_align: u29, ret_addr: usize) Error!usize,58 comptime resizeFn: fn (ptr: @TypeOf(pointer), buf: []u8, buf_align: u29, new_len: usize, len_align: u29, ret_addr: usize) Error!usize,
59 comptime freeFn: fn (ptr: @TypeOf(pointer), buf: []u8, buf_align: u29, ret_addr: usize) void,
54) Allocator {60) Allocator {
55 const Ptr = @TypeOf(pointer);61 const Ptr = @TypeOf(pointer);
56 assert(@typeInfo(Ptr) == .Pointer); // Must be a pointer62 const ptr_info = @typeInfo(Ptr);
57 assert(@typeInfo(Ptr).Pointer.size == .One); // Must be a single-item pointer63
64 assert(ptr_info == .Pointer); // Must be a pointer
65 assert(ptr_info.Pointer.size == .One); // Must be a single-item pointer
66
67 const alignment = ptr_info.Pointer.alignment;
68
58 const gen = struct {69 const gen = struct {
59 fn alloc(ptr: *c_void, len: usize, ptr_align: u29, len_align: u29, ret_addr: usize) Error![]u8 {70 fn alloc(ptr: *c_void, len: usize, ptr_align: u29, len_align: u29, ret_addr: usize) Error![]u8 {
60 const alignment = @typeInfo(Ptr).Pointer.alignment;
61 const self = @ptrCast(Ptr, @alignCast(alignment, ptr));71 const self = @ptrCast(Ptr, @alignCast(alignment, ptr));
62 return allocFn(self, len, ptr_align, len_align, ret_addr);72 return @call(.{ .modifier = .always_inline }, allocFn, .{ self, len, ptr_align, len_align, ret_addr });
63 }73 }
64 fn resize(ptr: *c_void, buf: []u8, buf_align: u29, new_len: usize, len_align: u29, ret_addr: usize) Error!usize {74 fn resize(ptr: *c_void, buf: []u8, buf_align: u29, new_len: usize, len_align: u29, ret_addr: usize) Error!usize {
65 const alignment = @typeInfo(Ptr).Pointer.alignment;75 assert(new_len != 0);
76 const self = @ptrCast(Ptr, @alignCast(alignment, ptr));
77 return @call(.{ .modifier = .always_inline }, resizeFn, .{ self, buf, buf_align, new_len, len_align, ret_addr });
78 }
79 fn free(ptr: *c_void, buf: []u8, buf_align: u29, ret_addr: usize) void {
66 const self = @ptrCast(Ptr, @alignCast(alignment, ptr));80 const self = @ptrCast(Ptr, @alignCast(alignment, ptr));
67 return resizeFn(self, buf, buf_align, new_len, len_align, ret_addr);81 @call(.{ .modifier = .always_inline }, freeFn, .{ self, buf, buf_align, ret_addr });
68 }82 }
69 };83 };
84
70 const vtable = VTable{85 const vtable = VTable{
71 .alloc = gen.alloc,86 .alloc = gen.alloc,
72 .resize = gen.resize,87 .resize = gen.resize,
88 .free = gen.free,
73 };89 };
7490
75 return .{91 return .{
...@@ -100,6 +116,56 @@ pub fn NoResize(comptime AllocatorType: type) type {...@@ -100,6 +116,56 @@ pub fn NoResize(comptime AllocatorType: type) type {
100 };116 };
101}117}
102118
119/// Set freeFn to `NoOpFree(AllocatorType).noOpFree` if free is a no-op.
120pub fn NoOpFree(comptime AllocatorType: type) type {
121 return struct {
122 pub fn noOpFree(
123 self: *AllocatorType,
124 buf: []u8,
125 buf_align: u29,
126 ret_addr: usize,
127 ) void {
128 _ = self;
129 _ = buf;
130 _ = buf_align;
131 _ = ret_addr;
132 }
133 };
134}
135
136/// Set freeFn to `PanicFree(AllocatorType).noOpFree` if free is not a supported operation.
137pub fn PanicFree(comptime AllocatorType: type) type {
138 return struct {
139 pub fn noOpFree(
140 self: *AllocatorType,
141 buf: []u8,
142 buf_align: u29,
143 ret_addr: usize,
144 ) void {
145 _ = self;
146 _ = buf;
147 _ = buf_align;
148 _ = ret_addr;
149 @panic("free is not a supported operation for the allocator: " ++ @typeName(AllocatorType));
150 }
151 };
152}
153
154/// This function is not intended to be called except from within the implementation of an Allocator
155pub inline fn rawAlloc(self: Allocator, len: usize, ptr_align: u29, len_align: u29, ret_addr: usize) Error![]u8 {
156 return self.vtable.alloc(self.ptr, len, ptr_align, len_align, ret_addr);
157}
158
159/// This function is not intended to be called except from within the implementation of an Allocator
160pub inline fn rawResize(self: Allocator, buf: []u8, buf_align: u29, new_len: usize, len_align: u29, ret_addr: usize) Error!usize {
161 return self.vtable.resize(self.ptr, buf, buf_align, new_len, len_align, ret_addr);
162}
163
164/// This function is not intended to be called except from within the implementation of an Allocator
165pub inline fn rawFree(self: Allocator, buf: []u8, buf_align: u29, ret_addr: usize) void {
166 return self.vtable.free(self.ptr, buf, buf_align, ret_addr);
167}
168
103/// Realloc is used to modify the size or alignment of an existing allocation,169/// Realloc is used to modify the size or alignment of an existing allocation,
104/// as well as to provide the allocator with an opportunity to move an allocation170/// as well as to provide the allocator with an opportunity to move an allocation
105/// to a better location.171/// to a better location.
...@@ -133,8 +199,7 @@ fn reallocBytes(...@@ -133,8 +199,7 @@ fn reallocBytes(
133 /// Guaranteed to be >= 1.199 /// Guaranteed to be >= 1.
134 /// Guaranteed to be a power of 2.200 /// Guaranteed to be a power of 2.
135 old_alignment: u29,201 old_alignment: u29,
136 /// If `new_byte_count` is 0 then this is a free and it is guaranteed that202 /// `new_byte_count` must be greater than zero
137 /// `old_mem.len != 0`.
138 new_byte_count: usize,203 new_byte_count: usize,
139 /// Guaranteed to be >= 1.204 /// Guaranteed to be >= 1.
140 /// Guaranteed to be a power of 2.205 /// Guaranteed to be a power of 2.
...@@ -147,18 +212,20 @@ fn reallocBytes(...@@ -147,18 +212,20 @@ fn reallocBytes(
147 return_address: usize,212 return_address: usize,
148) Error![]u8 {213) Error![]u8 {
149 if (old_mem.len == 0) {214 if (old_mem.len == 0) {
150 const new_mem = try self.vtable.alloc(self.ptr, new_byte_count, new_alignment, len_align, return_address);215 const new_mem = try self.rawAlloc(new_byte_count, new_alignment, len_align, return_address);
151 // TODO: https://github.com/ziglang/zig/issues/4298216 // TODO: https://github.com/ziglang/zig/issues/4298
152 @memset(new_mem.ptr, undefined, new_byte_count);217 @memset(new_mem.ptr, undefined, new_byte_count);
153 return new_mem;218 return new_mem;
154 }219 }
155220
221 assert(new_byte_count > 0); // `new_byte_count` must greater than zero, this is a resize not a free
222
156 if (mem.isAligned(@ptrToInt(old_mem.ptr), new_alignment)) {223 if (mem.isAligned(@ptrToInt(old_mem.ptr), new_alignment)) {
157 if (new_byte_count <= old_mem.len) {224 if (new_byte_count <= old_mem.len) {
158 const shrunk_len = self.shrinkBytes(old_mem, old_alignment, new_byte_count, len_align, return_address);225 const shrunk_len = self.shrinkBytes(old_mem, old_alignment, new_byte_count, len_align, return_address);
159 return old_mem.ptr[0..shrunk_len];226 return old_mem.ptr[0..shrunk_len];
160 }227 }
161 if (self.vtable.resize(self.ptr, old_mem, old_alignment, new_byte_count, len_align, return_address)) |resized_len| {228 if (self.rawResize(old_mem, old_alignment, new_byte_count, len_align, return_address)) |resized_len| {
162 assert(resized_len >= new_byte_count);229 assert(resized_len >= new_byte_count);
163 // TODO: https://github.com/ziglang/zig/issues/4298230 // TODO: https://github.com/ziglang/zig/issues/4298
164 @memset(old_mem.ptr + new_byte_count, undefined, resized_len - new_byte_count);231 @memset(old_mem.ptr + new_byte_count, undefined, resized_len - new_byte_count);
...@@ -184,11 +251,11 @@ fn moveBytes(...@@ -184,11 +251,11 @@ fn moveBytes(
184) Error![]u8 {251) Error![]u8 {
185 assert(old_mem.len > 0);252 assert(old_mem.len > 0);
186 assert(new_len > 0);253 assert(new_len > 0);
187 const new_mem = try self.vtable.alloc(self.ptr, new_len, new_alignment, len_align, return_address);254 const new_mem = try self.rawAlloc(new_len, new_alignment, len_align, return_address);
188 @memcpy(new_mem.ptr, old_mem.ptr, math.min(new_len, old_mem.len));255 @memcpy(new_mem.ptr, old_mem.ptr, math.min(new_len, old_mem.len));
189 // TODO https://github.com/ziglang/zig/issues/4298256 // TODO https://github.com/ziglang/zig/issues/4298
190 @memset(old_mem.ptr, undefined, old_mem.len);257 @memset(old_mem.ptr, undefined, old_mem.len);
191 _ = self.shrinkBytes(old_mem, old_align, 0, 0, return_address);258 self.rawFree(old_mem, old_align, return_address);
192 return new_mem;259 return new_mem;
193}260}
194261
...@@ -207,7 +274,7 @@ pub fn destroy(self: Allocator, ptr: anytype) void {...@@ -207,7 +274,7 @@ pub fn destroy(self: Allocator, ptr: anytype) void {
207 const T = info.child;274 const T = info.child;
208 if (@sizeOf(T) == 0) return;275 if (@sizeOf(T) == 0) return;
209 const non_const_ptr = @intToPtr([*]u8, @ptrToInt(ptr));276 const non_const_ptr = @intToPtr([*]u8, @ptrToInt(ptr));
210 _ = self.shrinkBytes(non_const_ptr[0..@sizeOf(T)], info.alignment, 0, 0, @returnAddress());277 self.rawFree(non_const_ptr[0..@sizeOf(T)], info.alignment, @returnAddress());
211}278}
212279
213/// Allocates an array of `n` items of type `T` and sets all the280/// Allocates an array of `n` items of type `T` and sets all the
...@@ -326,7 +393,7 @@ pub fn allocAdvancedWithRetAddr(...@@ -326,7 +393,7 @@ pub fn allocAdvancedWithRetAddr(
326 .exact => 0,393 .exact => 0,
327 .at_least => size_of_T,394 .at_least => size_of_T,
328 };395 };
329 const byte_slice = try self.vtable.alloc(self.ptr, byte_count, a, len_align, return_address);396 const byte_slice = try self.rawAlloc(byte_count, a, len_align, return_address);
330 switch (exact) {397 switch (exact) {
331 .exact => assert(byte_slice.len == byte_count),398 .exact => assert(byte_slice.len == byte_count),
332 .at_least => assert(byte_slice.len >= byte_count),399 .at_least => assert(byte_slice.len >= byte_count),
...@@ -351,7 +418,7 @@ pub fn resize(self: Allocator, old_mem: anytype, new_n: usize) Error!@TypeOf(old...@@ -351,7 +418,7 @@ pub fn resize(self: Allocator, old_mem: anytype, new_n: usize) Error!@TypeOf(old
351 }418 }
352 const old_byte_slice = mem.sliceAsBytes(old_mem);419 const old_byte_slice = mem.sliceAsBytes(old_mem);
353 const new_byte_count = math.mul(usize, @sizeOf(T), new_n) catch return Error.OutOfMemory;420 const new_byte_count = math.mul(usize, @sizeOf(T), new_n) catch return Error.OutOfMemory;
354 const rc = try self.vtable.resize(self.ptr, old_byte_slice, Slice.alignment, new_byte_count, 0, @returnAddress());421 const rc = try self.rawResize(old_byte_slice, Slice.alignment, new_byte_count, 0, @returnAddress());
355 assert(rc == new_byte_count);422 assert(rc == new_byte_count);
356 const new_byte_slice = old_byte_slice.ptr[0..new_byte_count];423 const new_byte_slice = old_byte_slice.ptr[0..new_byte_count];
357 return mem.bytesAsSlice(T, new_byte_slice);424 return mem.bytesAsSlice(T, new_byte_slice);
...@@ -465,6 +532,11 @@ pub fn alignedShrinkWithRetAddr(...@@ -465,6 +532,11 @@ pub fn alignedShrinkWithRetAddr(
465532
466 if (new_n == old_mem.len)533 if (new_n == old_mem.len)
467 return old_mem;534 return old_mem;
535 if (new_n == 0) {
536 self.free(old_mem);
537 return @as([*]align(new_alignment) T, undefined)[0..0];
538 }
539
468 assert(new_n < old_mem.len);540 assert(new_n < old_mem.len);
469 assert(new_alignment <= Slice.alignment);541 assert(new_alignment <= Slice.alignment);
470542
...@@ -489,7 +561,7 @@ pub fn free(self: Allocator, memory: anytype) void {...@@ -489,7 +561,7 @@ pub fn free(self: Allocator, memory: anytype) void {
489 const non_const_ptr = @intToPtr([*]u8, @ptrToInt(bytes.ptr));561 const non_const_ptr = @intToPtr([*]u8, @ptrToInt(bytes.ptr));
490 // TODO: https://github.com/ziglang/zig/issues/4298562 // TODO: https://github.com/ziglang/zig/issues/4298
491 @memset(non_const_ptr, undefined, bytes_len);563 @memset(non_const_ptr, undefined, bytes_len);
492 _ = self.shrinkBytes(non_const_ptr[0..bytes_len], Slice.alignment, 0, 0, @returnAddress());564 self.rawFree(non_const_ptr[0..bytes_len], Slice.alignment, @returnAddress());
493}565}
494566
495/// Copies `m` to newly allocated memory. Caller owns the memory.567/// Copies `m` to newly allocated memory. Caller owns the memory.
...@@ -520,5 +592,5 @@ pub fn shrinkBytes(...@@ -520,5 +592,5 @@ pub fn shrinkBytes(
520 return_address: usize,592 return_address: usize,
521) usize {593) usize {
522 assert(new_len <= buf.len);594 assert(new_len <= buf.len);
523 return self.vtable.resize(self.ptr, buf, buf_align, new_len, len_align, return_address) catch unreachable;595 return self.rawResize(buf, buf_align, new_len, len_align, return_address) catch unreachable;
524}596}
lib/std/testing/failing_allocator.zig+15-7
...@@ -41,7 +41,7 @@ pub const FailingAllocator = struct {...@@ -41,7 +41,7 @@ pub const FailingAllocator = struct {
41 }41 }
4242
43 pub fn allocator(self: *FailingAllocator) mem.Allocator {43 pub fn allocator(self: *FailingAllocator) mem.Allocator {
44 return mem.Allocator.init(self, alloc, resize);44 return mem.Allocator.init(self, alloc, resize, free);
45 }45 }
4646
47 fn alloc(47 fn alloc(
...@@ -54,7 +54,7 @@ pub const FailingAllocator = struct {...@@ -54,7 +54,7 @@ pub const FailingAllocator = struct {
54 if (self.index == self.fail_index) {54 if (self.index == self.fail_index) {
55 return error.OutOfMemory;55 return error.OutOfMemory;
56 }56 }
57 const result = try self.internal_allocator.vtable.alloc(self.internal_allocator.ptr, len, ptr_align, len_align, return_address);57 const result = try self.internal_allocator.rawAlloc(len, ptr_align, len_align, return_address);
58 self.allocated_bytes += result.len;58 self.allocated_bytes += result.len;
59 self.allocations += 1;59 self.allocations += 1;
60 self.index += 1;60 self.index += 1;
...@@ -69,18 +69,26 @@ pub const FailingAllocator = struct {...@@ -69,18 +69,26 @@ pub const FailingAllocator = struct {
69 len_align: u29,69 len_align: u29,
70 ra: usize,70 ra: usize,
71 ) error{OutOfMemory}!usize {71 ) error{OutOfMemory}!usize {
72 const r = self.internal_allocator.vtable.resize(self.internal_allocator.ptr, old_mem, old_align, new_len, len_align, ra) catch |e| {72 const r = self.internal_allocator.rawResize(old_mem, old_align, new_len, len_align, ra) catch |e| {
73 std.debug.assert(new_len > old_mem.len);73 std.debug.assert(new_len > old_mem.len);
74 return e;74 return e;
75 };75 };
76 if (new_len == 0) {76 if (r < old_mem.len) {
77 self.deallocations += 1;
78 self.freed_bytes += old_mem.len;
79 } else if (r < old_mem.len) {
80 self.freed_bytes += old_mem.len - r;77 self.freed_bytes += old_mem.len - r;
81 } else {78 } else {
82 self.allocated_bytes += r - old_mem.len;79 self.allocated_bytes += r - old_mem.len;
83 }80 }
84 return r;81 return r;
85 }82 }
83
84 fn free(
85 self: *FailingAllocator,
86 old_mem: []u8,
87 old_align: u29,
88 ra: usize,
89 ) void {
90 self.internal_allocator.rawFree(old_mem, old_align, ra);
91 self.deallocations += 1;
92 self.freed_bytes += old_mem.len;
93 }
86};94};
src/tracy.zig+13-7
...@@ -155,13 +155,10 @@ pub fn TracyAllocator(comptime name: ?[:0]const u8) type {...@@ -155,13 +155,10 @@ pub fn TracyAllocator(comptime name: ?[:0]const u8) type {
155 }155 }
156 }156 }
157157
158 if (resized_len != 0) {158 if (name) |n| {
159 // this was a shrink or a resize159 allocNamed(buf.ptr, resized_len, n);
160 if (name) |n| {160 } else {
161 allocNamed(buf.ptr, resized_len, n);161 alloc(buf.ptr, resized_len);
162 } else {
163 alloc(buf.ptr, resized_len);
164 }
165 }162 }
166163
167 return resized_len;164 return resized_len;
...@@ -172,6 +169,15 @@ pub fn TracyAllocator(comptime name: ?[:0]const u8) type {...@@ -172,6 +169,15 @@ pub fn TracyAllocator(comptime name: ?[:0]const u8) type {
172 return err;169 return err;
173 }170 }
174 }171 }
172
173 fn freeFn(self: *Self, buf: []u8, buf_align: u29, ret_addr: usize) void {
174 self.parent_allocator.rawFree(buf, buf_align, ret_addr);
175 if (name) |n| {
176 freeNamed(buf.ptr, n);
177 } else {
178 free(buf.ptr);
179 }
180 }
175 };181 };
176}182}
177183