authorgravatar for johnnymarler@gmail.comJonathan Marler <johnnymarler@gmail.com> 2020-04-17 14:15:36-06:00
committergravatar for johnnymarler@gmail.comJonathan Marler <johnnymarler@gmail.com> 2020-06-26 13:34:48-06:00
logdc9648f868ed8ad08f040753767c03976bbcf3b7
tree56025ae541deb21028ae9d96124bad9226987896
parent129a4fb251f8eab22eacf219fbf81006baec3251

new allocator interface

10 files changed, 624 insertions(+), 461 deletions(-)

lib/std/array_list.zig+4-2
...@@ -219,7 +219,8 @@ pub fn ArrayListAligned(comptime T: type, comptime alignment: ?u29) type {...@@ -219,7 +219,8 @@ pub fn ArrayListAligned(comptime T: type, comptime alignment: ?u29) type {
219 if (better_capacity >= new_capacity) break;219 if (better_capacity >= new_capacity) break;
220 }220 }
221221
222 const new_memory = try self.allocator.realloc(self.allocatedSlice(), better_capacity);222 const new_memory = try self.allocator.reallocAtLeast(self.allocatedSlice(), better_capacity);
223 assert(new_memory.len >= better_capacity);
223 self.items.ptr = new_memory.ptr;224 self.items.ptr = new_memory.ptr;
224 self.capacity = new_memory.len;225 self.capacity = new_memory.len;
225 }226 }
...@@ -441,7 +442,8 @@ pub fn ArrayListAlignedUnmanaged(comptime T: type, comptime alignment: ?u29) typ...@@ -441,7 +442,8 @@ pub fn ArrayListAlignedUnmanaged(comptime T: type, comptime alignment: ?u29) typ
441 if (better_capacity >= new_capacity) break;442 if (better_capacity >= new_capacity) break;
442 }443 }
443444
444 const new_memory = try allocator.realloc(self.allocatedSlice(), better_capacity);445 const new_memory = try allocator.reallocAtLeast(self.allocatedSlice(), better_capacity);
446 assert(new_memory.len >= better_capacity);
445 self.items.ptr = new_memory.ptr;447 self.items.ptr = new_memory.ptr;
446 self.capacity = new_memory.len;448 self.capacity = new_memory.len;
447 }449 }
lib/std/c.zig+11
...@@ -232,6 +232,17 @@ pub extern "c" fn setuid(uid: c_uint) c_int;...@@ -232,6 +232,17 @@ pub extern "c" fn setuid(uid: c_uint) c_int;
232232
233pub extern "c" fn aligned_alloc(alignment: usize, size: usize) ?*c_void;233pub extern "c" fn aligned_alloc(alignment: usize, size: usize) ?*c_void;
234pub extern "c" fn malloc(usize) ?*c_void;234pub extern "c" fn malloc(usize) ?*c_void;
235
236pub usingnamespace switch (builtin.os.tag) {
237 .linux, .freebsd, .kfreebsd, .netbsd, .openbsd => struct {
238 pub extern "c" fn malloc_usable_size(?*const c_void) usize;
239 },
240 .macosx, .ios, .watchos, .tvos => struct {
241 pub extern "c" fn malloc_size(?*const c_void) usize;
242 },
243 else => struct {},
244};
245
235pub extern "c" fn realloc(?*c_void, usize) ?*c_void;246pub extern "c" fn realloc(?*c_void, usize) ?*c_void;
236pub extern "c" fn free(*c_void) void;247pub extern "c" fn free(*c_void) void;
237pub extern "c" fn posix_memalign(memptr: **c_void, alignment: usize, size: usize) c_int;248pub extern "c" fn posix_memalign(memptr: **c_void, alignment: usize, size: usize) c_int;
lib/std/heap.zig+271-323
...@@ -15,48 +15,88 @@ pub const ArenaAllocator = @import("heap/arena_allocator.zig").ArenaAllocator;...@@ -15,48 +15,88 @@ pub const ArenaAllocator = @import("heap/arena_allocator.zig").ArenaAllocator;
1515
16const Allocator = mem.Allocator;16const Allocator = mem.Allocator;
1717
18pub const c_allocator = &c_allocator_state;18usingnamespace if (comptime @hasDecl(c, "malloc_size")) struct {
19 pub const supports_malloc_size = true;
20 pub const malloc_size = c.malloc_size;
21} else if (comptime @hasDecl(c, "malloc_usable_size")) struct {
22 pub const supports_malloc_size = true;
23 pub const malloc_size = c.malloc_usable_size;
24} else struct {
25 pub const supports_malloc_size = false;
26};
27
28pub const c_allocator = mem.getAllocatorPtr(&c_allocator_state);
19var c_allocator_state = Allocator{29var c_allocator_state = Allocator{
20 .reallocFn = cRealloc,30 .allocFn = cAlloc,
21 .shrinkFn = cShrink,31 .resizeFn = cResize,
22};32};
2333
24fn cRealloc(self: *Allocator, old_mem: []u8, old_align: u29, new_size: usize, new_align: u29) ![]u8 {34fn cAlloc(self: *Allocator, len: usize, ptr_align: u29, len_align: u29) Allocator.Error![]u8 {
25 assert(new_align <= @alignOf(c_longdouble));35 assert(ptr_align <= @alignOf(c_longdouble));
26 const old_ptr = if (old_mem.len == 0) null else @ptrCast(*c_void, old_mem.ptr);36 const ptr = @ptrCast([*]u8, c.malloc(len) orelse return error.OutOfMemory);
27 const buf = c.realloc(old_ptr, new_size) orelse return error.OutOfMemory;37 if (len_align == 0) {
28 return @ptrCast([*]u8, buf)[0..new_size];38 return ptr[0..len];
39 }
40 const full_len = init: {
41 if (comptime supports_malloc_size) {
42 const s = malloc_size(ptr);
43 assert(s >= len);
44 break :init s;
45 }
46 break :init len;
47 };
48 return ptr[0..mem.alignBackwardAnyAlign(full_len, len_align)];
29}49}
3050
31fn cShrink(self: *Allocator, old_mem: []u8, old_align: u29, new_size: usize, new_align: u29) []u8 {51fn cResize(self: *Allocator, buf: []u8, new_len: usize, len_align: u29) Allocator.Error!usize {
32 const old_ptr = @ptrCast(*c_void, old_mem.ptr);52 if (new_len == 0) {
33 const buf = c.realloc(old_ptr, new_size) orelse return old_mem[0..new_size];53 c.free(buf.ptr);
34 return @ptrCast([*]u8, buf)[0..new_size];54 return 0;
55 }
56 if (new_len <= buf.len) {
57 return mem.alignAllocLen(buf.len, new_len, len_align);
58 }
59 if (comptime supports_malloc_size) {
60 const full_len = malloc_size(buf.ptr);
61 if (new_len <= full_len) {
62 return mem.alignAllocLen(full_len, new_len, len_align);
63 }
64 }
65 // TODO: could we still use realloc? are there any cases where we can guarantee that realloc won't move memory?
66 return error.OutOfMemory;
35}67}
3668
37/// This allocator makes a syscall directly for every allocation and free.69/// This allocator makes a syscall directly for every allocation and free.
38/// Thread-safe and lock-free.70/// Thread-safe and lock-free.
39pub const page_allocator = if (std.Target.current.isWasm())71pub const page_allocator = if (std.Target.current.isWasm())
40 &wasm_page_allocator_state72 mem.getAllocatorPtr(&wasm_page_allocator_state)
41else if (std.Target.current.os.tag == .freestanding)73else if (std.Target.current.os.tag == .freestanding)
42 root.os.heap.page_allocator74 root.os.heap.page_allocator
43else75else
44 &page_allocator_state;76 mem.getAllocatorPtr(&page_allocator_state);
4577
46var page_allocator_state = Allocator{78var page_allocator_state = Allocator{
47 .reallocFn = PageAllocator.realloc,79 .allocFn = PageAllocator.alloc,
48 .shrinkFn = PageAllocator.shrink,80 .resizeFn = PageAllocator.resize,
49};81};
50var wasm_page_allocator_state = Allocator{82var wasm_page_allocator_state = Allocator{
51 .reallocFn = WasmPageAllocator.realloc,83 .allocFn = WasmPageAllocator.alloc,
52 .shrinkFn = WasmPageAllocator.shrink,84 .resizeFn = WasmPageAllocator.resize,
53};85};
5486
55pub const direct_allocator = @compileError("deprecated; use std.heap.page_allocator");87pub const direct_allocator = @compileError("deprecated; use std.heap.page_allocator");
5688
89/// Verifies that the adjusted length will still map to the full length
90pub fn alignPageAllocLen(full_len: usize, len: usize, len_align: u29) usize {
91 const aligned_len = mem.alignAllocLen(full_len, len, len_align);
92 assert(mem.alignForward(aligned_len, mem.page_size) == full_len);
93 return aligned_len;
94}
95
57const PageAllocator = struct {96const PageAllocator = struct {
58 fn alloc(allocator: *Allocator, n: usize, alignment: u29) error{OutOfMemory}![]u8 {97 fn alloc(allocator: *Allocator, n: usize, alignment: u29, len_align: u29) error{OutOfMemory}![]u8 {
59 if (n == 0) return &[0]u8{};98 assert(n > 0);
99 const alignedLen = mem.alignForward(n, mem.page_size);
60100
61 if (builtin.os.tag == .windows) {101 if (builtin.os.tag == .windows) {
62 const w = os.windows;102 const w = os.windows;
...@@ -68,21 +108,21 @@ const PageAllocator = struct {...@@ -68,21 +108,21 @@ const PageAllocator = struct {
68 // see https://devblogs.microsoft.com/oldnewthing/?p=42223108 // see https://devblogs.microsoft.com/oldnewthing/?p=42223
69 const addr = w.VirtualAlloc(109 const addr = w.VirtualAlloc(
70 null,110 null,
71 n,111 alignedLen,
72 w.MEM_COMMIT | w.MEM_RESERVE,112 w.MEM_COMMIT | w.MEM_RESERVE,
73 w.PAGE_READWRITE,113 w.PAGE_READWRITE,
74 ) catch return error.OutOfMemory;114 ) catch return error.OutOfMemory;
75115
76 // If the allocation is sufficiently aligned, use it.116 // If the allocation is sufficiently aligned, use it.
77 if (@ptrToInt(addr) & (alignment - 1) == 0) {117 if (@ptrToInt(addr) & (alignment - 1) == 0) {
78 return @ptrCast([*]u8, addr)[0..n];118 return @ptrCast([*]u8, addr)[0..alignPageAllocLen(alignedLen, n, len_align)];
79 }119 }
80120
81 // If it wasn't, actually do an explicitely aligned allocation.121 // If it wasn't, actually do an explicitely aligned allocation.
82 w.VirtualFree(addr, 0, w.MEM_RELEASE);122 w.VirtualFree(addr, 0, w.MEM_RELEASE);
83 const alloc_size = n + alignment;123 const alloc_size = n + alignment - mem.page_size;
84124
85 const final_addr = while (true) {125 while (true) {
86 // Reserve a range of memory large enough to find a sufficiently126 // Reserve a range of memory large enough to find a sufficiently
87 // aligned address.127 // aligned address.
88 const reserved_addr = w.VirtualAlloc(128 const reserved_addr = w.VirtualAlloc(
...@@ -102,48 +142,50 @@ const PageAllocator = struct {...@@ -102,48 +142,50 @@ const PageAllocator = struct {
102 // until it succeeds.142 // until it succeeds.
103 const ptr = w.VirtualAlloc(143 const ptr = w.VirtualAlloc(
104 @intToPtr(*c_void, aligned_addr),144 @intToPtr(*c_void, aligned_addr),
105 n,145 alignedLen,
106 w.MEM_COMMIT | w.MEM_RESERVE,146 w.MEM_COMMIT | w.MEM_RESERVE,
107 w.PAGE_READWRITE,147 w.PAGE_READWRITE,
108 ) catch continue;148 ) catch continue;
109149
110 return @ptrCast([*]u8, ptr)[0..n];150 return @ptrCast([*]u8, ptr)[0..alignPageAllocLen(alignedLen, n, len_align)];
111 };151 }
112
113 return @ptrCast([*]u8, final_addr)[0..n];
114 }152 }
115153
116 const alloc_size = if (alignment <= mem.page_size) n else n + alignment;154 const maxDropLen = alignment - std.math.min(alignment, mem.page_size);
155 const allocLen = if (maxDropLen <= alignedLen - n) alignedLen
156 else mem.alignForward(alignedLen + maxDropLen, mem.page_size);
117 const slice = os.mmap(157 const slice = os.mmap(
118 null,158 null,
119 mem.alignForward(alloc_size, mem.page_size),159 allocLen,
120 os.PROT_READ | os.PROT_WRITE,160 os.PROT_READ | os.PROT_WRITE,
121 os.MAP_PRIVATE | os.MAP_ANONYMOUS,161 os.MAP_PRIVATE | os.MAP_ANONYMOUS,
122 -1,162 -1,
123 0,163 0,
124 ) catch return error.OutOfMemory;164 ) catch return error.OutOfMemory;
125 if (alloc_size == n) return slice[0..n];165 assert(mem.isAligned(@ptrToInt(slice.ptr), mem.page_size));
126166
127 const aligned_addr = mem.alignForward(@ptrToInt(slice.ptr), alignment);167 const aligned_addr = mem.alignForward(@ptrToInt(slice.ptr), alignment);
128168
129 // Unmap the extra bytes that were only requested in order to guarantee169 // Unmap the extra bytes that were only requested in order to guarantee
130 // that the range of memory we were provided had a proper alignment in170 // that the range of memory we were provided had a proper alignment in
131 // it somewhere. The extra bytes could be at the beginning, or end, or both.171 // it somewhere. The extra bytes could be at the beginning, or end, or both.
132 const unused_start_len = aligned_addr - @ptrToInt(slice.ptr);172 const dropLen = aligned_addr - @ptrToInt(slice.ptr);
133 if (unused_start_len != 0) {173 if (dropLen != 0) {
134 os.munmap(slice[0..unused_start_len]);174 os.munmap(slice[0..dropLen]);
135 }175 }
136 const aligned_end_addr = mem.alignForward(aligned_addr + n, mem.page_size);176
137 const unused_end_len = @ptrToInt(slice.ptr) + slice.len - aligned_end_addr;177 // Unmap extra pages
138 if (unused_end_len != 0) {178 const alignedBufferLen = allocLen - dropLen;
139 os.munmap(@intToPtr([*]align(mem.page_size) u8, aligned_end_addr)[0..unused_end_len]);179 if (alignedBufferLen > alignedLen) {
180 os.munmap(@alignCast(mem.page_size, @intToPtr([*]u8, aligned_addr))[alignedLen..alignedBufferLen]);
140 }181 }
141182
142 return @intToPtr([*]u8, aligned_addr)[0..n];183 return @intToPtr([*]u8, aligned_addr)[0..alignPageAllocLen(alignedLen, n, len_align)];
143 }184 }
144185
145 fn shrink(allocator: *Allocator, old_mem_unaligned: []u8, old_align: u29, new_size: usize, new_align: u29) []u8 {186 fn resize(allocator: *Allocator, buf_unaligned: []u8, new_size: usize, len_align: u29) Allocator.Error!usize {
146 const old_mem = @alignCast(mem.page_size, old_mem_unaligned);187 const new_size_aligned = mem.alignForward(new_size, mem.page_size);
188
147 if (builtin.os.tag == .windows) {189 if (builtin.os.tag == .windows) {
148 const w = os.windows;190 const w = os.windows;
149 if (new_size == 0) {191 if (new_size == 0) {
...@@ -153,100 +195,45 @@ const PageAllocator = struct {...@@ -153,100 +195,45 @@ const PageAllocator = struct {
153 // is reserved in the initial allocation call to VirtualAlloc."195 // is reserved in the initial allocation call to VirtualAlloc."
154 // So we can only use MEM_RELEASE when actually releasing the196 // So we can only use MEM_RELEASE when actually releasing the
155 // whole allocation.197 // whole allocation.
156 w.VirtualFree(old_mem.ptr, 0, w.MEM_RELEASE);198 w.VirtualFree(buf_unaligned.ptr, 0, w.MEM_RELEASE);
157 } else {199 return 0;
158 const base_addr = @ptrToInt(old_mem.ptr);200 }
159 const old_addr_end = base_addr + old_mem.len;201 if (new_size < buf_unaligned.len) {
160 const new_addr_end = base_addr + new_size;202 const base_addr = @ptrToInt(buf_unaligned.ptr);
161 const new_addr_end_rounded = mem.alignForward(new_addr_end, mem.page_size);203 const old_addr_end = base_addr + buf_unaligned.len;
162 if (old_addr_end > new_addr_end_rounded) {204 const new_addr_end = mem.alignForward(base_addr + new_size, mem.page_size);
205 if (old_addr_end > new_addr_end) {
163 // For shrinking that is not releasing, we will only206 // For shrinking that is not releasing, we will only
164 // decommit the pages not needed anymore.207 // decommit the pages not needed anymore.
165 w.VirtualFree(208 w.VirtualFree(
166 @intToPtr(*c_void, new_addr_end_rounded),209 @intToPtr(*c_void, new_addr_end),
167 old_addr_end - new_addr_end_rounded,210 old_addr_end - new_addr_end,
168 w.MEM_DECOMMIT,211 w.MEM_DECOMMIT,
169 );212 );
170 }213 }
214 return alignPageAllocLen(new_size_aligned, new_size, len_align);
171 }215 }
172 return old_mem[0..new_size];216 if (new_size == buf_unaligned.len) {
173 }217 return alignPageAllocLen(new_size_aligned, new_size, len_align);
174 const base_addr = @ptrToInt(old_mem.ptr);
175 const old_addr_end = base_addr + old_mem.len;
176 const new_addr_end = base_addr + new_size;
177 const new_addr_end_rounded = mem.alignForward(new_addr_end, mem.page_size);
178 if (old_addr_end > new_addr_end_rounded) {
179 const ptr = @intToPtr([*]align(mem.page_size) u8, new_addr_end_rounded);
180 os.munmap(ptr[0 .. old_addr_end - new_addr_end_rounded]);
181 }
182 return old_mem[0..new_size];
183 }
184
185 fn realloc(allocator: *Allocator, old_mem_unaligned: []u8, old_align: u29, new_size: usize, new_align: u29) ![]u8 {
186 const old_mem = @alignCast(mem.page_size, old_mem_unaligned);
187 if (builtin.os.tag == .windows) {
188 if (old_mem.len == 0) {
189 return alloc(allocator, new_size, new_align);
190 }218 }
219 // new_size > buf_unaligned.len not implemented
220 return error.OutOfMemory;
221 }
191222
192 if (new_size <= old_mem.len and new_align <= old_align) {223 const buf_aligned_len = mem.alignForward(buf_unaligned.len, mem.page_size);
193 return shrink(allocator, old_mem, old_align, new_size, new_align);224 if (new_size_aligned == buf_aligned_len)
194 }225 return alignPageAllocLen(new_size_aligned, new_size, len_align);
195
196 const w = os.windows;
197 const base_addr = @ptrToInt(old_mem.ptr);
198
199 if (new_align > old_align and base_addr & (new_align - 1) != 0) {
200 // Current allocation doesn't satisfy the new alignment.
201 // For now we'll do a new one no matter what, but maybe
202 // there is something smarter to do instead.
203 const result = try alloc(allocator, new_size, new_align);
204 assert(old_mem.len != 0);
205 @memcpy(result.ptr, old_mem.ptr, std.math.min(old_mem.len, result.len));
206 w.VirtualFree(old_mem.ptr, 0, w.MEM_RELEASE);
207
208 return result;
209 }
210
211 const old_addr_end = base_addr + old_mem.len;
212 const old_addr_end_rounded = mem.alignForward(old_addr_end, mem.page_size);
213 const new_addr_end = base_addr + new_size;
214 const new_addr_end_rounded = mem.alignForward(new_addr_end, mem.page_size);
215 if (new_addr_end_rounded == old_addr_end_rounded) {
216 // The reallocation fits in the already allocated pages.
217 return @ptrCast([*]u8, old_mem.ptr)[0..new_size];
218 }
219 assert(new_addr_end_rounded > old_addr_end_rounded);
220
221 // We need to commit new pages.
222 const additional_size = new_addr_end - old_addr_end_rounded;
223 const realloc_addr = w.kernel32.VirtualAlloc(
224 @intToPtr(*c_void, old_addr_end_rounded),
225 additional_size,
226 w.MEM_COMMIT | w.MEM_RESERVE,
227 w.PAGE_READWRITE,
228 ) orelse {
229 // Committing new pages at the end of the existing allocation
230 // failed, we need to try a new one.
231 const new_alloc_mem = try alloc(allocator, new_size, new_align);
232 @memcpy(new_alloc_mem.ptr, old_mem.ptr, old_mem.len);
233 w.VirtualFree(old_mem.ptr, 0, w.MEM_RELEASE);
234
235 return new_alloc_mem;
236 };
237226
238 assert(@ptrToInt(realloc_addr) == old_addr_end_rounded);227 if (new_size_aligned < buf_aligned_len) {
239 return @ptrCast([*]u8, old_mem.ptr)[0..new_size];228 const ptr = @intToPtr([*]align(mem.page_size) u8, @ptrToInt(buf_unaligned.ptr) + new_size_aligned);
240 }229 os.munmap(ptr[0 .. buf_aligned_len - new_size_aligned]);
241 if (new_size <= old_mem.len and new_align <= old_align) {230 if (new_size_aligned == 0)
242 return shrink(allocator, old_mem, old_align, new_size, new_align);231 return 0;
243 }232 return alignPageAllocLen(new_size_aligned, new_size, len_align);
244 const result = try alloc(allocator, new_size, new_align);
245 if (old_mem.len != 0) {
246 @memcpy(result.ptr, old_mem.ptr, std.math.min(old_mem.len, result.len));
247 os.munmap(old_mem);
248 }233 }
249 return result;234
235 // TODO: call mremap
236 return error.OutOfMemory;
250 }237 }
251};238};
252239
...@@ -338,16 +325,24 @@ const WasmPageAllocator = struct {...@@ -338,16 +325,24 @@ const WasmPageAllocator = struct {
338 }325 }
339326
340 fn nPages(memsize: usize) usize {327 fn nPages(memsize: usize) usize {
341 return std.mem.alignForward(memsize, std.mem.page_size) / std.mem.page_size;328 return mem.alignForward(memsize, mem.page_size) / mem.page_size;
342 }329 }
343330
344 fn alloc(allocator: *Allocator, page_count: usize, alignment: u29) error{OutOfMemory}!usize {331 fn alloc(allocator: *Allocator, len: usize, alignment: u29, len_align: u29) error{OutOfMemory}![]u8 {
345 var idx = conventional.useRecycled(page_count);332 const page_count = nPages(len);
346 if (idx != FreeBlock.not_found) {333 const page_idx = try allocPages(page_count);
347 return idx;334 return @intToPtr([*]u8, page_idx * mem.page_size)
335 [0..alignPageAllocLen(page_count * mem.page_size, len, len_align)];
336 }
337 fn allocPages(page_count: usize) !usize {
338 {
339 const idx = conventional.useRecycled(page_count);
340 if (idx != FreeBlock.not_found) {
341 return idx;
342 }
348 }343 }
349344
350 idx = extended.useRecycled(page_count);345 const idx = extended.useRecycled(page_count);
351 if (idx != FreeBlock.not_found) {346 if (idx != FreeBlock.not_found) {
352 return idx + extendedOffset();347 return idx + extendedOffset();
353 }348 }
...@@ -360,51 +355,36 @@ const WasmPageAllocator = struct {...@@ -360,51 +355,36 @@ const WasmPageAllocator = struct {
360 return @intCast(usize, prev_page_count);355 return @intCast(usize, prev_page_count);
361 }356 }
362357
363 pub fn realloc(allocator: *Allocator, old_mem: []u8, old_align: u29, new_size: usize, new_align: u29) Allocator.Error![]u8 {358 fn freePages(start: usize, end: usize) void {
364 if (new_align > std.mem.page_size) {359 if (start < extendedOffset()) {
365 return error.OutOfMemory;360 conventional.recycle(start, std.math.min(extendedOffset(), end) - start);
366 }361 }
367362 if (end > extendedOffset()) {
368 if (nPages(new_size) == nPages(old_mem.len)) {363 var new_end = end;
369 return old_mem.ptr[0..new_size];364 if (!extended.isInitialized()) {
370 } else if (new_size < old_mem.len) {365 // Steal the last page from the memory currently being recycled
371 return shrink(allocator, old_mem, old_align, new_size, new_align);366 // TODO: would it be better if we use the first page instead?
372 } else {367 new_end -= 1;
373 const page_idx = try alloc(allocator, nPages(new_size), new_align);368
374 const new_mem = @intToPtr([*]u8, page_idx * std.mem.page_size)[0..new_size];369 extended.data = @intToPtr([*]u128, new_end * mem.page_size)[0 .. mem.page_size / @sizeOf(u128)];
375 std.mem.copy(u8, new_mem, old_mem);370 // Since this is the first page being freed and we consume it, assume *nothing* is free.
376 _ = shrink(allocator, old_mem, old_align, 0, 0);371 mem.set(u128, extended.data, PageStatus.none_free);
377 return new_mem;372 }
373 const clamped_start = std.math.max(extendedOffset(), start);
374 extended.recycle(clamped_start - extendedOffset(), new_end - clamped_start);
378 }375 }
379 }376 }
380377
381 pub fn shrink(allocator: *Allocator, old_mem: []u8, old_align: u29, new_size: usize, new_align: u29) []u8 {378 fn resize(allocator: *Allocator, buf: []u8, new_len: usize, len_align: u29) error{OutOfMemory}!usize {
382 @setCold(true);379 const aligned_len = mem.alignForward(buf.len, mem.page_size);
383 const free_start = nPages(@ptrToInt(old_mem.ptr) + new_size);380 if (new_len > aligned_len) return error.OutOfMemory;
384 var free_end = nPages(@ptrToInt(old_mem.ptr) + old_mem.len);381 const current_n = nPages(aligned_len);
385382 const new_n = nPages(new_len);
386 if (free_end > free_start) {383 if (new_n != current_n) {
387 if (free_start < extendedOffset()) {384 const base = nPages(@ptrToInt(buf.ptr));
388 const clamped_end = std.math.min(extendedOffset(), free_end);385 freePages(base + new_n, base + current_n);
389 conventional.recycle(free_start, clamped_end - free_start);
390 }
391
392 if (free_end > extendedOffset()) {
393 if (!extended.isInitialized()) {
394 // Steal the last page from the memory currently being recycled
395 // TODO: would it be better if we use the first page instead?
396 free_end -= 1;
397
398 extended.data = @intToPtr([*]u128, free_end * std.mem.page_size)[0 .. std.mem.page_size / @sizeOf(u128)];
399 // Since this is the first page being freed and we consume it, assume *nothing* is free.
400 std.mem.set(u128, extended.data, PageStatus.none_free);
401 }
402 const clamped_start = std.math.max(extendedOffset(), free_start);
403 extended.recycle(clamped_start - extendedOffset(), free_end - clamped_start);
404 }
405 }386 }
406387 return if (new_len == 0) 0 else alignPageAllocLen(new_n * mem.page_size, new_len, len_align);
407 return old_mem[0..new_size];
408 }388 }
409};389};
410390
...@@ -418,8 +398,8 @@ pub const HeapAllocator = switch (builtin.os.tag) {...@@ -418,8 +398,8 @@ pub const HeapAllocator = switch (builtin.os.tag) {
418 pub fn init() HeapAllocator {398 pub fn init() HeapAllocator {
419 return HeapAllocator{399 return HeapAllocator{
420 .allocator = Allocator{400 .allocator = Allocator{
421 .reallocFn = realloc,401 .allocFn = alloc,
422 .shrinkFn = shrink,402 .resizeFn = resize,
423 },403 },
424 .heap_handle = null,404 .heap_handle = null,
425 };405 };
...@@ -431,11 +411,14 @@ pub const HeapAllocator = switch (builtin.os.tag) {...@@ -431,11 +411,14 @@ pub const HeapAllocator = switch (builtin.os.tag) {
431 }411 }
432 }412 }
433413
434 fn alloc(allocator: *Allocator, n: usize, alignment: u29) error{OutOfMemory}![]u8 {414 fn getRecordPtr(buf: []u8) *align(1) usize {
415 return @intToPtr(*align(1) usize, @ptrToInt(buf.ptr) + buf.len);
416 }
417
418 fn alloc(allocator: *Allocator, n: usize, ptr_align: u29, len_align: u29) error{OutOfMemory}![]u8 {
435 const self = @fieldParentPtr(HeapAllocator, "allocator", allocator);419 const self = @fieldParentPtr(HeapAllocator, "allocator", allocator);
436 if (n == 0) return &[0]u8{};
437420
438 const amt = n + alignment + @sizeOf(usize);421 const amt = n + ptr_align - 1 + @sizeOf(usize);
439 const optional_heap_handle = @atomicLoad(?HeapHandle, &self.heap_handle, builtin.AtomicOrder.SeqCst);422 const optional_heap_handle = @atomicLoad(?HeapHandle, &self.heap_handle, builtin.AtomicOrder.SeqCst);
440 const heap_handle = optional_heap_handle orelse blk: {423 const heap_handle = optional_heap_handle orelse blk: {
441 const options = if (builtin.single_threaded) os.windows.HEAP_NO_SERIALIZE else 0;424 const options = if (builtin.single_threaded) os.windows.HEAP_NO_SERIALIZE else 0;
...@@ -446,66 +429,60 @@ pub const HeapAllocator = switch (builtin.os.tag) {...@@ -446,66 +429,60 @@ pub const HeapAllocator = switch (builtin.os.tag) {
446 };429 };
447 const ptr = os.windows.kernel32.HeapAlloc(heap_handle, 0, amt) orelse return error.OutOfMemory;430 const ptr = os.windows.kernel32.HeapAlloc(heap_handle, 0, amt) orelse return error.OutOfMemory;
448 const root_addr = @ptrToInt(ptr);431 const root_addr = @ptrToInt(ptr);
449 const adjusted_addr = mem.alignForward(root_addr, alignment);432 const aligned_addr = mem.alignForward(root_addr, ptr_align);
450 const record_addr = adjusted_addr + n;433 const return_len = init: {
451 @intToPtr(*align(1) usize, record_addr).* = root_addr;434 if (len_align == 0) break :init n;
452 return @intToPtr([*]u8, adjusted_addr)[0..n];435 const full_len = os.windows.kernel32.HeapSize(heap_handle, 0, ptr);
453 }436 assert(full_len != std.math.maxInt(usize));
454437 assert(full_len >= amt);
455 fn shrink(allocator: *Allocator, old_mem: []u8, old_align: u29, new_size: usize, new_align: u29) []u8 {438 break :init mem.alignBackwardAnyAlign(full_len - (aligned_addr - root_addr), len_align);
456 return realloc(allocator, old_mem, old_align, new_size, new_align) catch {
457 const old_adjusted_addr = @ptrToInt(old_mem.ptr);
458 const old_record_addr = old_adjusted_addr + old_mem.len;
459 const root_addr = @intToPtr(*align(1) usize, old_record_addr).*;
460 const old_ptr = @intToPtr(*c_void, root_addr);
461 const new_record_addr = old_record_addr - new_size + old_mem.len;
462 @intToPtr(*align(1) usize, new_record_addr).* = root_addr;
463 return old_mem[0..new_size];
464 };439 };
440 const buf = @intToPtr([*]u8, aligned_addr)[0..return_len];
441 getRecordPtr(buf).* = root_addr;
442 return buf;
465 }443 }
466444
467 fn realloc(allocator: *Allocator, old_mem: []u8, old_align: u29, new_size: usize, new_align: u29) ![]u8 {445 fn resize(allocator: *Allocator, buf: []u8, new_size: usize, len_align: u29) error{OutOfMemory}!usize {
468 if (old_mem.len == 0) return alloc(allocator, new_size, new_align);
469
470 const self = @fieldParentPtr(HeapAllocator, "allocator", allocator);446 const self = @fieldParentPtr(HeapAllocator, "allocator", allocator);
471 const old_adjusted_addr = @ptrToInt(old_mem.ptr);
472 const old_record_addr = old_adjusted_addr + old_mem.len;
473 const root_addr = @intToPtr(*align(1) usize, old_record_addr).*;
474 const old_ptr = @intToPtr(*c_void, root_addr);
475
476 if (new_size == 0) {447 if (new_size == 0) {
477 os.windows.HeapFree(self.heap_handle.?, 0, old_ptr);448 os.windows.HeapFree(self.heap_handle.?, 0, @intToPtr(*c_void ,getRecordPtr(buf).*));
478 return old_mem[0..0];449 return 0;
479 }450 }
480451
481 const amt = new_size + new_align + @sizeOf(usize);452 const root_addr = getRecordPtr(buf).*;
453 const align_offset = @ptrToInt(buf.ptr) - root_addr;
454 const amt = align_offset + new_size + @sizeOf(usize);
482 const new_ptr = os.windows.kernel32.HeapReAlloc(455 const new_ptr = os.windows.kernel32.HeapReAlloc(
483 self.heap_handle.?,456 self.heap_handle.?,
484 0,457 os.windows.HEAP_REALLOC_IN_PLACE_ONLY,
485 old_ptr,458 @intToPtr(*c_void, root_addr),
486 amt,459 amt,
487 ) orelse return error.OutOfMemory;460 ) orelse return error.OutOfMemory;
488 const offset = old_adjusted_addr - root_addr;461 assert(new_ptr == @intToPtr(*c_void, root_addr));
489 const new_root_addr = @ptrToInt(new_ptr);462 const return_len = init: {
490 var new_adjusted_addr = new_root_addr + offset;463 if (len_align == 0) break :init new_size;
491 const offset_is_valid = new_adjusted_addr + new_size + @sizeOf(usize) <= new_root_addr + amt;464 const full_len = os.windows.kernel32.HeapSize(self.heap_handle.?, 0, new_ptr);
492 const offset_is_aligned = new_adjusted_addr % new_align == 0;465 assert(full_len != std.math.maxInt(usize));
493 if (!offset_is_valid or !offset_is_aligned) {466 assert(full_len >= amt);
494 // If HeapReAlloc didn't happen to move the memory to the new alignment,467 break :init mem.alignBackwardAnyAlign(full_len - align_offset, len_align);
495 // or the memory starting at the old offset would be outside of the new allocation,468 };
496 // then we need to copy the memory to a valid aligned address and use that469 getRecordPtr(buf.ptr[0..return_len]).* = root_addr;
497 const new_aligned_addr = mem.alignForward(new_root_addr, new_align);470 return return_len;
498 @memcpy(@intToPtr([*]u8, new_aligned_addr), @intToPtr([*]u8, new_adjusted_addr), std.math.min(old_mem.len, new_size));
499 new_adjusted_addr = new_aligned_addr;
500 }
501 const new_record_addr = new_adjusted_addr + new_size;
502 @intToPtr(*align(1) usize, new_record_addr).* = new_root_addr;
503 return @intToPtr([*]u8, new_adjusted_addr)[0..new_size];
504 }471 }
505 },472 },
506 else => @compileError("Unsupported OS"),473 else => @compileError("Unsupported OS"),
507};474};
508475
476fn sliceContainsPtr(container: []u8, ptr: [*]u8) bool {
477 return @ptrToInt(ptr) >= @ptrToInt(container.ptr) and
478 @ptrToInt(ptr) < (@ptrToInt(container.ptr) + container.len);
479}
480
481fn sliceContainsSlice(container: []u8, slice: []u8) bool {
482 return @ptrToInt(slice.ptr) >= @ptrToInt(container.ptr) and
483 (@ptrToInt(slice.ptr) + slice.len) <= (@ptrToInt(container.ptr) + container.len);
484}
485
509pub const FixedBufferAllocator = struct {486pub const FixedBufferAllocator = struct {
510 allocator: Allocator,487 allocator: Allocator,
511 end_index: usize,488 end_index: usize,
...@@ -514,19 +491,33 @@ pub const FixedBufferAllocator = struct {...@@ -514,19 +491,33 @@ pub const FixedBufferAllocator = struct {
514 pub fn init(buffer: []u8) FixedBufferAllocator {491 pub fn init(buffer: []u8) FixedBufferAllocator {
515 return FixedBufferAllocator{492 return FixedBufferAllocator{
516 .allocator = Allocator{493 .allocator = Allocator{
517 .reallocFn = realloc,494 .allocFn = alloc,
518 .shrinkFn = shrink,495 .resizeFn = resize,
519 },496 },
520 .buffer = buffer,497 .buffer = buffer,
521 .end_index = 0,498 .end_index = 0,
522 };499 };
523 }500 }
524501
525 fn alloc(allocator: *Allocator, n: usize, alignment: u29) ![]u8 {502 pub fn ownsPtr(self: *FixedBufferAllocator, ptr: [*]u8) bool {
503 return sliceContainsPtr(self.buffer, ptr);
504 }
505
506 pub fn ownsSlice(self: *FixedBufferAllocator, slice: []u8) bool {
507 return sliceContainsSlice(self.buffer, slice);
508 }
509
510 // NOTE: this will not work in all cases, if the last allocation had an adjusted_index
511 // then we won't be able to determine what the last allocation was. This is because
512 // the alignForward operation done in alloc is not reverisible.
513 pub fn isLastAllocation(self: *FixedBufferAllocator, buf: []u8) bool {
514 return buf.ptr + buf.len == self.buffer.ptr + self.end_index;
515 }
516
517 fn alloc(allocator: *Allocator, n: usize, ptr_align: u29, len_align: u29) ![]u8 {
526 const self = @fieldParentPtr(FixedBufferAllocator, "allocator", allocator);518 const self = @fieldParentPtr(FixedBufferAllocator, "allocator", allocator);
527 const addr = @ptrToInt(self.buffer.ptr) + self.end_index;519 const aligned_addr = mem.alignForward(@ptrToInt(self.buffer.ptr) + self.end_index, ptr_align);
528 const adjusted_addr = mem.alignForward(addr, alignment);520 const adjusted_index = aligned_addr - @ptrToInt(self.buffer.ptr);
529 const adjusted_index = self.end_index + (adjusted_addr - addr);
530 const new_end_index = adjusted_index + n;521 const new_end_index = adjusted_index + n;
531 if (new_end_index > self.buffer.len) {522 if (new_end_index > self.buffer.len) {
532 return error.OutOfMemory;523 return error.OutOfMemory;
...@@ -534,33 +525,32 @@ pub const FixedBufferAllocator = struct {...@@ -534,33 +525,32 @@ pub const FixedBufferAllocator = struct {
534 const result = self.buffer[adjusted_index..new_end_index];525 const result = self.buffer[adjusted_index..new_end_index];
535 self.end_index = new_end_index;526 self.end_index = new_end_index;
536527
537 return result;528 return result[0..mem.alignAllocLen(result.len, n, len_align)];
538 }529 }
539530
540 fn realloc(allocator: *Allocator, old_mem: []u8, old_align: u29, new_size: usize, new_align: u29) ![]u8 {531 fn resize(allocator: *Allocator, buf: []u8, new_size: usize, len_align: u29) Allocator.Error!usize {
541 const self = @fieldParentPtr(FixedBufferAllocator, "allocator", allocator);532 const self = @fieldParentPtr(FixedBufferAllocator, "allocator", allocator);
542 assert(old_mem.len <= self.end_index);533 assert(self.ownsSlice(buf)); // sanity check
543 if (old_mem.ptr == self.buffer.ptr + self.end_index - old_mem.len and534
544 mem.alignForward(@ptrToInt(old_mem.ptr), new_align) == @ptrToInt(old_mem.ptr))535 if (!self.isLastAllocation(buf)) {
545 {536 if (new_size > buf.len)
546 const start_index = self.end_index - old_mem.len;537 return error.OutOfMemory;
547 const new_end_index = start_index + new_size;538 return if (new_size == 0) 0 else mem.alignAllocLen(buf.len, new_size, len_align);
548 if (new_end_index > self.buffer.len) return error.OutOfMemory;
549 const result = self.buffer[start_index..new_end_index];
550 self.end_index = new_end_index;
551 return result;
552 } else if (new_size <= old_mem.len and new_align <= old_align) {
553 // We can't do anything with the memory, so tell the client to keep it.
554 return error.OutOfMemory;
555 } else {
556 const result = try alloc(allocator, new_size, new_align);
557 @memcpy(result.ptr, old_mem.ptr, std.math.min(old_mem.len, result.len));
558 return result;
559 }539 }
560 }
561540
562 fn shrink(allocator: *Allocator, old_mem: []u8, old_align: u29, new_size: usize, new_align: u29) []u8 {541 if (new_size <= buf.len) {
563 return old_mem[0..new_size];542 const sub = buf.len - new_size;
543 self.end_index -= sub;
544 return if (new_size == 0) 0 else mem.alignAllocLen(buf.len - sub, new_size, len_align);
545 }
546
547 var add = new_size - buf.len;
548 if (add + self.end_index > self.buffer.len) {
549 //add = self.buffer.len - self.end_index;
550 return error.OutOfMemory;
551 }
552 self.end_index += add;
553 return mem.alignAllocLen(buf.len + add, new_size, len_align);
564 }554 }
565555
566 pub fn reset(self: *FixedBufferAllocator) void {556 pub fn reset(self: *FixedBufferAllocator) void {
...@@ -581,20 +571,20 @@ pub const ThreadSafeFixedBufferAllocator = blk: {...@@ -581,20 +571,20 @@ pub const ThreadSafeFixedBufferAllocator = blk: {
581 pub fn init(buffer: []u8) ThreadSafeFixedBufferAllocator {571 pub fn init(buffer: []u8) ThreadSafeFixedBufferAllocator {
582 return ThreadSafeFixedBufferAllocator{572 return ThreadSafeFixedBufferAllocator{
583 .allocator = Allocator{573 .allocator = Allocator{
584 .reallocFn = realloc,574 .allocFn = alloc,
585 .shrinkFn = shrink,575 .resizeFn = Allocator.noResize,
586 },576 },
587 .buffer = buffer,577 .buffer = buffer,
588 .end_index = 0,578 .end_index = 0,
589 };579 };
590 }580 }
591581
592 fn alloc(allocator: *Allocator, n: usize, alignment: u29) ![]u8 {582 fn alloc(allocator: *Allocator, n: usize, ptr_align: u29, len_align: u29) ![]u8 {
593 const self = @fieldParentPtr(ThreadSafeFixedBufferAllocator, "allocator", allocator);583 const self = @fieldParentPtr(ThreadSafeFixedBufferAllocator, "allocator", allocator);
594 var end_index = @atomicLoad(usize, &self.end_index, builtin.AtomicOrder.SeqCst);584 var end_index = @atomicLoad(usize, &self.end_index, builtin.AtomicOrder.SeqCst);
595 while (true) {585 while (true) {
596 const addr = @ptrToInt(self.buffer.ptr) + end_index;586 const addr = @ptrToInt(self.buffer.ptr) + end_index;
597 const adjusted_addr = mem.alignForward(addr, alignment);587 const adjusted_addr = mem.alignForward(addr, ptr_align);
598 const adjusted_index = end_index + (adjusted_addr - addr);588 const adjusted_index = end_index + (adjusted_addr - addr);
599 const new_end_index = adjusted_index + n;589 const new_end_index = adjusted_index + n;
600 if (new_end_index > self.buffer.len) {590 if (new_end_index > self.buffer.len) {
...@@ -604,21 +594,6 @@ pub const ThreadSafeFixedBufferAllocator = blk: {...@@ -604,21 +594,6 @@ pub const ThreadSafeFixedBufferAllocator = blk: {
604 }594 }
605 }595 }
606596
607 fn realloc(allocator: *Allocator, old_mem: []u8, old_align: u29, new_size: usize, new_align: u29) ![]u8 {
608 if (new_size <= old_mem.len and new_align <= old_align) {
609 // We can't do anything useful with the memory, tell the client to keep it.
610 return error.OutOfMemory;
611 } else {
612 const result = try alloc(allocator, new_size, new_align);
613 @memcpy(result.ptr, old_mem.ptr, std.math.min(old_mem.len, result.len));
614 return result;
615 }
616 }
617
618 fn shrink(allocator: *Allocator, old_mem: []u8, old_align: u29, new_size: usize, new_align: u29) []u8 {
619 return old_mem[0..new_size];
620 }
621
622 pub fn reset(self: *ThreadSafeFixedBufferAllocator) void {597 pub fn reset(self: *ThreadSafeFixedBufferAllocator) void {
623 self.end_index = 0;598 self.end_index = 0;
624 }599 }
...@@ -632,8 +607,8 @@ pub fn stackFallback(comptime size: usize, fallback_allocator: *Allocator) Stack...@@ -632,8 +607,8 @@ pub fn stackFallback(comptime size: usize, fallback_allocator: *Allocator) Stack
632 .fallback_allocator = fallback_allocator,607 .fallback_allocator = fallback_allocator,
633 .fixed_buffer_allocator = undefined,608 .fixed_buffer_allocator = undefined,
634 .allocator = Allocator{609 .allocator = Allocator{
635 .reallocFn = StackFallbackAllocator(size).realloc,610 .allocFn = StackFallbackAllocator(size).realloc,
636 .shrinkFn = StackFallbackAllocator(size).shrink,611 .resizeFn = StackFallbackAllocator(size).resize,
637 },612 },
638 };613 };
639}614}
...@@ -652,58 +627,19 @@ pub fn StackFallbackAllocator(comptime size: usize) type {...@@ -652,58 +627,19 @@ pub fn StackFallbackAllocator(comptime size: usize) type {
652 return &self.allocator;627 return &self.allocator;
653 }628 }
654629
655 fn realloc(allocator: *Allocator, old_mem: []u8, old_align: u29, new_size: usize, new_align: u29) ![]u8 {630 fn alloc(allocator: *Allocator, len: usize, ptr_align: u29, len_align: u29) error{OutOfMemory}![*]u8 {
656 const self = @fieldParentPtr(Self, "allocator", allocator);631 const self = @fieldParentPtr(Self, "allocator", allocator);
657 const in_buffer = @ptrToInt(old_mem.ptr) >= @ptrToInt(&self.buffer) and632 return FixedBufferAllocator.alloc(&self.fixed_buffer_allocator, len, ptr_align) catch
658 @ptrToInt(old_mem.ptr) < @ptrToInt(&self.buffer) + self.buffer.len;633 return fallback_allocator.alloc(len, ptr_align);
659 if (in_buffer) {
660 return FixedBufferAllocator.realloc(
661 &self.fixed_buffer_allocator.allocator,
662 old_mem,
663 old_align,
664 new_size,
665 new_align,
666 ) catch {
667 const result = try self.fallback_allocator.reallocFn(
668 self.fallback_allocator,
669 &[0]u8{},
670 undefined,
671 new_size,
672 new_align,
673 );
674 mem.copy(u8, result, old_mem);
675 return result;
676 };
677 }
678 return self.fallback_allocator.reallocFn(
679 self.fallback_allocator,
680 old_mem,
681 old_align,
682 new_size,
683 new_align,
684 );
685 }634 }
686635
687 fn shrink(allocator: *Allocator, old_mem: []u8, old_align: u29, new_size: usize, new_align: u29) []u8 {636 fn resize(self: *Allocator, buf: []u8, new_len: usize, len_align: u29) error{OutOfMemory}!void {
688 const self = @fieldParentPtr(Self, "allocator", allocator);637 const self = @fieldParentPtr(Self, "allocator", allocator);
689 const in_buffer = @ptrToInt(old_mem.ptr) >= @ptrToInt(&self.buffer) and638 if (self.fixed_buffer_allocator.ownsPtr(buf.ptr)) {
690 @ptrToInt(old_mem.ptr) < @ptrToInt(&self.buffer) + self.buffer.len;639 try self.fixed_buffer_allocator.callResizeFn(buf, new_len);
691 if (in_buffer) {640 } else {
692 return FixedBufferAllocator.shrink(641 try self.fallback_allocator.callResizeFn(buf, new_len);
693 &self.fixed_buffer_allocator.allocator,
694 old_mem,
695 old_align,
696 new_size,
697 new_align,
698 );
699 }642 }
700 return self.fallback_allocator.shrinkFn(
701 self.fallback_allocator,
702 old_mem,
703 old_align,
704 new_size,
705 new_align,
706 );
707 }643 }
708 };644 };
709}645}
...@@ -718,8 +654,8 @@ test "c_allocator" {...@@ -718,8 +654,8 @@ test "c_allocator" {
718654
719test "WasmPageAllocator internals" {655test "WasmPageAllocator internals" {
720 if (comptime std.Target.current.isWasm()) {656 if (comptime std.Target.current.isWasm()) {
721 const conventional_memsize = WasmPageAllocator.conventional.totalPages() * std.mem.page_size;657 const conventional_memsize = WasmPageAllocator.conventional.totalPages() * mem.page_size;
722 const initial = try page_allocator.alloc(u8, std.mem.page_size);658 const initial = try page_allocator.alloc(u8, mem.page_size);
723 std.debug.assert(@ptrToInt(initial.ptr) < conventional_memsize); // If this isn't conventional, the rest of these tests don't make sense. Also we have a serious memory leak in the test suite.659 std.debug.assert(@ptrToInt(initial.ptr) < conventional_memsize); // If this isn't conventional, the rest of these tests don't make sense. Also we have a serious memory leak in the test suite.
724660
725 var inplace = try page_allocator.realloc(initial, 1);661 var inplace = try page_allocator.realloc(initial, 1);
...@@ -799,7 +735,7 @@ test "ArenaAllocator" {...@@ -799,7 +735,7 @@ test "ArenaAllocator" {
799735
800var test_fixed_buffer_allocator_memory: [800000 * @sizeOf(u64)]u8 = undefined;736var test_fixed_buffer_allocator_memory: [800000 * @sizeOf(u64)]u8 = undefined;
801test "FixedBufferAllocator" {737test "FixedBufferAllocator" {
802 var fixed_buffer_allocator = FixedBufferAllocator.init(test_fixed_buffer_allocator_memory[0..]);738 var fixed_buffer_allocator = mem.sanityWrap(FixedBufferAllocator.init(test_fixed_buffer_allocator_memory[0..]));
803739
804 try testAllocator(&fixed_buffer_allocator.allocator);740 try testAllocator(&fixed_buffer_allocator.allocator);
805 try testAllocatorAligned(&fixed_buffer_allocator.allocator, 16);741 try testAllocatorAligned(&fixed_buffer_allocator.allocator, 16);
...@@ -865,7 +801,10 @@ test "ThreadSafeFixedBufferAllocator" {...@@ -865,7 +801,10 @@ test "ThreadSafeFixedBufferAllocator" {
865 try testAllocatorAlignedShrink(&fixed_buffer_allocator.allocator);801 try testAllocatorAlignedShrink(&fixed_buffer_allocator.allocator);
866}802}
867803
868fn testAllocator(allocator: *mem.Allocator) !void {804fn testAllocator(base_allocator: *mem.Allocator) !void {
805 var sanityAllocator = mem.sanityWrap(base_allocator);
806 const allocator = &sanityAllocator.allocator;
807
869 var slice = try allocator.alloc(*i32, 100);808 var slice = try allocator.alloc(*i32, 100);
870 testing.expect(slice.len == 100);809 testing.expect(slice.len == 100);
871 for (slice) |*item, i| {810 for (slice) |*item, i| {
...@@ -893,7 +832,10 @@ fn testAllocator(allocator: *mem.Allocator) !void {...@@ -893,7 +832,10 @@ fn testAllocator(allocator: *mem.Allocator) !void {
893 allocator.free(slice);832 allocator.free(slice);
894}833}
895834
896fn testAllocatorAligned(allocator: *mem.Allocator, comptime alignment: u29) !void {835fn testAllocatorAligned(base_allocator: *mem.Allocator, comptime alignment: u29) !void {
836 var sanityAllocator = mem.sanityWrap(base_allocator);
837 const allocator = &sanityAllocator.allocator;
838
897 // initial839 // initial
898 var slice = try allocator.alignedAlloc(u8, alignment, 10);840 var slice = try allocator.alignedAlloc(u8, alignment, 10);
899 testing.expect(slice.len == 10);841 testing.expect(slice.len == 10);
...@@ -917,7 +859,10 @@ fn testAllocatorAligned(allocator: *mem.Allocator, comptime alignment: u29) !voi...@@ -917,7 +859,10 @@ fn testAllocatorAligned(allocator: *mem.Allocator, comptime alignment: u29) !voi
917 testing.expect(slice.len == 0);859 testing.expect(slice.len == 0);
918}860}
919861
920fn testAllocatorLargeAlignment(allocator: *mem.Allocator) mem.Allocator.Error!void {862fn testAllocatorLargeAlignment(base_allocator: *mem.Allocator) mem.Allocator.Error!void {
863 var sanityAllocator = mem.sanityWrap(base_allocator);
864 const allocator = &sanityAllocator.allocator;
865
921 //Maybe a platform's page_size is actually the same as or866 //Maybe a platform's page_size is actually the same as or
922 // very near usize?867 // very near usize?
923 if (mem.page_size << 2 > maxInt(usize)) return;868 if (mem.page_size << 2 > maxInt(usize)) return;
...@@ -946,7 +891,10 @@ fn testAllocatorLargeAlignment(allocator: *mem.Allocator) mem.Allocator.Error!vo...@@ -946,7 +891,10 @@ fn testAllocatorLargeAlignment(allocator: *mem.Allocator) mem.Allocator.Error!vo
946 allocator.free(slice);891 allocator.free(slice);
947}892}
948893
949fn testAllocatorAlignedShrink(allocator: *mem.Allocator) mem.Allocator.Error!void {894fn testAllocatorAlignedShrink(base_allocator: *mem.Allocator) mem.Allocator.Error!void {
895 var sanityAllocator = mem.sanityWrap(base_allocator);
896 const allocator = &sanityAllocator.allocator;
897
950 var debug_buffer: [1000]u8 = undefined;898 var debug_buffer: [1000]u8 = undefined;
951 const debug_allocator = &FixedBufferAllocator.init(&debug_buffer).allocator;899 const debug_allocator = &FixedBufferAllocator.init(&debug_buffer).allocator;
952900
lib/std/heap/arena_allocator.zig+7-22
...@@ -20,8 +20,8 @@ pub const ArenaAllocator = struct {...@@ -20,8 +20,8 @@ pub const ArenaAllocator = struct {
20 pub fn promote(self: State, child_allocator: *Allocator) ArenaAllocator {20 pub fn promote(self: State, child_allocator: *Allocator) ArenaAllocator {
21 return .{21 return .{
22 .allocator = Allocator{22 .allocator = Allocator{
23 .reallocFn = realloc,23 .allocFn = alloc,
24 .shrinkFn = shrink,24 .resizeFn = Allocator.noResize,
25 },25 },
26 .child_allocator = child_allocator,26 .child_allocator = child_allocator,
27 .state = self,27 .state = self,
...@@ -61,38 +61,23 @@ pub const ArenaAllocator = struct {...@@ -61,38 +61,23 @@ pub const ArenaAllocator = struct {
61 return buf_node;61 return buf_node;
62 }62 }
6363
64 fn alloc(allocator: *Allocator, n: usize, alignment: u29) ![]u8 {64 fn alloc(allocator: *Allocator, n: usize, ptr_align: u29, len_align: u29) ![]u8 {
65 const self = @fieldParentPtr(ArenaAllocator, "allocator", allocator);65 const self = @fieldParentPtr(ArenaAllocator, "allocator", allocator);
6666
67 var cur_node = if (self.state.buffer_list.first) |first_node| first_node else try self.createNode(0, n + alignment);67 var cur_node = if (self.state.buffer_list.first) |first_node| first_node else try self.createNode(0, n + ptr_align);
68 while (true) {68 while (true) {
69 const cur_buf = cur_node.data[@sizeOf(BufNode)..];69 const cur_buf = cur_node.data[@sizeOf(BufNode)..];
70 const addr = @ptrToInt(cur_buf.ptr) + self.state.end_index;70 const addr = @ptrToInt(cur_buf.ptr) + self.state.end_index;
71 const adjusted_addr = mem.alignForward(addr, alignment);71 const adjusted_addr = mem.alignForward(addr, ptr_align);
72 const adjusted_index = self.state.end_index + (adjusted_addr - addr);72 const adjusted_index = self.state.end_index + (adjusted_addr - addr);
73 const new_end_index = adjusted_index + n;73 const new_end_index = adjusted_index + n;
74 if (new_end_index > cur_buf.len) {74 if (new_end_index > cur_buf.len) {
75 cur_node = try self.createNode(cur_buf.len, n + alignment);75 cur_node = try self.createNode(cur_buf.len, n + ptr_align);
76 continue;76 continue;
77 }77 }
78 const result = cur_buf[adjusted_index..new_end_index];78 const result = cur_buf[adjusted_index..new_end_index];
79 self.state.end_index = new_end_index;79 self.state.end_index = new_end_index;
80 return result;80 return result[0..mem.alignAllocLen(result.len, n, len_align)];
81 }81 }
82 }82 }
83
84 fn realloc(allocator: *Allocator, old_mem: []u8, old_align: u29, new_size: usize, new_align: u29) ![]u8 {
85 if (new_size <= old_mem.len and new_align <= new_size) {
86 // We can't do anything with the memory, so tell the client to keep it.
87 return error.OutOfMemory;
88 } else {
89 const result = try alloc(allocator, new_size, new_align);
90 @memcpy(result.ptr, old_mem.ptr, std.math.min(old_mem.len, result.len));
91 return result;
92 }
93 }
94
95 fn shrink(allocator: *Allocator, old_mem: []u8, old_align: u29, new_size: usize, new_align: u29) []u8 {
96 return old_mem[0..new_size];
97 }
98};83};
lib/std/heap/logging_allocator.zig+37-24
...@@ -15,39 +15,45 @@ pub fn LoggingAllocator(comptime OutStreamType: type) type {...@@ -15,39 +15,45 @@ pub fn LoggingAllocator(comptime OutStreamType: type) type {
15 pub fn init(parent_allocator: *Allocator, out_stream: OutStreamType) Self {15 pub fn init(parent_allocator: *Allocator, out_stream: OutStreamType) Self {
16 return Self{16 return Self{
17 .allocator = Allocator{17 .allocator = Allocator{
18 .reallocFn = realloc,18 .allocFn = alloc,
19 .shrinkFn = shrink,19 .resizeFn = resize,
20 },20 },
21 .parent_allocator = parent_allocator,21 .parent_allocator = parent_allocator,
22 .out_stream = out_stream,22 .out_stream = out_stream,
23 };23 };
24 }24 }
2525
26 fn realloc(allocator: *Allocator, old_mem: []u8, old_align: u29, new_size: usize, new_align: u29) ![]u8 {26 fn alloc(allocator: *Allocator, len: usize, ptr_align: u29, len_align: u29) error{OutOfMemory}![]u8 {
27 const self = @fieldParentPtr(Self, "allocator", allocator);27 const self = @fieldParentPtr(Self, "allocator", allocator);
28 if (old_mem.len == 0) {28 self.out_stream.print("alloc : {}", .{len}) catch {};
29 self.out_stream.print("allocation of {} ", .{new_size}) catch {};29 const result = self.parent_allocator.callAllocFn(len, ptr_align, len_align);
30 } else {
31 self.out_stream.print("resize from {} to {} ", .{ old_mem.len, new_size }) catch {};
32 }
33 const result = self.parent_allocator.reallocFn(self.parent_allocator, old_mem, old_align, new_size, new_align);
34 if (result) |buff| {30 if (result) |buff| {
35 self.out_stream.print("success!\n", .{}) catch {};31 self.out_stream.print(" success!\n", .{}) catch {};
36 } else |err| {32 } else |err| {
37 self.out_stream.print("failure!\n", .{}) catch {};33 self.out_stream.print(" failure!\n", .{}) catch {};
38 }34 }
39 return result;35 return result;
40 }36 }
4137
42 fn shrink(allocator: *Allocator, old_mem: []u8, old_align: u29, new_size: usize, new_align: u29) []u8 {38 fn resize(allocator: *Allocator, buf: []u8, new_len: usize, len_align: u29) error{OutOfMemory}!usize {
43 const self = @fieldParentPtr(Self, "allocator", allocator);39 const self = @fieldParentPtr(Self, "allocator", allocator);
44 const result = self.parent_allocator.shrinkFn(self.parent_allocator, old_mem, old_align, new_size, new_align);40 if (new_len == 0) {
45 if (new_size == 0) {41 self.out_stream.print("free : {}\n", .{buf.len}) catch {};
46 self.out_stream.print("free of {} bytes success!\n", .{old_mem.len}) catch {};42 } else if (new_len <= buf.len) {
43 self.out_stream.print("shrink: {} to {}\n", .{buf.len, new_len}) catch {};
47 } else {44 } else {
48 self.out_stream.print("shrink from {} bytes to {} bytes success!\n", .{ old_mem.len, new_size }) catch {};45 self.out_stream.print("expand: {} to {}", .{ buf.len, new_len }) catch {};
46 }
47 if (self.parent_allocator.callResizeFn(buf, new_len, len_align)) |resized_len| {
48 if (new_len > buf.len) {
49 self.out_stream.print(" success!\n", .{}) catch {};
50 }
51 return resized_len;
52 } else |e| {
53 std.debug.assert(new_len > buf.len);
54 self.out_stream.print(" failure!\n", .{}) catch {};
55 return e;
49 }56 }
50 return result;
51 }57 }
52 };58 };
53}59}
...@@ -60,17 +66,24 @@ pub fn loggingAllocator(...@@ -60,17 +66,24 @@ pub fn loggingAllocator(
60}66}
6167
62test "LoggingAllocator" {68test "LoggingAllocator" {
63 var buf: [255]u8 = undefined;69 var log_buf: [255]u8 = undefined;
64 var fbs = std.io.fixedBufferStream(&buf);70 var fbs = std.io.fixedBufferStream(&log_buf);
6571
66 const allocator = &loggingAllocator(std.testing.allocator, fbs.outStream()).allocator;72 var allocator_buf: [10]u8 = undefined;
73 var fixedBufferAllocator = std.mem.sanityWrap(std.heap.FixedBufferAllocator.init(&allocator_buf));
74 const allocator = &loggingAllocator(&fixedBufferAllocator.allocator, fbs.outStream()).allocator;
6775
68 const ptr = try allocator.alloc(u8, 10);76 var a = try allocator.alloc(u8, 10);
69 allocator.free(ptr);77 a.len = allocator.shrinkBytes(a, 5, 0);
78 std.debug.assert(a.len == 5);
79 std.testing.expectError(error.OutOfMemory, allocator.callResizeFn(a, 20, 0));
80 allocator.free(a);
7081
71 std.testing.expectEqualSlices(u8,82 std.testing.expectEqualSlices(u8,
72 \\allocation of 10 success!83 \\alloc : 10 success!
73 \\free of 10 bytes success!84 \\shrink: 10 to 5
85 \\expand: 5 to 20 failure!
86 \\free : 5
74 \\87 \\
75 , fbs.getWritten());88 , fbs.getWritten());
76}89}
lib/std/mem.zig+263-56
...@@ -16,6 +16,52 @@ pub const page_size = switch (builtin.arch) {...@@ -16,6 +16,52 @@ pub const page_size = switch (builtin.arch) {
16pub const Allocator = struct {16pub const Allocator = struct {
17 pub const Error = error{OutOfMemory};17 pub const Error = error{OutOfMemory};
1818
19 /// Attempt to allocate at least `len` bytes aligned to `ptr_align`.
20 ///
21 /// If `len_align` is `0`, then the length returned MUST be exactly `len` bytes,
22 /// otherwise, the length must be aligned to `len_align`.
23 ///
24 /// `len` must be greater than or equal to `len_align` and must be aligned by `len_align`.
25 allocFn: fn (self: *Allocator, len: usize, ptr_align: u29, len_align: u29) Error![]u8,
26
27 /// Attempt to expand or shrink memory in place. `buf.len` must equal the most recent
28 /// length returned by `allocFn` or `resizeFn`.
29 ///
30 /// Passing a `new_len` of 0 frees and invalidates the buffer such that it can no
31 /// longer be passed to `resizeFn`.
32 ///
33 /// error.OutOfMemory can only be returned if `new_len` is greater than `buf.len`.
34 /// If `buf` cannot be expanded to accomodate `new_len`, then the allocation MUST be
35 /// unmodified and error.OutOfMemory MUST be returned.
36 ///
37 /// If `len_align` is `0`, then the length returned MUST be exactly `len` bytes,
38 /// otherwise, the length must be aligned to `len_align`.
39 ///
40 /// `new_len` must be greater than or equal to `len_align` and must be aligned by `len_align`.
41 resizeFn: fn (self: *Allocator, buf: []u8, new_len: usize, len_align: u29) Error!usize,
42
43 pub fn callAllocFn(self: *Allocator, new_len: usize, alignment: u29, len_align: u29) Error![]u8 {
44 return self.allocFn(self, new_len, alignment, len_align);
45 }
46
47 pub fn callResizeFn(self: *Allocator, buf: []u8, new_len: usize, len_align: u29) Error!usize {
48 return self.resizeFn(self, buf, new_len, len_align);
49 }
50
51 /// Set to resizeFn if in-place resize is not supported.
52 pub fn noResize(self: *Allocator, buf: []u8, new_len: usize, len_align: u29) Error!usize {
53 if (new_len > buf.len)
54 return error.OutOfMemory;
55 return new_len;
56 }
57
58 /// Call `resizeFn`, but caller guarantees that `new_len` <= `buf.len` meaning
59 /// error.OutOfMemory should be impossible.
60 pub fn shrinkBytes(self: *Allocator, buf: []u8, new_len: usize, len_align: u29) usize {
61 assert(new_len <= buf.len);
62 return self.callResizeFn(buf, new_len, len_align) catch unreachable;
63 }
64
19 /// Realloc is used to modify the size or alignment of an existing allocation,65 /// Realloc is used to modify the size or alignment of an existing allocation,
20 /// as well as to provide the allocator with an opportunity to move an allocation66 /// as well as to provide the allocator with an opportunity to move an allocation
21 /// to a better location.67 /// to a better location.
...@@ -24,7 +70,7 @@ pub const Allocator = struct {...@@ -24,7 +70,7 @@ pub const Allocator = struct {
24 /// When the size/alignment is less than or equal to the previous allocation,70 /// When the size/alignment is less than or equal to the previous allocation,
25 /// this function returns `error.OutOfMemory` when the allocator decides the client71 /// this function returns `error.OutOfMemory` when the allocator decides the client
26 /// would be better off keeping the extra alignment/size. Clients will call72 /// would be better off keeping the extra alignment/size. Clients will call
27 /// `shrinkFn` when they require the allocator to track a new alignment/size,73 /// `callResizeFn` when they require the allocator to track a new alignment/size,
28 /// and so this function should only return success when the allocator considers74 /// and so this function should only return success when the allocator considers
29 /// the reallocation desirable from the allocator's perspective.75 /// the reallocation desirable from the allocator's perspective.
30 /// As an example, `std.ArrayList` tracks a "capacity", and therefore can handle76 /// As an example, `std.ArrayList` tracks a "capacity", and therefore can handle
...@@ -37,16 +83,15 @@ pub const Allocator = struct {...@@ -37,16 +83,15 @@ pub const Allocator = struct {
37 /// as `old_mem` was when `reallocFn` is called. The bytes of83 /// as `old_mem` was when `reallocFn` is called. The bytes of
38 /// `return_value[old_mem.len..]` have undefined values.84 /// `return_value[old_mem.len..]` have undefined values.
39 /// The returned slice must have its pointer aligned at least to `new_alignment` bytes.85 /// The returned slice must have its pointer aligned at least to `new_alignment` bytes.
40 reallocFn: fn (86 fn reallocBytes(
41 self: *Allocator,87 self: *Allocator,
42 /// Guaranteed to be the same as what was returned from most recent call to88 /// Guaranteed to be the same as what was returned from most recent call to
43 /// `reallocFn` or `shrinkFn`.89 /// `allocFn` or `resizeFn`.
44 /// If `old_mem.len == 0` then this is a new allocation and `new_byte_count`90 /// If `old_mem.len == 0` then this is a new allocation and `new_byte_count`
45 /// is guaranteed to be >= 1.91 /// is guaranteed to be >= 1.
46 old_mem: []u8,92 old_mem: []u8,
47 /// If `old_mem.len == 0` then this is `undefined`, otherwise:93 /// If `old_mem.len == 0` then this is `undefined`, otherwise:
48 /// Guaranteed to be the same as what was returned from most recent call to94 /// Guaranteed to be the same as what was passed to `allocFn`.
49 /// `reallocFn` or `shrinkFn`.
50 /// Guaranteed to be >= 1.95 /// Guaranteed to be >= 1.
51 /// Guaranteed to be a power of 2.96 /// Guaranteed to be a power of 2.
52 old_alignment: u29,97 old_alignment: u29,
...@@ -57,23 +102,49 @@ pub const Allocator = struct {...@@ -57,23 +102,49 @@ pub const Allocator = struct {
57 /// Guaranteed to be a power of 2.102 /// Guaranteed to be a power of 2.
58 /// Returned slice's pointer must have this alignment.103 /// Returned slice's pointer must have this alignment.
59 new_alignment: u29,104 new_alignment: u29,
60 ) Error![]u8,105 /// 0 indicates the length of the slice returned MUST match `new_byte_count` exactly
106 /// non-zero means the length of the returned slice must be aligned by `len_align`
107 /// `new_len` must be aligned by `len_align`
108 len_align: u29,
109 ) Error![]u8 {
110 if (old_mem.len == 0) {
111 const new_mem = try self.callAllocFn(new_byte_count, new_alignment, len_align);
112 @memset(new_mem.ptr, undefined, new_byte_count);
113 return new_mem;
114 }
61115
62 /// This function deallocates memory. It must succeed.116 if (isAligned(@ptrToInt(old_mem.ptr), new_alignment)) {
63 shrinkFn: fn (117 if (new_byte_count <= old_mem.len) {
64 self: *Allocator,118 const shrunk_len = self.shrinkBytes(old_mem, new_byte_count, len_align);
65 /// Guaranteed to be the same as what was returned from most recent call to119 if (shrunk_len < old_mem.len) {
66 /// `reallocFn` or `shrinkFn`.120 @memset(old_mem.ptr + shrunk_len, undefined, old_mem.len - shrunk_len);
67 old_mem: []u8,121 }
68 /// Guaranteed to be the same as what was returned from most recent call to122 return old_mem.ptr[0..shrunk_len];
69 /// `reallocFn` or `shrinkFn`.123 }
70 old_alignment: u29,124 if (self.callResizeFn(old_mem, new_byte_count, len_align)) |resized_len| {
71 /// Guaranteed to be less than or equal to `old_mem.len`.125 assert(resized_len >= new_byte_count);
72 new_byte_count: usize,126 @memset(old_mem.ptr + new_byte_count, undefined, resized_len - new_byte_count);
73 /// If `new_byte_count == 0` then this is `undefined`, otherwise:127 return old_mem.ptr[0..resized_len];
74 /// Guaranteed to be less than or equal to `old_alignment`.128 } else |_| { }
75 new_alignment: u29,129 }
76 ) []u8,130 if (new_byte_count <= old_mem.len and new_alignment <= old_alignment) {
131 return error.OutOfMemory;
132 }
133 return self.moveBytes(old_mem, new_byte_count, new_alignment, len_align);
134 }
135
136 /// Move the given memory to a new location in the given allocator to accomodate a new
137 /// size and alignment.
138 fn moveBytes(self: *Allocator, old_mem: []u8, new_len: usize, new_alignment: u29, len_align: u29) Error![]u8 {
139 assert(old_mem.len > 0);
140 assert(new_len > 0);
141 const new_mem = try self.callAllocFn(new_len, new_alignment, len_align);
142 @memcpy(new_mem.ptr, old_mem.ptr, std.math.min(new_len, old_mem.len));
143 // DISABLED TO AVOID BUGS IN TRANSLATE C
144 //@memset(old_mem.ptr, undefined, old_mem.len);
145 _ = self.shrinkBytes(old_mem, 0, 0);
146 return new_mem;
147 }
77148
78 /// Returns a pointer to undefined memory.149 /// Returns a pointer to undefined memory.
79 /// Call `destroy` with the result to free the memory.150 /// Call `destroy` with the result to free the memory.
...@@ -89,8 +160,7 @@ pub const Allocator = struct {...@@ -89,8 +160,7 @@ pub const Allocator = struct {
89 const T = @TypeOf(ptr).Child;160 const T = @TypeOf(ptr).Child;
90 if (@sizeOf(T) == 0) return;161 if (@sizeOf(T) == 0) return;
91 const non_const_ptr = @intToPtr([*]u8, @ptrToInt(ptr));162 const non_const_ptr = @intToPtr([*]u8, @ptrToInt(ptr));
92 const shrink_result = self.shrinkFn(self, non_const_ptr[0..@sizeOf(T)], @alignOf(T), 0, 1);163 _ = self.shrinkBytes(non_const_ptr[0..@sizeOf(T)], 0, 0);
93 assert(shrink_result.len == 0);
94 }164 }
95165
96 /// Allocates an array of `n` items of type `T` and sets all the166 /// Allocates an array of `n` items of type `T` and sets all the
...@@ -150,9 +220,21 @@ pub const Allocator = struct {...@@ -150,9 +220,21 @@ pub const Allocator = struct {
150 /// null means naturally aligned220 /// null means naturally aligned
151 comptime alignment: ?u29,221 comptime alignment: ?u29,
152 n: usize,222 n: usize,
223 ) Error![]align(alignment orelse @alignOf(T)) T {
224 return self.alignedAlloc2(T, alignment, n, .exact);
225 }
226
227 const Exact = enum {exact,atLeast};
228 pub fn alignedAlloc2(
229 self: *Allocator,
230 comptime T: type,
231 /// null means naturally aligned
232 comptime alignment: ?u29,
233 n: usize,
234 exact: Exact,
153 ) Error![]align(alignment orelse @alignOf(T)) T {235 ) Error![]align(alignment orelse @alignOf(T)) T {
154 const a = if (alignment) |a| blk: {236 const a = if (alignment) |a| blk: {
155 if (a == @alignOf(T)) return alignedAlloc(self, T, null, n);237 if (a == @alignOf(T)) return alignedAlloc2(self, T, null, n, exact);
156 break :blk a;238 break :blk a;
157 } else @alignOf(T);239 } else @alignOf(T);
158240
...@@ -161,15 +243,16 @@ pub const Allocator = struct {...@@ -161,15 +243,16 @@ pub const Allocator = struct {
161 }243 }
162244
163 const byte_count = math.mul(usize, @sizeOf(T), n) catch return Error.OutOfMemory;245 const byte_count = math.mul(usize, @sizeOf(T), n) catch return Error.OutOfMemory;
164 const byte_slice = try self.reallocFn(self, &[0]u8{}, undefined, byte_count, a);246 // TODO The `if (alignment == null)` blocks are workarounds for zig not being able to
165 assert(byte_slice.len == byte_count);247 // access certain type information about T without creating a circular dependency in async
248 // functions that heap-allocate their own frame with @Frame(func).
249 const sizeOfT = if (alignment == null) @intCast(u29, @divExact(byte_count, n)) else @sizeOf(T);
250 const byte_slice = try self.callAllocFn(byte_count, a, if (exact == .exact) @as(u29, 0) else sizeOfT);
251 assert(if (exact == .exact) byte_slice.len == byte_count else byte_slice.len >= byte_count);
166 @memset(byte_slice.ptr, undefined, byte_slice.len);252 @memset(byte_slice.ptr, undefined, byte_slice.len);
167 if (alignment == null) {253 if (alignment == null) {
168 // TODO This is a workaround for zig not being able to successfully do254 // This if block is a workaround (see comment above)
169 // @bytesToSlice(T, @alignCast(a, byte_slice)) without resolving alignment of T,255 return @intToPtr([*]T, @ptrToInt(byte_slice.ptr))[0..@divExact(byte_slice.len, @sizeOf(T))];
170 // which causes a circular dependency in async functions which try to heap-allocate
171 // their own frame with @Frame(func).
172 return @intToPtr([*]T, @ptrToInt(byte_slice.ptr))[0..n];
173 } else {256 } else {
174 return mem.bytesAsSlice(T, @alignCast(a, byte_slice));257 return mem.bytesAsSlice(T, @alignCast(a, byte_slice));
175 }258 }
...@@ -190,7 +273,15 @@ pub const Allocator = struct {...@@ -190,7 +273,15 @@ pub const Allocator = struct {
190 break :t Error![]align(Slice.alignment) Slice.child;273 break :t Error![]align(Slice.alignment) Slice.child;
191 } {274 } {
192 const old_alignment = @typeInfo(@TypeOf(old_mem)).Pointer.alignment;275 const old_alignment = @typeInfo(@TypeOf(old_mem)).Pointer.alignment;
193 return self.alignedRealloc(old_mem, old_alignment, new_n);276 return self.alignedRealloc2(old_mem, old_alignment, new_n, .exact);
277 }
278
279 pub fn reallocAtLeast(self: *Allocator, old_mem: var, new_n: usize) t: {
280 const Slice = @typeInfo(@TypeOf(old_mem)).Pointer;
281 break :t Error![]align(Slice.alignment) Slice.child;
282 } {
283 const old_alignment = @typeInfo(@TypeOf(old_mem)).Pointer.alignment;
284 return self.alignedRealloc2(old_mem, old_alignment, new_n, .atLeast);
194 }285 }
195286
196 /// This is the same as `realloc`, except caller may additionally request287 /// This is the same as `realloc`, except caller may additionally request
...@@ -201,11 +292,24 @@ pub const Allocator = struct {...@@ -201,11 +292,24 @@ pub const Allocator = struct {
201 old_mem: var,292 old_mem: var,
202 comptime new_alignment: u29,293 comptime new_alignment: u29,
203 new_n: usize,294 new_n: usize,
295 ) Error![]align(new_alignment) @typeInfo(@TypeOf(old_mem)).Pointer.child {
296 return self.alignedRealloc2(old_mem, new_alignment, new_n, .exact);
297 }
298
299 /// This is the same as `realloc`, except caller may additionally request
300 /// a new alignment, which can be larger, smaller, or the same as the old
301 /// allocation.
302 pub fn alignedRealloc2(
303 self: *Allocator,
304 old_mem: var,
305 comptime new_alignment: u29,
306 new_n: usize,
307 exact: Exact,
204 ) Error![]align(new_alignment) @typeInfo(@TypeOf(old_mem)).Pointer.child {308 ) Error![]align(new_alignment) @typeInfo(@TypeOf(old_mem)).Pointer.child {
205 const Slice = @typeInfo(@TypeOf(old_mem)).Pointer;309 const Slice = @typeInfo(@TypeOf(old_mem)).Pointer;
206 const T = Slice.child;310 const T = Slice.child;
207 if (old_mem.len == 0) {311 if (old_mem.len == 0) {
208 return self.alignedAlloc(T, new_alignment, new_n);312 return self.alignedAlloc2(T, new_alignment, new_n, exact);
209 }313 }
210 if (new_n == 0) {314 if (new_n == 0) {
211 self.free(old_mem);315 self.free(old_mem);
...@@ -215,12 +319,9 @@ pub const Allocator = struct {...@@ -215,12 +319,9 @@ pub const Allocator = struct {
215 const old_byte_slice = mem.sliceAsBytes(old_mem);319 const old_byte_slice = mem.sliceAsBytes(old_mem);
216 const byte_count = math.mul(usize, @sizeOf(T), new_n) catch return Error.OutOfMemory;320 const byte_count = math.mul(usize, @sizeOf(T), new_n) catch return Error.OutOfMemory;
217 // Note: can't set shrunk memory to undefined as memory shouldn't be modified on realloc failure321 // Note: can't set shrunk memory to undefined as memory shouldn't be modified on realloc failure
218 const byte_slice = try self.reallocFn(self, old_byte_slice, Slice.alignment, byte_count, new_alignment);322 const new_byte_slice = try self.reallocBytes(old_byte_slice, Slice.alignment, byte_count, new_alignment,
219 assert(byte_slice.len == byte_count);323 if (exact == .exact) @as(u29, 0) else @sizeOf(T));
220 if (new_n > old_mem.len) {324 return mem.bytesAsSlice(T, @alignCast(new_alignment, new_byte_slice));
221 @memset(byte_slice.ptr + old_byte_slice.len, undefined, byte_slice.len - old_byte_slice.len);
222 }
223 return mem.bytesAsSlice(T, @alignCast(new_alignment, byte_slice));
224 }325 }
225326
226 /// Prefer calling realloc to shrink if you can tolerate failure, such as327 /// Prefer calling realloc to shrink if you can tolerate failure, such as
...@@ -248,12 +349,9 @@ pub const Allocator = struct {...@@ -248,12 +349,9 @@ pub const Allocator = struct {
248 const Slice = @typeInfo(@TypeOf(old_mem)).Pointer;349 const Slice = @typeInfo(@TypeOf(old_mem)).Pointer;
249 const T = Slice.child;350 const T = Slice.child;
250351
251 if (new_n == 0) {352 if (new_n == old_mem.len)
252 self.free(old_mem);353 return old_mem;
253 return old_mem[0..0];354 assert(new_n < old_mem.len);
254 }
255
256 assert(new_n <= old_mem.len);
257 assert(new_alignment <= Slice.alignment);355 assert(new_alignment <= Slice.alignment);
258356
259 // Here we skip the overflow checking on the multiplication because357 // Here we skip the overflow checking on the multiplication because
...@@ -262,9 +360,8 @@ pub const Allocator = struct {...@@ -262,9 +360,8 @@ pub const Allocator = struct {
262360
263 const old_byte_slice = mem.sliceAsBytes(old_mem);361 const old_byte_slice = mem.sliceAsBytes(old_mem);
264 @memset(old_byte_slice.ptr + byte_count, undefined, old_byte_slice.len - byte_count);362 @memset(old_byte_slice.ptr + byte_count, undefined, old_byte_slice.len - byte_count);
265 const byte_slice = self.shrinkFn(self, old_byte_slice, Slice.alignment, byte_count, new_alignment);363 _ = self.shrinkBytes(old_byte_slice, byte_count, 0);
266 assert(byte_slice.len == byte_count);364 return old_mem[0..new_n];
267 return mem.bytesAsSlice(T, @alignCast(new_alignment, byte_slice));
268 }365 }
269366
270 /// Free an array allocated with `alloc`. To free a single item,367 /// Free an array allocated with `alloc`. To free a single item,
...@@ -276,8 +373,7 @@ pub const Allocator = struct {...@@ -276,8 +373,7 @@ pub const Allocator = struct {
276 if (bytes_len == 0) return;373 if (bytes_len == 0) return;
277 const non_const_ptr = @intToPtr([*]u8, @ptrToInt(bytes.ptr));374 const non_const_ptr = @intToPtr([*]u8, @ptrToInt(bytes.ptr));
278 @memset(non_const_ptr, undefined, bytes_len);375 @memset(non_const_ptr, undefined, bytes_len);
279 const shrink_result = self.shrinkFn(self, non_const_ptr[0..bytes_len], Slice.alignment, 0, 1);376 _ = self.shrinkBytes(non_const_ptr[0..bytes_len], 0, 0);
280 assert(shrink_result.len == 0);
281 }377 }
282378
283 /// Copies `m` to newly allocated memory. Caller owns the memory.379 /// Copies `m` to newly allocated memory. Caller owns the memory.
...@@ -296,16 +392,107 @@ pub const Allocator = struct {...@@ -296,16 +392,107 @@ pub const Allocator = struct {
296 }392 }
297};393};
298394
395/// Given a pointer to an allocator, return the *Allocator for it. `allocatorStatePtr` can
396/// either be a `*Allocator`, in which case it is returned as-is, otherwise, the address of
397/// the `allocator` field is returned.
398pub fn getAllocatorPtr(allocatorStatePtr: var) *Allocator {
399 // allocator must be a pointer or else this function will return a copy of the allocator which
400 // is not what this is for
401 const T = @TypeOf(allocatorStatePtr);
402 switch (@typeInfo(T)) {
403 .Pointer => {},
404 else => @compileError("getAllocatorPtr expects a pointer to an allocator but got: " ++ @typeName(T)),
405 }
406 if (T == *Allocator)
407 return allocatorStatePtr;
408 return &allocatorStatePtr.allocator;
409}
410
411/// Detects and asserts if the std.mem.Allocator interface is violated
412pub fn SanityAllocator(comptime T: type) type { return struct {
413 const Self = @This();
414 allocator: Allocator,
415 underlying_allocator: T,
416 pub fn init(allocator: T) @This() {
417 return .{
418 .allocator = .{
419 .allocFn = alloc,
420 .resizeFn = resize,
421 },
422 .underlying_allocator = allocator,
423 };
424 }
425 fn getUnderlyingAllocatorPtr(self: *@This()) *Allocator {
426 if (T == *Allocator) return self.underlying_allocator;
427 return getAllocatorPtr(&self.underlying_allocator);
428 }
429 pub fn alloc(allocator: *Allocator, n: usize, ptr_align: u29, len_align: u29) Allocator.Error![]u8 {
430 assert(n > 0);
431 assert(mem.isValidAlign(ptr_align));
432 if (len_align != 0) {
433 assert(mem.isAlignedAnyAlign(n, len_align));
434 assert(n >= len_align);
435 }
436
437 const self = @fieldParentPtr(@This(), "allocator", allocator);
438 const result = try self.getUnderlyingAllocatorPtr().callAllocFn(n, ptr_align, len_align);
439 if (len_align == 0) {
440 assert(result.len == n);
441 } else {
442 assert(result.len >= n);
443 assert(mem.isAlignedAnyAlign(result.len, len_align));
444 }
445 return result;
446 }
447 pub fn resize(allocator: *Allocator, buf: []u8, new_len: usize, len_align: u29) Allocator.Error!usize {
448 assert(buf.len > 0);
449 if (len_align != 0) {
450 assert(mem.isAlignedAnyAlign(new_len, len_align));
451 assert(new_len >= len_align);
452 }
453 const self = @fieldParentPtr(@This(), "allocator", allocator);
454 const result = try self.getUnderlyingAllocatorPtr().callResizeFn(buf, new_len, len_align);
455 if (len_align == 0) {
456 assert(result == new_len);
457 } else {
458 assert(result >= new_len);
459 assert(mem.isAlignedAnyAlign(result, len_align));
460 }
461 return result;
462 }
463 pub usingnamespace if (T == *Allocator or !@hasDecl(T, "reset")) struct {} else struct {
464 pub fn reset(self: *Self) void {
465 self.underlying_allocator.reset();
466 }
467 };
468};}
469
470pub fn sanityWrap(allocator: var) SanityAllocator(@TypeOf(allocator)) {
471 return SanityAllocator(@TypeOf(allocator)).init(allocator);
472}
473
474/// An allocator helper function. Adjusts an allocation length satisfy `len_align`.
475/// `full_len` should be the full capacity of the allocation which may be greater
476/// than the `len` that was requsted. This function should only be used by allocators
477/// that are unaffected by `len_align`.
478pub fn alignAllocLen(full_len: usize, alloc_len: usize, len_align: u29) usize {
479 assert(alloc_len > 0);
480 assert(alloc_len >= len_align);
481 assert(full_len >= alloc_len);
482 if (len_align == 0)
483 return alloc_len;
484 const adjusted = alignBackwardAnyAlign(full_len, len_align);
485 assert(adjusted >= alloc_len);
486 return adjusted;
487}
488
299var failAllocator = Allocator{489var failAllocator = Allocator{
300 .reallocFn = failAllocatorRealloc,490 .allocFn = failAllocatorAlloc,
301 .shrinkFn = failAllocatorShrink,491 .resizeFn = Allocator.noResize,
302};492};
303fn failAllocatorRealloc(self: *Allocator, old_mem: []u8, old_align: u29, new_size: usize, new_align: u29) ![]u8 {493fn failAllocatorAlloc(self: *Allocator, n: usize, alignment: u29, len_align: u29) Allocator.Error![]u8 {
304 return error.OutOfMemory;494 return error.OutOfMemory;
305}495}
306fn failAllocatorShrink(self: *Allocator, old_mem: []u8, old_align: u29, new_size: usize, new_align: u29) []u8 {
307 @panic("failAllocatorShrink should never be called because it cannot allocate");
308}
309496
310test "mem.Allocator basics" {497test "mem.Allocator basics" {
311 testing.expectError(error.OutOfMemory, failAllocator.alloc(u8, 1));498 testing.expectError(error.OutOfMemory, failAllocator.alloc(u8, 1));
...@@ -2190,6 +2377,13 @@ test "alignForward" {...@@ -2190,6 +2377,13 @@ test "alignForward" {
2190 testing.expect(alignForward(17, 8) == 24);2377 testing.expect(alignForward(17, 8) == 24);
2191}2378}
21922379
2380pub fn alignBackwardAnyAlign(i: usize, alignment: usize) usize {
2381 if (@popCount(usize, alignment) == 1)
2382 return alignBackward(i, alignment);
2383 assert(alignment != 0);
2384 return i - @mod(i, alignment);
2385}
2386
2193/// Round an address up to the previous aligned address2387/// Round an address up to the previous aligned address
2194/// The alignment must be a power of 2 and greater than 0.2388/// The alignment must be a power of 2 and greater than 0.
2195pub fn alignBackward(addr: usize, alignment: usize) usize {2389pub fn alignBackward(addr: usize, alignment: usize) usize {
...@@ -2206,6 +2400,19 @@ pub fn alignBackwardGeneric(comptime T: type, addr: T, alignment: T) T {...@@ -2206,6 +2400,19 @@ pub fn alignBackwardGeneric(comptime T: type, addr: T, alignment: T) T {
2206 return addr & ~(alignment - 1);2400 return addr & ~(alignment - 1);
2207}2401}
22082402
2403/// Returns whether `alignment` is a valid alignment, meaning it is
2404/// a positive power of 2.
2405pub fn isValidAlign(alignment: u29) bool {
2406 return @popCount(u29, alignment) == 1;
2407}
2408
2409pub fn isAlignedAnyAlign(i: usize, alignment: usize) bool {
2410 if (@popCount(usize, alignment) == 1)
2411 return isAligned(i, alignment);
2412 assert(alignment != 0);
2413 return 0 == @mod(i, alignment);
2414}
2415
2209/// Given an address and an alignment, return true if the address is a multiple of the alignment2416/// Given an address and an alignment, return true if the address is a multiple of the alignment
2210/// The alignment must be a power of 2 and greater than 0.2417/// The alignment must be a power of 2 and greater than 0.
2211pub fn isAligned(addr: usize, alignment: usize) bool {2418pub fn isAligned(addr: usize, alignment: usize) bool {
lib/std/os/windows/bits.zig+1
...@@ -593,6 +593,7 @@ pub const FILE_CURRENT = 1;...@@ -593,6 +593,7 @@ pub const FILE_CURRENT = 1;
593pub const FILE_END = 2;593pub const FILE_END = 2;
594594
595pub const HEAP_CREATE_ENABLE_EXECUTE = 0x00040000;595pub const HEAP_CREATE_ENABLE_EXECUTE = 0x00040000;
596pub const HEAP_REALLOC_IN_PLACE_ONLY = 0x00000010;
596pub const HEAP_GENERATE_EXCEPTIONS = 0x00000004;597pub const HEAP_GENERATE_EXCEPTIONS = 0x00000004;
597pub const HEAP_NO_SERIALIZE = 0x00000001;598pub const HEAP_NO_SERIALIZE = 0x00000001;
598599
lib/std/testing.zig+1-1
...@@ -11,7 +11,7 @@ pub var allocator_instance = LeakCountAllocator.init(&base_allocator_instance.al...@@ -11,7 +11,7 @@ pub var allocator_instance = LeakCountAllocator.init(&base_allocator_instance.al
11pub const failing_allocator = &failing_allocator_instance.allocator;11pub const failing_allocator = &failing_allocator_instance.allocator;
12pub var failing_allocator_instance = FailingAllocator.init(&base_allocator_instance.allocator, 0);12pub var failing_allocator_instance = FailingAllocator.init(&base_allocator_instance.allocator, 0);
1313
14pub var base_allocator_instance = std.heap.ThreadSafeFixedBufferAllocator.init(allocator_mem[0..]);14pub var base_allocator_instance = std.mem.sanityWrap(std.heap.ThreadSafeFixedBufferAllocator.init(allocator_mem[0..]));
15var allocator_mem: [2 * 1024 * 1024]u8 = undefined;15var allocator_mem: [2 * 1024 * 1024]u8 = undefined;
1616
17/// This function is intended to be used only in tests. It prints diagnostics to stderr17/// This function is intended to be used only in tests. It prints diagnostics to stderr
lib/std/testing/failing_allocator.zig+18-23
...@@ -39,43 +39,38 @@ pub const FailingAllocator = struct {...@@ -39,43 +39,38 @@ pub const FailingAllocator = struct {
39 .allocations = 0,39 .allocations = 0,
40 .deallocations = 0,40 .deallocations = 0,
41 .allocator = mem.Allocator{41 .allocator = mem.Allocator{
42 .reallocFn = realloc,42 .allocFn = alloc,
43 .shrinkFn = shrink,43 .resizeFn = resize,
44 },44 },
45 };45 };
46 }46 }
4747
48 fn realloc(allocator: *mem.Allocator, old_mem: []u8, old_align: u29, new_size: usize, new_align: u29) ![]u8 {48 fn alloc(allocator: *std.mem.Allocator, len: usize, ptr_align: u29, len_align: u29) error{OutOfMemory}![]u8 {
49 const self = @fieldParentPtr(FailingAllocator, "allocator", allocator);49 const self = @fieldParentPtr(FailingAllocator, "allocator", allocator);
50 if (self.index == self.fail_index) {50 if (self.index == self.fail_index) {
51 return error.OutOfMemory;51 return error.OutOfMemory;
52 }52 }
53 const result = try self.internal_allocator.reallocFn(53 const result = try self.internal_allocator.callAllocFn(len, ptr_align, len_align);
54 self.internal_allocator,54 self.allocated_bytes += result.len;
55 old_mem,55 self.allocations += 1;
56 old_align,
57 new_size,
58 new_align,
59 );
60 if (new_size < old_mem.len) {
61 self.freed_bytes += old_mem.len - new_size;
62 if (new_size == 0)
63 self.deallocations += 1;
64 } else if (new_size > old_mem.len) {
65 self.allocated_bytes += new_size - old_mem.len;
66 if (old_mem.len == 0)
67 self.allocations += 1;
68 }
69 self.index += 1;56 self.index += 1;
70 return result;57 return result;
71 }58 }
7259
73 fn shrink(allocator: *mem.Allocator, old_mem: []u8, old_align: u29, new_size: usize, new_align: u29) []u8 {60 fn resize(allocator: *std.mem.Allocator, old_mem: []u8, new_len: usize, len_align: u29) error{OutOfMemory}!usize {
74 const self = @fieldParentPtr(FailingAllocator, "allocator", allocator);61 const self = @fieldParentPtr(FailingAllocator, "allocator", allocator);
75 const r = self.internal_allocator.shrinkFn(self.internal_allocator, old_mem, old_align, new_size, new_align);62 const r = self.internal_allocator.callResizeFn(old_mem, new_len, len_align) catch |e| {
76 self.freed_bytes += old_mem.len - r.len;63 std.debug.assert(new_len > old_mem.len);
77 if (new_size == 0)64 return e;
65 };
66 if (new_len == 0) {
78 self.deallocations += 1;67 self.deallocations += 1;
68 self.freed_bytes += old_mem.len;
69 } else if (r < old_mem.len) {
70 self.freed_bytes += old_mem.len - r;
71 } else {
72 self.allocated_bytes += r - old_mem.len;
73 }
79 return r;74 return r;
80 }75 }
81};76};
lib/std/testing/leak_count_allocator.zig+11-10
...@@ -14,23 +14,21 @@ pub const LeakCountAllocator = struct {...@@ -14,23 +14,21 @@ pub const LeakCountAllocator = struct {
14 return .{14 return .{
15 .count = 0,15 .count = 0,
16 .allocator = .{16 .allocator = .{
17 .reallocFn = realloc,17 .allocFn = alloc,
18 .shrinkFn = shrink,18 .resizeFn = resize,
19 },19 },
20 .internal_allocator = allocator,20 .internal_allocator = allocator,
21 };21 };
22 }22 }
2323
24 fn realloc(allocator: *std.mem.Allocator, old_mem: []u8, old_align: u29, new_size: usize, new_align: u29) ![]u8 {24 fn alloc(allocator: *std.mem.Allocator, len: usize, ptr_align: u29, len_align: u29) error{OutOfMemory}![]u8 {
25 const self = @fieldParentPtr(LeakCountAllocator, "allocator", allocator);25 const self = @fieldParentPtr(LeakCountAllocator, "allocator", allocator);
26 var data = try self.internal_allocator.reallocFn(self.internal_allocator, old_mem, old_align, new_size, new_align);26 const ptr = try self.internal_allocator.callAllocFn(len, ptr_align, len_align);
27 if (old_mem.len == 0) {27 self.count += 1;
28 self.count += 1;28 return ptr;
29 }
30 return data;
31 }29 }
3230
33 fn shrink(allocator: *std.mem.Allocator, old_mem: []u8, old_align: u29, new_size: usize, new_align: u29) []u8 {31 fn resize(allocator: *std.mem.Allocator, old_mem: []u8, new_size: usize, len_align: u29) error{OutOfMemory}!usize {
34 const self = @fieldParentPtr(LeakCountAllocator, "allocator", allocator);32 const self = @fieldParentPtr(LeakCountAllocator, "allocator", allocator);
35 if (new_size == 0) {33 if (new_size == 0) {
36 if (self.count == 0) {34 if (self.count == 0) {
...@@ -38,7 +36,10 @@ pub const LeakCountAllocator = struct {...@@ -38,7 +36,10 @@ pub const LeakCountAllocator = struct {
38 }36 }
39 self.count -= 1;37 self.count -= 1;
40 }38 }
41 return self.internal_allocator.shrinkFn(self.internal_allocator, old_mem, old_align, new_size, new_align);39 return self.internal_allocator.callResizeFn(old_mem, new_size, len_align) catch |e| {
40 std.debug.assert(new_size > old_mem.len);
41 return e;
42 };
42 }43 }
4344
44 pub fn validate(self: LeakCountAllocator) !void {45 pub fn validate(self: LeakCountAllocator) !void {