authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2022-11-11 22:48:06-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2022-11-29 23:46:02-07:00
loge2e60f5ff9942902e97aebfdf234bb6a0f821dfe
treed2d9275197b00c6f998a60bda8e8a13b8d537127
parent3dcea95ffe97ee93af50bf8906e53c7c7a7ec84e

std.heap.WasmAllocator: redo

The previous version had a fatal flaw: it did ensureCapacity(1) on the freelist when allocating, but I neglected to consider that you could free() twice in a row. Silly! This strategy allocates an intrusive freelist node with every allocation, big or small. It also does not have the problems with resize because in this case we can push the upper areas of freed stuff into the corresponding freelist.

3 files changed, 136 insertions(+), 160 deletions(-)

lib/std/heap.zig+5-37
......@@ -16,6 +16,7 @@ pub const LogToWriterAllocator = @import("heap/log_to_writer_allocator.zig").Log
1616pub const logToWriterAllocator = @import("heap/log_to_writer_allocator.zig").logToWriterAllocator;
1717pub const ArenaAllocator = @import("heap/arena_allocator.zig").ArenaAllocator;
1818pub const GeneralPurposeAllocator = @import("heap/general_purpose_allocator.zig").GeneralPurposeAllocator;
19pub const WasmAllocator = @import("heap/WasmAllocator.zig");
1920pub const WasmPageAllocator = @import("heap/WasmPageAllocator.zig");
2021pub const PageAllocator = @import("heap/PageAllocator.zig");
2122
......@@ -565,43 +566,6 @@ test "raw_c_allocator" {
565566 }
566567}
567568
568test "WasmPageAllocator internals" {
569 if (comptime builtin.target.isWasm()) {
570 const conventional_memsize = WasmPageAllocator.conventional.totalPages() * mem.page_size;
571 const initial = try page_allocator.alloc(u8, mem.page_size);
572 try testing.expect(@ptrToInt(initial.ptr) < conventional_memsize); // If this isn't conventional, the rest of these tests don't make sense. Also we have a serious memory leak in the test suite.
573
574 var inplace = try page_allocator.realloc(initial, 1);
575 try testing.expectEqual(initial.ptr, inplace.ptr);
576 inplace = try page_allocator.realloc(inplace, 4);
577 try testing.expectEqual(initial.ptr, inplace.ptr);
578 page_allocator.free(inplace);
579
580 const reuse = try page_allocator.alloc(u8, 1);
581 try testing.expectEqual(initial.ptr, reuse.ptr);
582 page_allocator.free(reuse);
583
584 // This segment may span conventional and extended which has really complex rules so we're just ignoring it for now.
585 const padding = try page_allocator.alloc(u8, conventional_memsize);
586 page_allocator.free(padding);
587
588 const extended = try page_allocator.alloc(u8, conventional_memsize);
589 try testing.expect(@ptrToInt(extended.ptr) >= conventional_memsize);
590
591 const use_small = try page_allocator.alloc(u8, 1);
592 try testing.expectEqual(initial.ptr, use_small.ptr);
593 page_allocator.free(use_small);
594
595 inplace = try page_allocator.realloc(extended, 1);
596 try testing.expectEqual(extended.ptr, inplace.ptr);
597 page_allocator.free(inplace);
598
599 const reuse_extended = try page_allocator.alloc(u8, conventional_memsize);
600 try testing.expectEqual(extended.ptr, reuse_extended.ptr);
601 page_allocator.free(reuse_extended);
602 }
603}
604
605569test "PageAllocator" {
606570 const allocator = page_allocator;
607571 try testAllocator(allocator);
......@@ -875,4 +839,8 @@ test {
875839 _ = ScopedLoggingAllocator;
876840 _ = ArenaAllocator;
877841 _ = GeneralPurposeAllocator;
842 if (comptime builtin.target.isWasm()) {
843 _ = WasmAllocator;
844 _ = WasmPageAllocator;
845 }
878846}
lib/std/heap/WasmAllocator.zig+92-123
......@@ -24,83 +24,59 @@ pub const Error = Allocator.Error;
2424
2525const max_usize = math.maxInt(usize);
2626const ushift = math.Log2Int(usize);
27const bigpage_size = 512 * 1024;
27const bigpage_size = 64 * 1024;
2828const pages_per_bigpage = bigpage_size / wasm.page_size;
2929const bigpage_count = max_usize / bigpage_size;
3030
31/// We have a small size class for all sizes up to 512kb.
32const size_class_count = math.log2(bigpage_size);
31/// Because of storing free list pointers, the minimum size class is 3.
32const min_class = math.log2(math.ceilPowerOfTwoAssert(usize, 1 + @sizeOf(usize)));
33const size_class_count = math.log2(bigpage_size) - min_class;
3334/// 0 - 1 bigpage
3435/// 1 - 2 bigpages
3536/// 2 - 4 bigpages
3637/// etc.
3738const big_size_class_count = math.log2(bigpage_count);
3839
39const FreeList = struct {
40 /// Each element is the address of a freed pointer.
41 ptr: [*]usize,
42 len: usize,
43 cap: usize,
44
45 const init: FreeList = .{
46 .ptr = undefined,
47 .len = 0,
48 .cap = 0,
49 };
50};
51
52const Bucket = struct {
53 ptr: usize,
54 end: usize,
55
56 const init: Bucket = .{
57 .ptr = 0,
58 .end = 0,
59 };
60};
61
62var next_addrs = [1]Bucket{Bucket.init} ** size_class_count;
63var frees = [1]FreeList{FreeList.init} ** size_class_count;
64var big_frees = [1]FreeList{FreeList.init} ** big_size_class_count;
40var next_addrs = [1]usize{0} ** size_class_count;
41/// For each size class, points to the freed pointer.
42var frees = [1]usize{0} ** size_class_count;
43/// For each big size class, points to the freed pointer.
44var big_frees = [1]usize{0} ** big_size_class_count;
6545
6646fn alloc(ctx: *anyopaque, len: usize, alignment: u29, len_align: u29, ra: usize) Error![]u8 {
6747 _ = ctx;
6848 _ = len_align;
6949 _ = ra;
7050 if (alignment > wasm.page_size) return error.OutOfMemory; // calm down
71 const aligned_len = @max(len, alignment);
72 const slot_size = math.ceilPowerOfTwo(usize, aligned_len) catch return error.OutOfMemory;
73 const class = math.log2(slot_size);
51 // Make room for the freelist next pointer.
52 const actual_len = @max(len +| @sizeOf(usize), alignment);
53 const slot_size = math.ceilPowerOfTwo(usize, actual_len) catch return error.OutOfMemory;
54 const class = math.log2(slot_size) - min_class;
7455 if (class < size_class_count) {
7556 const addr = a: {
76 const free_list = &frees[class];
77 if (free_list.len > 0) {
78 free_list.len -= 1;
79 break :a free_list.ptr[free_list.len];
57 const top_free_ptr = frees[class];
58 if (top_free_ptr != 0) {
59 const node = @intToPtr(*usize, top_free_ptr + (slot_size - @sizeOf(usize)));
60 frees[class] = node.*;
61 break :a top_free_ptr;
8062 }
8163
82 // This prevents memory allocation within free().
83 try ensureFreeListCapacity(free_list);
84
8564 const next_addr = next_addrs[class];
86 if (next_addr.ptr == next_addr.end) {
65 if (next_addr % wasm.page_size == 0) {
8766 const addr = try allocBigPages(1);
8867 //std.debug.print("allocated fresh slot_size={d} class={d} addr=0x{x}\n", .{
8968 // slot_size, class, addr,
9069 //});
91 next_addrs[class] = .{
92 .ptr = addr + slot_size,
93 .end = addr + bigpage_size,
94 };
70 next_addrs[class] = addr + slot_size;
9571 break :a addr;
9672 } else {
97 next_addrs[class].ptr = next_addr.ptr + slot_size;
98 break :a next_addr.ptr;
73 next_addrs[class] = next_addr + slot_size;
74 break :a next_addr;
9975 }
10076 };
10177 return @intToPtr([*]u8, addr)[0..len];
10278 }
103 const bigpages_needed = (aligned_len + (bigpage_size - 1)) / bigpage_size;
79 const bigpages_needed = bigPagesNeeded(actual_len);
10480 const addr = try allocBigPages(bigpages_needed);
10581 return @intToPtr([*]u8, addr)[0..len];
10682}
......@@ -113,39 +89,49 @@ fn resize(
11389 len_align: u29,
11490 ra: usize,
11591) ?usize {
92 _ = ctx;
93 _ = len_align;
94 _ = ra;
11695 // We don't want to move anything from one size class to another. But we can recover bytes
11796 // in between powers of two.
118 const old_aligned_len = @max(buf.len, buf_align);
119 const new_aligned_len = @max(new_len, buf_align);
120 const old_small_slot_size = math.ceilPowerOfTwoAssert(usize, old_aligned_len);
121 const old_small_class = math.log2(old_small_slot_size);
97 const old_actual_len = @max(buf.len + @sizeOf(usize), buf_align);
98 const new_actual_len = @max(new_len +| @sizeOf(usize), buf_align);
99 const old_small_slot_size = math.ceilPowerOfTwoAssert(usize, old_actual_len);
100 const old_small_class = math.log2(old_small_slot_size) - min_class;
122101 if (old_small_class < size_class_count) {
123 const new_small_slot_size = math.ceilPowerOfTwo(usize, new_aligned_len) catch return null;
124 //std.debug.print("resize: old_small_slot_size={d} new_small_slot_size={d}\n", .{
125 // old_small_slot_size, new_small_slot_size,
126 //});
127 if (old_small_slot_size != new_small_slot_size) {
128 if (new_aligned_len >= old_aligned_len) {
129 return null;
130 }
131 // TODO this panic is a design flaw in the Allocator interface that
132 // should be addressed.
133 const new = alloc(ctx, new_len, buf_align, len_align, ra) catch @panic("out of memory");
134 @memcpy(new.ptr, buf.ptr, buf.len);
102 const new_small_slot_size = math.ceilPowerOfTwo(usize, new_actual_len) catch return null;
103 if (old_small_slot_size == new_small_slot_size) return new_len;
104 if (new_actual_len >= old_actual_len) return null;
105 const new_small_class = math.log2(new_small_slot_size) - min_class;
106 assert(new_small_class < old_small_class);
107 // Split the small allocation into frees.
108 var class = old_small_class - 1;
109 while (true) {
110 const slot_size = @as(usize, 1) << @intCast(ushift, class + min_class);
111 const upper_addr = @ptrToInt(buf.ptr) + slot_size;
112 const node = @intToPtr(*usize, upper_addr + (slot_size - @sizeOf(usize)));
113 node.* = frees[class];
114 frees[class] = upper_addr;
115 if (class == new_small_class) break;
116 class -= 1;
135117 }
136118 } else {
137 const old_bigpages_needed = (old_aligned_len + (bigpage_size - 1)) / bigpage_size;
119 const old_bigpages_needed = bigPagesNeeded(old_actual_len);
138120 const old_big_slot_size = math.ceilPowerOfTwoAssert(usize, old_bigpages_needed);
139 const new_bigpages_needed = (new_aligned_len + (bigpage_size - 1)) / bigpage_size;
121 const new_bigpages_needed = bigPagesNeeded(new_actual_len);
140122 const new_big_slot_size = math.ceilPowerOfTwo(usize, new_bigpages_needed) catch return null;
141 if (old_big_slot_size != new_big_slot_size) {
142 if (new_aligned_len >= old_aligned_len) {
143 return null;
144 }
145 // TODO this panic is a design flaw in the Allocator interface that
146 // should be addressed.
147 const new = alloc(ctx, new_len, buf_align, len_align, ra) catch @panic("out of memory");
148 @memcpy(new.ptr, buf.ptr, buf.len);
123 if (old_big_slot_size == new_big_slot_size) return new_len;
124 if (new_actual_len >= old_actual_len) return null;
125
126 const new_small_slot_size = math.ceilPowerOfTwoAssert(usize, new_actual_len);
127 if (new_small_slot_size < size_class_count) {
128 const new_small_class = math.log2(new_small_slot_size) - min_class;
129 // TODO: push the big allocation into the free list
130 _ = new_small_class;
131 } else {
132 const new_big_class = math.log2(new_big_slot_size);
133 // TODO: push the upper area into the free list
134 _ = new_big_class;
149135 }
150136 }
151137 return new_len;
......@@ -159,67 +145,47 @@ fn free(
159145) void {
160146 _ = ctx;
161147 _ = return_address;
162 const aligned_len = @max(buf.len, buf_align);
163 const slot_size = math.ceilPowerOfTwoAssert(usize, aligned_len);
164 const class = math.log2(slot_size);
148 const actual_len = @max(buf.len + @sizeOf(usize), buf_align);
149 const slot_size = math.ceilPowerOfTwoAssert(usize, actual_len);
150 const class = math.log2(slot_size) - min_class;
151 const addr = @ptrToInt(buf.ptr);
165152 if (class < size_class_count) {
166 const free_list = &frees[class];
167 assert(free_list.len < free_list.cap);
168 free_list.ptr[free_list.len] = @ptrToInt(buf.ptr);
169 free_list.len += 1;
153 const node = @intToPtr(*usize, addr + (slot_size - @sizeOf(usize)));
154 node.* = frees[class];
155 frees[class] = addr;
170156 } else {
171 const bigpages_needed = (aligned_len + (bigpage_size - 1)) / bigpage_size;
172 const big_slot_size = math.ceilPowerOfTwoAssert(usize, bigpages_needed);
173 const big_class = math.log2(big_slot_size);
174 const free_list = &big_frees[big_class];
175 assert(free_list.len < free_list.cap);
176 free_list.ptr[free_list.len] = @ptrToInt(buf.ptr);
177 free_list.len += 1;
157 const bigpages_needed = bigPagesNeeded(actual_len);
158 const pow2_pages = math.ceilPowerOfTwoAssert(usize, bigpages_needed);
159 const big_slot_size_bytes = pow2_pages * bigpage_size;
160 const node = @intToPtr(*usize, addr + (big_slot_size_bytes - @sizeOf(usize)));
161 const big_class = math.log2(pow2_pages);
162 node.* = big_frees[big_class];
163 big_frees[big_class] = addr;
178164 }
179165}
180166
181fn allocBigPages(n: usize) !usize {
182 const slot_size = math.ceilPowerOfTwoAssert(usize, n);
183 const class = math.log2(slot_size);
167inline fn bigPagesNeeded(byte_count: usize) usize {
168 return (byte_count + (bigpage_size + (@sizeOf(usize) - 1))) / bigpage_size;
169}
184170
185 const free_list = &big_frees[class];
186 if (free_list.len > 0) {
187 free_list.len -= 1;
188 return free_list.ptr[free_list.len];
171fn allocBigPages(n: usize) !usize {
172 const pow2_pages = math.ceilPowerOfTwoAssert(usize, n);
173 const slot_size_bytes = pow2_pages * bigpage_size;
174 const class = math.log2(pow2_pages);
175
176 const top_free_ptr = big_frees[class];
177 if (top_free_ptr != 0) {
178 const node = @intToPtr(*usize, top_free_ptr + (slot_size_bytes - @sizeOf(usize)));
179 big_frees[class] = node.*;
180 return top_free_ptr;
189181 }
190182
191 //std.debug.print("ensureFreeListCapacity slot_size={d} big_class={d}\n", .{
192 // slot_size, class,
193 //});
194 // This prevents memory allocation within free().
195 try ensureFreeListCapacity(free_list);
196
197 const page_index = @wasmMemoryGrow(0, slot_size * pages_per_bigpage);
183 const page_index = @wasmMemoryGrow(0, pow2_pages * pages_per_bigpage);
198184 if (page_index <= 0) return error.OutOfMemory;
199185 const addr = @intCast(u32, page_index) * wasm.page_size;
200 //std.debug.print("got 0x{x}..0x{x} from memory.grow\n", .{
201 // addr, addr + wasm.page_size * slot_size * pages_per_bigpage,
202 //});
203186 return addr;
204187}
205188
206fn ensureFreeListCapacity(free_list: *FreeList) Allocator.Error!void {
207 if (free_list.len < free_list.cap) return;
208 const old_bigpage_count = free_list.cap / bigpage_size;
209 free_list.cap = math.maxInt(usize); // Prevent recursive calls.
210 const new_bigpage_count = @max(old_bigpage_count * 2, 1);
211 const addr = try allocBigPages(new_bigpage_count);
212 //std.debug.print("allocated {d} big pages: 0x{x}\n", .{ new_bigpage_count, addr });
213 const new_ptr = @intToPtr([*]usize, addr);
214 @memcpy(
215 @ptrCast([*]u8, new_ptr),
216 @ptrCast([*]u8, free_list.ptr),
217 @sizeOf(usize) * free_list.len,
218 );
219 free_list.ptr = new_ptr;
220 free_list.cap = new_bigpage_count * (bigpage_size / @sizeOf(usize));
221}
222
223189const test_ally = Allocator{
224190 .ptr = undefined,
225191 .vtable = &vtable,
......@@ -315,8 +281,6 @@ test "large object - grow" {
315281 try std.testing.expect(slice1.ptr == old.ptr);
316282
317283 slice1 = try test_ally.realloc(slice1, bigpage_size * 2);
318 try std.testing.expect(slice1.ptr == old.ptr);
319
320284 slice1 = try test_ally.realloc(slice1, bigpage_size * 2 + 1);
321285}
322286
......@@ -370,3 +334,8 @@ test "objects of size 1024 and 2048" {
370334 test_ally.free(slice);
371335 test_ally.free(slice2);
372336}
337
338test "standard allocator tests" {
339 try std.heap.testAllocator(test_ally);
340 try std.heap.testAllocatorAligned(test_ally);
341}
lib/std/heap/WasmPageAllocator.zig+39
......@@ -1,3 +1,4 @@
1const WasmPageAllocator = @This();
12const std = @import("../std.zig");
23const builtin = @import("builtin");
34const Allocator = std.mem.Allocator;
......@@ -194,3 +195,41 @@ fn free(
194195 const base = nPages(@ptrToInt(buf.ptr));
195196 freePages(base, base + current_n);
196197}
198
199test "internals" {
200 const page_allocator = std.heap.page_allocator;
201 const testing = std.testing;
202
203 const conventional_memsize = WasmPageAllocator.conventional.totalPages() * mem.page_size;
204 const initial = try page_allocator.alloc(u8, mem.page_size);
205 try testing.expect(@ptrToInt(initial.ptr) < conventional_memsize); // If this isn't conventional, the rest of these tests don't make sense. Also we have a serious memory leak in the test suite.
206
207 var inplace = try page_allocator.realloc(initial, 1);
208 try testing.expectEqual(initial.ptr, inplace.ptr);
209 inplace = try page_allocator.realloc(inplace, 4);
210 try testing.expectEqual(initial.ptr, inplace.ptr);
211 page_allocator.free(inplace);
212
213 const reuse = try page_allocator.alloc(u8, 1);
214 try testing.expectEqual(initial.ptr, reuse.ptr);
215 page_allocator.free(reuse);
216
217 // This segment may span conventional and extended which has really complex rules so we're just ignoring it for now.
218 const padding = try page_allocator.alloc(u8, conventional_memsize);
219 page_allocator.free(padding);
220
221 const ext = try page_allocator.alloc(u8, conventional_memsize);
222 try testing.expect(@ptrToInt(ext.ptr) >= conventional_memsize);
223
224 const use_small = try page_allocator.alloc(u8, 1);
225 try testing.expectEqual(initial.ptr, use_small.ptr);
226 page_allocator.free(use_small);
227
228 inplace = try page_allocator.realloc(ext, 1);
229 try testing.expectEqual(ext.ptr, inplace.ptr);
230 page_allocator.free(inplace);
231
232 const reuse_extended = try page_allocator.alloc(u8, conventional_memsize);
233 try testing.expectEqual(ext.ptr, reuse_extended.ptr);
234 page_allocator.free(reuse_extended);
235}