authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2022-11-11 22:48:06-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2022-11-29 23:46:02-07:00
loge2e60f5ff9942902e97aebfdf234bb6a0f821dfe
treed2d9275197b00c6f998a60bda8e8a13b8d537127
parent3dcea95ffe97ee93af50bf8906e53c7c7a7ec84e

std.heap.WasmAllocator: redo

The previous version had a fatal flaw: it did ensureCapacity(1) on the freelist when allocating, but I neglected to consider that you could free() twice in a row. Silly! This strategy allocates an intrusive freelist node with every allocation, big or small. It also does not have the problems with resize because in this case we can push the upper areas of freed stuff into the corresponding freelist.

3 files changed, 136 insertions(+), 160 deletions(-)

lib/std/heap.zig+5-37
...@@ -16,6 +16,7 @@ pub const LogToWriterAllocator = @import("heap/log_to_writer_allocator.zig").Log...@@ -16,6 +16,7 @@ pub const LogToWriterAllocator = @import("heap/log_to_writer_allocator.zig").Log
16pub const logToWriterAllocator = @import("heap/log_to_writer_allocator.zig").logToWriterAllocator;16pub const logToWriterAllocator = @import("heap/log_to_writer_allocator.zig").logToWriterAllocator;
17pub const ArenaAllocator = @import("heap/arena_allocator.zig").ArenaAllocator;17pub const ArenaAllocator = @import("heap/arena_allocator.zig").ArenaAllocator;
18pub const GeneralPurposeAllocator = @import("heap/general_purpose_allocator.zig").GeneralPurposeAllocator;18pub const GeneralPurposeAllocator = @import("heap/general_purpose_allocator.zig").GeneralPurposeAllocator;
19pub const WasmAllocator = @import("heap/WasmAllocator.zig");
19pub const WasmPageAllocator = @import("heap/WasmPageAllocator.zig");20pub const WasmPageAllocator = @import("heap/WasmPageAllocator.zig");
20pub const PageAllocator = @import("heap/PageAllocator.zig");21pub const PageAllocator = @import("heap/PageAllocator.zig");
2122
...@@ -565,43 +566,6 @@ test "raw_c_allocator" {...@@ -565,43 +566,6 @@ test "raw_c_allocator" {
565 }566 }
566}567}
567568
568test "WasmPageAllocator internals" {
569 if (comptime builtin.target.isWasm()) {
570 const conventional_memsize = WasmPageAllocator.conventional.totalPages() * mem.page_size;
571 const initial = try page_allocator.alloc(u8, mem.page_size);
572 try testing.expect(@ptrToInt(initial.ptr) < conventional_memsize); // If this isn't conventional, the rest of these tests don't make sense. Also we have a serious memory leak in the test suite.
573
574 var inplace = try page_allocator.realloc(initial, 1);
575 try testing.expectEqual(initial.ptr, inplace.ptr);
576 inplace = try page_allocator.realloc(inplace, 4);
577 try testing.expectEqual(initial.ptr, inplace.ptr);
578 page_allocator.free(inplace);
579
580 const reuse = try page_allocator.alloc(u8, 1);
581 try testing.expectEqual(initial.ptr, reuse.ptr);
582 page_allocator.free(reuse);
583
584 // This segment may span conventional and extended which has really complex rules so we're just ignoring it for now.
585 const padding = try page_allocator.alloc(u8, conventional_memsize);
586 page_allocator.free(padding);
587
588 const extended = try page_allocator.alloc(u8, conventional_memsize);
589 try testing.expect(@ptrToInt(extended.ptr) >= conventional_memsize);
590
591 const use_small = try page_allocator.alloc(u8, 1);
592 try testing.expectEqual(initial.ptr, use_small.ptr);
593 page_allocator.free(use_small);
594
595 inplace = try page_allocator.realloc(extended, 1);
596 try testing.expectEqual(extended.ptr, inplace.ptr);
597 page_allocator.free(inplace);
598
599 const reuse_extended = try page_allocator.alloc(u8, conventional_memsize);
600 try testing.expectEqual(extended.ptr, reuse_extended.ptr);
601 page_allocator.free(reuse_extended);
602 }
603}
604
605test "PageAllocator" {569test "PageAllocator" {
606 const allocator = page_allocator;570 const allocator = page_allocator;
607 try testAllocator(allocator);571 try testAllocator(allocator);
...@@ -875,4 +839,8 @@ test {...@@ -875,4 +839,8 @@ test {
875 _ = ScopedLoggingAllocator;839 _ = ScopedLoggingAllocator;
876 _ = ArenaAllocator;840 _ = ArenaAllocator;
877 _ = GeneralPurposeAllocator;841 _ = GeneralPurposeAllocator;
842 if (comptime builtin.target.isWasm()) {
843 _ = WasmAllocator;
844 _ = WasmPageAllocator;
845 }
878}846}
lib/std/heap/WasmAllocator.zig+92-123
...@@ -24,83 +24,59 @@ pub const Error = Allocator.Error;...@@ -24,83 +24,59 @@ pub const Error = Allocator.Error;
2424
25const max_usize = math.maxInt(usize);25const max_usize = math.maxInt(usize);
26const ushift = math.Log2Int(usize);26const ushift = math.Log2Int(usize);
27const bigpage_size = 512 * 1024;27const bigpage_size = 64 * 1024;
28const pages_per_bigpage = bigpage_size / wasm.page_size;28const pages_per_bigpage = bigpage_size / wasm.page_size;
29const bigpage_count = max_usize / bigpage_size;29const bigpage_count = max_usize / bigpage_size;
3030
31/// We have a small size class for all sizes up to 512kb.31/// Because of storing free list pointers, the minimum size class is 3.
32const size_class_count = math.log2(bigpage_size);32const min_class = math.log2(math.ceilPowerOfTwoAssert(usize, 1 + @sizeOf(usize)));
33const size_class_count = math.log2(bigpage_size) - min_class;
33/// 0 - 1 bigpage34/// 0 - 1 bigpage
34/// 1 - 2 bigpages35/// 1 - 2 bigpages
35/// 2 - 4 bigpages36/// 2 - 4 bigpages
36/// etc.37/// etc.
37const big_size_class_count = math.log2(bigpage_count);38const big_size_class_count = math.log2(bigpage_count);
3839
39const FreeList = struct {40var next_addrs = [1]usize{0} ** size_class_count;
40 /// Each element is the address of a freed pointer.41/// For each size class, points to the freed pointer.
41 ptr: [*]usize,42var frees = [1]usize{0} ** size_class_count;
42 len: usize,43/// For each big size class, points to the freed pointer.
43 cap: usize,44var big_frees = [1]usize{0} ** big_size_class_count;
44
45 const init: FreeList = .{
46 .ptr = undefined,
47 .len = 0,
48 .cap = 0,
49 };
50};
51
52const Bucket = struct {
53 ptr: usize,
54 end: usize,
55
56 const init: Bucket = .{
57 .ptr = 0,
58 .end = 0,
59 };
60};
61
62var next_addrs = [1]Bucket{Bucket.init} ** size_class_count;
63var frees = [1]FreeList{FreeList.init} ** size_class_count;
64var big_frees = [1]FreeList{FreeList.init} ** big_size_class_count;
6545
66fn alloc(ctx: *anyopaque, len: usize, alignment: u29, len_align: u29, ra: usize) Error![]u8 {46fn alloc(ctx: *anyopaque, len: usize, alignment: u29, len_align: u29, ra: usize) Error![]u8 {
67 _ = ctx;47 _ = ctx;
68 _ = len_align;48 _ = len_align;
69 _ = ra;49 _ = ra;
70 if (alignment > wasm.page_size) return error.OutOfMemory; // calm down50 if (alignment > wasm.page_size) return error.OutOfMemory; // calm down
71 const aligned_len = @max(len, alignment);51 // Make room for the freelist next pointer.
72 const slot_size = math.ceilPowerOfTwo(usize, aligned_len) catch return error.OutOfMemory;52 const actual_len = @max(len +| @sizeOf(usize), alignment);
73 const class = math.log2(slot_size);53 const slot_size = math.ceilPowerOfTwo(usize, actual_len) catch return error.OutOfMemory;
54 const class = math.log2(slot_size) - min_class;
74 if (class < size_class_count) {55 if (class < size_class_count) {
75 const addr = a: {56 const addr = a: {
76 const free_list = &frees[class];57 const top_free_ptr = frees[class];
77 if (free_list.len > 0) {58 if (top_free_ptr != 0) {
78 free_list.len -= 1;59 const node = @intToPtr(*usize, top_free_ptr + (slot_size - @sizeOf(usize)));
79 break :a free_list.ptr[free_list.len];60 frees[class] = node.*;
61 break :a top_free_ptr;
80 }62 }
8163
82 // This prevents memory allocation within free().
83 try ensureFreeListCapacity(free_list);
84
85 const next_addr = next_addrs[class];64 const next_addr = next_addrs[class];
86 if (next_addr.ptr == next_addr.end) {65 if (next_addr % wasm.page_size == 0) {
87 const addr = try allocBigPages(1);66 const addr = try allocBigPages(1);
88 //std.debug.print("allocated fresh slot_size={d} class={d} addr=0x{x}\n", .{67 //std.debug.print("allocated fresh slot_size={d} class={d} addr=0x{x}\n", .{
89 // slot_size, class, addr,68 // slot_size, class, addr,
90 //});69 //});
91 next_addrs[class] = .{70 next_addrs[class] = addr + slot_size;
92 .ptr = addr + slot_size,
93 .end = addr + bigpage_size,
94 };
95 break :a addr;71 break :a addr;
96 } else {72 } else {
97 next_addrs[class].ptr = next_addr.ptr + slot_size;73 next_addrs[class] = next_addr + slot_size;
98 break :a next_addr.ptr;74 break :a next_addr;
99 }75 }
100 };76 };
101 return @intToPtr([*]u8, addr)[0..len];77 return @intToPtr([*]u8, addr)[0..len];
102 }78 }
103 const bigpages_needed = (aligned_len + (bigpage_size - 1)) / bigpage_size;79 const bigpages_needed = bigPagesNeeded(actual_len);
104 const addr = try allocBigPages(bigpages_needed);80 const addr = try allocBigPages(bigpages_needed);
105 return @intToPtr([*]u8, addr)[0..len];81 return @intToPtr([*]u8, addr)[0..len];
106}82}
...@@ -113,39 +89,49 @@ fn resize(...@@ -113,39 +89,49 @@ fn resize(
113 len_align: u29,89 len_align: u29,
114 ra: usize,90 ra: usize,
115) ?usize {91) ?usize {
92 _ = ctx;
93 _ = len_align;
94 _ = ra;
116 // We don't want to move anything from one size class to another. But we can recover bytes95 // We don't want to move anything from one size class to another. But we can recover bytes
117 // in between powers of two.96 // in between powers of two.
118 const old_aligned_len = @max(buf.len, buf_align);97 const old_actual_len = @max(buf.len + @sizeOf(usize), buf_align);
119 const new_aligned_len = @max(new_len, buf_align);98 const new_actual_len = @max(new_len +| @sizeOf(usize), buf_align);
120 const old_small_slot_size = math.ceilPowerOfTwoAssert(usize, old_aligned_len);99 const old_small_slot_size = math.ceilPowerOfTwoAssert(usize, old_actual_len);
121 const old_small_class = math.log2(old_small_slot_size);100 const old_small_class = math.log2(old_small_slot_size) - min_class;
122 if (old_small_class < size_class_count) {101 if (old_small_class < size_class_count) {
123 const new_small_slot_size = math.ceilPowerOfTwo(usize, new_aligned_len) catch return null;102 const new_small_slot_size = math.ceilPowerOfTwo(usize, new_actual_len) catch return null;
124 //std.debug.print("resize: old_small_slot_size={d} new_small_slot_size={d}\n", .{103 if (old_small_slot_size == new_small_slot_size) return new_len;
125 // old_small_slot_size, new_small_slot_size,104 if (new_actual_len >= old_actual_len) return null;
126 //});105 const new_small_class = math.log2(new_small_slot_size) - min_class;
127 if (old_small_slot_size != new_small_slot_size) {106 assert(new_small_class < old_small_class);
128 if (new_aligned_len >= old_aligned_len) {107 // Split the small allocation into frees.
129 return null;108 var class = old_small_class - 1;
130 }109 while (true) {
131 // TODO this panic is a design flaw in the Allocator interface that110 const slot_size = @as(usize, 1) << @intCast(ushift, class + min_class);
132 // should be addressed.111 const upper_addr = @ptrToInt(buf.ptr) + slot_size;
133 const new = alloc(ctx, new_len, buf_align, len_align, ra) catch @panic("out of memory");112 const node = @intToPtr(*usize, upper_addr + (slot_size - @sizeOf(usize)));
134 @memcpy(new.ptr, buf.ptr, buf.len);113 node.* = frees[class];
114 frees[class] = upper_addr;
115 if (class == new_small_class) break;
116 class -= 1;
135 }117 }
136 } else {118 } else {
137 const old_bigpages_needed = (old_aligned_len + (bigpage_size - 1)) / bigpage_size;119 const old_bigpages_needed = bigPagesNeeded(old_actual_len);
138 const old_big_slot_size = math.ceilPowerOfTwoAssert(usize, old_bigpages_needed);120 const old_big_slot_size = math.ceilPowerOfTwoAssert(usize, old_bigpages_needed);
139 const new_bigpages_needed = (new_aligned_len + (bigpage_size - 1)) / bigpage_size;121 const new_bigpages_needed = bigPagesNeeded(new_actual_len);
140 const new_big_slot_size = math.ceilPowerOfTwo(usize, new_bigpages_needed) catch return null;122 const new_big_slot_size = math.ceilPowerOfTwo(usize, new_bigpages_needed) catch return null;
141 if (old_big_slot_size != new_big_slot_size) {123 if (old_big_slot_size == new_big_slot_size) return new_len;
142 if (new_aligned_len >= old_aligned_len) {124 if (new_actual_len >= old_actual_len) return null;
143 return null;125
144 }126 const new_small_slot_size = math.ceilPowerOfTwoAssert(usize, new_actual_len);
145 // TODO this panic is a design flaw in the Allocator interface that127 if (new_small_slot_size < size_class_count) {
146 // should be addressed.128 const new_small_class = math.log2(new_small_slot_size) - min_class;
147 const new = alloc(ctx, new_len, buf_align, len_align, ra) catch @panic("out of memory");129 // TODO: push the big allocation into the free list
148 @memcpy(new.ptr, buf.ptr, buf.len);130 _ = new_small_class;
131 } else {
132 const new_big_class = math.log2(new_big_slot_size);
133 // TODO: push the upper area into the free list
134 _ = new_big_class;
149 }135 }
150 }136 }
151 return new_len;137 return new_len;
...@@ -159,67 +145,47 @@ fn free(...@@ -159,67 +145,47 @@ fn free(
159) void {145) void {
160 _ = ctx;146 _ = ctx;
161 _ = return_address;147 _ = return_address;
162 const aligned_len = @max(buf.len, buf_align);148 const actual_len = @max(buf.len + @sizeOf(usize), buf_align);
163 const slot_size = math.ceilPowerOfTwoAssert(usize, aligned_len);149 const slot_size = math.ceilPowerOfTwoAssert(usize, actual_len);
164 const class = math.log2(slot_size);150 const class = math.log2(slot_size) - min_class;
151 const addr = @ptrToInt(buf.ptr);
165 if (class < size_class_count) {152 if (class < size_class_count) {
166 const free_list = &frees[class];153 const node = @intToPtr(*usize, addr + (slot_size - @sizeOf(usize)));
167 assert(free_list.len < free_list.cap);154 node.* = frees[class];
168 free_list.ptr[free_list.len] = @ptrToInt(buf.ptr);155 frees[class] = addr;
169 free_list.len += 1;
170 } else {156 } else {
171 const bigpages_needed = (aligned_len + (bigpage_size - 1)) / bigpage_size;157 const bigpages_needed = bigPagesNeeded(actual_len);
172 const big_slot_size = math.ceilPowerOfTwoAssert(usize, bigpages_needed);158 const pow2_pages = math.ceilPowerOfTwoAssert(usize, bigpages_needed);
173 const big_class = math.log2(big_slot_size);159 const big_slot_size_bytes = pow2_pages * bigpage_size;
174 const free_list = &big_frees[big_class];160 const node = @intToPtr(*usize, addr + (big_slot_size_bytes - @sizeOf(usize)));
175 assert(free_list.len < free_list.cap);161 const big_class = math.log2(pow2_pages);
176 free_list.ptr[free_list.len] = @ptrToInt(buf.ptr);162 node.* = big_frees[big_class];
177 free_list.len += 1;163 big_frees[big_class] = addr;
178 }164 }
179}165}
180166
181fn allocBigPages(n: usize) !usize {167inline fn bigPagesNeeded(byte_count: usize) usize {
182 const slot_size = math.ceilPowerOfTwoAssert(usize, n);168 return (byte_count + (bigpage_size + (@sizeOf(usize) - 1))) / bigpage_size;
183 const class = math.log2(slot_size);169}
184170
185 const free_list = &big_frees[class];171fn allocBigPages(n: usize) !usize {
186 if (free_list.len > 0) {172 const pow2_pages = math.ceilPowerOfTwoAssert(usize, n);
187 free_list.len -= 1;173 const slot_size_bytes = pow2_pages * bigpage_size;
188 return free_list.ptr[free_list.len];174 const class = math.log2(pow2_pages);
175
176 const top_free_ptr = big_frees[class];
177 if (top_free_ptr != 0) {
178 const node = @intToPtr(*usize, top_free_ptr + (slot_size_bytes - @sizeOf(usize)));
179 big_frees[class] = node.*;
180 return top_free_ptr;
189 }181 }
190182
191 //std.debug.print("ensureFreeListCapacity slot_size={d} big_class={d}\n", .{183 const page_index = @wasmMemoryGrow(0, pow2_pages * pages_per_bigpage);
192 // slot_size, class,
193 //});
194 // This prevents memory allocation within free().
195 try ensureFreeListCapacity(free_list);
196
197 const page_index = @wasmMemoryGrow(0, slot_size * pages_per_bigpage);
198 if (page_index <= 0) return error.OutOfMemory;184 if (page_index <= 0) return error.OutOfMemory;
199 const addr = @intCast(u32, page_index) * wasm.page_size;185 const addr = @intCast(u32, page_index) * wasm.page_size;
200 //std.debug.print("got 0x{x}..0x{x} from memory.grow\n", .{
201 // addr, addr + wasm.page_size * slot_size * pages_per_bigpage,
202 //});
203 return addr;186 return addr;
204}187}
205188
206fn ensureFreeListCapacity(free_list: *FreeList) Allocator.Error!void {
207 if (free_list.len < free_list.cap) return;
208 const old_bigpage_count = free_list.cap / bigpage_size;
209 free_list.cap = math.maxInt(usize); // Prevent recursive calls.
210 const new_bigpage_count = @max(old_bigpage_count * 2, 1);
211 const addr = try allocBigPages(new_bigpage_count);
212 //std.debug.print("allocated {d} big pages: 0x{x}\n", .{ new_bigpage_count, addr });
213 const new_ptr = @intToPtr([*]usize, addr);
214 @memcpy(
215 @ptrCast([*]u8, new_ptr),
216 @ptrCast([*]u8, free_list.ptr),
217 @sizeOf(usize) * free_list.len,
218 );
219 free_list.ptr = new_ptr;
220 free_list.cap = new_bigpage_count * (bigpage_size / @sizeOf(usize));
221}
222
223const test_ally = Allocator{189const test_ally = Allocator{
224 .ptr = undefined,190 .ptr = undefined,
225 .vtable = &vtable,191 .vtable = &vtable,
...@@ -315,8 +281,6 @@ test "large object - grow" {...@@ -315,8 +281,6 @@ test "large object - grow" {
315 try std.testing.expect(slice1.ptr == old.ptr);281 try std.testing.expect(slice1.ptr == old.ptr);
316282
317 slice1 = try test_ally.realloc(slice1, bigpage_size * 2);283 slice1 = try test_ally.realloc(slice1, bigpage_size * 2);
318 try std.testing.expect(slice1.ptr == old.ptr);
319
320 slice1 = try test_ally.realloc(slice1, bigpage_size * 2 + 1);284 slice1 = try test_ally.realloc(slice1, bigpage_size * 2 + 1);
321}285}
322286
...@@ -370,3 +334,8 @@ test "objects of size 1024 and 2048" {...@@ -370,3 +334,8 @@ test "objects of size 1024 and 2048" {
370 test_ally.free(slice);334 test_ally.free(slice);
371 test_ally.free(slice2);335 test_ally.free(slice2);
372}336}
337
338test "standard allocator tests" {
339 try std.heap.testAllocator(test_ally);
340 try std.heap.testAllocatorAligned(test_ally);
341}
lib/std/heap/WasmPageAllocator.zig+39
...@@ -1,3 +1,4 @@...@@ -1,3 +1,4 @@
1const WasmPageAllocator = @This();
1const std = @import("../std.zig");2const std = @import("../std.zig");
2const builtin = @import("builtin");3const builtin = @import("builtin");
3const Allocator = std.mem.Allocator;4const Allocator = std.mem.Allocator;
...@@ -194,3 +195,41 @@ fn free(...@@ -194,3 +195,41 @@ fn free(
194 const base = nPages(@ptrToInt(buf.ptr));195 const base = nPages(@ptrToInt(buf.ptr));
195 freePages(base, base + current_n);196 freePages(base, base + current_n);
196}197}
198
199test "internals" {
200 const page_allocator = std.heap.page_allocator;
201 const testing = std.testing;
202
203 const conventional_memsize = WasmPageAllocator.conventional.totalPages() * mem.page_size;
204 const initial = try page_allocator.alloc(u8, mem.page_size);
205 try testing.expect(@ptrToInt(initial.ptr) < conventional_memsize); // If this isn't conventional, the rest of these tests don't make sense. Also we have a serious memory leak in the test suite.
206
207 var inplace = try page_allocator.realloc(initial, 1);
208 try testing.expectEqual(initial.ptr, inplace.ptr);
209 inplace = try page_allocator.realloc(inplace, 4);
210 try testing.expectEqual(initial.ptr, inplace.ptr);
211 page_allocator.free(inplace);
212
213 const reuse = try page_allocator.alloc(u8, 1);
214 try testing.expectEqual(initial.ptr, reuse.ptr);
215 page_allocator.free(reuse);
216
217 // This segment may span conventional and extended which has really complex rules so we're just ignoring it for now.
218 const padding = try page_allocator.alloc(u8, conventional_memsize);
219 page_allocator.free(padding);
220
221 const ext = try page_allocator.alloc(u8, conventional_memsize);
222 try testing.expect(@ptrToInt(ext.ptr) >= conventional_memsize);
223
224 const use_small = try page_allocator.alloc(u8, 1);
225 try testing.expectEqual(initial.ptr, use_small.ptr);
226 page_allocator.free(use_small);
227
228 inplace = try page_allocator.realloc(ext, 1);
229 try testing.expectEqual(ext.ptr, inplace.ptr);
230 page_allocator.free(inplace);
231
232 const reuse_extended = try page_allocator.alloc(u8, conventional_memsize);
233 try testing.expectEqual(ext.ptr, reuse_extended.ptr);
234 page_allocator.free(reuse_extended);
235}