| ... | ... | @@ -23,15 +23,18 @@ pub const vtable = Allocator.VTable{ |
| 23 | 23 | pub const Error = Allocator.Error; |
| 24 | 24 | |
| 25 | 25 | const max_usize = math.maxInt(usize); |
| 26 | const ushift = math.Log2Int(usize); |
| 26 | 27 | const bigpage_size = 512 * 1024; |
| 27 | 28 | const pages_per_bigpage = bigpage_size / wasm.page_size; |
| 28 | 29 | const bigpage_count = max_usize / bigpage_size; |
| 29 | 30 | |
| 30 | | //// This has a length of 1024 usizes. |
| 31 | | //var bigpages_used = [1]usize{0} ** (bigpage_count / @bitSizeOf(usize)); |
| 32 | | |
| 33 | 31 | /// We have a small size class for all sizes up to 512kb. |
| 34 | 32 | const size_class_count = math.log2(bigpage_size); |
| 33 | /// 0 - 1 bigpage |
| 34 | /// 1 - 2 bigpages |
| 35 | /// 2 - 4 bigpages |
| 36 | /// etc. |
| 37 | const big_size_class_count = math.log2(bigpage_count); |
| 35 | 38 | |
| 36 | 39 | const FreeList = struct { |
| 37 | 40 | /// Each element is the address of a freed pointer. |
| ... | ... | @@ -46,20 +49,26 @@ const FreeList = struct { |
| 46 | 49 | }; |
| 47 | 50 | }; |
| 48 | 51 | |
| 49 | | var next_addrs = [1]usize{0} ** size_class_count; |
| 50 | | var frees = [1]FreeList{FreeList.init} ** size_class_count; |
| 51 | | var bigpage_free_list: FreeList = .{ |
| 52 | | .ptr = &bigpage_free_buf, |
| 53 | | .len = 0, |
| 54 | | .cap = bigpage_free_buf.len, |
| 52 | const Bucket = struct { |
| 53 | ptr: usize, |
| 54 | end: usize, |
| 55 | |
| 56 | const init: Bucket = .{ |
| 57 | .ptr = 0, |
| 58 | .end = 0, |
| 59 | }; |
| 55 | 60 | }; |
| 56 | | var bigpage_free_buf: [16]usize = undefined; |
| 61 | |
| 62 | var next_addrs = [1]Bucket{Bucket.init} ** size_class_count; |
| 63 | var frees = [1]FreeList{FreeList.init} ** size_class_count; |
| 64 | var big_frees = [1]FreeList{FreeList.init} ** big_size_class_count; |
| 57 | 65 | |
| 58 | 66 | fn alloc(ctx: *anyopaque, len: usize, alignment: u29, len_align: u29, ra: usize) Error![]u8 { |
| 59 | 67 | _ = ctx; |
| 60 | 68 | _ = len_align; |
| 61 | 69 | _ = ra; |
| 62 | | const slot_size = math.ceilPowerOfTwoAssert(usize, @max(len, alignment)); |
| 70 | const aligned_len = @max(len, alignment); |
| 71 | const slot_size = math.ceilPowerOfTwoAssert(usize, aligned_len); |
| 63 | 72 | const class = math.log2(slot_size); |
| 64 | 73 | if (class < size_class_count) { |
| 65 | 74 | const addr = a: { |
| ... | ... | @@ -69,55 +78,30 @@ fn alloc(ctx: *anyopaque, len: usize, alignment: u29, len_align: u29, ra: usize) |
| 69 | 78 | break :a free_list.ptr[free_list.len]; |
| 70 | 79 | } |
| 71 | 80 | |
| 72 | | // Ensure unused capacity in the corresponding free list. |
| 73 | 81 | // This prevents memory allocation within free(). |
| 74 | | if (free_list.len >= free_list.cap) { |
| 75 | | const old_bigpage_count = free_list.cap / bigpage_size; |
| 76 | | if (bigpage_free_list.cap - bigpage_free_list.len < old_bigpage_count) { |
| 77 | | return error.OutOfMemory; |
| 78 | | } |
| 79 | | const new_bigpage_count = old_bigpage_count + 1; |
| 80 | | const addr = try allocBigPages(new_bigpage_count); |
| 81 | | const new_ptr = @intToPtr([*]usize, addr); |
| 82 | | const old_ptr = free_list.ptr; |
| 83 | | @memcpy( |
| 84 | | @ptrCast([*]u8, new_ptr), |
| 85 | | @ptrCast([*]u8, old_ptr), |
| 86 | | @sizeOf(usize) * free_list.len, |
| 87 | | ); |
| 88 | | free_list.ptr = new_ptr; |
| 89 | | free_list.cap = new_bigpage_count * (bigpage_size / @sizeOf(usize)); |
| 90 | | |
| 91 | | var i: usize = 0; |
| 92 | | while (i < old_bigpage_count) : (i += 1) { |
| 93 | | bigpage_free_list.ptr[bigpage_free_list.len] = @ptrToInt(old_ptr) + |
| 94 | | i * bigpage_size; |
| 95 | | bigpage_free_list.len += 1; |
| 96 | | } |
| 97 | | } |
| 82 | try ensureFreeListCapacity(free_list); |
| 98 | 83 | |
| 99 | 84 | const next_addr = next_addrs[class]; |
| 100 | | if (next_addr % bigpage_size == 0) { |
| 101 | | //std.debug.print("alloc big page len={d} class={d} slot_size={d}\n", .{ |
| 102 | | // len, class, slot_size, |
| 103 | | //}); |
| 85 | if (next_addr.ptr == next_addr.end) { |
| 104 | 86 | const addr = try allocBigPages(1); |
| 105 | | next_addrs[class] = addr + slot_size; |
| 87 | //std.debug.print("allocated fresh slot_size={d} class={d} addr=0x{x}\n", .{ |
| 88 | // slot_size, class, addr, |
| 89 | //}); |
| 90 | next_addrs[class] = .{ |
| 91 | .ptr = addr + slot_size, |
| 92 | .end = addr + bigpage_size, |
| 93 | }; |
| 106 | 94 | break :a addr; |
| 107 | 95 | } else { |
| 108 | | //std.debug.print("easy! len={d} class={d} slot_size={d}\n", .{ |
| 109 | | // len, class, slot_size, |
| 110 | | //}); |
| 111 | | next_addrs[class] = next_addr + slot_size; |
| 112 | | break :a next_addr; |
| 96 | next_addrs[class].ptr = next_addr.ptr + slot_size; |
| 97 | break :a next_addr.ptr; |
| 113 | 98 | } |
| 114 | 99 | }; |
| 115 | 100 | return @intToPtr([*]u8, addr)[0..len]; |
| 116 | | } else { |
| 117 | | std.debug.panic("big alloc: len={d} align={d} slot_size={d} class={d}", .{ |
| 118 | | len, alignment, slot_size, class, |
| 119 | | }); |
| 120 | 101 | } |
| 102 | const bigpages_needed = (aligned_len + (bigpage_size - 1)) / bigpage_size; |
| 103 | const addr = try allocBigPages(bigpages_needed); |
| 104 | return @intToPtr([*]u8, addr)[0..len]; |
| 121 | 105 | } |
| 122 | 106 | |
| 123 | 107 | fn resize( |
| ... | ... | @@ -145,29 +129,65 @@ fn free( |
| 145 | 129 | ) void { |
| 146 | 130 | _ = ctx; |
| 147 | 131 | _ = return_address; |
| 148 | | const class_size = @max(buf.len, buf_align); |
| 149 | | const class = math.log2(class_size); |
| 132 | const aligned_len = @max(buf.len, buf_align); |
| 133 | const slot_size = math.ceilPowerOfTwoAssert(usize, aligned_len); |
| 134 | const class = math.log2(slot_size); |
| 150 | 135 | if (class < size_class_count) { |
| 151 | 136 | const free_list = &frees[class]; |
| 152 | 137 | assert(free_list.len < free_list.cap); |
| 153 | 138 | free_list.ptr[free_list.len] = @ptrToInt(buf.ptr); |
| 154 | 139 | free_list.len += 1; |
| 155 | 140 | } else { |
| 156 | | std.debug.panic("big free: len={d} align={d}", .{ |
| 157 | | buf.len, buf_align, |
| 158 | | }); |
| 141 | const bigpages_needed = (aligned_len + (bigpage_size - 1)) / bigpage_size; |
| 142 | const big_slot_size = math.ceilPowerOfTwoAssert(usize, bigpages_needed); |
| 143 | const big_class = math.log2(big_slot_size); |
| 144 | const free_list = &big_frees[big_class]; |
| 145 | assert(free_list.len < free_list.cap); |
| 146 | free_list.ptr[free_list.len] = @ptrToInt(buf.ptr); |
| 147 | free_list.len += 1; |
| 159 | 148 | } |
| 160 | 149 | } |
| 161 | 150 | |
| 162 | | inline fn allocBigPages(n: usize) !usize { |
| 163 | | if (n == 1 and bigpage_free_list.len > 0) { |
| 164 | | bigpage_free_list.len -= 1; |
| 165 | | return bigpage_free_list.ptr[bigpage_free_list.len]; |
| 151 | fn allocBigPages(n: usize) !usize { |
| 152 | const slot_size = math.ceilPowerOfTwoAssert(usize, n); |
| 153 | const class = math.log2(slot_size); |
| 154 | |
| 155 | const free_list = &big_frees[class]; |
| 156 | if (free_list.len > 0) { |
| 157 | free_list.len -= 1; |
| 158 | return free_list.ptr[free_list.len]; |
| 166 | 159 | } |
| 167 | | const page_index = @wasmMemoryGrow(0, n * pages_per_bigpage); |
| 168 | | if (page_index <= 0) |
| 169 | | return error.OutOfMemory; |
| 170 | | return @intCast(u32, page_index) * wasm.page_size; |
| 160 | |
| 161 | //std.debug.print("ensureFreeListCapacity slot_size={d} big_class={d}\n", .{ |
| 162 | // slot_size, class, |
| 163 | //}); |
| 164 | // This prevents memory allocation within free(). |
| 165 | try ensureFreeListCapacity(free_list); |
| 166 | |
| 167 | const page_index = @wasmMemoryGrow(0, slot_size * pages_per_bigpage); |
| 168 | if (page_index <= 0) return error.OutOfMemory; |
| 169 | const addr = @intCast(u32, page_index) * wasm.page_size; |
| 170 | //std.debug.print("got 0x{x}..0x{x} from memory.grow\n", .{ |
| 171 | // addr, addr + wasm.page_size * slot_size * pages_per_bigpage, |
| 172 | //}); |
| 173 | return addr; |
| 174 | } |
| 175 | |
| 176 | fn ensureFreeListCapacity(free_list: *FreeList) Allocator.Error!void { |
| 177 | if (free_list.len < free_list.cap) return; |
| 178 | const old_bigpage_count = free_list.cap / bigpage_size; |
| 179 | free_list.cap = math.maxInt(usize); // Prevent recursive calls. |
| 180 | const new_bigpage_count = @max(old_bigpage_count * 2, 1); |
| 181 | const addr = try allocBigPages(new_bigpage_count); |
| 182 | //std.debug.print("allocated {d} big pages: 0x{x}\n", .{ new_bigpage_count, addr }); |
| 183 | const new_ptr = @intToPtr([*]usize, addr); |
| 184 | @memcpy( |
| 185 | @ptrCast([*]u8, new_ptr), |
| 186 | @ptrCast([*]u8, free_list.ptr), |
| 187 | @sizeOf(usize) * free_list.len, |
| 188 | ); |
| 189 | free_list.ptr = new_ptr; |
| 190 | free_list.cap = new_bigpage_count * (bigpage_size / @sizeOf(usize)); |
| 171 | 191 | } |
| 172 | 192 | |
| 173 | 193 | const test_ally = Allocator{ |
| ... | ... | @@ -207,16 +227,10 @@ test "small allocations - free in reverse order" { |
| 207 | 227 | } |
| 208 | 228 | |
| 209 | 229 | test "large allocations" { |
| 210 | | std.debug.print("alloc ptr1\n", .{}); |
| 211 | 230 | const ptr1 = try test_ally.alloc(u64, 42768); |
| 212 | | std.debug.print("alloc ptr2\n", .{}); |
| 213 | 231 | const ptr2 = try test_ally.alloc(u64, 52768); |
| 214 | | std.debug.print("free ptr1\n", .{}); |
| 215 | 232 | test_ally.free(ptr1); |
| 216 | | std.debug.print("alloc ptr3\n", .{}); |
| 217 | 233 | const ptr3 = try test_ally.alloc(u64, 62768); |
| 218 | | std.debug.print("free ptr3\n", .{}); |
| 219 | 234 | test_ally.free(ptr3); |
| 220 | | std.debug.print("free ptr2\n", .{}); |
| 221 | 235 | test_ally.free(ptr2); |
| 222 | 236 | } |
Andrew Kelley <andrew@ziglang.org> 2022-11-10 21:08:29-07:00
lib