| ... | @@ -23,15 +23,18 @@ pub const vtable = Allocator.VTable{ | ... | @@ -23,15 +23,18 @@ pub const vtable = Allocator.VTable{ |
| 23 | pub const Error = Allocator.Error; | 23 | pub const Error = Allocator.Error; |
| 24 | | 24 | |
| 25 | const max_usize = math.maxInt(usize); | 25 | const max_usize = math.maxInt(usize); |
| | 26 | const ushift = math.Log2Int(usize); |
| 26 | const bigpage_size = 512 * 1024; | 27 | const bigpage_size = 512 * 1024; |
| 27 | const pages_per_bigpage = bigpage_size / wasm.page_size; | 28 | const pages_per_bigpage = bigpage_size / wasm.page_size; |
| 28 | const bigpage_count = max_usize / bigpage_size; | 29 | const bigpage_count = max_usize / bigpage_size; |
| 29 | | 30 | |
| 30 | //// This has a length of 1024 usizes. | | |
| 31 | //var bigpages_used = [1]usize{0} ** (bigpage_count / @bitSizeOf(usize)); | | |
| 32 | | | |
| 33 | /// We have a small size class for all sizes up to 512kb. | 31 | /// We have a small size class for all sizes up to 512kb. |
| 34 | const size_class_count = math.log2(bigpage_size); | 32 | const size_class_count = math.log2(bigpage_size); |
| | 33 | /// 0 - 1 bigpage |
| | 34 | /// 1 - 2 bigpages |
| | 35 | /// 2 - 4 bigpages |
| | 36 | /// etc. |
| | 37 | const big_size_class_count = math.log2(bigpage_count); |
| 35 | | 38 | |
| 36 | const FreeList = struct { | 39 | const FreeList = struct { |
| 37 | /// Each element is the address of a freed pointer. | 40 | /// Each element is the address of a freed pointer. |
| ... | @@ -46,20 +49,26 @@ const FreeList = struct { | ... | @@ -46,20 +49,26 @@ const FreeList = struct { |
| 46 | }; | 49 | }; |
| 47 | }; | 50 | }; |
| 48 | | 51 | |
| 49 | var next_addrs = [1]usize{0} ** size_class_count; | 52 | const Bucket = struct { |
| 50 | var frees = [1]FreeList{FreeList.init} ** size_class_count; | 53 | ptr: usize, |
| 51 | var bigpage_free_list: FreeList = .{ | 54 | end: usize, |
| 52 | .ptr = &bigpage_free_buf, | 55 | |
| 53 | .len = 0, | 56 | const init: Bucket = .{ |
| 54 | .cap = bigpage_free_buf.len, | 57 | .ptr = 0, |
| | 58 | .end = 0, |
| | 59 | }; |
| 55 | }; | 60 | }; |
| 56 | var bigpage_free_buf: [16]usize = undefined; | 61 | |
| | 62 | var next_addrs = [1]Bucket{Bucket.init} ** size_class_count; |
| | 63 | var frees = [1]FreeList{FreeList.init} ** size_class_count; |
| | 64 | var big_frees = [1]FreeList{FreeList.init} ** big_size_class_count; |
| 57 | | 65 | |
| 58 | fn alloc(ctx: *anyopaque, len: usize, alignment: u29, len_align: u29, ra: usize) Error![]u8 { | 66 | fn alloc(ctx: *anyopaque, len: usize, alignment: u29, len_align: u29, ra: usize) Error![]u8 { |
| 59 | _ = ctx; | 67 | _ = ctx; |
| 60 | _ = len_align; | 68 | _ = len_align; |
| 61 | _ = ra; | 69 | _ = ra; |
| 62 | const slot_size = math.ceilPowerOfTwoAssert(usize, @max(len, alignment)); | 70 | const aligned_len = @max(len, alignment); |
| | 71 | const slot_size = math.ceilPowerOfTwoAssert(usize, aligned_len); |
| 63 | const class = math.log2(slot_size); | 72 | const class = math.log2(slot_size); |
| 64 | if (class < size_class_count) { | 73 | if (class < size_class_count) { |
| 65 | const addr = a: { | 74 | const addr = a: { |
| ... | @@ -69,55 +78,30 @@ fn alloc(ctx: *anyopaque, len: usize, alignment: u29, len_align: u29, ra: usize) | ... | @@ -69,55 +78,30 @@ fn alloc(ctx: *anyopaque, len: usize, alignment: u29, len_align: u29, ra: usize) |
| 69 | break :a free_list.ptr[free_list.len]; | 78 | break :a free_list.ptr[free_list.len]; |
| 70 | } | 79 | } |
| 71 | | 80 | |
| 72 | // Ensure unused capacity in the corresponding free list. | | |
| 73 | // This prevents memory allocation within free(). | 81 | // This prevents memory allocation within free(). |
| 74 | if (free_list.len >= free_list.cap) { | 82 | try ensureFreeListCapacity(free_list); |
| 75 | const old_bigpage_count = free_list.cap / bigpage_size; | | |
| 76 | if (bigpage_free_list.cap - bigpage_free_list.len < old_bigpage_count) { | | |
| 77 | return error.OutOfMemory; | | |
| 78 | } | | |
| 79 | const new_bigpage_count = old_bigpage_count + 1; | | |
| 80 | const addr = try allocBigPages(new_bigpage_count); | | |
| 81 | const new_ptr = @intToPtr([*]usize, addr); | | |
| 82 | const old_ptr = free_list.ptr; | | |
| 83 | @memcpy( | | |
| 84 | @ptrCast([*]u8, new_ptr), | | |
| 85 | @ptrCast([*]u8, old_ptr), | | |
| 86 | @sizeOf(usize) * free_list.len, | | |
| 87 | ); | | |
| 88 | free_list.ptr = new_ptr; | | |
| 89 | free_list.cap = new_bigpage_count * (bigpage_size / @sizeOf(usize)); | | |
| 90 | | | |
| 91 | var i: usize = 0; | | |
| 92 | while (i < old_bigpage_count) : (i += 1) { | | |
| 93 | bigpage_free_list.ptr[bigpage_free_list.len] = @ptrToInt(old_ptr) + | | |
| 94 | i * bigpage_size; | | |
| 95 | bigpage_free_list.len += 1; | | |
| 96 | } | | |
| 97 | } | | |
| 98 | | 83 | |
| 99 | const next_addr = next_addrs[class]; | 84 | const next_addr = next_addrs[class]; |
| 100 | if (next_addr % bigpage_size == 0) { | 85 | if (next_addr.ptr == next_addr.end) { |
| 101 | //std.debug.print("alloc big page len={d} class={d} slot_size={d}\n", .{ | | |
| 102 | // len, class, slot_size, | | |
| 103 | //}); | | |
| 104 | const addr = try allocBigPages(1); | 86 | const addr = try allocBigPages(1); |
| 105 | next_addrs[class] = addr + slot_size; | 87 | //std.debug.print("allocated fresh slot_size={d} class={d} addr=0x{x}\n", .{ |
| | 88 | // slot_size, class, addr, |
| | 89 | //}); |
| | 90 | next_addrs[class] = .{ |
| | 91 | .ptr = addr + slot_size, |
| | 92 | .end = addr + bigpage_size, |
| | 93 | }; |
| 106 | break :a addr; | 94 | break :a addr; |
| 107 | } else { | 95 | } else { |
| 108 | //std.debug.print("easy! len={d} class={d} slot_size={d}\n", .{ | 96 | next_addrs[class].ptr = next_addr.ptr + slot_size; |
| 109 | // len, class, slot_size, | 97 | break :a next_addr.ptr; |
| 110 | //}); | | |
| 111 | next_addrs[class] = next_addr + slot_size; | | |
| 112 | break :a next_addr; | | |
| 113 | } | 98 | } |
| 114 | }; | 99 | }; |
| 115 | return @intToPtr([*]u8, addr)[0..len]; | 100 | return @intToPtr([*]u8, addr)[0..len]; |
| 116 | } else { | | |
| 117 | std.debug.panic("big alloc: len={d} align={d} slot_size={d} class={d}", .{ | | |
| 118 | len, alignment, slot_size, class, | | |
| 119 | }); | | |
| 120 | } | 101 | } |
| | 102 | const bigpages_needed = (aligned_len + (bigpage_size - 1)) / bigpage_size; |
| | 103 | const addr = try allocBigPages(bigpages_needed); |
| | 104 | return @intToPtr([*]u8, addr)[0..len]; |
| 121 | } | 105 | } |
| 122 | | 106 | |
| 123 | fn resize( | 107 | fn resize( |
| ... | @@ -145,29 +129,65 @@ fn free( | ... | @@ -145,29 +129,65 @@ fn free( |
| 145 | ) void { | 129 | ) void { |
| 146 | _ = ctx; | 130 | _ = ctx; |
| 147 | _ = return_address; | 131 | _ = return_address; |
| 148 | const class_size = @max(buf.len, buf_align); | 132 | const aligned_len = @max(buf.len, buf_align); |
| 149 | const class = math.log2(class_size); | 133 | const slot_size = math.ceilPowerOfTwoAssert(usize, aligned_len); |
| | 134 | const class = math.log2(slot_size); |
| 150 | if (class < size_class_count) { | 135 | if (class < size_class_count) { |
| 151 | const free_list = &frees[class]; | 136 | const free_list = &frees[class]; |
| 152 | assert(free_list.len < free_list.cap); | 137 | assert(free_list.len < free_list.cap); |
| 153 | free_list.ptr[free_list.len] = @ptrToInt(buf.ptr); | 138 | free_list.ptr[free_list.len] = @ptrToInt(buf.ptr); |
| 154 | free_list.len += 1; | 139 | free_list.len += 1; |
| 155 | } else { | 140 | } else { |
| 156 | std.debug.panic("big free: len={d} align={d}", .{ | 141 | const bigpages_needed = (aligned_len + (bigpage_size - 1)) / bigpage_size; |
| 157 | buf.len, buf_align, | 142 | const big_slot_size = math.ceilPowerOfTwoAssert(usize, bigpages_needed); |
| 158 | }); | 143 | const big_class = math.log2(big_slot_size); |
| | 144 | const free_list = &big_frees[big_class]; |
| | 145 | assert(free_list.len < free_list.cap); |
| | 146 | free_list.ptr[free_list.len] = @ptrToInt(buf.ptr); |
| | 147 | free_list.len += 1; |
| 159 | } | 148 | } |
| 160 | } | 149 | } |
| 161 | | 150 | |
| 162 | inline fn allocBigPages(n: usize) !usize { | 151 | fn allocBigPages(n: usize) !usize { |
| 163 | if (n == 1 and bigpage_free_list.len > 0) { | 152 | const slot_size = math.ceilPowerOfTwoAssert(usize, n); |
| 164 | bigpage_free_list.len -= 1; | 153 | const class = math.log2(slot_size); |
| 165 | return bigpage_free_list.ptr[bigpage_free_list.len]; | 154 | |
| | 155 | const free_list = &big_frees[class]; |
| | 156 | if (free_list.len > 0) { |
| | 157 | free_list.len -= 1; |
| | 158 | return free_list.ptr[free_list.len]; |
| 166 | } | 159 | } |
| 167 | const page_index = @wasmMemoryGrow(0, n * pages_per_bigpage); | 160 | |
| 168 | if (page_index <= 0) | 161 | //std.debug.print("ensureFreeListCapacity slot_size={d} big_class={d}\n", .{ |
| 169 | return error.OutOfMemory; | 162 | // slot_size, class, |
| 170 | return @intCast(u32, page_index) * wasm.page_size; | 163 | //}); |
| | 164 | // This prevents memory allocation within free(). |
| | 165 | try ensureFreeListCapacity(free_list); |
| | 166 | |
| | 167 | const page_index = @wasmMemoryGrow(0, slot_size * pages_per_bigpage); |
| | 168 | if (page_index <= 0) return error.OutOfMemory; |
| | 169 | const addr = @intCast(u32, page_index) * wasm.page_size; |
| | 170 | //std.debug.print("got 0x{x}..0x{x} from memory.grow\n", .{ |
| | 171 | // addr, addr + wasm.page_size * slot_size * pages_per_bigpage, |
| | 172 | //}); |
| | 173 | return addr; |
| | 174 | } |
| | 175 | |
| | 176 | fn ensureFreeListCapacity(free_list: *FreeList) Allocator.Error!void { |
| | 177 | if (free_list.len < free_list.cap) return; |
| | 178 | const old_bigpage_count = free_list.cap / bigpage_size; |
| | 179 | free_list.cap = math.maxInt(usize); // Prevent recursive calls. |
| | 180 | const new_bigpage_count = @max(old_bigpage_count * 2, 1); |
| | 181 | const addr = try allocBigPages(new_bigpage_count); |
| | 182 | //std.debug.print("allocated {d} big pages: 0x{x}\n", .{ new_bigpage_count, addr }); |
| | 183 | const new_ptr = @intToPtr([*]usize, addr); |
| | 184 | @memcpy( |
| | 185 | @ptrCast([*]u8, new_ptr), |
| | 186 | @ptrCast([*]u8, free_list.ptr), |
| | 187 | @sizeOf(usize) * free_list.len, |
| | 188 | ); |
| | 189 | free_list.ptr = new_ptr; |
| | 190 | free_list.cap = new_bigpage_count * (bigpage_size / @sizeOf(usize)); |
| 171 | } | 191 | } |
| 172 | | 192 | |
| 173 | const test_ally = Allocator{ | 193 | const test_ally = Allocator{ |
| ... | @@ -207,16 +227,10 @@ test "small allocations - free in reverse order" { | ... | @@ -207,16 +227,10 @@ test "small allocations - free in reverse order" { |
| 207 | } | 227 | } |
| 208 | | 228 | |
| 209 | test "large allocations" { | 229 | test "large allocations" { |
| 210 | std.debug.print("alloc ptr1\n", .{}); | | |
| 211 | const ptr1 = try test_ally.alloc(u64, 42768); | 230 | const ptr1 = try test_ally.alloc(u64, 42768); |
| 212 | std.debug.print("alloc ptr2\n", .{}); | | |
| 213 | const ptr2 = try test_ally.alloc(u64, 52768); | 231 | const ptr2 = try test_ally.alloc(u64, 52768); |
| 214 | std.debug.print("free ptr1\n", .{}); | | |
| 215 | test_ally.free(ptr1); | 232 | test_ally.free(ptr1); |
| 216 | std.debug.print("alloc ptr3\n", .{}); | | |
| 217 | const ptr3 = try test_ally.alloc(u64, 62768); | 233 | const ptr3 = try test_ally.alloc(u64, 62768); |
| 218 | std.debug.print("free ptr3\n", .{}); | | |
| 219 | test_ally.free(ptr3); | 234 | test_ally.free(ptr3); |
| 220 | std.debug.print("free ptr2\n", .{}); | | |
| 221 | test_ally.free(ptr2); | 235 | test_ally.free(ptr2); |
| 222 | } | 236 | } |
Andrew Kelley <andrew@ziglang.org> 2022-11-10 21:08:29-07:00
lib