| ... | ... | @@ -257,7 +257,6 @@ const WasmPageAllocator = struct { |
| 257 | 257 | var heap_base_wannabe: [256]u8 align(16) = undefined; |
| 258 | 258 | |
| 259 | 259 | const FreeBlock = struct { |
| 260 | | offset: usize = 0, |
| 261 | 260 | packed_data: std.PackedIntSlice(u1) = std.PackedIntSlice(u1).init(&[_]u8{}, 0), |
| 262 | 261 | block_data: []u128 = &[_]u128{}, |
| 263 | 262 | |
| ... | ... | @@ -275,11 +274,20 @@ const WasmPageAllocator = struct { |
| 275 | 274 | fn setBits(self: *FreeBlock, start_idx: usize, len: usize, val: u1) void { |
| 276 | 275 | var i: usize = 0; |
| 277 | 276 | while (i < len) : (i += 1) { |
| 278 | | self.packed_data.set(i + start_idx + self.offset, val); |
| 277 | self.packed_data.set(i + start_idx, val); |
| 279 | 278 | } |
| 280 | 279 | } |
| 281 | 280 | |
| 282 | | fn useRecycled(self: *FreeBlock, num_pages: usize) ?[*]u8 { |
| 281 | // Use '0xFFFFFFFF' as a _missing_ sentinel |
| 282 | // This saves ~50 bytes compared to returning a nullable |
| 283 | |
| 284 | // We can guarantee that conventional memory never gets this big, |
| 285 | // and wasm32 would not be able to address this block (32 GB > usize). |
| 286 | |
| 287 | // Revisit if this is settled: https://github.com/ziglang/zig/issues/3806 |
| 288 | const not_found = std.math.maxInt(usize); |
| 289 | |
| 290 | fn useRecycled(self: *FreeBlock, num_pages: usize) usize { |
| 283 | 291 | @setCold(true); |
| 284 | 292 | for (self.block_data) |segment, i| { |
| 285 | 293 | const spills_into_next = @bitCast(i128, segment) < 0; |
| ... | ... | @@ -294,13 +302,13 @@ const WasmPageAllocator = struct { |
| 294 | 302 | count += 1; |
| 295 | 303 | if (count >= num_pages) { |
| 296 | 304 | self.setBits(j, num_pages, 0); |
| 297 | | return @intToPtr([*]u8, (j + self.offset) * std.mem.page_size); |
| 305 | return j; |
| 298 | 306 | } |
| 299 | 307 | } |
| 300 | 308 | j += count; |
| 301 | 309 | } |
| 302 | 310 | } |
| 303 | | return null; |
| 311 | return not_found; |
| 304 | 312 | } |
| 305 | 313 | |
| 306 | 314 | fn recycle(self: *FreeBlock, start_idx: usize, len: usize) void { |
| ... | ... | @@ -311,22 +319,31 @@ const WasmPageAllocator = struct { |
| 311 | 319 | var conventional = FreeBlock{}; |
| 312 | 320 | var extended = FreeBlock{}; |
| 313 | 321 | |
| 322 | fn extendedOffset() usize { |
| 323 | return conventional.totalPages(); |
| 324 | } |
| 325 | |
| 314 | 326 | fn nPages(memsize: usize) usize { |
| 315 | 327 | return std.mem.alignForward(memsize, std.mem.page_size) / std.mem.page_size; |
| 316 | 328 | } |
| 317 | 329 | |
| 318 | | fn alloc(allocator: *Allocator, n: usize, alignment: u29) error{OutOfMemory}![]u8 { |
| 319 | | const n_pages = nPages(n); |
| 320 | | const page = conventional.useRecycled(n_pages) orelse extended.useRecycled(n_pages) orelse blk: { |
| 321 | | const prev_page_count = @"llvm.wasm.memory.grow.i32"(0, @intCast(u32, n_pages)); |
| 322 | | if (prev_page_count < 0) { |
| 323 | | return error.OutOfMemory; |
| 324 | | } |
| 330 | fn alloc(allocator: *Allocator, page_count: usize, alignment: u29) error{OutOfMemory}!usize { |
| 331 | var idx = conventional.useRecycled(page_count); |
| 332 | if (idx != FreeBlock.not_found) { |
| 333 | return idx; |
| 334 | } |
| 325 | 335 | |
| 326 | | break :blk @intToPtr([*]u8, @intCast(usize, prev_page_count) * std.mem.page_size); |
| 327 | | }; |
| 336 | idx = extended.useRecycled(page_count); |
| 337 | if (idx != FreeBlock.not_found) { |
| 338 | return idx + extendedOffset(); |
| 339 | } |
| 328 | 340 | |
| 329 | | return page[0..n]; |
| 341 | const prev_page_count = @"llvm.wasm.memory.grow.i32"(0, @intCast(u32, page_count)); |
| 342 | if (prev_page_count <= 0) { |
| 343 | return error.OutOfMemory; |
| 344 | } |
| 345 | |
| 346 | return @intCast(usize, prev_page_count); |
| 330 | 347 | } |
| 331 | 348 | |
| 332 | 349 | pub fn realloc(allocator: *Allocator, old_mem: []u8, old_align: u29, new_size: usize, new_align: u29) Allocator.Error![]u8 { |
| ... | ... | @@ -339,10 +356,11 @@ const WasmPageAllocator = struct { |
| 339 | 356 | } else if (new_size < old_mem.len) { |
| 340 | 357 | return shrink(allocator, old_mem, old_align, new_size, new_align); |
| 341 | 358 | } else { |
| 342 | | const new_mem = try alloc(allocator, new_size, new_align); |
| 359 | const page_idx = try alloc(allocator, nPages(new_size), new_align); |
| 360 | const new_mem = @intToPtr([*]u8, page_idx * std.mem.page_size)[0..new_size]; |
| 343 | 361 | std.mem.copy(u8, new_mem, old_mem); |
| 344 | 362 | _ = shrink(allocator, old_mem, old_align, 0, 0); |
| 345 | | return new_mem[0..new_size]; |
| 363 | return new_mem; |
| 346 | 364 | } |
| 347 | 365 | } |
| 348 | 366 | |
| ... | ... | @@ -352,22 +370,19 @@ const WasmPageAllocator = struct { |
| 352 | 370 | |
| 353 | 371 | if (free_end > free_start) { |
| 354 | 372 | if (conventional.totalPages() == 0) { |
| 355 | | conventional.offset = 0; |
| 356 | 373 | //conventional.initData(__heap_base[0..@intCast(usize, @"llvm.wasm.memory.size.i32"(0) * std.mem.page_size)]); |
| 357 | 374 | conventional.initData(heap_base_wannabe[0..]); |
| 358 | 375 | } |
| 359 | 376 | |
| 360 | | if (free_start < conventional.totalPages()) { |
| 377 | if (free_start < extendedOffset()) { |
| 361 | 378 | conventional.recycle(free_start, free_end - free_start); |
| 362 | 379 | } else { |
| 363 | 380 | if (extended.totalPages() == 0) { |
| 364 | | extended.offset = conventional.offset + conventional.totalPages(); |
| 365 | | |
| 366 | 381 | // Steal the last page from the memory currently being recycled |
| 367 | 382 | free_end -= 1; |
| 368 | 383 | extended.initData(@intToPtr([*]align(16) u8, free_end * std.mem.page_size)[0..std.mem.page_size]); |
| 369 | 384 | } |
| 370 | | extended.recycle(free_start, free_end - free_start); |
| 385 | extended.recycle(free_start - extendedOffset(), free_end - free_start); |
| 371 | 386 | } |
| 372 | 387 | } |
| 373 | 388 | |
Benjamin Feng <contact@fengb.me> 2019-12-04 18:12:25-06:00
lib