| ... | ... | @@ -254,79 +254,160 @@ const PageAllocator = struct { |
| 254 | 254 | extern fn @"llvm.wasm.memory.size.i32"(u32) u32; |
| 255 | 255 | extern fn @"llvm.wasm.memory.grow.i32"(u32, u32) i32; |
| 256 | 256 | |
| 257 | | /// TODO: make this re-use freed pages, and cooperate with other callers of these global intrinsics |
| 258 | | /// by better utilizing the return value of grow() |
| 259 | 257 | const WasmPageAllocator = struct { |
| 260 | | var start_ptr: [*]u8 = undefined; |
| 261 | | var num_pages: usize = 0; |
| 262 | | var end_index: usize = 0; |
| 263 | | |
| 264 | 258 | comptime { |
| 265 | | if (builtin.arch != .wasm32) { |
| 259 | if (!std.Target.current.isWasm()) { |
| 266 | 260 | @compileError("WasmPageAllocator is only available for wasm32 arch"); |
| 267 | 261 | } |
| 268 | 262 | } |
| 269 | 263 | |
| 270 | | fn alloc(allocator: *Allocator, size: usize, alignment: u29) ![]u8 { |
| 271 | | const addr = @ptrToInt(start_ptr) + end_index; |
| 272 | | const adjusted_addr = mem.alignForward(addr, alignment); |
| 273 | | const adjusted_index = end_index + (adjusted_addr - addr); |
| 274 | | const new_end_index = adjusted_index + size; |
| 264 | const PageStatus = enum(u1) { |
| 265 | used = 0, |
| 266 | free = 1, |
| 267 | |
| 268 | pub const none_free: u8 = 0; |
| 269 | }; |
| 275 | 270 | |
| 276 | | if (new_end_index > num_pages * mem.page_size) { |
| 277 | | const required_memory = new_end_index - (num_pages * mem.page_size); |
| 271 | const FreeBlock = struct { |
| 272 | data: []u128, |
| 278 | 273 | |
| 279 | | var inner_num_pages: usize = required_memory / mem.page_size; |
| 280 | | if (required_memory % mem.page_size != 0) { |
| 281 | | inner_num_pages += 1; |
| 274 | const Io = std.packed_int_array.PackedIntIo(u1, .Little); |
| 275 | |
| 276 | fn totalPages(self: FreeBlock) usize { |
| 277 | return self.data.len * 128; |
| 278 | } |
| 279 | |
| 280 | fn isInitialized(self: FreeBlock) bool { |
| 281 | return self.data.len > 0; |
| 282 | } |
| 283 | |
| 284 | fn getBit(self: FreeBlock, idx: usize) PageStatus { |
| 285 | const bit_offset = 0; |
| 286 | return @intToEnum(PageStatus, Io.get(@sliceToBytes(self.data), idx, bit_offset)); |
| 287 | } |
| 288 | |
| 289 | fn setBits(self: FreeBlock, start_idx: usize, len: usize, val: PageStatus) void { |
| 290 | const bit_offset = 0; |
| 291 | var i: usize = 0; |
| 292 | while (i < len) : (i += 1) { |
| 293 | Io.set(@sliceToBytes(self.data), start_idx + i, bit_offset, @enumToInt(val)); |
| 282 | 294 | } |
| 295 | } |
| 296 | |
| 297 | // Use '0xFFFFFFFF' as a _missing_ sentinel |
| 298 | // This saves ~50 bytes compared to returning a nullable |
| 299 | |
| 300 | // We can guarantee that conventional memory never gets this big, |
| 301 | // and wasm32 would not be able to address this memory (32 GB > usize). |
| 302 | |
| 303 | // Revisit if this is settled: https://github.com/ziglang/zig/issues/3806 |
| 304 | const not_found = std.math.maxInt(usize); |
| 283 | 305 | |
| 284 | | const prev_page = @"llvm.wasm.memory.grow.i32"(0, @intCast(u32, inner_num_pages)); |
| 285 | | if (prev_page == -1) { |
| 286 | | return error.OutOfMemory; |
| 306 | fn useRecycled(self: FreeBlock, num_pages: usize) usize { |
| 307 | @setCold(true); |
| 308 | for (self.data) |segment, i| { |
| 309 | const spills_into_next = @bitCast(i128, segment) < 0; |
| 310 | const has_enough_bits = @popCount(u128, segment) >= num_pages; |
| 311 | |
| 312 | if (!spills_into_next and !has_enough_bits) continue; |
| 313 | |
| 314 | var j: usize = i * 128; |
| 315 | while (j < (i + 1) * 128) : (j += 1) { |
| 316 | var count: usize = 0; |
| 317 | while (j + count < self.totalPages() and self.getBit(j + count) == .free) { |
| 318 | count += 1; |
| 319 | if (count >= num_pages) { |
| 320 | self.setBits(j, num_pages, .used); |
| 321 | return j; |
| 322 | } |
| 323 | } |
| 324 | j += count; |
| 325 | } |
| 287 | 326 | } |
| 327 | return not_found; |
| 328 | } |
| 288 | 329 | |
| 289 | | num_pages += inner_num_pages; |
| 330 | fn recycle(self: FreeBlock, start_idx: usize, len: usize) void { |
| 331 | self.setBits(start_idx, len, .free); |
| 290 | 332 | } |
| 333 | }; |
| 291 | 334 | |
| 292 | | const result = start_ptr[adjusted_index..new_end_index]; |
| 293 | | end_index = new_end_index; |
| 335 | var _conventional_data = [_]u128{0} ** 16; |
| 336 | // Marking `conventional` as const saves ~40 bytes |
| 337 | const conventional = FreeBlock{ .data = &_conventional_data }; |
| 338 | var extended = FreeBlock{ .data = &[_]u128{} }; |
| 294 | 339 | |
| 295 | | return result; |
| 340 | fn extendedOffset() usize { |
| 341 | return conventional.totalPages(); |
| 296 | 342 | } |
| 297 | 343 | |
| 298 | | // Check if memory is the last "item" and is aligned correctly |
| 299 | | fn is_last_item(memory: []u8, alignment: u29) bool { |
| 300 | | return memory.ptr == start_ptr + end_index - memory.len and mem.alignForward(@ptrToInt(memory.ptr), alignment) == @ptrToInt(memory.ptr); |
| 344 | fn nPages(memsize: usize) usize { |
| 345 | return std.mem.alignForward(memsize, std.mem.page_size) / std.mem.page_size; |
| 301 | 346 | } |
| 302 | 347 | |
| 303 | | fn realloc(allocator: *Allocator, old_mem: []u8, old_align: u29, new_size: usize, new_align: u29) ![]u8 { |
| 304 | | // Initialize start_ptr at the first realloc |
| 305 | | if (num_pages == 0) { |
| 306 | | start_ptr = @intToPtr([*]u8, @intCast(usize, @"llvm.wasm.memory.size.i32"(0)) * mem.page_size); |
| 348 | fn alloc(allocator: *Allocator, page_count: usize, alignment: u29) error{OutOfMemory}!usize { |
| 349 | var idx = conventional.useRecycled(page_count); |
| 350 | if (idx != FreeBlock.not_found) { |
| 351 | return idx; |
| 307 | 352 | } |
| 308 | 353 | |
| 309 | | if (is_last_item(old_mem, new_align)) { |
| 310 | | const start_index = end_index - old_mem.len; |
| 311 | | const new_end_index = start_index + new_size; |
| 354 | idx = extended.useRecycled(page_count); |
| 355 | if (idx != FreeBlock.not_found) { |
| 356 | return idx + extendedOffset(); |
| 357 | } |
| 312 | 358 | |
| 313 | | if (new_end_index > num_pages * mem.page_size) { |
| 314 | | _ = try alloc(allocator, new_end_index - end_index, new_align); |
| 315 | | } |
| 316 | | const result = start_ptr[start_index..new_end_index]; |
| 359 | const prev_page_count = @"llvm.wasm.memory.grow.i32"(0, @intCast(u32, page_count)); |
| 360 | if (prev_page_count <= 0) { |
| 361 | return error.OutOfMemory; |
| 362 | } |
| 317 | 363 | |
| 318 | | end_index = new_end_index; |
| 319 | | return result; |
| 320 | | } else if (new_size <= old_mem.len and new_align <= old_align) { |
| 364 | return @intCast(usize, prev_page_count); |
| 365 | } |
| 366 | |
| 367 | pub fn realloc(allocator: *Allocator, old_mem: []u8, old_align: u29, new_size: usize, new_align: u29) Allocator.Error![]u8 { |
| 368 | if (new_align > std.mem.page_size) { |
| 321 | 369 | return error.OutOfMemory; |
| 370 | } |
| 371 | |
| 372 | if (nPages(new_size) == nPages(old_mem.len)) { |
| 373 | return old_mem.ptr[0..new_size]; |
| 374 | } else if (new_size < old_mem.len) { |
| 375 | return shrink(allocator, old_mem, old_align, new_size, new_align); |
| 322 | 376 | } else { |
| 323 | | const result = try alloc(allocator, new_size, new_align); |
| 324 | | mem.copy(u8, result, old_mem); |
| 325 | | return result; |
| 377 | const page_idx = try alloc(allocator, nPages(new_size), new_align); |
| 378 | const new_mem = @intToPtr([*]u8, page_idx * std.mem.page_size)[0..new_size]; |
| 379 | std.mem.copy(u8, new_mem, old_mem); |
| 380 | _ = shrink(allocator, old_mem, old_align, 0, 0); |
| 381 | return new_mem; |
| 326 | 382 | } |
| 327 | 383 | } |
| 328 | 384 | |
| 329 | | fn shrink(allocator: *Allocator, old_mem: []u8, old_align: u29, new_size: usize, new_align: u29) []u8 { |
| 385 | pub fn shrink(allocator: *Allocator, old_mem: []u8, old_align: u29, new_size: usize, new_align: u29) []u8 { |
| 386 | @setCold(true); |
| 387 | const free_start = nPages(@ptrToInt(old_mem.ptr) + new_size); |
| 388 | var free_end = nPages(@ptrToInt(old_mem.ptr) + old_mem.len); |
| 389 | |
| 390 | if (free_end > free_start) { |
| 391 | if (free_start < extendedOffset()) { |
| 392 | const clamped_end = std.math.min(extendedOffset(), free_end); |
| 393 | conventional.recycle(free_start, clamped_end - free_start); |
| 394 | } |
| 395 | |
| 396 | if (free_end > extendedOffset()) { |
| 397 | if (!extended.isInitialized()) { |
| 398 | // Steal the last page from the memory currently being recycled |
| 399 | // TODO: would it be better if we use the first page instead? |
| 400 | free_end -= 1; |
| 401 | |
| 402 | extended.data = @intToPtr([*]u128, free_end * std.mem.page_size)[0 .. std.mem.page_size / @sizeOf(u128)]; |
| 403 | // Since this is the first page being freed and we consume it, assume *nothing* is free. |
| 404 | std.mem.set(u128, extended.data, PageStatus.none_free); |
| 405 | } |
| 406 | const clamped_start = std.math.max(extendedOffset(), free_start); |
| 407 | extended.recycle(clamped_start - extendedOffset(), free_end - clamped_start); |
| 408 | } |
| 409 | } |
| 410 | |
| 330 | 411 | return old_mem[0..new_size]; |
| 331 | 412 | } |
| 332 | 413 | }; |
| ... | ... | @@ -724,12 +805,51 @@ test "c_allocator" { |
| 724 | 805 | } |
| 725 | 806 | } |
| 726 | 807 | |
| 808 | test "WasmPageAllocator internals" { |
| 809 | if (comptime std.Target.current.isWasm()) { |
| 810 | const conventional_memsize = WasmPageAllocator.conventional.totalPages() * std.mem.page_size; |
| 811 | const initial = try page_allocator.alloc(u8, std.mem.page_size); |
| 812 | std.debug.assert(@ptrToInt(initial.ptr) < conventional_memsize); // If this isn't conventional, the rest of these tests don't make sense. Also we have a serious memory leak in the test suite. |
| 813 | |
| 814 | var inplace = try page_allocator.realloc(initial, 1); |
| 815 | testing.expectEqual(initial.ptr, inplace.ptr); |
| 816 | inplace = try page_allocator.realloc(inplace, 4); |
| 817 | testing.expectEqual(initial.ptr, inplace.ptr); |
| 818 | page_allocator.free(inplace); |
| 819 | |
| 820 | const reuse = try page_allocator.alloc(u8, 1); |
| 821 | testing.expectEqual(initial.ptr, reuse.ptr); |
| 822 | page_allocator.free(reuse); |
| 823 | |
| 824 | // This segment may span conventional and extended which has really complex rules so we're just ignoring it for now. |
| 825 | const padding = try page_allocator.alloc(u8, conventional_memsize); |
| 826 | page_allocator.free(padding); |
| 827 | |
| 828 | const extended = try page_allocator.alloc(u8, conventional_memsize); |
| 829 | testing.expect(@ptrToInt(extended.ptr) >= conventional_memsize); |
| 830 | |
| 831 | const use_small = try page_allocator.alloc(u8, 1); |
| 832 | testing.expectEqual(initial.ptr, use_small.ptr); |
| 833 | page_allocator.free(use_small); |
| 834 | |
| 835 | inplace = try page_allocator.realloc(extended, 1); |
| 836 | testing.expectEqual(extended.ptr, inplace.ptr); |
| 837 | page_allocator.free(inplace); |
| 838 | |
| 839 | const reuse_extended = try page_allocator.alloc(u8, conventional_memsize); |
| 840 | testing.expectEqual(extended.ptr, reuse_extended.ptr); |
| 841 | page_allocator.free(reuse_extended); |
| 842 | } |
| 843 | } |
| 844 | |
| 727 | 845 | test "PageAllocator" { |
| 728 | 846 | const allocator = page_allocator; |
| 729 | 847 | try testAllocator(allocator); |
| 730 | 848 | try testAllocatorAligned(allocator, 16); |
| 731 | | try testAllocatorLargeAlignment(allocator); |
| 732 | | try testAllocatorAlignedShrink(allocator); |
| 849 | if (!std.Target.current.isWasm()) { |
| 850 | try testAllocatorLargeAlignment(allocator); |
| 851 | try testAllocatorAlignedShrink(allocator); |
| 852 | } |
| 733 | 853 | |
| 734 | 854 | if (builtin.os == .windows) { |
| 735 | 855 | // Trying really large alignment. As mentionned in the implementation, |
| ... | ... | @@ -766,7 +886,7 @@ test "ArenaAllocator" { |
| 766 | 886 | try testAllocatorAlignedShrink(&arena_allocator.allocator); |
| 767 | 887 | } |
| 768 | 888 | |
| 769 | | var test_fixed_buffer_allocator_memory: [80000 * @sizeOf(u64)]u8 = undefined; |
| 889 | var test_fixed_buffer_allocator_memory: [800000 * @sizeOf(u64)]u8 = undefined; |
| 770 | 890 | test "FixedBufferAllocator" { |
| 771 | 891 | var fixed_buffer_allocator = FixedBufferAllocator.init(test_fixed_buffer_allocator_memory[0..]); |
| 772 | 892 | |
Andrew Kelley <andrew@ziglang.org> 2019-12-10 10:58:50-05:00
GitHub <noreply@github.com> 2019-12-10 10:58:50-05:00
Signed by PGP key 4AEE18F83AFDEB23
lib