| ... | ... | @@ -261,7 +261,9 @@ const Node = struct { |
| 261 | 261 | return @as([*]u8, @ptrCast(node))[0..size.toInt()]; |
| 262 | 262 | } |
| 263 | 263 | |
| 264 | | fn endResize(node: *Node, size: usize) void { |
| 264 | fn endResize(node: *Node, size: usize, prev_size: usize) void { |
| 265 | assert(size >= prev_size); // nodes must not shrink |
| 266 | assert(@atomicLoad(Size, &node.size, .unordered).toInt() == prev_size); |
| 265 | 267 | return @atomicStore(Size, &node.size, .fromInt(size), .release); // syncs with acquire in `beginResize` |
| 266 | 268 | } |
| 267 | 269 | |
| ... | ... | @@ -302,6 +304,8 @@ fn stealFreeList(arena: *ArenaAllocator) ?*Node { |
| 302 | 304 | |
| 303 | 305 | fn pushFreeList(arena: *ArenaAllocator, first: *Node, last: *Node) void { |
| 304 | 306 | assert(first != last.next); |
| 307 | assert(first != first.next); |
| 308 | assert(last != last.next); |
| 305 | 309 | while (@cmpxchgWeak( |
| 306 | 310 | ?*Node, |
| 307 | 311 | &arena.state.free_list, |
| ... | ... | @@ -364,7 +368,7 @@ fn alloc(ctx: *anyopaque, n: usize, alignment: Alignment, ret_addr: usize) ?[*]u |
| 364 | 368 | const node = first_node orelse break :resize; |
| 365 | 369 | const allocated_slice = node.beginResize() orelse break :resize; |
| 366 | 370 | var size = allocated_slice.len; |
| 367 | | defer node.endResize(size); |
| 371 | defer node.endResize(size, allocated_slice.len); |
| 368 | 372 | |
| 369 | 373 | const buf = allocated_slice[@sizeOf(Node)..]; |
| 370 | 374 | const end_index = @atomicLoad(usize, &node.end_index, .monotonic); |
| ... | ... | @@ -406,92 +410,81 @@ fn alloc(ctx: *anyopaque, n: usize, alignment: Alignment, ret_addr: usize) ?[*]u |
| 406 | 410 | // Also this avoids the ABA problem; stealing the list with an atomic |
| 407 | 411 | // swap doesn't introduce any potentially stale `next` pointers. |
| 408 | 412 | |
| 409 | | const free_list = arena.stealFreeList(); |
| 410 | | var first_free: ?*Node = free_list; |
| 411 | | var last_free: ?*Node = free_list; |
| 412 | | defer { |
| 413 | | // Push remaining stolen free list back onto `arena.state.free_list`. |
| 414 | | if (first_free) |first| { |
| 415 | | const last = last_free.?; |
| 416 | | assert(last.next == null); // optimize for no new nodes added during steal |
| 417 | | arena.pushFreeList(first, last); |
| 418 | | } |
| 419 | | } |
| 413 | const free_list = arena.stealFreeList() orelse break :from_free_list; |
| 420 | 414 | |
| 421 | | const candidate: ?*Node, const prev: ?*Node = candidate: { |
| 415 | const first_free: *Node, const last_free: *Node, const node: *Node, const prev: ?*Node = find: { |
| 422 | 416 | var best_fit_prev: ?*Node = null; |
| 423 | 417 | var best_fit: ?*Node = null; |
| 424 | 418 | var best_fit_diff: usize = std.math.maxInt(usize); |
| 425 | 419 | |
| 426 | 420 | var it_prev: ?*Node = null; |
| 427 | | var it = free_list; |
| 421 | var it: ?*Node = free_list; |
| 428 | 422 | while (it) |node| : ({ |
| 429 | | it_prev = it; |
| 423 | it_prev = node; |
| 430 | 424 | it = node.next; |
| 431 | 425 | }) { |
| 432 | | last_free = node; |
| 433 | 426 | assert(!node.size.resizing); |
| 434 | 427 | const buf = node.allocatedSliceUnsafe()[@sizeOf(Node)..]; |
| 435 | 428 | const aligned_index = alignedIndex(buf.ptr, 0, alignment); |
| 436 | 429 | |
| 437 | | if (aligned_index + n <= buf.len) { |
| 438 | | break :candidate .{ node, it_prev }; |
| 439 | | } |
| 440 | | |
| 441 | | const diff = aligned_index + n - buf.len; |
| 442 | | if (diff <= best_fit_diff) { |
| 430 | const diff = aligned_index + n -| buf.len; |
| 431 | if (diff < best_fit_diff) { |
| 443 | 432 | best_fit_prev = it_prev; |
| 444 | 433 | best_fit = node; |
| 445 | 434 | best_fit_diff = diff; |
| 446 | 435 | } |
| 447 | | } else { |
| 448 | | // Ideally we want to use all nodes in `free_list` eventually, |
| 449 | | // so even if none fit we'll try to resize the one that was the |
| 450 | | // closest to being large enough. |
| 451 | | if (best_fit) |node| { |
| 452 | | const allocated_slice = node.allocatedSliceUnsafe(); |
| 453 | | const buf = allocated_slice[@sizeOf(Node)..]; |
| 454 | | const aligned_index = alignedIndex(buf.ptr, 0, alignment); |
| 455 | | const new_size = mem.alignForward(usize, @sizeOf(Node) + aligned_index + n, 2); |
| 456 | | |
| 457 | | if (arena.child_allocator.rawResize(allocated_slice, .of(Node), new_size, @returnAddress())) { |
| 458 | | node.size = .fromInt(new_size); |
| 459 | | break :candidate .{ node, best_fit_prev }; |
| 460 | | } |
| 461 | | } |
| 462 | | break :from_free_list; |
| 463 | 436 | } |
| 437 | |
| 438 | break :find .{ free_list, it_prev.?, best_fit.?, best_fit_prev }; |
| 464 | 439 | }; |
| 465 | 440 | |
| 466 | | { |
| 467 | | var it = last_free; |
| 468 | | while (it) |node| : (it = node.next) { |
| 469 | | last_free = node; |
| 441 | const aligned_index, const need_resize = aligned_index: { |
| 442 | const buf = node.allocatedSliceUnsafe()[@sizeOf(Node)..]; |
| 443 | const aligned_index = alignedIndex(buf.ptr, 0, alignment); |
| 444 | break :aligned_index .{ aligned_index, aligned_index + n > buf.len }; |
| 445 | }; |
| 446 | |
| 447 | if (need_resize) { |
| 448 | // Ideally we want to use all nodes in `free_list` eventually, |
| 449 | // so even if none fit we'll try to resize the one that was the |
| 450 | // closest to being large enough. |
| 451 | const new_size = mem.alignForward(usize, @sizeOf(Node) + aligned_index + n, 2); |
| 452 | if (arena.child_allocator.rawResize(node.allocatedSliceUnsafe(), .of(Node), new_size, @returnAddress())) { |
| 453 | node.size = .fromInt(new_size); |
| 454 | } else { |
| 455 | arena.pushFreeList(first_free, last_free); |
| 456 | break :from_free_list; // we couldn't find a fitting free node |
| 470 | 457 | } |
| 471 | 458 | } |
| 472 | 459 | |
| 473 | | const node = candidate orelse break :from_free_list; |
| 474 | | const old_next = node.next; |
| 475 | | |
| 476 | 460 | const buf = node.allocatedSliceUnsafe()[@sizeOf(Node)..]; |
| 477 | | const aligned_index = alignedIndex(buf.ptr, 0, alignment); |
| 461 | const old_next = node.next; |
| 478 | 462 | |
| 479 | 463 | node.end_index = aligned_index + n; |
| 480 | 464 | node.next = first_node; |
| 481 | 465 | |
| 482 | 466 | switch (arena.tryPushNode(node)) { |
| 483 | 467 | .success => { |
| 484 | | // finish removing node from free list |
| 468 | // Finish removing node from free list. |
| 485 | 469 | if (prev) |p| p.next = old_next; |
| 486 | | if (node == first_free) first_free = old_next; |
| 487 | | if (node == last_free) last_free = prev; |
| 470 | |
| 471 | // Push remaining stolen free list back onto `arena.state.free_list`. |
| 472 | const new_first_free = if (node == first_free) old_next else first_free; |
| 473 | const new_last_free = if (node == last_free) prev else last_free; |
| 474 | if (new_first_free) |first| { |
| 475 | const last = new_last_free.?; |
| 476 | arena.pushFreeList(first, last); |
| 477 | } |
| 478 | |
| 488 | 479 | return buf[aligned_index..][0..n].ptr; |
| 489 | 480 | }, |
| 490 | 481 | .failure => |old_first_node| { |
| 491 | | cur_first_node = old_first_node; |
| 492 | 482 | // restore free list to as we found it |
| 493 | 483 | node.next = old_next; |
| 494 | | continue :retry; |
| 484 | arena.pushFreeList(first_free, last_free); |
| 485 | |
| 486 | cur_first_node = old_first_node; |
| 487 | continue :retry; // there's a new first node; retry! |
| 495 | 488 | }, |
| 496 | 489 | } |
| 497 | 490 | } |
| ... | ... | @@ -503,16 +496,17 @@ fn alloc(ctx: *anyopaque, n: usize, alignment: Alignment, ret_addr: usize) ?[*]u |
| 503 | 496 | @branchHint(.cold); |
| 504 | 497 | } |
| 505 | 498 | |
| 506 | | const size: usize = size: { |
| 499 | const size: Node.Size = size: { |
| 507 | 500 | const min_size = @sizeOf(Node) + alignment.toByteUnits() + n; |
| 508 | 501 | const big_enough_size = prev_size + min_size + 16; |
| 509 | | break :size mem.alignForward(usize, big_enough_size + big_enough_size / 2, 2); |
| 502 | const size = mem.alignForward(usize, big_enough_size + big_enough_size / 2, 2); |
| 503 | break :size .fromInt(size); |
| 510 | 504 | }; |
| 511 | | const ptr = arena.child_allocator.rawAlloc(size, .of(Node), @returnAddress()) orelse |
| 505 | const ptr = arena.child_allocator.rawAlloc(size.toInt(), .of(Node), @returnAddress()) orelse |
| 512 | 506 | return null; |
| 513 | 507 | const new_node: *Node = @ptrCast(@alignCast(ptr)); |
| 514 | 508 | new_node.* = .{ |
| 515 | | .size = .fromInt(size), |
| 509 | .size = size, |
| 516 | 510 | .end_index = undefined, // set below |
| 517 | 511 | .next = undefined, // set below |
| 518 | 512 | }; |
Justus Klausecker <justus@klausecker.de> 2026-03-05 16:26:01+01:00
lib