| ... | ... | @@ -207,17 +207,20 @@ pub fn reset(arena: *ArenaAllocator, mode: ResetMode) bool { |
| 207 | 207 | |
| 208 | 208 | /// Concurrent accesses to node pointers generally have to have acquire/release |
| 209 | 209 | /// semantics to guarantee that newly allocated notes are in a valid state when |
| 210 | | /// being inserted into a list. Exceptions are possible, e.g. a CAS loop that |
| 210 | /// being inserted into a list. Exceptions are possible, e.g. a cmpxchg loop that |
| 211 | 211 | /// never accesses the node returned on failure can use monotonic semantics on |
| 212 | 212 | /// failure, but must still use release semantics on success to protect the node |
| 213 | 213 | /// it's trying to push. |
| 214 | 214 | const Node = struct { |
| 215 | 215 | /// Only meant to be accessed indirectly via the methods supplied by this type, |
| 216 | 216 | /// except if the node is owned by the thread accessing it. |
| 217 | | /// Must always be an even number to accomodate `resize` bit. |
| 217 | /// Must always be an even number to accommodate `resize` bit. |
| 218 | 218 | size: Size, |
| 219 | | /// Concurrent accesses to `end_index` can be monotonic as long as its value |
| 220 | | /// is compared to a version of `size` before using it to access memory. |
| 219 | /// Any increase of `end_index` has to use acquire semantics; |
| 220 | /// any decrease of `end_index` that invalidates (formerly) active allocations |
| 221 | /// has to use release semantics. |
| 222 | /// This guarantees that all accesses to memory that's about to be freed |
| 223 | /// happen-before the free is published. |
| 221 | 224 | /// Since `size` can only grow and never shrink, memory access depending on |
| 222 | 225 | /// any `end_index` <= any `size` can never be OOB. |
| 223 | 226 | end_index: usize, |
| ... | ... | @@ -319,11 +322,6 @@ fn pushFreeList(arena: *ArenaAllocator, first: *Node, last: *Node) void { |
| 319 | 322 | } |
| 320 | 323 | } |
| 321 | 324 | |
| 322 | | fn sliceContainsSlice(container: []u8, slice: []u8) bool { |
| 323 | | return @intFromPtr(slice.ptr) >= @intFromPtr(container.ptr) and |
| 324 | | @intFromPtr(slice.ptr + slice.len) <= @intFromPtr(container.ptr + container.len); |
| 325 | | } |
| 326 | | |
| 327 | 325 | fn alignedIndex(buf_ptr: [*]u8, end_index: usize, alignment: Alignment) usize { |
| 328 | 326 | // Wrapping arithmetic to avoid overflows since `end_index` isn't bounded by |
| 329 | 327 | // `size`. This is always ok since the max alignment in byte units is also |
| ... | ... | @@ -357,10 +355,17 @@ fn alloc(ctx: *anyopaque, n: usize, alignment: Alignment, ret_addr: usize) ?[*]u |
| 357 | 355 | // with a single cmpxchg afterwards, which may fail. |
| 358 | 356 | |
| 359 | 357 | const alignable = n + alignment.toByteUnits() - 1; |
| 360 | | const end_index = @atomicRmw(usize, &node.end_index, .Add, alignable, .monotonic); |
| 358 | const end_index = @atomicRmw(usize, &node.end_index, .Add, alignable, .acquire); // acquire any memory that may have been freed |
| 361 | 359 | const aligned_index = alignedIndex(buf.ptr, end_index, alignment); |
| 362 | 360 | assert(end_index + alignable >= aligned_index + n); |
| 363 | | _ = @cmpxchgStrong(usize, &node.end_index, end_index + alignable, aligned_index + n, .monotonic, .monotonic); |
| 361 | _ = @cmpxchgStrong( |
| 362 | usize, |
| 363 | &node.end_index, |
| 364 | end_index + alignable, |
| 365 | aligned_index + n, |
| 366 | .monotonic, // no need to release alignment padding; there's no one accessing it! |
| 367 | .monotonic, |
| 368 | ); |
| 364 | 369 | |
| 365 | 370 | if (aligned_index + n > buf.len) break :first_node .{ node, buf.len }; |
| 366 | 371 | return buf[aligned_index..][0..n].ptr; |
| ... | ... | @@ -382,7 +387,7 @@ fn alloc(ctx: *anyopaque, n: usize, alignment: Alignment, ret_addr: usize) ?[*]u |
| 382 | 387 | const new_size = mem.alignForward(usize, @sizeOf(Node) + aligned_index + n, 2); |
| 383 | 388 | |
| 384 | 389 | if (new_size <= allocated_slice.len) { |
| 385 | | // a `resize` or `free` call managed to sneak in and we need to |
| 390 | // A `resize` or `free` call managed to sneak in and we need to |
| 386 | 391 | // guarantee that `size` is only ever increased; retry! |
| 387 | 392 | continue :retry; |
| 388 | 393 | } |
| ... | ... | @@ -390,14 +395,16 @@ fn alloc(ctx: *anyopaque, n: usize, alignment: Alignment, ret_addr: usize) ?[*]u |
| 390 | 395 | if (arena.child_allocator.rawResize(allocated_slice, .of(Node), new_size, @returnAddress())) { |
| 391 | 396 | size = new_size; |
| 392 | 397 | |
| 393 | | if (@cmpxchgStrong( // strong because a spurious failure could result in suboptimal usage of this node |
| 398 | // strong because a spurious failure could result in suboptimal |
| 399 | // usage of this node |
| 400 | if (null == @cmpxchgStrong( |
| 394 | 401 | usize, |
| 395 | 402 | &node.end_index, |
| 396 | 403 | end_index, |
| 397 | 404 | aligned_index + n, |
| 405 | .acquire, // acquire any memory that may have been freed |
| 398 | 406 | .monotonic, |
| 399 | | .monotonic, |
| 400 | | ) == null) { |
| 407 | )) { |
| 401 | 408 | const new_buf = allocated_slice.ptr[0..new_size][@sizeOf(Node)..]; |
| 402 | 409 | return new_buf[aligned_index..][0..n].ptr; |
| 403 | 410 | } |
| ... | ... | @@ -548,40 +555,48 @@ fn resize(ctx: *anyopaque, memory: []u8, alignment: Alignment, new_len: usize, r |
| 548 | 555 | assert(new_len > 0); |
| 549 | 556 | |
| 550 | 557 | const node = arena.loadFirstNode().?; |
| 551 | | const buf = node.loadBuf(); |
| 552 | | |
| 553 | | if (!sliceContainsSlice(buf, memory)) { |
| 554 | | // Not within current node. |
| 555 | | return new_len <= memory.len; |
| 556 | | } |
| 558 | const buf_ptr = @as([*]u8, @ptrCast(node)) + @sizeOf(Node); |
| 557 | 559 | |
| 558 | 560 | const cur_end_index = @atomicLoad(usize, &node.end_index, .monotonic); |
| 559 | 561 | |
| 560 | | if (buf.ptr + cur_end_index != memory.ptr + memory.len) { |
| 561 | | // It's not the most recent allocation, so it cannot be expanded. |
| 562 | if (buf_ptr + cur_end_index != memory.ptr + memory.len) { |
| 563 | // It's not the most recent allocation, so it cannot be expanded, |
| 564 | // but it's fine if they want to make it smaller. |
| 562 | 565 | return new_len <= memory.len; |
| 563 | 566 | } |
| 564 | 567 | |
| 565 | | const new_end_index: usize = new_end_index: { |
| 566 | | if (memory.len >= new_len) { |
| 567 | | break :new_end_index cur_end_index - (memory.len - new_len); |
| 568 | | } |
| 569 | | if (buf.len - cur_end_index >= new_len - memory.len) { |
| 570 | | break :new_end_index cur_end_index + (new_len - memory.len); |
| 571 | | } |
| 572 | | return false; |
| 573 | | }; |
| 574 | | assert(buf.ptr + new_end_index == memory.ptr + new_len); |
| 568 | if (new_len <= memory.len) { |
| 569 | const new_end_index = cur_end_index - (memory.len - new_len); |
| 570 | assert(buf_ptr + new_end_index == memory.ptr + new_len); |
| 571 | |
| 572 | _ = @cmpxchgStrong( |
| 573 | usize, |
| 574 | &node.end_index, |
| 575 | cur_end_index, |
| 576 | new_end_index, |
| 577 | .release, // release freed memory |
| 578 | .monotonic, |
| 579 | ); |
| 580 | return true; // Shrinking allocations should always succeed. |
| 581 | } |
| 575 | 582 | |
| 576 | | return null == @cmpxchgStrong( |
| 577 | | usize, |
| 578 | | &node.end_index, |
| 579 | | cur_end_index, |
| 580 | | new_end_index, |
| 581 | | .monotonic, |
| 582 | | .monotonic, |
| 583 | | ) or |
| 584 | | new_len <= memory.len; // Shrinking allocations should always succeed. |
| 583 | // Saturating arithmetic because `end_index` is not guaranteed to be `<= size`. |
| 584 | // The allocation we're trying to resize *could* belong to a different node! |
| 585 | if (node.loadBuf().len -| cur_end_index >= new_len - memory.len) { |
| 586 | const new_end_index = cur_end_index + (new_len - memory.len); |
| 587 | assert(buf_ptr + new_end_index == memory.ptr + new_len); |
| 588 | |
| 589 | return null == @cmpxchgStrong( |
| 590 | usize, |
| 591 | &node.end_index, |
| 592 | cur_end_index, |
| 593 | new_end_index, |
| 594 | .acquire, // acquire any memory that may have been freed |
| 595 | .monotonic, |
| 596 | ); |
| 597 | } |
| 598 | |
| 599 | return false; |
| 585 | 600 | } |
| 586 | 601 | |
| 587 | 602 | fn remap(ctx: *anyopaque, memory: []u8, alignment: Alignment, new_len: usize, ret_addr: usize) ?[*]u8 { |
| ... | ... | @@ -596,29 +611,24 @@ fn free(ctx: *anyopaque, memory: []u8, alignment: Alignment, ret_addr: usize) vo |
| 596 | 611 | assert(memory.len > 0); |
| 597 | 612 | |
| 598 | 613 | const node = arena.loadFirstNode().?; |
| 599 | | const buf = node.loadBuf(); |
| 600 | | |
| 601 | | if (!sliceContainsSlice(buf, memory)) { |
| 602 | | // Not within current node; we cannot free it. |
| 603 | | return; |
| 604 | | } |
| 614 | const buf_ptr = @as([*]u8, @ptrCast(node)) + @sizeOf(Node); |
| 605 | 615 | |
| 606 | 616 | const cur_end_index = @atomicLoad(usize, &node.end_index, .monotonic); |
| 607 | 617 | |
| 608 | | if (buf.ptr + cur_end_index != memory.ptr + memory.len) { |
| 618 | if (buf_ptr + cur_end_index != memory.ptr + memory.len) { |
| 609 | 619 | // Not the most recent allocation; we cannot free it. |
| 610 | 620 | return; |
| 611 | 621 | } |
| 612 | 622 | |
| 613 | 623 | const new_end_index = cur_end_index - memory.len; |
| 614 | | assert(buf.ptr + new_end_index == memory.ptr); |
| 624 | assert(buf_ptr + new_end_index == memory.ptr); |
| 615 | 625 | |
| 616 | 626 | _ = @cmpxchgStrong( |
| 617 | 627 | usize, |
| 618 | 628 | &node.end_index, |
| 619 | 629 | cur_end_index, |
| 620 | 630 | new_end_index, |
| 621 | | .monotonic, |
| 631 | .release, // release freed memory |
| 622 | 632 | .monotonic, |
| 623 | 633 | ); |
| 624 | 634 | } |
Alex Rønne Petersen <alex@alexrp.com> 2026-03-25 18:27:03+01:00