Commit 1df1f936b3f4b6f2e8e97248d5779cac1f60a911 | zig

author	Robbie Lyman <me@robbielyman.com> 2026-09-09 14:38:09-04:00
committer	Andrew Kelley <andrew@ziglang.org> 2026-10-09 23:14:28+02:00
log	1df1f936b3f4b6f2e8e97248d5779cac1f60a911
tree	25ba8d52a9cbe9beab28ed36b96c2dd73c9e3419
parent	d69634c10d2e4ca4af834a0644136ac1af72cd26

feat(MultiArrayList): Pointer Stability

This commit builds on #19523 to add pointer stability to MultiArrayList,
following the lead of the design for HashMap and ArrayList. Since
initializing MultiArrayList with a struct literal is much more common
within the compiler than for ArrayList, I have chosen to make the field
a default init, although I think this is not the best design.

Similar to ArrayList, I have chosen to interpret "pointer stability"
broadly, meaning that existing pointers to (fields of) elements in the
MultiArrayList continue to be valid pointers to the same object. Thus
`appendAssumeCapacity()` does not invalidate pointers, but `pop()` and
`swap()` do.

Closes #19327.

2 files changed, 96 insertions(+), 5 deletions(-)

lib
- std
  - multi_array_list.zig
  - zig
    TokenSmith.zig

lib/std/multi_array_list.zig+95-5

 @@ -26,11 +26,13 @@ pub fn MultiArrayList(comptime T: type) type {
         bytes: [*]u8 = undefined,
         len: usize = 0,
         capacity: usize = 0,
+        pointer_stability: std.debug.SafetyLock = .{},
         pub const empty: Self = .{
             .bytes = undefined,
             .len = 0,
             .capacity = 0,
+            .pointer_stability = .{},
         };
         /// Initialize with capacity to hold exactly `num` elements.
 @@ -80,13 +82,16 @@ pub fn MultiArrayList(comptime T: type) type {
             ptrs: [field_names.len][*]u8,
             len: usize,
             capacity: usize,
+            pointer_stability: std.debug.SafetyLock,
             pub const empty: Slice = .{
                 .ptrs = undefined,
                 .len = 0,
                 .capacity = 0,
+                .pointer_stability = .{},
             };
+            /// The returned pointer may be invalidated by various operations to this Slice.
             pub fn items(self: Slice, comptime field: Field) []FieldType(field) {
                 const F = FieldType(field);
                 if (self.capacity == 0) {
 @@ -100,7 +105,9 @@ pub fn MultiArrayList(comptime T: type) type {
                 return casted_ptr[0..self.len];
             }
+            /// Invalidates pre-existing pointers to fields of the element at index `index`.
             pub fn set(self: *Slice, index: usize, elem: T) void {
+                self.pointer_stability.assertUnlocked();
                 const e = switch (@typeInfo(T)) {
                     .@"struct" => elem,
                     .@"union" => Elem.fromT(elem),
 @@ -123,25 +130,31 @@ pub fn MultiArrayList(comptime T: type) type {
                 };
             }
+            /// Invalidates pre-existing pointers to fields of the element at indices `a` and `b`.
             pub fn swap(self: Slice, a: usize, b: usize) void {
+                self.pointer_stability.assertUnlocked();
                 inline for (@typeInfo(Field).@"enum".field_names) |field_name| {
                     const its = self.items(@field(Field, field_name));
                     std.mem.swap(@FieldType(T, field_name), &its[a], &its[b]);
                 }
             }
+            /// The returned MultiArrayList inherits the locked/unlocked state of this Slice
             pub fn toMultiArrayList(self: Slice) Self {
                 if (self.ptrs.len == 0 or self.capacity == 0) {
-                    return .{};
+                    return .empty;
                 }
                 return .{
                     .bytes = self.ptrs[sizes.fields[0]],
                     .len = self.len,
                     .capacity = self.capacity,
+                    .pointer_stability = self.pointer_stability,
                 };
             }
+            /// Invalidates all element pointers
             pub fn deinit(self: *Slice, gpa: Allocator) void {
+                self.pointer_stability.assertUnlocked();
                 var other = self.toMultiArrayList();
                 other.deinit(gpa);
                 self.* = undefined;
 @@ -160,9 +173,27 @@ pub fn MultiArrayList(comptime T: type) type {
                     .ptrs = ptrs,
                     .len = len,
                     .capacity = len,
+                    .pointer_stability = s.pointer_stability,
                 };
             }
+            /// Puts the Slice into a state where any method call that
+            /// would cause an existing item pointer to become invalidated will
+            /// instead trigger an assertion.
+            ///
+            /// `lockPointers` may be called multiple times. This allows multiple
+            /// independent users of the hash map to keep it locked simultaneously.
+            ///
+            /// `unlockPointers` returns the Slice to the previous state.
+            pub fn lockPointers(self: *Slice) void {
+                self.pointer_stability.lockShared();
+            }
+            /// Undoes one call to `lockPointers`.
+            pub fn unlockPointers(self: *Slice) void {
+                self.pointer_stability.unlockShared();
+            }
             /// This function is used in the debugger pretty formatters in lib/lldb/ to fetch the
             /// child field order and entry type to facilitate fancy debug printing for this type.
             fn dbHelper(self: *Slice, child: *Elem, field: *Field, entry: *Entry) void {
 @@ -219,12 +250,32 @@ pub fn MultiArrayList(comptime T: type) type {
         };
         /// Release all allocated memory.
+        /// Invalidates all element pointers.
         pub fn deinit(self: *Self, gpa: Allocator) void {
             gpa.free(self.allocatedBytes());
             self.* = undefined;
         }
+        /// Puts the MultiArrayList into a state where any method call that
+        /// would cause an existing item pointer to become invalidated will
+        /// instead trigger an assertion.
+        ///
+        /// `lockPointers` may be called multiple times. This allows multiple
+        /// independent users of the hash map to keep it locked simultaneously.
+        ///
+        /// `unlockPointers` returns the MultiArrayList to the previous state.
+        pub fn lockPointers(self: *Self) void {
+            self.pointer_stability.lockShared();
+        }
+        /// Undoes one call to `lockPointers`.
+        pub fn unlockPointers(self: *Self) void {
+            self.pointer_stability.unlockShared();
+        }
         /// The caller owns the returned memory. Empties this MultiArrayList.
+        /// Does not invalidate element pointers.
+        /// The returned Slice inherits its locked/unlocked state from this MultiArrayList.
         pub fn toOwnedSlice(self: *Self) Slice {
             const result = self.slice();
             self.* = .empty;
 @@ -234,11 +285,14 @@ pub fn MultiArrayList(comptime T: type) type {
         /// Compute pointers to the start of each field of the array.
         /// If you need to access multiple fields, calling this may
         /// be more efficient than calling `items()` multiple times.
+        /// Does not invalidate element pointers.
+        /// The returned Slice inherits its locked/unlocked state from this MultiArrayList.
         pub fn slice(self: Self) Slice {
             var result: Slice = .{
                 .ptrs = undefined,
                 .len = self.len,
                 .capacity = self.capacity,
+                .pointer_stability = self.pointer_stability,
             };
             var ptr: [*]u8 = self.bytes;
             for (sizes.bytes, sizes.fields) |field_size, i| {
 @@ -251,12 +305,15 @@ pub fn MultiArrayList(comptime T: type) type {
         /// Get the slice of values for a specified field.
         /// If you need multiple fields, consider calling slice()
         /// instead.
+        /// The returned pointer may be invalidated by various operations to this MultiArrayList.
         pub fn items(self: Self, comptime field: Field) []FieldType(field) {
             return self.slice().items(field);
         }
         /// Overwrite one array element with new data.
+        /// Invalidates previously existing pointers to fields of that element.
         pub fn set(self: *Self, index: usize, elem: T) void {
+            self.pointer_stability.assertUnlocked();
             var slices = self.slice();
             slices.set(index, elem);
         }
 @@ -266,13 +323,17 @@ pub fn MultiArrayList(comptime T: type) type {
             return self.slice().get(index);
         }
+        /// Invalidates previously existing pointers to fields of the element
+        /// at indices `a` and `b`.
         pub fn swap(self: Self, a: usize, b: usize) void {
+            self.pointer_stability.assertUnlocked();
             return self.slice().swap(a, b);
         }
         /// Extend the list by 1 element.
         ///
         /// Allocates more memory as necessary.
+        /// Invalidates element pointers if more memory is needed.
         pub fn append(self: *Self, gpa: Allocator, elem: T) Allocator.Error!void {
             try self.ensureUnusedCapacity(gpa, 1);
             self.appendAssumeCapacity(elem);
 @@ -281,6 +342,7 @@ pub fn MultiArrayList(comptime T: type) type {
         /// Extend the list by 1 element.
         ///
         /// Asserts that capacity is sufficient to hold an additional item.
+        /// Never invalidates element pointers.
         pub fn appendAssumeCapacity(self: *Self, elem: T) void {
             assert(self.len < self.capacity);
             self.len += 1;
 @@ -291,6 +353,7 @@ pub fn MultiArrayList(comptime T: type) type {
         ///
         /// If capacity is not sufficient to hold an additional
         /// item, returns `error.OutOfMemory`.
+        /// Never invalidates element pointers.
         pub fn appendBounded(self: *Self, elem: T) error{OutOfMemory}!void {
             if (self.capacity - self.len < 1) return error.OutOfMemory;
             return appendAssumeCapacity(self, elem);
 @@ -300,6 +363,7 @@ pub fn MultiArrayList(comptime T: type) type {
         /// index with uninitialized data.
         ///
         /// Allocates more memory as necessary.
+        /// Invalidates element pointers if more memory is needed.
         pub fn addOne(self: *Self, gpa: Allocator) Allocator.Error!usize {
             try self.ensureUnusedCapacity(gpa, 1);
             return self.addOneAssumeCapacity();
 @@ -309,6 +373,7 @@ pub fn MultiArrayList(comptime T: type) type {
         /// index with uninitialized data.
         ///
         /// Asserts that capacity is sufficient to hold an additional item.
+        /// Never invalidates element pointers.
         pub fn addOneAssumeCapacity(self: *Self) usize {
             assert(self.len < self.capacity);
             const index = self.len;
 @@ -321,6 +386,7 @@ pub fn MultiArrayList(comptime T: type) type {
         ///
         /// If capacity is not sufficient to hold an additional
         /// item, returns `error.OutOfMemory`.
+        /// Never invalidates element pointers.
         pub fn addOneBounded(self: *Self) error{OutOfMemory}!usize {
             if (self.capacity - self.len < 1) return error.OutOfMemory;
             return addOneAssumeCapacity(self);
 @@ -329,6 +395,7 @@ pub fn MultiArrayList(comptime T: type) type {
         /// Remove and return the last element from the list, or return `null` if list is empty.
         /// Invalidates pointers to fields of the removed element.
         pub fn pop(self: *Self) ?T {
+            self.pointer_stability.assertUnlocked();
             if (self.len == 0) return null;
             const val = self.get(self.len - 1);
             self.len -= 1;
 @@ -340,6 +407,7 @@ pub fn MultiArrayList(comptime T: type) type {
         /// sets the given index to the specified element.
         ///
         /// Allocates more memory as necessary.
+        /// Invalidates pointers to shifted elements.
         pub fn insert(self: *Self, gpa: Allocator, index: usize, elem: T) !void {
             try self.ensureUnusedCapacity(gpa, 1);
             self.insertAssumeCapacity(index, elem);
 @@ -350,9 +418,11 @@ pub fn MultiArrayList(comptime T: type) type {
         /// sets the given index to the specified element.
         ///
         /// Asserts that capacity is sufficient to hold an additional item.
+        /// Invalidates pointers to shifted elements.
         pub fn insertAssumeCapacity(self: *Self, index: usize, elem: T) void {
             assert(self.len < self.capacity);
             assert(index <= self.len);
+            self.pointer_stability.assertUnlocked();
             self.len += 1;
             const entry = switch (@typeInfo(T)) {
                 .@"struct" => elem,
 @@ -376,6 +446,7 @@ pub fn MultiArrayList(comptime T: type) type {
         ///
         /// If capacity is not sufficient to hold an additional
         /// item, returns `error.OutOfMemory`.
+        /// Invalidates pointers to shifted elements.
         pub fn insertBounded(self: *Self, index: usize, elem: T) error{OutOfMemory}!void {
             if (self.capacity - self.len < 1) return error.OutOfMemory;
             return insertAssumeCapacity(self, index, elem);
 @@ -384,7 +455,9 @@ pub fn MultiArrayList(comptime T: type) type {
         /// Remove the specified item from the list, swapping the last
         /// item in the list into its position. Fast, but does not
         /// retain list ordering.
+        /// Invalidates pointers to fields of the item at the end of the list.
         pub fn swapRemove(self: *Self, index: usize) void {
+            self.pointer_stability.assertUnlocked();
             const slices = self.slice();
             inline for (field_names, 0..) |_, i| {
                 const field_slice = slices.items(@as(Field, @fromBackingInt(@intCast(i))));
 @@ -396,7 +469,9 @@ pub fn MultiArrayList(comptime T: type) type {
         /// Remove the specified item from the list, shifting items
         /// after it to preserve order.
+        /// Invalidates pointers to shifted elements.
         pub fn orderedRemove(self: *Self, index: usize) void {
+            self.pointer_stability.assertUnlocked();
             const slices = self.slice();
             inline for (field_names, 0..) |_, field_index| {
                 const field_slice = slices.items(@as(Field, @fromBackingInt(@intCast(field_index))));
 @@ -423,6 +498,7 @@ pub fn MultiArrayList(comptime T: type) type {
         /// Invalidates element pointers beyond the first deleted index.
         pub fn orderedRemoveMany(self: *Self, sorted_indexes: []const usize) void {
             if (sorted_indexes.len == 0) return;
+            self.pointer_stability.assertUnlocked();
             const slices = self.slice();
             var shift: usize = 1;
             for (sorted_indexes[0 .. sorted_indexes.len - 1], sorted_indexes[1..]) |removed, end| {
 @@ -447,6 +523,7 @@ pub fn MultiArrayList(comptime T: type) type {
         /// Adjust the list's length to `new_len`.
         /// Does not initialize added items, if any.
+        /// Invalidates element pointers if additional memory is required.
         pub fn resize(self: *Self, gpa: Allocator, new_len: usize) Allocator.Error!void {
             try self.ensureTotalCapacity(gpa, new_len);
             self.len = new_len;
 @@ -455,11 +532,12 @@ pub fn MultiArrayList(comptime T: type) type {
         /// Attempt to reduce allocated capacity to `new_len`.
         /// If `new_len` is greater than zero, this may fail to reduce the capacity,
         /// but the data remains intact and the length is updated to new_len.
+        /// Invalidates element pointers if additional memory is required.
         pub fn shrinkAndFree(self: *Self, gpa: Allocator, new_len: usize) void {
             if (new_len == 0) return clearAndFree(self, gpa);
             assert(new_len <= self.capacity);
             assert(new_len <= self.len);
+            self.pointer_stability.assertUnlocked();
             const other_bytes = gpa.alignedAlloc(u8, sizes.big_align, capacityInBytes(new_len)) catch {
                 const self_slice = self.slice();
 @@ -476,7 +554,7 @@ pub fn MultiArrayList(comptime T: type) type {
                 self.len = new_len;
                 return;
             };
-            var other = Self{
+            var other: Self = .{
                 .bytes = other_bytes.ptr,
                 .capacity = new_len,
                 .len = new_len,
 @@ -494,7 +572,9 @@ pub fn MultiArrayList(comptime T: type) type {
             self.* = other;
         }
+        /// Invalidates all element pointers.
         pub fn clearAndFree(self: *Self, gpa: Allocator) void {
+            self.pointer_stability.assertUnlocked();
             gpa.free(self.allocatedBytes());
             self.* = .empty;
         }
 @@ -503,11 +583,13 @@ pub fn MultiArrayList(comptime T: type) type {
         /// Invalidates pointers to elements `items[new_len..]`.
         /// Keeps capacity the same.
         pub fn shrinkRetainingCapacity(self: *Self, new_len: usize) void {
+            self.pointer_stability.assertUnlocked();
             self.len = new_len;
         }
         /// Invalidates all element pointers.
         pub fn clearRetainingCapacity(self: *Self) void {
+            self.pointer_stability.assertUnlocked();
             self.len = 0;
         }
 @@ -542,6 +624,7 @@ pub fn MultiArrayList(comptime T: type) type {
         /// `new_capacity` must be greater or equal to `len`.
         pub fn setCapacity(self: *Self, gpa: Allocator, new_capacity: usize) Allocator.Error!void {
             assert(new_capacity >= self.len);
+            self.pointer_stability.assertUnlocked();
             const new_bytes = try gpa.alignedAlloc(u8, sizes.big_align, capacityInBytes(new_capacity));
             if (self.len == 0) {
                 gpa.free(self.allocatedBytes());
 @@ -549,7 +632,7 @@ pub fn MultiArrayList(comptime T: type) type {
                 self.capacity = new_capacity;
                 return;
             }
-            var other = Self{
+            var other: Self = .{
                 .bytes = new_bytes.ptr,
                 .capacity = new_capacity,
                 .len = self.len,
 @@ -568,8 +651,9 @@ pub fn MultiArrayList(comptime T: type) type {
         /// Create a copy of this list with a new backing store,
         /// using the specified allocator.
+        /// The returned MultiArrayList is in an unlocked state.
         pub fn clone(self: Self, gpa: Allocator) Allocator.Error!Self {
-            var result = Self{};
+            var result: Self = .{};
             errdefer result.deinit(gpa);
             try result.ensureTotalCapacity(gpa, self.len);
             result.len = self.len;
 @@ -584,9 +668,11 @@ pub fn MultiArrayList(comptime T: type) type {
             return result;
         }
+        /// Invalidates element pointers.
         /// `ctx` has the following method:
         /// `fn lessThan(ctx: @TypeOf(ctx), a_index: usize, b_index: usize) bool`
         fn sortInternal(self: Self, a: usize, b: usize, ctx: anytype, comptime mode: std.sort.Mode) void {
+            self.pointer_stability.assertUnlocked();
             const sort_context: struct {
                 sub_ctx: @TypeOf(ctx),
                 slice: Slice,
 @@ -616,6 +702,7 @@ pub fn MultiArrayList(comptime T: type) type {
         }
         /// This function guarantees a stable sort, i.e the relative order of equal elements is preserved during sorting.
+        /// Invalidates element pointers.
         /// Read more about stable sorting here: https://en.wikipedia.org/wiki/Sorting_algorithm#Stability
         /// If this guarantee does not matter, `sortUnstable` might be a faster alternative.
         /// `ctx` has the following method:
 @@ -625,6 +712,7 @@ pub fn MultiArrayList(comptime T: type) type {
         }
         /// Sorts only the subsection of items between indices `a` and `b` (excluding `b`)
+        /// Invalidates element pointers.
         /// This function guarantees a stable sort, i.e the relative order of equal elements is preserved during sorting.
         /// Read more about stable sorting here: https://en.wikipedia.org/wiki/Sorting_algorithm#Stability
         /// If this guarantee does not matter, `sortSpanUnstable` might be a faster alternative.
 @@ -635,6 +723,7 @@ pub fn MultiArrayList(comptime T: type) type {
         }
         /// This function does NOT guarantee a stable sort, i.e the relative order of equal elements may change during sorting.
+        /// Invalidates element pointers.
         /// Due to the weaker guarantees of this function, this may be faster than the stable `sort` method.
         /// Read more about stable sorting here: https://en.wikipedia.org/wiki/Sorting_algorithm#Stability
         /// `ctx` has the following method:
 @@ -644,6 +733,7 @@ pub fn MultiArrayList(comptime T: type) type {
         }
         /// Sorts only the subsection of items between indices `a` and `b` (excluding `b`)
+        /// Invalidates element pointers.
         /// This function does NOT guarantee a stable sort, i.e the relative order of equal elements may change during sorting.
         /// Due to the weaker guarantees of this function, this may be faster than the stable `sortSpan` method.
         /// Read more about stable sorting here: https://en.wikipedia.org/wiki/Sorting_algorithm#Stability

lib/std/zig/TokenSmith.zig+1

...	@@ -246,6 +246,7 @@ pub fn list(t: *TokenSmith) TokenList.Slice {	...	@@ -246,6 +246,7 @@ pub fn list(t: *TokenSmith) TokenList.Slice {
246	.ptrs = undefined,	246	.ptrs = undefined,
247	.len = t.tags_len,	247	.len = t.tags_len,
248	.capacity = t.tags_len,	248	.capacity = t.tags_len,
		249	.pointer_stability = .{},
249	};	250	};
250	comptime std.debug.assert(slice.ptrs.len == 2);	251	comptime std.debug.assert(slice.ptrs.len == 2);
251	slice.ptrs[@backingInt(TokenList.Field.tag)] = @ptrCast(&t.tag_buf);	252	slice.ptrs[@backingInt(TokenList.Field.tag)] = @ptrCast(&t.tag_buf);