From 1df1f936b3f4b6f2e8e97248d5779cac1f60a911 Mon Sep 17 00:00:00 2001 From: Robbie Lyman Date: Wed, 9 Sep 2026 14:38:09 -0400 Subject: [PATCH] feat(MultiArrayList): Pointer Stability This commit builds on #19523 to add pointer stability to MultiArrayList, following the lead of the design for HashMap and ArrayList. Since initializing MultiArrayList with a struct literal is much more common within the compiler than for ArrayList, I have chosen to make the field a default init, although I think this is not the best design. Similar to ArrayList, I have chosen to interpret "pointer stability" broadly, meaning that existing pointers to (fields of) elements in the MultiArrayList continue to be valid pointers to the same object. Thus `appendAssumeCapacity()` does not invalidate pointers, but `pop()` and `swap()` do. Closes #19327. --- lib/std/multi_array_list.zig | 100 +++++++++++++++++++++++++++++++++-- lib/std/zig/TokenSmith.zig | 1 + 2 files changed, 96 insertions(+), 5 deletions(-) diff --git a/lib/std/multi_array_list.zig b/lib/std/multi_array_list.zig index 26c722bc6cd98d267027a63c13cd76dea009e2a9..99840c6938d2ce815d641bfab5ed8a3d1e9b9c0c 100644 --- a/lib/std/multi_array_list.zig +++ b/lib/std/multi_array_list.zig @@ -26,11 +26,13 @@ pub fn MultiArrayList(comptime T: type) type { bytes: [*]u8 = undefined, len: usize = 0, capacity: usize = 0, + pointer_stability: std.debug.SafetyLock = .{}, pub const empty: Self = .{ .bytes = undefined, .len = 0, .capacity = 0, + .pointer_stability = .{}, }; /// Initialize with capacity to hold exactly `num` elements. @@ -80,13 +82,16 @@ pub fn MultiArrayList(comptime T: type) type { ptrs: [field_names.len][*]u8, len: usize, capacity: usize, + pointer_stability: std.debug.SafetyLock, pub const empty: Slice = .{ .ptrs = undefined, .len = 0, .capacity = 0, + .pointer_stability = .{}, }; + /// The returned pointer may be invalidated by various operations to this Slice. pub fn items(self: Slice, comptime field: Field) []FieldType(field) { const F = FieldType(field); if (self.capacity == 0) { @@ -100,7 +105,9 @@ pub fn MultiArrayList(comptime T: type) type { return casted_ptr[0..self.len]; } + /// Invalidates pre-existing pointers to fields of the element at index `index`. pub fn set(self: *Slice, index: usize, elem: T) void { + self.pointer_stability.assertUnlocked(); const e = switch (@typeInfo(T)) { .@"struct" => elem, .@"union" => Elem.fromT(elem), @@ -123,25 +130,31 @@ pub fn MultiArrayList(comptime T: type) type { }; } + /// Invalidates pre-existing pointers to fields of the element at indices `a` and `b`. pub fn swap(self: Slice, a: usize, b: usize) void { + self.pointer_stability.assertUnlocked(); inline for (@typeInfo(Field).@"enum".field_names) |field_name| { const its = self.items(@field(Field, field_name)); std.mem.swap(@FieldType(T, field_name), &its[a], &its[b]); } } + /// The returned MultiArrayList inherits the locked/unlocked state of this Slice pub fn toMultiArrayList(self: Slice) Self { if (self.ptrs.len == 0 or self.capacity == 0) { - return .{}; + return .empty; } return .{ .bytes = self.ptrs[sizes.fields[0]], .len = self.len, .capacity = self.capacity, + .pointer_stability = self.pointer_stability, }; } + /// Invalidates all element pointers pub fn deinit(self: *Slice, gpa: Allocator) void { + self.pointer_stability.assertUnlocked(); var other = self.toMultiArrayList(); other.deinit(gpa); self.* = undefined; @@ -160,9 +173,27 @@ pub fn MultiArrayList(comptime T: type) type { .ptrs = ptrs, .len = len, .capacity = len, + .pointer_stability = s.pointer_stability, }; } + /// Puts the Slice into a state where any method call that + /// would cause an existing item pointer to become invalidated will + /// instead trigger an assertion. + /// + /// `lockPointers` may be called multiple times. This allows multiple + /// independent users of the hash map to keep it locked simultaneously. + /// + /// `unlockPointers` returns the Slice to the previous state. + pub fn lockPointers(self: *Slice) void { + self.pointer_stability.lockShared(); + } + + /// Undoes one call to `lockPointers`. + pub fn unlockPointers(self: *Slice) void { + self.pointer_stability.unlockShared(); + } + /// This function is used in the debugger pretty formatters in lib/lldb/ to fetch the /// child field order and entry type to facilitate fancy debug printing for this type. fn dbHelper(self: *Slice, child: *Elem, field: *Field, entry: *Entry) void { @@ -219,12 +250,32 @@ pub fn MultiArrayList(comptime T: type) type { }; /// Release all allocated memory. + /// Invalidates all element pointers. pub fn deinit(self: *Self, gpa: Allocator) void { gpa.free(self.allocatedBytes()); self.* = undefined; } + /// Puts the MultiArrayList into a state where any method call that + /// would cause an existing item pointer to become invalidated will + /// instead trigger an assertion. + /// + /// `lockPointers` may be called multiple times. This allows multiple + /// independent users of the hash map to keep it locked simultaneously. + /// + /// `unlockPointers` returns the MultiArrayList to the previous state. + pub fn lockPointers(self: *Self) void { + self.pointer_stability.lockShared(); + } + + /// Undoes one call to `lockPointers`. + pub fn unlockPointers(self: *Self) void { + self.pointer_stability.unlockShared(); + } + /// The caller owns the returned memory. Empties this MultiArrayList. + /// Does not invalidate element pointers. + /// The returned Slice inherits its locked/unlocked state from this MultiArrayList. pub fn toOwnedSlice(self: *Self) Slice { const result = self.slice(); self.* = .empty; @@ -234,11 +285,14 @@ pub fn MultiArrayList(comptime T: type) type { /// Compute pointers to the start of each field of the array. /// If you need to access multiple fields, calling this may /// be more efficient than calling `items()` multiple times. + /// Does not invalidate element pointers. + /// The returned Slice inherits its locked/unlocked state from this MultiArrayList. pub fn slice(self: Self) Slice { var result: Slice = .{ .ptrs = undefined, .len = self.len, .capacity = self.capacity, + .pointer_stability = self.pointer_stability, }; var ptr: [*]u8 = self.bytes; for (sizes.bytes, sizes.fields) |field_size, i| { @@ -251,12 +305,15 @@ pub fn MultiArrayList(comptime T: type) type { /// Get the slice of values for a specified field. /// If you need multiple fields, consider calling slice() /// instead. + /// The returned pointer may be invalidated by various operations to this MultiArrayList. pub fn items(self: Self, comptime field: Field) []FieldType(field) { return self.slice().items(field); } /// Overwrite one array element with new data. + /// Invalidates previously existing pointers to fields of that element. pub fn set(self: *Self, index: usize, elem: T) void { + self.pointer_stability.assertUnlocked(); var slices = self.slice(); slices.set(index, elem); } @@ -266,13 +323,17 @@ pub fn MultiArrayList(comptime T: type) type { return self.slice().get(index); } + /// Invalidates previously existing pointers to fields of the element + /// at indices `a` and `b`. pub fn swap(self: Self, a: usize, b: usize) void { + self.pointer_stability.assertUnlocked(); return self.slice().swap(a, b); } /// Extend the list by 1 element. /// /// Allocates more memory as necessary. + /// Invalidates element pointers if more memory is needed. pub fn append(self: *Self, gpa: Allocator, elem: T) Allocator.Error!void { try self.ensureUnusedCapacity(gpa, 1); self.appendAssumeCapacity(elem); @@ -281,6 +342,7 @@ pub fn MultiArrayList(comptime T: type) type { /// Extend the list by 1 element. /// /// Asserts that capacity is sufficient to hold an additional item. + /// Never invalidates element pointers. pub fn appendAssumeCapacity(self: *Self, elem: T) void { assert(self.len < self.capacity); self.len += 1; @@ -291,6 +353,7 @@ pub fn MultiArrayList(comptime T: type) type { /// /// If capacity is not sufficient to hold an additional /// item, returns `error.OutOfMemory`. + /// Never invalidates element pointers. pub fn appendBounded(self: *Self, elem: T) error{OutOfMemory}!void { if (self.capacity - self.len < 1) return error.OutOfMemory; return appendAssumeCapacity(self, elem); @@ -300,6 +363,7 @@ pub fn MultiArrayList(comptime T: type) type { /// index with uninitialized data. /// /// Allocates more memory as necessary. + /// Invalidates element pointers if more memory is needed. pub fn addOne(self: *Self, gpa: Allocator) Allocator.Error!usize { try self.ensureUnusedCapacity(gpa, 1); return self.addOneAssumeCapacity(); @@ -309,6 +373,7 @@ pub fn MultiArrayList(comptime T: type) type { /// index with uninitialized data. /// /// Asserts that capacity is sufficient to hold an additional item. + /// Never invalidates element pointers. pub fn addOneAssumeCapacity(self: *Self) usize { assert(self.len < self.capacity); const index = self.len; @@ -321,6 +386,7 @@ pub fn MultiArrayList(comptime T: type) type { /// /// If capacity is not sufficient to hold an additional /// item, returns `error.OutOfMemory`. + /// Never invalidates element pointers. pub fn addOneBounded(self: *Self) error{OutOfMemory}!usize { if (self.capacity - self.len < 1) return error.OutOfMemory; return addOneAssumeCapacity(self); @@ -329,6 +395,7 @@ pub fn MultiArrayList(comptime T: type) type { /// Remove and return the last element from the list, or return `null` if list is empty. /// Invalidates pointers to fields of the removed element. pub fn pop(self: *Self) ?T { + self.pointer_stability.assertUnlocked(); if (self.len == 0) return null; const val = self.get(self.len - 1); self.len -= 1; @@ -340,6 +407,7 @@ pub fn MultiArrayList(comptime T: type) type { /// sets the given index to the specified element. /// /// Allocates more memory as necessary. + /// Invalidates pointers to shifted elements. pub fn insert(self: *Self, gpa: Allocator, index: usize, elem: T) !void { try self.ensureUnusedCapacity(gpa, 1); self.insertAssumeCapacity(index, elem); @@ -350,9 +418,11 @@ pub fn MultiArrayList(comptime T: type) type { /// sets the given index to the specified element. /// /// Asserts that capacity is sufficient to hold an additional item. + /// Invalidates pointers to shifted elements. pub fn insertAssumeCapacity(self: *Self, index: usize, elem: T) void { assert(self.len < self.capacity); assert(index <= self.len); + self.pointer_stability.assertUnlocked(); self.len += 1; const entry = switch (@typeInfo(T)) { .@"struct" => elem, @@ -376,6 +446,7 @@ pub fn MultiArrayList(comptime T: type) type { /// /// If capacity is not sufficient to hold an additional /// item, returns `error.OutOfMemory`. + /// Invalidates pointers to shifted elements. pub fn insertBounded(self: *Self, index: usize, elem: T) error{OutOfMemory}!void { if (self.capacity - self.len < 1) return error.OutOfMemory; return insertAssumeCapacity(self, index, elem); @@ -384,7 +455,9 @@ pub fn MultiArrayList(comptime T: type) type { /// Remove the specified item from the list, swapping the last /// item in the list into its position. Fast, but does not /// retain list ordering. + /// Invalidates pointers to fields of the item at the end of the list. pub fn swapRemove(self: *Self, index: usize) void { + self.pointer_stability.assertUnlocked(); const slices = self.slice(); inline for (field_names, 0..) |_, i| { const field_slice = slices.items(@as(Field, @fromBackingInt(@intCast(i)))); @@ -396,7 +469,9 @@ pub fn MultiArrayList(comptime T: type) type { /// Remove the specified item from the list, shifting items /// after it to preserve order. + /// Invalidates pointers to shifted elements. pub fn orderedRemove(self: *Self, index: usize) void { + self.pointer_stability.assertUnlocked(); const slices = self.slice(); inline for (field_names, 0..) |_, field_index| { const field_slice = slices.items(@as(Field, @fromBackingInt(@intCast(field_index)))); @@ -423,6 +498,7 @@ pub fn MultiArrayList(comptime T: type) type { /// Invalidates element pointers beyond the first deleted index. pub fn orderedRemoveMany(self: *Self, sorted_indexes: []const usize) void { if (sorted_indexes.len == 0) return; + self.pointer_stability.assertUnlocked(); const slices = self.slice(); var shift: usize = 1; for (sorted_indexes[0 .. sorted_indexes.len - 1], sorted_indexes[1..]) |removed, end| { @@ -447,6 +523,7 @@ pub fn MultiArrayList(comptime T: type) type { /// Adjust the list's length to `new_len`. /// Does not initialize added items, if any. + /// Invalidates element pointers if additional memory is required. pub fn resize(self: *Self, gpa: Allocator, new_len: usize) Allocator.Error!void { try self.ensureTotalCapacity(gpa, new_len); self.len = new_len; @@ -455,11 +532,12 @@ pub fn MultiArrayList(comptime T: type) type { /// Attempt to reduce allocated capacity to `new_len`. /// If `new_len` is greater than zero, this may fail to reduce the capacity, /// but the data remains intact and the length is updated to new_len. + /// Invalidates element pointers if additional memory is required. pub fn shrinkAndFree(self: *Self, gpa: Allocator, new_len: usize) void { if (new_len == 0) return clearAndFree(self, gpa); - assert(new_len <= self.capacity); assert(new_len <= self.len); + self.pointer_stability.assertUnlocked(); const other_bytes = gpa.alignedAlloc(u8, sizes.big_align, capacityInBytes(new_len)) catch { const self_slice = self.slice(); @@ -476,7 +554,7 @@ pub fn MultiArrayList(comptime T: type) type { self.len = new_len; return; }; - var other = Self{ + var other: Self = .{ .bytes = other_bytes.ptr, .capacity = new_len, .len = new_len, @@ -494,7 +572,9 @@ pub fn MultiArrayList(comptime T: type) type { self.* = other; } + /// Invalidates all element pointers. pub fn clearAndFree(self: *Self, gpa: Allocator) void { + self.pointer_stability.assertUnlocked(); gpa.free(self.allocatedBytes()); self.* = .empty; } @@ -503,11 +583,13 @@ pub fn MultiArrayList(comptime T: type) type { /// Invalidates pointers to elements `items[new_len..]`. /// Keeps capacity the same. pub fn shrinkRetainingCapacity(self: *Self, new_len: usize) void { + self.pointer_stability.assertUnlocked(); self.len = new_len; } /// Invalidates all element pointers. pub fn clearRetainingCapacity(self: *Self) void { + self.pointer_stability.assertUnlocked(); self.len = 0; } @@ -542,6 +624,7 @@ pub fn MultiArrayList(comptime T: type) type { /// `new_capacity` must be greater or equal to `len`. pub fn setCapacity(self: *Self, gpa: Allocator, new_capacity: usize) Allocator.Error!void { assert(new_capacity >= self.len); + self.pointer_stability.assertUnlocked(); const new_bytes = try gpa.alignedAlloc(u8, sizes.big_align, capacityInBytes(new_capacity)); if (self.len == 0) { gpa.free(self.allocatedBytes()); @@ -549,7 +632,7 @@ pub fn MultiArrayList(comptime T: type) type { self.capacity = new_capacity; return; } - var other = Self{ + var other: Self = .{ .bytes = new_bytes.ptr, .capacity = new_capacity, .len = self.len, @@ -568,8 +651,9 @@ pub fn MultiArrayList(comptime T: type) type { /// Create a copy of this list with a new backing store, /// using the specified allocator. + /// The returned MultiArrayList is in an unlocked state. pub fn clone(self: Self, gpa: Allocator) Allocator.Error!Self { - var result = Self{}; + var result: Self = .{}; errdefer result.deinit(gpa); try result.ensureTotalCapacity(gpa, self.len); result.len = self.len; @@ -584,9 +668,11 @@ pub fn MultiArrayList(comptime T: type) type { return result; } + /// Invalidates element pointers. /// `ctx` has the following method: /// `fn lessThan(ctx: @TypeOf(ctx), a_index: usize, b_index: usize) bool` fn sortInternal(self: Self, a: usize, b: usize, ctx: anytype, comptime mode: std.sort.Mode) void { + self.pointer_stability.assertUnlocked(); const sort_context: struct { sub_ctx: @TypeOf(ctx), slice: Slice, @@ -616,6 +702,7 @@ pub fn MultiArrayList(comptime T: type) type { } /// This function guarantees a stable sort, i.e the relative order of equal elements is preserved during sorting. + /// Invalidates element pointers. /// Read more about stable sorting here: https://en.wikipedia.org/wiki/Sorting_algorithm#Stability /// If this guarantee does not matter, `sortUnstable` might be a faster alternative. /// `ctx` has the following method: @@ -625,6 +712,7 @@ pub fn MultiArrayList(comptime T: type) type { } /// Sorts only the subsection of items between indices `a` and `b` (excluding `b`) + /// Invalidates element pointers. /// This function guarantees a stable sort, i.e the relative order of equal elements is preserved during sorting. /// Read more about stable sorting here: https://en.wikipedia.org/wiki/Sorting_algorithm#Stability /// If this guarantee does not matter, `sortSpanUnstable` might be a faster alternative. @@ -635,6 +723,7 @@ pub fn MultiArrayList(comptime T: type) type { } /// This function does NOT guarantee a stable sort, i.e the relative order of equal elements may change during sorting. + /// Invalidates element pointers. /// Due to the weaker guarantees of this function, this may be faster than the stable `sort` method. /// Read more about stable sorting here: https://en.wikipedia.org/wiki/Sorting_algorithm#Stability /// `ctx` has the following method: @@ -644,6 +733,7 @@ pub fn MultiArrayList(comptime T: type) type { } /// Sorts only the subsection of items between indices `a` and `b` (excluding `b`) + /// Invalidates element pointers. /// This function does NOT guarantee a stable sort, i.e the relative order of equal elements may change during sorting. /// Due to the weaker guarantees of this function, this may be faster than the stable `sortSpan` method. /// Read more about stable sorting here: https://en.wikipedia.org/wiki/Sorting_algorithm#Stability diff --git a/lib/std/zig/TokenSmith.zig b/lib/std/zig/TokenSmith.zig index 3f8cb2bec05dd15b564643633d484c30e8d60842..cdc30da0f2f9cba86bf0b508d1f5b627f25feb9a 100644 --- a/lib/std/zig/TokenSmith.zig +++ b/lib/std/zig/TokenSmith.zig @@ -246,6 +246,7 @@ pub fn list(t: *TokenSmith) TokenList.Slice { .ptrs = undefined, .len = t.tags_len, .capacity = t.tags_len, + .pointer_stability = .{}, }; comptime std.debug.assert(slice.ptrs.len == 2); slice.ptrs[@backingInt(TokenList.Field.tag)] = @ptrCast(&t.tag_buf); -- 2.54.0