authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-02-03 19:55:09-08:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-02-06 14:23:23-08:00
log7eeef5fb2b9dc78679f4091e2a8173d07968b3e5
treea5edee99fe9ea1e5c241ec0e7b21268d65a8ff8b
parentdd2fa4f75d3d2b1214fde22081f0b88850d1b55d

std.mem.Allocator: introduce `remap` function to the interface

This one changes the size of an allocation, allowing it to be relocated. However, the implementation will still return `null` if it would be equivalent to new = alloc memcpy(new, old) free(old) Mainly this prepares for taking advantage of `mremap` which I thought would be a bigger deal but apparently is only available on Linux. Still, we should use it on Linux.

7 files changed, 389 insertions(+), 217 deletions(-)

lib/std/array_list.zig+18-19
...@@ -105,21 +105,19 @@ pub fn ArrayListAligned(comptime T: type, comptime alignment: ?u29) type {...@@ -105,21 +105,19 @@ pub fn ArrayListAligned(comptime T: type, comptime alignment: ?u29) type {
105 return result;105 return result;
106 }106 }
107107
108 /// The caller owns the returned memory. Empties this ArrayList,108 /// The caller owns the returned memory. Empties this ArrayList.
109 /// Its capacity is cleared, making deinit() safe but unnecessary to call.109 /// Its capacity is cleared, making `deinit` safe but unnecessary to call.
110 pub fn toOwnedSlice(self: *Self) Allocator.Error!Slice {110 pub fn toOwnedSlice(self: *Self) Allocator.Error!Slice {
111 const allocator = self.allocator;111 const allocator = self.allocator;
112112
113 const old_memory = self.allocatedSlice();113 const old_memory = self.allocatedSlice();
114 if (allocator.resize(old_memory, self.items.len)) {114 if (allocator.remap(old_memory, self.items.len)) |new_items| {
115 const result = self.items;
116 self.* = init(allocator);115 self.* = init(allocator);
117 return result;116 return new_items;
118 }117 }
119118
120 const new_memory = try allocator.alignedAlloc(T, alignment, self.items.len);119 const new_memory = try allocator.alignedAlloc(T, alignment, self.items.len);
121 @memcpy(new_memory, self.items);120 @memcpy(new_memory, self.items);
122 @memset(self.items, undefined);
123 self.clearAndFree();121 self.clearAndFree();
124 return new_memory;122 return new_memory;
125 }123 }
...@@ -185,8 +183,9 @@ pub fn ArrayListAligned(comptime T: type, comptime alignment: ?u29) type {...@@ -185,8 +183,9 @@ pub fn ArrayListAligned(comptime T: type, comptime alignment: ?u29) type {
185 // extra capacity.183 // extra capacity.
186 const new_capacity = growCapacity(self.capacity, new_len);184 const new_capacity = growCapacity(self.capacity, new_len);
187 const old_memory = self.allocatedSlice();185 const old_memory = self.allocatedSlice();
188 if (self.allocator.resize(old_memory, new_capacity)) {186 if (self.allocator.remap(old_memory, new_capacity)) |new_memory| {
189 self.capacity = new_capacity;187 self.items.ptr = new_memory.ptr;
188 self.capacity = new_memory.len;
190 return addManyAtAssumeCapacity(self, index, count);189 return addManyAtAssumeCapacity(self, index, count);
191 }190 }
192191
...@@ -468,8 +467,9 @@ pub fn ArrayListAligned(comptime T: type, comptime alignment: ?u29) type {...@@ -468,8 +467,9 @@ pub fn ArrayListAligned(comptime T: type, comptime alignment: ?u29) type {
468 // the allocator implementation would pointlessly copy our467 // the allocator implementation would pointlessly copy our
469 // extra capacity.468 // extra capacity.
470 const old_memory = self.allocatedSlice();469 const old_memory = self.allocatedSlice();
471 if (self.allocator.resize(old_memory, new_capacity)) {470 if (self.allocator.remap(old_memory, new_capacity)) |new_memory| {
472 self.capacity = new_capacity;471 self.items.ptr = new_memory.ptr;
472 self.capacity = new_memory.len;
473 } else {473 } else {
474 const new_memory = try self.allocator.alignedAlloc(T, alignment, new_capacity);474 const new_memory = try self.allocator.alignedAlloc(T, alignment, new_capacity);
475 @memcpy(new_memory[0..self.items.len], self.items);475 @memcpy(new_memory[0..self.items.len], self.items);
...@@ -707,15 +707,13 @@ pub fn ArrayListAlignedUnmanaged(comptime T: type, comptime alignment: ?u29) typ...@@ -707,15 +707,13 @@ pub fn ArrayListAlignedUnmanaged(comptime T: type, comptime alignment: ?u29) typ
707 /// Its capacity is cleared, making deinit() safe but unnecessary to call.707 /// Its capacity is cleared, making deinit() safe but unnecessary to call.
708 pub fn toOwnedSlice(self: *Self, allocator: Allocator) Allocator.Error!Slice {708 pub fn toOwnedSlice(self: *Self, allocator: Allocator) Allocator.Error!Slice {
709 const old_memory = self.allocatedSlice();709 const old_memory = self.allocatedSlice();
710 if (allocator.resize(old_memory, self.items.len)) {710 if (allocator.remap(old_memory, self.items.len)) |new_items| {
711 const result = self.items;
712 self.* = .empty;711 self.* = .empty;
713 return result;712 return new_items;
714 }713 }
715714
716 const new_memory = try allocator.alignedAlloc(T, alignment, self.items.len);715 const new_memory = try allocator.alignedAlloc(T, alignment, self.items.len);
717 @memcpy(new_memory, self.items);716 @memcpy(new_memory, self.items);
718 @memset(self.items, undefined);
719 self.clearAndFree(allocator);717 self.clearAndFree(allocator);
720 return new_memory;718 return new_memory;
721 }719 }
...@@ -1031,9 +1029,9 @@ pub fn ArrayListAlignedUnmanaged(comptime T: type, comptime alignment: ?u29) typ...@@ -1031,9 +1029,9 @@ pub fn ArrayListAlignedUnmanaged(comptime T: type, comptime alignment: ?u29) typ
1031 }1029 }
10321030
1033 const old_memory = self.allocatedSlice();1031 const old_memory = self.allocatedSlice();
1034 if (allocator.resize(old_memory, new_len)) {1032 if (allocator.remap(old_memory, new_len)) |new_items| {
1035 self.capacity = new_len;1033 self.capacity = new_items.len;
1036 self.items.len = new_len;1034 self.items = new_items;
1037 return;1035 return;
1038 }1036 }
10391037
...@@ -1099,8 +1097,9 @@ pub fn ArrayListAlignedUnmanaged(comptime T: type, comptime alignment: ?u29) typ...@@ -1099,8 +1097,9 @@ pub fn ArrayListAlignedUnmanaged(comptime T: type, comptime alignment: ?u29) typ
1099 // the allocator implementation would pointlessly copy our1097 // the allocator implementation would pointlessly copy our
1100 // extra capacity.1098 // extra capacity.
1101 const old_memory = self.allocatedSlice();1099 const old_memory = self.allocatedSlice();
1102 if (allocator.resize(old_memory, new_capacity)) {1100 if (allocator.remap(old_memory, new_capacity)) |new_memory| {
1103 self.capacity = new_capacity;1101 self.items.ptr = new_memory.ptr;
1102 self.capacity = new_memory.len;
1104 } else {1103 } else {
1105 const new_memory = try allocator.alignedAlloc(T, alignment, new_capacity);1104 const new_memory = try allocator.alignedAlloc(T, alignment, new_capacity);
1106 @memcpy(new_memory[0..self.items.len], self.items);1105 @memcpy(new_memory[0..self.items.len], self.items);
lib/std/heap/FixedBufferAllocator.zig+21-9
...@@ -9,7 +9,7 @@ end_index: usize,...@@ -9,7 +9,7 @@ end_index: usize,
9buffer: []u8,9buffer: []u8,
1010
11pub fn init(buffer: []u8) FixedBufferAllocator {11pub fn init(buffer: []u8) FixedBufferAllocator {
12 return FixedBufferAllocator{12 return .{
13 .buffer = buffer,13 .buffer = buffer,
14 .end_index = 0,14 .end_index = 0,
15 };15 };
...@@ -22,6 +22,7 @@ pub fn allocator(self: *FixedBufferAllocator) Allocator {...@@ -22,6 +22,7 @@ pub fn allocator(self: *FixedBufferAllocator) Allocator {
22 .vtable = &.{22 .vtable = &.{
23 .alloc = alloc,23 .alloc = alloc,
24 .resize = resize,24 .resize = resize,
25 .remap = remap,
25 .free = free,26 .free = free,
26 },27 },
27 };28 };
...@@ -36,6 +37,7 @@ pub fn threadSafeAllocator(self: *FixedBufferAllocator) Allocator {...@@ -36,6 +37,7 @@ pub fn threadSafeAllocator(self: *FixedBufferAllocator) Allocator {
36 .vtable = &.{37 .vtable = &.{
37 .alloc = threadSafeAlloc,38 .alloc = threadSafeAlloc,
38 .resize = Allocator.noResize,39 .resize = Allocator.noResize,
40 .remap = Allocator.noRemap,
39 .free = Allocator.noFree,41 .free = Allocator.noFree,
40 },42 },
41 };43 };
...@@ -57,10 +59,10 @@ pub fn isLastAllocation(self: *FixedBufferAllocator, buf: []u8) bool {...@@ -57,10 +59,10 @@ pub fn isLastAllocation(self: *FixedBufferAllocator, buf: []u8) bool {
57 return buf.ptr + buf.len == self.buffer.ptr + self.end_index;59 return buf.ptr + buf.len == self.buffer.ptr + self.end_index;
58}60}
5961
60pub fn alloc(ctx: *anyopaque, n: usize, log2_ptr_align: u8, ra: usize) ?[*]u8 {62pub fn alloc(ctx: *anyopaque, n: usize, alignment: mem.Alignment, ra: usize) ?[*]u8 {
61 const self: *FixedBufferAllocator = @ptrCast(@alignCast(ctx));63 const self: *FixedBufferAllocator = @ptrCast(@alignCast(ctx));
62 _ = ra;64 _ = ra;
63 const ptr_align = @as(usize, 1) << @as(Allocator.Log2Align, @intCast(log2_ptr_align));65 const ptr_align = alignment.toByteUnits();
64 const adjust_off = mem.alignPointerOffset(self.buffer.ptr + self.end_index, ptr_align) orelse return null;66 const adjust_off = mem.alignPointerOffset(self.buffer.ptr + self.end_index, ptr_align) orelse return null;
65 const adjusted_index = self.end_index + adjust_off;67 const adjusted_index = self.end_index + adjust_off;
66 const new_end_index = adjusted_index + n;68 const new_end_index = adjusted_index + n;
...@@ -72,12 +74,12 @@ pub fn alloc(ctx: *anyopaque, n: usize, log2_ptr_align: u8, ra: usize) ?[*]u8 {...@@ -72,12 +74,12 @@ pub fn alloc(ctx: *anyopaque, n: usize, log2_ptr_align: u8, ra: usize) ?[*]u8 {
72pub fn resize(74pub fn resize(
73 ctx: *anyopaque,75 ctx: *anyopaque,
74 buf: []u8,76 buf: []u8,
75 log2_buf_align: u8,77 alignment: mem.Alignment,
76 new_size: usize,78 new_size: usize,
77 return_address: usize,79 return_address: usize,
78) bool {80) bool {
79 const self: *FixedBufferAllocator = @ptrCast(@alignCast(ctx));81 const self: *FixedBufferAllocator = @ptrCast(@alignCast(ctx));
80 _ = log2_buf_align;82 _ = alignment;
81 _ = return_address;83 _ = return_address;
82 assert(@inComptime() or self.ownsSlice(buf));84 assert(@inComptime() or self.ownsSlice(buf));
8385
...@@ -99,14 +101,24 @@ pub fn resize(...@@ -99,14 +101,24 @@ pub fn resize(
99 return true;101 return true;
100}102}
101103
104pub fn remap(
105 context: *anyopaque,
106 memory: []u8,
107 alignment: mem.Alignment,
108 new_len: usize,
109 return_address: usize,
110) ?[*]u8 {
111 return if (resize(context, memory, alignment, new_len, return_address)) memory.ptr else null;
112}
113
102pub fn free(114pub fn free(
103 ctx: *anyopaque,115 ctx: *anyopaque,
104 buf: []u8,116 buf: []u8,
105 log2_buf_align: u8,117 alignment: mem.Alignment,
106 return_address: usize,118 return_address: usize,
107) void {119) void {
108 const self: *FixedBufferAllocator = @ptrCast(@alignCast(ctx));120 const self: *FixedBufferAllocator = @ptrCast(@alignCast(ctx));
109 _ = log2_buf_align;121 _ = alignment;
110 _ = return_address;122 _ = return_address;
111 assert(@inComptime() or self.ownsSlice(buf));123 assert(@inComptime() or self.ownsSlice(buf));
112124
...@@ -115,10 +127,10 @@ pub fn free(...@@ -115,10 +127,10 @@ pub fn free(
115 }127 }
116}128}
117129
118fn threadSafeAlloc(ctx: *anyopaque, n: usize, log2_ptr_align: u8, ra: usize) ?[*]u8 {130fn threadSafeAlloc(ctx: *anyopaque, n: usize, alignment: mem.Alignment, ra: usize) ?[*]u8 {
119 const self: *FixedBufferAllocator = @ptrCast(@alignCast(ctx));131 const self: *FixedBufferAllocator = @ptrCast(@alignCast(ctx));
120 _ = ra;132 _ = ra;
121 const ptr_align = @as(usize, 1) << @as(Allocator.Log2Align, @intCast(log2_ptr_align));133 const ptr_align = alignment.toByteUnits();
122 var end_index = @atomicLoad(usize, &self.end_index, .seq_cst);134 var end_index = @atomicLoad(usize, &self.end_index, .seq_cst);
123 while (true) {135 while (true) {
124 const adjust_off = mem.alignPointerOffset(self.buffer.ptr + end_index, ptr_align) orelse return null;136 const adjust_off = mem.alignPointerOffset(self.buffer.ptr + end_index, ptr_align) orelse return null;
lib/std/heap/PageAllocator.zig+60-39
...@@ -12,18 +12,18 @@ const page_size_min = std.heap.page_size_min;...@@ -12,18 +12,18 @@ const page_size_min = std.heap.page_size_min;
12pub const vtable: Allocator.VTable = .{12pub const vtable: Allocator.VTable = .{
13 .alloc = alloc,13 .alloc = alloc,
14 .resize = resize,14 .resize = resize,
15 .remap = remap,
15 .free = free,16 .free = free,
16};17};
1718
18fn alloc(context: *anyopaque, n: usize, log2_align: u8, ra: usize) ?[*]u8 {19fn alloc(context: *anyopaque, n: usize, alignment: mem.Alignment, ra: usize) ?[*]u8 {
19 const requested_alignment: mem.Alignment = @enumFromInt(log2_align);
20 _ = context;20 _ = context;
21 _ = ra;21 _ = ra;
22 assert(n > 0);22 assert(n > 0);
2323
24 const page_size = std.heap.pageSize();24 const page_size = std.heap.pageSize();
25 if (n >= maxInt(usize) - page_size) return null;25 if (n >= maxInt(usize) - page_size) return null;
26 const alignment_bytes = requested_alignment.toByteUnits();26 const alignment_bytes = alignment.toByteUnits();
2727
28 if (native_os == .windows) {28 if (native_os == .windows) {
29 // According to official documentation, VirtualAlloc aligns to page29 // According to official documentation, VirtualAlloc aligns to page
...@@ -103,22 +103,52 @@ fn alloc(context: *anyopaque, n: usize, log2_align: u8, ra: usize) ?[*]u8 {...@@ -103,22 +103,52 @@ fn alloc(context: *anyopaque, n: usize, log2_align: u8, ra: usize) ?[*]u8 {
103103
104fn resize(104fn resize(
105 context: *anyopaque,105 context: *anyopaque,
106 buf_unaligned: []u8,106 memory: []u8,
107 log2_buf_align: u8,107 alignment: mem.Alignment,
108 new_size: usize,108 new_len: usize,
109 return_address: usize,109 return_address: usize,
110) bool {110) bool {
111 _ = context;111 _ = context;
112 _ = log2_buf_align;112 _ = alignment;
113 _ = return_address;113 _ = return_address;
114 return realloc(memory, new_len, false) != null;
115}
116
117pub fn remap(
118 context: *anyopaque,
119 memory: []u8,
120 alignment: mem.Alignment,
121 new_len: usize,
122 return_address: usize,
123) ?[*]u8 {
124 _ = context;
125 _ = alignment;
126 _ = return_address;
127 return realloc(memory, new_len, true);
128}
129
130fn free(context: *anyopaque, slice: []u8, alignment: mem.Alignment, return_address: usize) void {
131 _ = context;
132 _ = alignment;
133 _ = return_address;
134
135 if (native_os == .windows) {
136 windows.VirtualFree(slice.ptr, 0, windows.MEM_RELEASE);
137 } else {
138 const buf_aligned_len = mem.alignForward(usize, slice.len, std.heap.pageSize());
139 posix.munmap(@alignCast(slice.ptr[0..buf_aligned_len]));
140 }
141}
142
143fn realloc(memory: []u8, new_len: usize, may_move: bool) ?[*]u8 {
114 const page_size = std.heap.pageSize();144 const page_size = std.heap.pageSize();
115 const new_size_aligned = mem.alignForward(usize, new_size, page_size);145 const new_size_aligned = mem.alignForward(usize, new_len, page_size);
116146
117 if (native_os == .windows) {147 if (native_os == .windows) {
118 if (new_size <= buf_unaligned.len) {148 if (new_len <= memory.len) {
119 const base_addr = @intFromPtr(buf_unaligned.ptr);149 const base_addr = @intFromPtr(memory.ptr);
120 const old_addr_end = base_addr + buf_unaligned.len;150 const old_addr_end = base_addr + memory.len;
121 const new_addr_end = mem.alignForward(usize, base_addr + new_size, page_size);151 const new_addr_end = mem.alignForward(usize, base_addr + new_len, page_size);
122 if (old_addr_end > new_addr_end) {152 if (old_addr_end > new_addr_end) {
123 // For shrinking that is not releasing, we will only decommit153 // For shrinking that is not releasing, we will only decommit
124 // the pages not needed anymore.154 // the pages not needed anymore.
...@@ -128,40 +158,31 @@ fn resize(...@@ -128,40 +158,31 @@ fn resize(
128 windows.MEM_DECOMMIT,158 windows.MEM_DECOMMIT,
129 );159 );
130 }160 }
131 return true;161 return memory.ptr;
132 }162 }
133 const old_size_aligned = mem.alignForward(usize, buf_unaligned.len, page_size);163 const old_size_aligned = mem.alignForward(usize, memory.len, page_size);
134 if (new_size_aligned <= old_size_aligned) {164 if (new_size_aligned <= old_size_aligned) {
135 return true;165 return memory.ptr;
136 }166 }
137 return false;167 return null;
138 }168 }
139169
140 const buf_aligned_len = mem.alignForward(usize, buf_unaligned.len, page_size);170 const page_aligned_len = mem.alignForward(usize, memory.len, page_size);
141 if (new_size_aligned == buf_aligned_len)171 if (new_size_aligned == page_aligned_len)
142 return true;172 return memory.ptr;
143173
144 if (new_size_aligned < buf_aligned_len) {174 const mremap_available = false; // native_os == .linux;
145 const ptr = buf_unaligned.ptr + new_size_aligned;175 if (mremap_available) {
146 // TODO: if the next_mmap_addr_hint is within the unmapped range, update it176 // TODO: if the next_mmap_addr_hint is within the remapped range, update it
147 posix.munmap(@alignCast(ptr[0 .. buf_aligned_len - new_size_aligned]));177 return posix.mremap(memory, new_len, .{ .MAYMOVE = may_move }, null) catch return null;
148 return true;
149 }178 }
150179
151 // TODO: call mremap180 if (new_size_aligned < page_aligned_len) {
152 // TODO: if the next_mmap_addr_hint is within the remapped range, update it181 const ptr = memory.ptr + new_size_aligned;
153 return false;182 // TODO: if the next_mmap_addr_hint is within the unmapped range, update it
154}183 posix.munmap(@alignCast(ptr[0 .. page_aligned_len - new_size_aligned]));
155184 return memory.ptr;
156fn free(context: *anyopaque, slice: []u8, log2_buf_align: u8, return_address: usize) void {
157 _ = context;
158 _ = log2_buf_align;
159 _ = return_address;
160
161 if (native_os == .windows) {
162 windows.VirtualFree(slice.ptr, 0, windows.MEM_RELEASE);
163 } else {
164 const buf_aligned_len = mem.alignForward(usize, slice.len, std.heap.pageSize());
165 posix.munmap(@alignCast(slice.ptr[0..buf_aligned_len]));
166 }185 }
186
187 return null;
167}188}
lib/std/heap/arena_allocator.zig+25-17
...@@ -29,12 +29,14 @@ pub const ArenaAllocator = struct {...@@ -29,12 +29,14 @@ pub const ArenaAllocator = struct {
29 .vtable = &.{29 .vtable = &.{
30 .alloc = alloc,30 .alloc = alloc,
31 .resize = resize,31 .resize = resize,
32 .remap = remap,
32 .free = free,33 .free = free,
33 },34 },
34 };35 };
35 }36 }
3637
37 const BufNode = std.SinglyLinkedList(usize).Node;38 const BufNode = std.SinglyLinkedList(usize).Node;
39 const BufNode_alignment: mem.Alignment = .fromByteUnits(@alignOf(BufNode));
3840
39 pub fn init(child_allocator: Allocator) ArenaAllocator {41 pub fn init(child_allocator: Allocator) ArenaAllocator {
40 return (State{}).promote(child_allocator);42 return (State{}).promote(child_allocator);
...@@ -47,9 +49,8 @@ pub const ArenaAllocator = struct {...@@ -47,9 +49,8 @@ pub const ArenaAllocator = struct {
47 while (it) |node| {49 while (it) |node| {
48 // this has to occur before the free because the free frees node50 // this has to occur before the free because the free frees node
49 const next_it = node.next;51 const next_it = node.next;
50 const align_bits = std.math.log2_int(usize, @alignOf(BufNode));
51 const alloc_buf = @as([*]u8, @ptrCast(node))[0..node.data];52 const alloc_buf = @as([*]u8, @ptrCast(node))[0..node.data];
52 self.child_allocator.rawFree(alloc_buf, align_bits, @returnAddress());53 self.child_allocator.rawFree(alloc_buf, BufNode_alignment, @returnAddress());
53 it = next_it;54 it = next_it;
54 }55 }
55 }56 }
...@@ -120,7 +121,6 @@ pub const ArenaAllocator = struct {...@@ -120,7 +121,6 @@ pub const ArenaAllocator = struct {
120 return true;121 return true;
121 }122 }
122 const total_size = requested_capacity + @sizeOf(BufNode);123 const total_size = requested_capacity + @sizeOf(BufNode);
123 const align_bits = std.math.log2_int(usize, @alignOf(BufNode));
124 // Free all nodes except for the last one124 // Free all nodes except for the last one
125 var it = self.state.buffer_list.first;125 var it = self.state.buffer_list.first;
126 const maybe_first_node = while (it) |node| {126 const maybe_first_node = while (it) |node| {
...@@ -129,7 +129,7 @@ pub const ArenaAllocator = struct {...@@ -129,7 +129,7 @@ pub const ArenaAllocator = struct {
129 if (next_it == null)129 if (next_it == null)
130 break node;130 break node;
131 const alloc_buf = @as([*]u8, @ptrCast(node))[0..node.data];131 const alloc_buf = @as([*]u8, @ptrCast(node))[0..node.data];
132 self.child_allocator.rawFree(alloc_buf, align_bits, @returnAddress());132 self.child_allocator.rawFree(alloc_buf, BufNode_alignment, @returnAddress());
133 it = next_it;133 it = next_it;
134 } else null;134 } else null;
135 std.debug.assert(maybe_first_node == null or maybe_first_node.?.next == null);135 std.debug.assert(maybe_first_node == null or maybe_first_node.?.next == null);
...@@ -141,16 +141,16 @@ pub const ArenaAllocator = struct {...@@ -141,16 +141,16 @@ pub const ArenaAllocator = struct {
141 if (first_node.data == total_size)141 if (first_node.data == total_size)
142 return true;142 return true;
143 const first_alloc_buf = @as([*]u8, @ptrCast(first_node))[0..first_node.data];143 const first_alloc_buf = @as([*]u8, @ptrCast(first_node))[0..first_node.data];
144 if (self.child_allocator.rawResize(first_alloc_buf, align_bits, total_size, @returnAddress())) {144 if (self.child_allocator.rawResize(first_alloc_buf, BufNode_alignment, total_size, @returnAddress())) {
145 // successful resize145 // successful resize
146 first_node.data = total_size;146 first_node.data = total_size;
147 } else {147 } else {
148 // manual realloc148 // manual realloc
149 const new_ptr = self.child_allocator.rawAlloc(total_size, align_bits, @returnAddress()) orelse {149 const new_ptr = self.child_allocator.rawAlloc(total_size, BufNode_alignment, @returnAddress()) orelse {
150 // we failed to preheat the arena properly, signal this to the user.150 // we failed to preheat the arena properly, signal this to the user.
151 return false;151 return false;
152 };152 };
153 self.child_allocator.rawFree(first_alloc_buf, align_bits, @returnAddress());153 self.child_allocator.rawFree(first_alloc_buf, BufNode_alignment, @returnAddress());
154 const node: *BufNode = @ptrCast(@alignCast(new_ptr));154 const node: *BufNode = @ptrCast(@alignCast(new_ptr));
155 node.* = .{ .data = total_size };155 node.* = .{ .data = total_size };
156 self.state.buffer_list.first = node;156 self.state.buffer_list.first = node;
...@@ -163,8 +163,7 @@ pub const ArenaAllocator = struct {...@@ -163,8 +163,7 @@ pub const ArenaAllocator = struct {
163 const actual_min_size = minimum_size + (@sizeOf(BufNode) + 16);163 const actual_min_size = minimum_size + (@sizeOf(BufNode) + 16);
164 const big_enough_len = prev_len + actual_min_size;164 const big_enough_len = prev_len + actual_min_size;
165 const len = big_enough_len + big_enough_len / 2;165 const len = big_enough_len + big_enough_len / 2;
166 const log2_align = comptime std.math.log2_int(usize, @alignOf(BufNode));166 const ptr = self.child_allocator.rawAlloc(len, BufNode_alignment, @returnAddress()) orelse
167 const ptr = self.child_allocator.rawAlloc(len, log2_align, @returnAddress()) orelse
168 return null;167 return null;
169 const buf_node: *BufNode = @ptrCast(@alignCast(ptr));168 const buf_node: *BufNode = @ptrCast(@alignCast(ptr));
170 buf_node.* = .{ .data = len };169 buf_node.* = .{ .data = len };
...@@ -173,11 +172,11 @@ pub const ArenaAllocator = struct {...@@ -173,11 +172,11 @@ pub const ArenaAllocator = struct {
173 return buf_node;172 return buf_node;
174 }173 }
175174
176 fn alloc(ctx: *anyopaque, n: usize, log2_ptr_align: u8, ra: usize) ?[*]u8 {175 fn alloc(ctx: *anyopaque, n: usize, alignment: mem.Alignment, ra: usize) ?[*]u8 {
177 const self: *ArenaAllocator = @ptrCast(@alignCast(ctx));176 const self: *ArenaAllocator = @ptrCast(@alignCast(ctx));
178 _ = ra;177 _ = ra;
179178
180 const ptr_align = @as(usize, 1) << @as(Allocator.Log2Align, @intCast(log2_ptr_align));179 const ptr_align = alignment.toByteUnits();
181 var cur_node = if (self.state.buffer_list.first) |first_node|180 var cur_node = if (self.state.buffer_list.first) |first_node|
182 first_node181 first_node
183 else182 else
...@@ -197,8 +196,7 @@ pub const ArenaAllocator = struct {...@@ -197,8 +196,7 @@ pub const ArenaAllocator = struct {
197 }196 }
198197
199 const bigger_buf_size = @sizeOf(BufNode) + new_end_index;198 const bigger_buf_size = @sizeOf(BufNode) + new_end_index;
200 const log2_align = comptime std.math.log2_int(usize, @alignOf(BufNode));199 if (self.child_allocator.rawResize(cur_alloc_buf, BufNode_alignment, bigger_buf_size, @returnAddress())) {
201 if (self.child_allocator.rawResize(cur_alloc_buf, log2_align, bigger_buf_size, @returnAddress())) {
202 cur_node.data = bigger_buf_size;200 cur_node.data = bigger_buf_size;
203 } else {201 } else {
204 // Allocate a new node if that's not possible202 // Allocate a new node if that's not possible
...@@ -207,9 +205,9 @@ pub const ArenaAllocator = struct {...@@ -207,9 +205,9 @@ pub const ArenaAllocator = struct {
207 }205 }
208 }206 }
209207
210 fn resize(ctx: *anyopaque, buf: []u8, log2_buf_align: u8, new_len: usize, ret_addr: usize) bool {208 fn resize(ctx: *anyopaque, buf: []u8, alignment: mem.Alignment, new_len: usize, ret_addr: usize) bool {
211 const self: *ArenaAllocator = @ptrCast(@alignCast(ctx));209 const self: *ArenaAllocator = @ptrCast(@alignCast(ctx));
212 _ = log2_buf_align;210 _ = alignment;
213 _ = ret_addr;211 _ = ret_addr;
214212
215 const cur_node = self.state.buffer_list.first orelse return false;213 const cur_node = self.state.buffer_list.first orelse return false;
...@@ -231,8 +229,18 @@ pub const ArenaAllocator = struct {...@@ -231,8 +229,18 @@ pub const ArenaAllocator = struct {
231 }229 }
232 }230 }
233231
234 fn free(ctx: *anyopaque, buf: []u8, log2_buf_align: u8, ret_addr: usize) void {232 fn remap(
235 _ = log2_buf_align;233 context: *anyopaque,
234 memory: []u8,
235 alignment: mem.Alignment,
236 new_len: usize,
237 return_address: usize,
238 ) ?[*]u8 {
239 return if (resize(context, memory, alignment, new_len, return_address)) memory.ptr else null;
240 }
241
242 fn free(ctx: *anyopaque, buf: []u8, alignment: mem.Alignment, ret_addr: usize) void {
243 _ = alignment;
236 _ = ret_addr;244 _ = ret_addr;
237245
238 const self: *ArenaAllocator = @ptrCast(@alignCast(ctx));246 const self: *ArenaAllocator = @ptrCast(@alignCast(ctx));
lib/std/heap/general_purpose_allocator.zig+86-58
...@@ -226,7 +226,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {...@@ -226,7 +226,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
226 requested_size: if (config.enable_memory_limit) usize else void,226 requested_size: if (config.enable_memory_limit) usize else void,
227 stack_addresses: [trace_n][stack_n]usize,227 stack_addresses: [trace_n][stack_n]usize,
228 freed: if (config.retain_metadata) bool else void,228 freed: if (config.retain_metadata) bool else void,
229 log2_ptr_align: if (config.never_unmap and config.retain_metadata) u8 else void,229 alignment: if (config.never_unmap and config.retain_metadata) mem.Alignment else void,
230230
231 const trace_n = if (config.retain_metadata) traces_per_slot else 1;231 const trace_n = if (config.retain_metadata) traces_per_slot else 1;
232232
...@@ -281,11 +281,11 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {...@@ -281,11 +281,11 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
281 return sizes[0..slot_count];281 return sizes[0..slot_count];
282 }282 }
283283
284 fn log2PtrAligns(bucket: *BucketHeader, size_class: usize) []u8 {284 fn log2PtrAligns(bucket: *BucketHeader, size_class: usize) []mem.Alignment {
285 if (!config.safety) @compileError("requested size is only stored when safety is enabled");285 if (!config.safety) @compileError("requested size is only stored when safety is enabled");
286 const aligns_ptr = @as([*]u8, @ptrCast(bucket)) + bucketAlignsStart(size_class);286 const aligns_ptr = @as([*]u8, @ptrCast(bucket)) + bucketAlignsStart(size_class);
287 const slot_count = @divExact(page_size, size_class);287 const slot_count = @divExact(page_size, size_class);
288 return aligns_ptr[0..slot_count];288 return @ptrCast(aligns_ptr[0..slot_count]);
289 }289 }
290290
291 fn stackTracePtr(291 fn stackTracePtr(
...@@ -326,6 +326,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {...@@ -326,6 +326,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
326 .vtable = &.{326 .vtable = &.{
327 .alloc = alloc,327 .alloc = alloc,
328 .resize = resize,328 .resize = resize,
329 .remap = remap,
329 .free = free,330 .free = free,
330 },331 },
331 };332 };
...@@ -455,7 +456,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {...@@ -455,7 +456,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
455 var it = self.large_allocations.iterator();456 var it = self.large_allocations.iterator();
456 while (it.next()) |large| {457 while (it.next()) |large| {
457 if (large.value_ptr.freed) {458 if (large.value_ptr.freed) {
458 self.backing_allocator.rawFree(large.value_ptr.bytes, large.value_ptr.log2_ptr_align, @returnAddress());459 self.backing_allocator.rawFree(large.value_ptr.bytes, large.value_ptr.alignment, @returnAddress());
459 }460 }
460 }461 }
461 }462 }
...@@ -583,10 +584,11 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {...@@ -583,10 +584,11 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
583 fn resizeLarge(584 fn resizeLarge(
584 self: *Self,585 self: *Self,
585 old_mem: []u8,586 old_mem: []u8,
586 log2_old_align: u8,587 alignment: mem.Alignment,
587 new_size: usize,588 new_size: usize,
588 ret_addr: usize,589 ret_addr: usize,
589 ) bool {590 may_move: bool,
591 ) ?[*]u8 {
590 const entry = self.large_allocations.getEntry(@intFromPtr(old_mem.ptr)) orelse {592 const entry = self.large_allocations.getEntry(@intFromPtr(old_mem.ptr)) orelse {
591 if (config.safety) {593 if (config.safety) {
592 @panic("Invalid free");594 @panic("Invalid free");
...@@ -628,30 +630,37 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {...@@ -628,30 +630,37 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
628 if (config.enable_memory_limit) {630 if (config.enable_memory_limit) {
629 const new_req_bytes = prev_req_bytes + new_size - entry.value_ptr.requested_size;631 const new_req_bytes = prev_req_bytes + new_size - entry.value_ptr.requested_size;
630 if (new_req_bytes > prev_req_bytes and new_req_bytes > self.requested_memory_limit) {632 if (new_req_bytes > prev_req_bytes and new_req_bytes > self.requested_memory_limit) {
631 return false;633 return null;
632 }634 }
633 self.total_requested_bytes = new_req_bytes;635 self.total_requested_bytes = new_req_bytes;
634 }636 }
635637
636 if (!self.backing_allocator.rawResize(old_mem, log2_old_align, new_size, ret_addr)) {638 const opt_resized_ptr = if (may_move)
639 self.backing_allocator.rawRemap(old_mem, alignment, new_size, ret_addr)
640 else if (self.backing_allocator.rawResize(old_mem, alignment, new_size, ret_addr))
641 old_mem.ptr
642 else
643 null;
644
645 const resized_ptr = opt_resized_ptr orelse {
637 if (config.enable_memory_limit) {646 if (config.enable_memory_limit) {
638 self.total_requested_bytes = prev_req_bytes;647 self.total_requested_bytes = prev_req_bytes;
639 }648 }
640 return false;649 return null;
641 }650 };
642651
643 if (config.enable_memory_limit) {652 if (config.enable_memory_limit) {
644 entry.value_ptr.requested_size = new_size;653 entry.value_ptr.requested_size = new_size;
645 }654 }
646655
647 if (config.verbose_log) {656 if (config.verbose_log) {
648 log.info("large resize {d} bytes at {*} to {d}", .{657 log.info("large resize {d} bytes at {*} to {d} at {*}", .{
649 old_mem.len, old_mem.ptr, new_size,658 old_mem.len, old_mem.ptr, new_size, resized_ptr,
650 });659 });
651 }660 }
652 entry.value_ptr.bytes = old_mem.ptr[0..new_size];661 entry.value_ptr.bytes = resized_ptr[0..new_size];
653 entry.value_ptr.captureStackTrace(ret_addr, .alloc);662 entry.value_ptr.captureStackTrace(ret_addr, .alloc);
654 return true;663 return resized_ptr;
655 }664 }
656665
657 /// This function assumes the object is in the large object storage regardless666 /// This function assumes the object is in the large object storage regardless
...@@ -659,7 +668,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {...@@ -659,7 +668,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
659 fn freeLarge(668 fn freeLarge(
660 self: *Self,669 self: *Self,
661 old_mem: []u8,670 old_mem: []u8,
662 log2_old_align: u8,671 alignment: mem.Alignment,
663 ret_addr: usize,672 ret_addr: usize,
664 ) void {673 ) void {
665 const entry = self.large_allocations.getEntry(@intFromPtr(old_mem.ptr)) orelse {674 const entry = self.large_allocations.getEntry(@intFromPtr(old_mem.ptr)) orelse {
...@@ -695,7 +704,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {...@@ -695,7 +704,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
695 }704 }
696705
697 if (!config.never_unmap) {706 if (!config.never_unmap) {
698 self.backing_allocator.rawFree(old_mem, log2_old_align, ret_addr);707 self.backing_allocator.rawFree(old_mem, alignment, ret_addr);
699 }708 }
700709
701 if (config.enable_memory_limit) {710 if (config.enable_memory_limit) {
...@@ -719,22 +728,42 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {...@@ -719,22 +728,42 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
719 }728 }
720729
721 fn resize(730 fn resize(
722 ctx: *anyopaque,731 context: *anyopaque,
732 memory: []u8,
733 alignment: mem.Alignment,
734 new_len: usize,
735 return_address: usize,
736 ) bool {
737 return realloc(context, memory, alignment, new_len, return_address, false) != null;
738 }
739
740 fn remap(
741 context: *anyopaque,
742 memory: []u8,
743 alignment: mem.Alignment,
744 new_len: usize,
745 return_address: usize,
746 ) ?[*]u8 {
747 return realloc(context, memory, alignment, new_len, return_address, true);
748 }
749
750 fn realloc(
751 context: *anyopaque,
723 old_mem: []u8,752 old_mem: []u8,
724 log2_old_align_u8: u8,753 alignment: mem.Alignment,
725 new_size: usize,754 new_len: usize,
726 ret_addr: usize,755 ret_addr: usize,
727 ) bool {756 may_move: bool,
728 const self: *Self = @ptrCast(@alignCast(ctx));757 ) ?[*]u8 {
729 const log2_old_align = @as(Allocator.Log2Align, @intCast(log2_old_align_u8));758 const self: *Self = @ptrCast(@alignCast(context));
730 self.mutex.lock();759 self.mutex.lock();
731 defer self.mutex.unlock();760 defer self.mutex.unlock();
732761
733 assert(old_mem.len != 0);762 assert(old_mem.len != 0);
734763
735 const aligned_size = @max(old_mem.len, @as(usize, 1) << log2_old_align);764 const aligned_size = @max(old_mem.len, alignment.toByteUnits());
736 if (aligned_size > largest_bucket_object_size) {765 if (aligned_size > largest_bucket_object_size) {
737 return self.resizeLarge(old_mem, log2_old_align, new_size, ret_addr);766 return self.resizeLarge(old_mem, alignment, new_len, ret_addr, may_move);
738 }767 }
739 const size_class_hint = math.ceilPowerOfTwoAssert(usize, aligned_size);768 const size_class_hint = math.ceilPowerOfTwoAssert(usize, aligned_size);
740769
...@@ -758,7 +787,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {...@@ -758,7 +787,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
758 }787 }
759 }788 }
760 }789 }
761 return self.resizeLarge(old_mem, log2_old_align, new_size, ret_addr);790 return self.resizeLarge(old_mem, alignment, new_len, ret_addr, may_move);
762 };791 };
763 const byte_offset = @intFromPtr(old_mem.ptr) - @intFromPtr(bucket.page);792 const byte_offset = @intFromPtr(old_mem.ptr) - @intFromPtr(bucket.page);
764 const slot_index = @as(SlotIndex, @intCast(byte_offset / size_class));793 const slot_index = @as(SlotIndex, @intCast(byte_offset / size_class));
...@@ -779,8 +808,8 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {...@@ -779,8 +808,8 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
779 if (config.safety) {808 if (config.safety) {
780 const requested_size = bucket.requestedSizes(size_class)[slot_index];809 const requested_size = bucket.requestedSizes(size_class)[slot_index];
781 if (requested_size == 0) @panic("Invalid free");810 if (requested_size == 0) @panic("Invalid free");
782 const log2_ptr_align = bucket.log2PtrAligns(size_class)[slot_index];811 const slot_alignment = bucket.log2PtrAligns(size_class)[slot_index];
783 if (old_mem.len != requested_size or log2_old_align != log2_ptr_align) {812 if (old_mem.len != requested_size or alignment != slot_alignment) {
784 var addresses: [stack_n]usize = [1]usize{0} ** stack_n;813 var addresses: [stack_n]usize = [1]usize{0} ** stack_n;
785 var free_stack_trace = StackTrace{814 var free_stack_trace = StackTrace{
786 .instruction_addresses = &addresses,815 .instruction_addresses = &addresses,
...@@ -795,10 +824,10 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {...@@ -795,10 +824,10 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
795 free_stack_trace,824 free_stack_trace,
796 });825 });
797 }826 }
798 if (log2_old_align != log2_ptr_align) {827 if (alignment != slot_alignment) {
799 log.err("Allocation alignment {d} does not match resize alignment {d}. Allocation: {} Resize: {}", .{828 log.err("Allocation alignment {d} does not match resize alignment {d}. Allocation: {} Resize: {}", .{
800 @as(usize, 1) << @as(math.Log2Int(usize), @intCast(log2_ptr_align)),829 slot_alignment.toByteUnits(),
801 @as(usize, 1) << @as(math.Log2Int(usize), @intCast(log2_old_align)),830 alignment.toByteUnits(),
802 bucketStackTrace(bucket, size_class, slot_index, .alloc),831 bucketStackTrace(bucket, size_class, slot_index, .alloc),
803 free_stack_trace,832 free_stack_trace,
804 });833 });
...@@ -807,52 +836,51 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {...@@ -807,52 +836,51 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
807 }836 }
808 const prev_req_bytes = self.total_requested_bytes;837 const prev_req_bytes = self.total_requested_bytes;
809 if (config.enable_memory_limit) {838 if (config.enable_memory_limit) {
810 const new_req_bytes = prev_req_bytes + new_size - old_mem.len;839 const new_req_bytes = prev_req_bytes + new_len - old_mem.len;
811 if (new_req_bytes > prev_req_bytes and new_req_bytes > self.requested_memory_limit) {840 if (new_req_bytes > prev_req_bytes and new_req_bytes > self.requested_memory_limit) {
812 return false;841 return null;
813 }842 }
814 self.total_requested_bytes = new_req_bytes;843 self.total_requested_bytes = new_req_bytes;
815 }844 }
816845
817 const new_aligned_size = @max(new_size, @as(usize, 1) << log2_old_align);846 const new_aligned_size = @max(new_len, alignment.toByteUnits());
818 const new_size_class = math.ceilPowerOfTwoAssert(usize, new_aligned_size);847 const new_size_class = math.ceilPowerOfTwoAssert(usize, new_aligned_size);
819 if (new_size_class <= size_class) {848 if (new_size_class <= size_class) {
820 if (old_mem.len > new_size) {849 if (old_mem.len > new_len) {
821 @memset(old_mem[new_size..], undefined);850 @memset(old_mem[new_len..], undefined);
822 }851 }
823 if (config.verbose_log) {852 if (config.verbose_log) {
824 log.info("small resize {d} bytes at {*} to {d}", .{853 log.info("small resize {d} bytes at {*} to {d}", .{
825 old_mem.len, old_mem.ptr, new_size,854 old_mem.len, old_mem.ptr, new_len,
826 });855 });
827 }856 }
828 if (config.safety) {857 if (config.safety) {
829 bucket.requestedSizes(size_class)[slot_index] = @intCast(new_size);858 bucket.requestedSizes(size_class)[slot_index] = @intCast(new_len);
830 }859 }
831 return true;860 return old_mem.ptr;
832 }861 }
833862
834 if (config.enable_memory_limit) {863 if (config.enable_memory_limit) {
835 self.total_requested_bytes = prev_req_bytes;864 self.total_requested_bytes = prev_req_bytes;
836 }865 }
837 return false;866 return null;
838 }867 }
839868
840 fn free(869 fn free(
841 ctx: *anyopaque,870 ctx: *anyopaque,
842 old_mem: []u8,871 old_mem: []u8,
843 log2_old_align_u8: u8,872 alignment: mem.Alignment,
844 ret_addr: usize,873 ret_addr: usize,
845 ) void {874 ) void {
846 const self: *Self = @ptrCast(@alignCast(ctx));875 const self: *Self = @ptrCast(@alignCast(ctx));
847 const log2_old_align = @as(Allocator.Log2Align, @intCast(log2_old_align_u8));
848 self.mutex.lock();876 self.mutex.lock();
849 defer self.mutex.unlock();877 defer self.mutex.unlock();
850878
851 assert(old_mem.len != 0);879 assert(old_mem.len != 0);
852880
853 const aligned_size = @max(old_mem.len, @as(usize, 1) << log2_old_align);881 const aligned_size = @max(old_mem.len, alignment.toByteUnits());
854 if (aligned_size > largest_bucket_object_size) {882 if (aligned_size > largest_bucket_object_size) {
855 self.freeLarge(old_mem, log2_old_align, ret_addr);883 self.freeLarge(old_mem, alignment, ret_addr);
856 return;884 return;
857 }885 }
858 const size_class_hint = math.ceilPowerOfTwoAssert(usize, aligned_size);886 const size_class_hint = math.ceilPowerOfTwoAssert(usize, aligned_size);
...@@ -877,7 +905,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {...@@ -877,7 +905,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
877 }905 }
878 }906 }
879 }907 }
880 self.freeLarge(old_mem, log2_old_align, ret_addr);908 self.freeLarge(old_mem, alignment, ret_addr);
881 return;909 return;
882 };910 };
883 const byte_offset = @intFromPtr(old_mem.ptr) - @intFromPtr(bucket.page);911 const byte_offset = @intFromPtr(old_mem.ptr) - @intFromPtr(bucket.page);
...@@ -900,8 +928,8 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {...@@ -900,8 +928,8 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
900 if (config.safety) {928 if (config.safety) {
901 const requested_size = bucket.requestedSizes(size_class)[slot_index];929 const requested_size = bucket.requestedSizes(size_class)[slot_index];
902 if (requested_size == 0) @panic("Invalid free");930 if (requested_size == 0) @panic("Invalid free");
903 const log2_ptr_align = bucket.log2PtrAligns(size_class)[slot_index];931 const slot_alignment = bucket.log2PtrAligns(size_class)[slot_index];
904 if (old_mem.len != requested_size or log2_old_align != log2_ptr_align) {932 if (old_mem.len != requested_size or alignment != slot_alignment) {
905 var addresses: [stack_n]usize = [1]usize{0} ** stack_n;933 var addresses: [stack_n]usize = [1]usize{0} ** stack_n;
906 var free_stack_trace = StackTrace{934 var free_stack_trace = StackTrace{
907 .instruction_addresses = &addresses,935 .instruction_addresses = &addresses,
...@@ -916,10 +944,10 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {...@@ -916,10 +944,10 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
916 free_stack_trace,944 free_stack_trace,
917 });945 });
918 }946 }
919 if (log2_old_align != log2_ptr_align) {947 if (alignment != slot_alignment) {
920 log.err("Allocation alignment {d} does not match free alignment {d}. Allocation: {} Free: {}", .{948 log.err("Allocation alignment {d} does not match free alignment {d}. Allocation: {} Free: {}", .{
921 @as(usize, 1) << @as(math.Log2Int(usize), @intCast(log2_ptr_align)),949 slot_alignment.toByteUnits(),
922 @as(usize, 1) << @as(math.Log2Int(usize), @intCast(log2_old_align)),950 alignment.toByteUnits(),
923 bucketStackTrace(bucket, size_class, slot_index, .alloc),951 bucketStackTrace(bucket, size_class, slot_index, .alloc),
924 free_stack_trace,952 free_stack_trace,
925 });953 });
...@@ -981,24 +1009,24 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {...@@ -981,24 +1009,24 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
981 return true;1009 return true;
982 }1010 }
9831011
984 fn alloc(ctx: *anyopaque, len: usize, log2_ptr_align: u8, ret_addr: usize) ?[*]u8 {1012 fn alloc(ctx: *anyopaque, len: usize, alignment: mem.Alignment, ret_addr: usize) ?[*]u8 {
985 const self: *Self = @ptrCast(@alignCast(ctx));1013 const self: *Self = @ptrCast(@alignCast(ctx));
986 self.mutex.lock();1014 self.mutex.lock();
987 defer self.mutex.unlock();1015 defer self.mutex.unlock();
988 if (!self.isAllocationAllowed(len)) return null;1016 if (!self.isAllocationAllowed(len)) return null;
989 return allocInner(self, len, @as(Allocator.Log2Align, @intCast(log2_ptr_align)), ret_addr) catch return null;1017 return allocInner(self, len, alignment, ret_addr) catch return null;
990 }1018 }
9911019
992 fn allocInner(1020 fn allocInner(
993 self: *Self,1021 self: *Self,
994 len: usize,1022 len: usize,
995 log2_ptr_align: Allocator.Log2Align,1023 alignment: mem.Alignment,
996 ret_addr: usize,1024 ret_addr: usize,
997 ) Allocator.Error![*]u8 {1025 ) Allocator.Error![*]u8 {
998 const new_aligned_size = @max(len, @as(usize, 1) << @as(Allocator.Log2Align, @intCast(log2_ptr_align)));1026 const new_aligned_size = @max(len, alignment.toByteUnits());
999 if (new_aligned_size > largest_bucket_object_size) {1027 if (new_aligned_size > largest_bucket_object_size) {
1000 try self.large_allocations.ensureUnusedCapacity(self.backing_allocator, 1);1028 try self.large_allocations.ensureUnusedCapacity(self.backing_allocator, 1);
1001 const ptr = self.backing_allocator.rawAlloc(len, log2_ptr_align, ret_addr) orelse1029 const ptr = self.backing_allocator.rawAlloc(len, alignment, ret_addr) orelse
1002 return error.OutOfMemory;1030 return error.OutOfMemory;
1003 const slice = ptr[0..len];1031 const slice = ptr[0..len];
10041032
...@@ -1016,7 +1044,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {...@@ -1016,7 +1044,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
1016 if (config.retain_metadata) {1044 if (config.retain_metadata) {
1017 gop.value_ptr.freed = false;1045 gop.value_ptr.freed = false;
1018 if (config.never_unmap) {1046 if (config.never_unmap) {
1019 gop.value_ptr.log2_ptr_align = log2_ptr_align;1047 gop.value_ptr.alignment = alignment;
1020 }1048 }
1021 }1049 }
10221050
...@@ -1030,7 +1058,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {...@@ -1030,7 +1058,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
1030 const slot = try self.allocSlot(new_size_class, ret_addr);1058 const slot = try self.allocSlot(new_size_class, ret_addr);
1031 if (config.safety) {1059 if (config.safety) {
1032 slot.bucket.requestedSizes(new_size_class)[slot.slot_index] = @intCast(len);1060 slot.bucket.requestedSizes(new_size_class)[slot.slot_index] = @intCast(len);
1033 slot.bucket.log2PtrAligns(new_size_class)[slot.slot_index] = log2_ptr_align;1061 slot.bucket.log2PtrAligns(new_size_class)[slot.slot_index] = alignment;
1034 }1062 }
1035 if (config.verbose_log) {1063 if (config.verbose_log) {
1036 log.info("small alloc {d} bytes at {*}", .{ len, slot.ptr });1064 log.info("small alloc {d} bytes at {*}", .{ len, slot.ptr });
...@@ -1150,7 +1178,7 @@ test "realloc" {...@@ -1150,7 +1178,7 @@ test "realloc" {
1150}1178}
11511179
1152test "shrink" {1180test "shrink" {
1153 var gpa = GeneralPurposeAllocator(test_config){};1181 var gpa: GeneralPurposeAllocator(test_config) = .{};
1154 defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak");1182 defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak");
1155 const allocator = gpa.allocator();1183 const allocator = gpa.allocator();
11561184
...@@ -1214,7 +1242,7 @@ test "realloc small object to large object" {...@@ -1214,7 +1242,7 @@ test "realloc small object to large object" {
1214}1242}
12151243
1216test "shrink large object to large object" {1244test "shrink large object to large object" {
1217 var gpa = GeneralPurposeAllocator(test_config){};1245 var gpa: GeneralPurposeAllocator(test_config) = .{};
1218 defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak");1246 defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak");
1219 const allocator = gpa.allocator();1247 const allocator = gpa.allocator();
12201248
lib/std/mem/Allocator.zig+149-65
...@@ -6,19 +6,21 @@ const math = std.math;...@@ -6,19 +6,21 @@ const math = std.math;
6const mem = std.mem;6const mem = std.mem;
7const Allocator = @This();7const Allocator = @This();
8const builtin = @import("builtin");8const builtin = @import("builtin");
9const Alignment = std.mem.Alignment;
910
10pub const Error = error{OutOfMemory};11pub const Error = error{OutOfMemory};
11pub const Log2Align = math.Log2Int(usize);12pub const Log2Align = math.Log2Int(usize);
1213
13/// The type erased pointer to the allocator implementation.14/// The type erased pointer to the allocator implementation.
14/// Any comparison of this field may result in illegal behavior, since it may be set to15///
15/// `undefined` in cases where the allocator implementation does not have any associated16/// Any comparison of this field may result in illegal behavior, since it may
16/// state.17/// be set to `undefined` in cases where the allocator implementation does not
18/// have any associated state.
17ptr: *anyopaque,19ptr: *anyopaque,
18vtable: *const VTable,20vtable: *const VTable,
1921
20pub const VTable = struct {22pub const VTable = struct {
21 /// Allocate exactly `len` bytes aligned to `1 << ptr_align`, or return `null`23 /// Allocate exactly `len` bytes aligned to `alignment`, or return `null`
22 /// indicating the allocation failed.24 /// indicating the allocation failed.
23 ///25 ///
24 /// `ret_addr` is optionally provided as the first return address of the26 /// `ret_addr` is optionally provided as the first return address of the
...@@ -27,12 +29,14 @@ pub const VTable = struct {...@@ -27,12 +29,14 @@ pub const VTable = struct {
27 ///29 ///
28 /// The returned slice of memory must have been `@memset` to `undefined`30 /// The returned slice of memory must have been `@memset` to `undefined`
29 /// by the allocator implementation.31 /// by the allocator implementation.
30 alloc: *const fn (ctx: *anyopaque, len: usize, ptr_align: u8, ret_addr: usize) ?[*]u8,32 alloc: *const fn (*anyopaque, len: usize, alignment: Alignment, ret_addr: usize) ?[*]u8,
3133
32 /// Attempt to expand or shrink memory in place. `buf.len` must equal the34 /// Attempt to expand or shrink memory in place.
33 /// length requested from the most recent successful call to `alloc` or35 ///
34 /// `resize`. `buf_align` must equal the same value that was passed as the36 /// `memory.len` must equal the length requested from the most recent
35 /// `ptr_align` parameter to the original `alloc` call.37 /// successful call to `alloc` or `resize`. `alignment` must equal the same
38 /// value that was passed as the `alignment` parameter to the original
39 /// `alloc` call.
36 ///40 ///
37 /// A result of `true` indicates the resize was successful and the41 /// A result of `true` indicates the resize was successful and the
38 /// allocation now has the same address but a size of `new_len`. `false`42 /// allocation now has the same address but a size of `new_len`. `false`
...@@ -44,72 +48,114 @@ pub const VTable = struct {...@@ -44,72 +48,114 @@ pub const VTable = struct {
44 /// `ret_addr` is optionally provided as the first return address of the48 /// `ret_addr` is optionally provided as the first return address of the
45 /// allocation call stack. If the value is `0` it means no return address49 /// allocation call stack. If the value is `0` it means no return address
46 /// has been provided.50 /// has been provided.
47 resize: *const fn (ctx: *anyopaque, buf: []u8, buf_align: u8, new_len: usize, ret_addr: usize) bool,51 resize: *const fn (*anyopaque, memory: []u8, alignment: Alignment, new_len: usize, ret_addr: usize) bool,
4852
49 /// Free and invalidate a buffer.53 /// Attempt to expand or shrink memory, allowing relocation.
54 ///
55 /// `memory.len` must equal the length requested from the most recent
56 /// successful call to `alloc` or `resize`. `alignment` must equal the same
57 /// value that was passed as the `alignment` parameter to the original
58 /// `alloc` call.
59 ///
60 /// A non-`null` return value indicates the resize was successful. The
61 /// allocation may have same address, or may have been relocated. In either
62 /// case, the allocation now has size of `new_len`. A `null` return value
63 /// indicates that the resize would be equivalent to allocating new memory,
64 /// copying the bytes from the old memory, and then freeing the old memory.
65 /// In such case, it is more efficient for the caller to perform the copy.
50 ///66 ///
51 /// `buf.len` must equal the most recent length returned by `alloc` or67 /// `new_len` must be greater than zero.
68 ///
69 /// `ret_addr` is optionally provided as the first return address of the
70 /// allocation call stack. If the value is `0` it means no return address
71 /// has been provided.
72 remap: *const fn (*anyopaque, memory: []u8, alignment: Alignment, new_len: usize, ret_addr: usize) ?[*]u8,
73
74 /// Free and invalidate a region of memory.
75 ///
76 /// `memory.len` must equal the most recent length returned by `alloc` or
52 /// given to a successful `resize` call.77 /// given to a successful `resize` call.
53 ///78 ///
54 /// `buf_align` must equal the same value that was passed as the79 /// `alignment` must equal the same value that was passed as the
55 /// `ptr_align` parameter to the original `alloc` call.80 /// `alignment` parameter to the original `alloc` call.
56 ///81 ///
57 /// `ret_addr` is optionally provided as the first return address of the82 /// `ret_addr` is optionally provided as the first return address of the
58 /// allocation call stack. If the value is `0` it means no return address83 /// allocation call stack. If the value is `0` it means no return address
59 /// has been provided.84 /// has been provided.
60 free: *const fn (ctx: *anyopaque, buf: []u8, buf_align: u8, ret_addr: usize) void,85 free: *const fn (*anyopaque, memory: []u8, alignment: Alignment, ret_addr: usize) void,
61};86};
6287
63pub fn noResize(88pub fn noResize(
64 self: *anyopaque,89 self: *anyopaque,
65 buf: []u8,90 memory: []u8,
66 log2_buf_align: u8,91 alignment: Alignment,
67 new_len: usize,92 new_len: usize,
68 ret_addr: usize,93 ret_addr: usize,
69) bool {94) bool {
70 _ = self;95 _ = self;
71 _ = buf;96 _ = memory;
72 _ = log2_buf_align;97 _ = alignment;
73 _ = new_len;98 _ = new_len;
74 _ = ret_addr;99 _ = ret_addr;
75 return false;100 return false;
76}101}
77102
103pub fn noRemap(
104 self: *anyopaque,
105 memory: []u8,
106 alignment: Alignment,
107 new_len: usize,
108 ret_addr: usize,
109) ?[*]u8 {
110 _ = self;
111 _ = memory;
112 _ = alignment;
113 _ = new_len;
114 _ = ret_addr;
115 return null;
116}
117
78pub fn noFree(118pub fn noFree(
79 self: *anyopaque,119 self: *anyopaque,
80 buf: []u8,120 memory: []u8,
81 log2_buf_align: u8,121 alignment: Alignment,
82 ret_addr: usize,122 ret_addr: usize,
83) void {123) void {
84 _ = self;124 _ = self;
85 _ = buf;125 _ = memory;
86 _ = log2_buf_align;126 _ = alignment;
87 _ = ret_addr;127 _ = ret_addr;
88}128}
89129
90/// This function is not intended to be called except from within the130/// This function is not intended to be called except from within the
91/// implementation of an Allocator131/// implementation of an Allocator
92pub inline fn rawAlloc(self: Allocator, len: usize, ptr_align: u8, ret_addr: usize) ?[*]u8 {132pub inline fn rawAlloc(a: Allocator, len: usize, alignment: Alignment, ret_addr: usize) ?[*]u8 {
93 return self.vtable.alloc(self.ptr, len, ptr_align, ret_addr);133 return a.vtable.alloc(a.ptr, len, alignment, ret_addr);
94}134}
95135
96/// This function is not intended to be called except from within the136/// This function is not intended to be called except from within the
97/// implementation of an Allocator137/// implementation of an Allocator.
98pub inline fn rawResize(self: Allocator, buf: []u8, log2_buf_align: u8, new_len: usize, ret_addr: usize) bool {138pub inline fn rawResize(a: Allocator, memory: []u8, alignment: Alignment, new_len: usize, ret_addr: usize) bool {
99 return self.vtable.resize(self.ptr, buf, log2_buf_align, new_len, ret_addr);139 return a.vtable.resize(a.ptr, memory, alignment, new_len, ret_addr);
140}
141
142/// This function is not intended to be called except from within the
143/// implementation of an Allocator.
144pub inline fn rawRemap(a: Allocator, memory: []u8, alignment: Alignment, new_len: usize, ret_addr: usize) ?[*]u8 {
145 return a.vtable.remap(a.ptr, memory, alignment, new_len, ret_addr);
100}146}
101147
102/// This function is not intended to be called except from within the148/// This function is not intended to be called except from within the
103/// implementation of an Allocator149/// implementation of an Allocator
104pub inline fn rawFree(self: Allocator, buf: []u8, log2_buf_align: u8, ret_addr: usize) void {150pub inline fn rawFree(a: Allocator, memory: []u8, alignment: Alignment, ret_addr: usize) void {
105 return self.vtable.free(self.ptr, buf, log2_buf_align, ret_addr);151 return a.vtable.free(a.ptr, memory, alignment, ret_addr);
106}152}
107153
108/// Returns a pointer to undefined memory.154/// Returns a pointer to undefined memory.
109/// Call `destroy` with the result to free the memory.155/// Call `destroy` with the result to free the memory.
110pub fn create(self: Allocator, comptime T: type) Error!*T {156pub fn create(a: Allocator, comptime T: type) Error!*T {
111 if (@sizeOf(T) == 0) return @as(*T, @ptrFromInt(math.maxInt(usize)));157 if (@sizeOf(T) == 0) return @as(*T, @ptrFromInt(math.maxInt(usize)));
112 const ptr: *T = @ptrCast(try self.allocBytesWithAlignment(@alignOf(T), @sizeOf(T), @returnAddress()));158 const ptr: *T = @ptrCast(try a.allocBytesWithAlignment(@alignOf(T), @sizeOf(T), @returnAddress()));
113 return ptr;159 return ptr;
114}160}
115161
...@@ -121,7 +167,7 @@ pub fn destroy(self: Allocator, ptr: anytype) void {...@@ -121,7 +167,7 @@ pub fn destroy(self: Allocator, ptr: anytype) void {
121 const T = info.child;167 const T = info.child;
122 if (@sizeOf(T) == 0) return;168 if (@sizeOf(T) == 0) return;
123 const non_const_ptr = @as([*]u8, @ptrCast(@constCast(ptr)));169 const non_const_ptr = @as([*]u8, @ptrCast(@constCast(ptr)));
124 self.rawFree(non_const_ptr[0..@sizeOf(T)], log2a(info.alignment), @returnAddress());170 self.rawFree(non_const_ptr[0..@sizeOf(T)], .fromByteUnits(info.alignment), @returnAddress());
125}171}
126172
127/// Allocates an array of `n` items of type `T` and sets all the173/// Allocates an array of `n` items of type `T` and sets all the
...@@ -224,36 +270,88 @@ fn allocBytesWithAlignment(self: Allocator, comptime alignment: u29, byte_count:...@@ -224,36 +270,88 @@ fn allocBytesWithAlignment(self: Allocator, comptime alignment: u29, byte_count:
224 return @as([*]align(alignment) u8, @ptrFromInt(ptr));270 return @as([*]align(alignment) u8, @ptrFromInt(ptr));
225 }271 }
226272
227 const byte_ptr = self.rawAlloc(byte_count, log2a(alignment), return_address) orelse return Error.OutOfMemory;273 const byte_ptr = self.rawAlloc(byte_count, .fromByteUnits(alignment), return_address) orelse return Error.OutOfMemory;
228 // TODO: https://github.com/ziglang/zig/issues/4298274 // TODO: https://github.com/ziglang/zig/issues/4298
229 @memset(byte_ptr[0..byte_count], undefined);275 @memset(byte_ptr[0..byte_count], undefined);
230 return @alignCast(byte_ptr);276 return @alignCast(byte_ptr);
231}277}
232278
233/// Requests to modify the size of an allocation. It is guaranteed to not move279/// Request to modify the size of an allocation.
234/// the pointer, however the allocator implementation may refuse the resize280///
235/// request by returning `false`.281/// It is guaranteed to not move the pointer, however the allocator
236pub fn resize(self: Allocator, old_mem: anytype, new_n: usize) bool {282/// implementation may refuse the resize request by returning `false`.
237 const Slice = @typeInfo(@TypeOf(old_mem)).pointer;283///
284/// `allocation` may be an empty slice, in which case a new allocation is made.
285///
286/// `new_len` may be zero, in which case the allocation is freed.
287pub fn resize(self: Allocator, allocation: anytype, new_len: usize) bool {
288 const Slice = @typeInfo(@TypeOf(allocation)).pointer;
238 const T = Slice.child;289 const T = Slice.child;
239 if (new_n == 0) {290 const alignment = Slice.alignment;
240 self.free(old_mem);291 if (new_len == 0) {
292 self.free(allocation);
241 return true;293 return true;
242 }294 }
243 if (old_mem.len == 0) {295 if (allocation.len == 0) {
244 return false;296 return false;
245 }297 }
246 const old_byte_slice = mem.sliceAsBytes(old_mem);298 const old_memory = mem.sliceAsBytes(allocation);
299 // I would like to use saturating multiplication here, but LLVM cannot lower it
300 // on WebAssembly: https://github.com/ziglang/zig/issues/9660
301 //const new_len_bytes = new_len *| @sizeOf(T);
302 const new_len_bytes = math.mul(usize, @sizeOf(T), new_len) catch return false;
303 return self.rawResize(old_memory, .fromByteUnits(alignment), new_len_bytes, @returnAddress());
304}
305
306/// Request to modify the size of an allocation, allowing relocation.
307///
308/// A non-`null` return value indicates the resize was successful. The
309/// allocation may have same address, or may have been relocated. In either
310/// case, the allocation now has size of `new_len`. A `null` return value
311/// indicates that the resize would be equivalent to allocating new memory,
312/// copying the bytes from the old memory, and then freeing the old memory.
313/// In such case, it is more efficient for the caller to perform those
314/// operations.
315///
316/// `allocation` may be an empty slice, in which case a new allocation is made.
317///
318/// `new_len` may be zero, in which case the allocation is freed.
319pub fn remap(self: Allocator, allocation: anytype, new_len: usize) t: {
320 const Slice = @typeInfo(@TypeOf(allocation)).pointer;
321 break :t ?[]align(Slice.alignment) Slice.child;
322} {
323 const Slice = @typeInfo(@TypeOf(allocation)).pointer;
324 const T = Slice.child;
325 const alignment = Slice.alignment;
326 if (new_len == 0) {
327 self.free(allocation);
328 return allocation[0..0];
329 }
330 if (allocation.len == 0) {
331 return null;
332 }
333 const old_memory = mem.sliceAsBytes(allocation);
247 // I would like to use saturating multiplication here, but LLVM cannot lower it334 // I would like to use saturating multiplication here, but LLVM cannot lower it
248 // on WebAssembly: https://github.com/ziglang/zig/issues/9660335 // on WebAssembly: https://github.com/ziglang/zig/issues/9660
249 //const new_byte_count = new_n *| @sizeOf(T);336 //const new_len_bytes = new_len *| @sizeOf(T);
250 const new_byte_count = math.mul(usize, @sizeOf(T), new_n) catch return false;337 const new_len_bytes = math.mul(usize, @sizeOf(T), new_len) catch return null;
251 return self.rawResize(old_byte_slice, log2a(Slice.alignment), new_byte_count, @returnAddress());338 const new_ptr = self.rawRemap(old_memory, .fromByteUnits(alignment), new_len_bytes, @returnAddress()) orelse return null;
339 const new_memory: []align(alignment) u8 = @alignCast(new_ptr[0..new_len_bytes]);
340 return mem.bytesAsSlice(T, new_memory);
252}341}
253342
254/// This function requests a new byte size for an existing allocation, which343/// This function requests a new byte size for an existing allocation, which
255/// can be larger, smaller, or the same size as the old memory allocation.344/// can be larger, smaller, or the same size as the old memory allocation.
345///
256/// If `new_n` is 0, this is the same as `free` and it always succeeds.346/// If `new_n` is 0, this is the same as `free` and it always succeeds.
347///
348/// `old_mem` may have length zero, which makes a new allocation.
349///
350/// This function only fails on out-of-memory conditions, unlike:
351/// * `remap` which returns `null` when the `Allocator` implementation cannot
352/// do the realloc more efficiently than the caller
353/// * `resize` which returns `false` when the `Allocator` implementation cannot
354/// change the size without relocating the allocation.
257pub fn realloc(self: Allocator, old_mem: anytype, new_n: usize) t: {355pub fn realloc(self: Allocator, old_mem: anytype, new_n: usize) t: {
258 const Slice = @typeInfo(@TypeOf(old_mem)).pointer;356 const Slice = @typeInfo(@TypeOf(old_mem)).pointer;
259 break :t Error![]align(Slice.alignment) Slice.child;357 break :t Error![]align(Slice.alignment) Slice.child;
...@@ -284,18 +382,18 @@ pub fn reallocAdvanced(...@@ -284,18 +382,18 @@ pub fn reallocAdvanced(
284 const old_byte_slice = mem.sliceAsBytes(old_mem);382 const old_byte_slice = mem.sliceAsBytes(old_mem);
285 const byte_count = math.mul(usize, @sizeOf(T), new_n) catch return Error.OutOfMemory;383 const byte_count = math.mul(usize, @sizeOf(T), new_n) catch return Error.OutOfMemory;
286 // Note: can't set shrunk memory to undefined as memory shouldn't be modified on realloc failure384 // Note: can't set shrunk memory to undefined as memory shouldn't be modified on realloc failure
287 if (self.rawResize(old_byte_slice, log2a(Slice.alignment), byte_count, return_address)) {385 if (self.rawRemap(old_byte_slice, .fromByteUnits(Slice.alignment), byte_count, return_address)) |p| {
288 const new_bytes: []align(Slice.alignment) u8 = @alignCast(old_byte_slice.ptr[0..byte_count]);386 const new_bytes: []align(Slice.alignment) u8 = @alignCast(p[0..byte_count]);
289 return mem.bytesAsSlice(T, new_bytes);387 return mem.bytesAsSlice(T, new_bytes);
290 }388 }
291389
292 const new_mem = self.rawAlloc(byte_count, log2a(Slice.alignment), return_address) orelse390 const new_mem = self.rawAlloc(byte_count, .fromByteUnits(Slice.alignment), return_address) orelse
293 return error.OutOfMemory;391 return error.OutOfMemory;
294 const copy_len = @min(byte_count, old_byte_slice.len);392 const copy_len = @min(byte_count, old_byte_slice.len);
295 @memcpy(new_mem[0..copy_len], old_byte_slice[0..copy_len]);393 @memcpy(new_mem[0..copy_len], old_byte_slice[0..copy_len]);
296 // TODO https://github.com/ziglang/zig/issues/4298394 // TODO https://github.com/ziglang/zig/issues/4298
297 @memset(old_byte_slice, undefined);395 @memset(old_byte_slice, undefined);
298 self.rawFree(old_byte_slice, log2a(Slice.alignment), return_address);396 self.rawFree(old_byte_slice, .fromByteUnits(Slice.alignment), return_address);
299397
300 const new_bytes: []align(Slice.alignment) u8 = @alignCast(new_mem[0..byte_count]);398 const new_bytes: []align(Slice.alignment) u8 = @alignCast(new_mem[0..byte_count]);
301 return mem.bytesAsSlice(T, new_bytes);399 return mem.bytesAsSlice(T, new_bytes);
...@@ -312,7 +410,7 @@ pub fn free(self: Allocator, memory: anytype) void {...@@ -312,7 +410,7 @@ pub fn free(self: Allocator, memory: anytype) void {
312 const non_const_ptr = @constCast(bytes.ptr);410 const non_const_ptr = @constCast(bytes.ptr);
313 // TODO: https://github.com/ziglang/zig/issues/4298411 // TODO: https://github.com/ziglang/zig/issues/4298
314 @memset(non_const_ptr[0..bytes_len], undefined);412 @memset(non_const_ptr[0..bytes_len], undefined);
315 self.rawFree(non_const_ptr[0..bytes_len], log2a(Slice.alignment), @returnAddress());413 self.rawFree(non_const_ptr[0..bytes_len], .fromByteUnits(Slice.alignment), @returnAddress());
316}414}
317415
318/// Copies `m` to newly allocated memory. Caller owns the memory.416/// Copies `m` to newly allocated memory. Caller owns the memory.
...@@ -329,17 +427,3 @@ pub fn dupeZ(allocator: Allocator, comptime T: type, m: []const T) Error![:0]T {...@@ -329,17 +427,3 @@ pub fn dupeZ(allocator: Allocator, comptime T: type, m: []const T) Error![:0]T {
329 new_buf[m.len] = 0;427 new_buf[m.len] = 0;
330 return new_buf[0..m.len :0];428 return new_buf[0..m.len :0];
331}429}
332
333/// TODO replace callsites with `@log2` after this proposal is implemented:
334/// https://github.com/ziglang/zig/issues/13642
335inline fn log2a(x: anytype) switch (@typeInfo(@TypeOf(x))) {
336 .int => math.Log2Int(@TypeOf(x)),
337 .comptime_int => comptime_int,
338 else => @compileError("int please"),
339} {
340 switch (@typeInfo(@TypeOf(x))) {
341 .int => return math.log2_int(@TypeOf(x), x),
342 .comptime_int => return math.log2(x),
343 else => @compileError("bad"),
344 }
345}
lib/std/testing/failing_allocator.zig+30-10
...@@ -62,6 +62,7 @@ pub const FailingAllocator = struct {...@@ -62,6 +62,7 @@ pub const FailingAllocator = struct {
62 .vtable = &.{62 .vtable = &.{
63 .alloc = alloc,63 .alloc = alloc,
64 .resize = resize,64 .resize = resize,
65 .remap = remap,
65 .free = free,66 .free = free,
66 },67 },
67 };68 };
...@@ -70,7 +71,7 @@ pub const FailingAllocator = struct {...@@ -70,7 +71,7 @@ pub const FailingAllocator = struct {
70 fn alloc(71 fn alloc(
71 ctx: *anyopaque,72 ctx: *anyopaque,
72 len: usize,73 len: usize,
73 log2_ptr_align: u8,74 alignment: mem.Alignment,
74 return_address: usize,75 return_address: usize,
75 ) ?[*]u8 {76 ) ?[*]u8 {
76 const self: *FailingAllocator = @ptrCast(@alignCast(ctx));77 const self: *FailingAllocator = @ptrCast(@alignCast(ctx));
...@@ -86,7 +87,7 @@ pub const FailingAllocator = struct {...@@ -86,7 +87,7 @@ pub const FailingAllocator = struct {
86 }87 }
87 return null;88 return null;
88 }89 }
89 const result = self.internal_allocator.rawAlloc(len, log2_ptr_align, return_address) orelse90 const result = self.internal_allocator.rawAlloc(len, alignment, return_address) orelse
90 return null;91 return null;
91 self.allocated_bytes += len;92 self.allocated_bytes += len;
92 self.allocations += 1;93 self.allocations += 1;
...@@ -96,33 +97,52 @@ pub const FailingAllocator = struct {...@@ -96,33 +97,52 @@ pub const FailingAllocator = struct {
9697
97 fn resize(98 fn resize(
98 ctx: *anyopaque,99 ctx: *anyopaque,
99 old_mem: []u8,100 memory: []u8,
100 log2_old_align: u8,101 alignment: mem.Alignment,
101 new_len: usize,102 new_len: usize,
102 ra: usize,103 ra: usize,
103 ) bool {104 ) bool {
104 const self: *FailingAllocator = @ptrCast(@alignCast(ctx));105 const self: *FailingAllocator = @ptrCast(@alignCast(ctx));
105 if (self.resize_index == self.resize_fail_index)106 if (self.resize_index == self.resize_fail_index)
106 return false;107 return false;
107 if (!self.internal_allocator.rawResize(old_mem, log2_old_align, new_len, ra))108 if (!self.internal_allocator.rawResize(memory, alignment, new_len, ra))
108 return false;109 return false;
109 if (new_len < old_mem.len) {110 if (new_len < memory.len) {
110 self.freed_bytes += old_mem.len - new_len;111 self.freed_bytes += memory.len - new_len;
111 } else {112 } else {
112 self.allocated_bytes += new_len - old_mem.len;113 self.allocated_bytes += new_len - memory.len;
113 }114 }
114 self.resize_index += 1;115 self.resize_index += 1;
115 return true;116 return true;
116 }117 }
117118
119 fn remap(
120 ctx: *anyopaque,
121 memory: []u8,
122 alignment: mem.Alignment,
123 new_len: usize,
124 ra: usize,
125 ) ?[*]u8 {
126 const self: *FailingAllocator = @ptrCast(@alignCast(ctx));
127 if (self.resize_index == self.resize_fail_index) return null;
128 const new_ptr = self.internal_allocator.rawRemap(memory, alignment, new_len, ra) orelse return null;
129 if (new_len < memory.len) {
130 self.freed_bytes += memory.len - new_len;
131 } else {
132 self.allocated_bytes += new_len - memory.len;
133 }
134 self.resize_index += 1;
135 return new_ptr;
136 }
137
118 fn free(138 fn free(
119 ctx: *anyopaque,139 ctx: *anyopaque,
120 old_mem: []u8,140 old_mem: []u8,
121 log2_old_align: u8,141 alignment: mem.Alignment,
122 ra: usize,142 ra: usize,
123 ) void {143 ) void {
124 const self: *FailingAllocator = @ptrCast(@alignCast(ctx));144 const self: *FailingAllocator = @ptrCast(@alignCast(ctx));
125 self.internal_allocator.rawFree(old_mem, log2_old_align, ra);145 self.internal_allocator.rawFree(old_mem, alignment, ra);
126 self.deallocations += 1;146 self.deallocations += 1;
127 self.freed_bytes += old_mem.len;147 self.freed_bytes += old_mem.len;
128 }148 }